rbbt-util 5.42.0 → 5.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/annotations/util.rb +1 -1
- data/lib/rbbt/association/database.rb +2 -1
- data/lib/rbbt/association/index.rb +1 -0
- data/lib/rbbt/association/util.rb +13 -3
- data/lib/rbbt/entity.rb +6 -5
- data/lib/rbbt/hpc/orchestrate/batches.rb +1 -1
- data/lib/rbbt/hpc/orchestrate/chains.rb +3 -4
- data/lib/rbbt/knowledge_base/traverse.rb +24 -16
- data/lib/rbbt/persist/tsv/tokyocabinet.rb +1 -1
- data/lib/rbbt/resource/path.rb +7 -2
- data/lib/rbbt/resource.rb +1 -1
- data/lib/rbbt/tsv/parallel/traverse.rb +44 -0
- data/lib/rbbt/tsv/parser.rb +7 -6
- data/lib/rbbt/tsv/serializers.rb +1 -1
- data/lib/rbbt/tsv/util.rb +12 -0
- data/lib/rbbt/util/log/progress.rb +1 -1
- data/lib/rbbt/util/misc/bgzf.rb +5 -4
- data/lib/rbbt/util/misc/development.rb +4 -4
- data/lib/rbbt/util/misc.rb +1 -1
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +8 -3
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +4 -0
- data/lib/rbbt/workflow/util/trace.rb +2 -2
- data/lib/rbbt/workflow.rb +2 -2
- data/test/rbbt/association/test_database.rb +0 -2
- data/test/rbbt/association/test_open.rb +1 -1
- data/test/rbbt/entity/test_identifiers.rb +7 -47
- data/test/rbbt/hpc/orchestrate/test_batches.rb +2 -45
- data/test/rbbt/hpc/orchestrate/test_chains.rb +13 -44
- data/test/rbbt/hpc/orchestrate/test_rules.rb +2 -35
- data/test/rbbt/hpc/test_batch.rb +18 -22
- data/test/rbbt/hpc/test_hpc_test_workflows.rb +0 -0
- data/test/rbbt/hpc/test_pbs.rb +1 -1
- data/test/rbbt/hpc/test_slurm.rb +6 -10
- data/test/rbbt/knowledge_base/test_enrichment.rb +5 -3
- data/test/rbbt/knowledge_base/test_entity.rb +26 -6
- data/test/rbbt/knowledge_base/test_query.rb +1 -1
- data/test/rbbt/knowledge_base/test_registry.rb +34 -41
- data/test/rbbt/knowledge_base/test_traverse.rb +106 -58
- data/test/rbbt/persist/test_tsv.rb +4 -4
- data/test/rbbt/persist/tsv/test_lmdb.rb +7 -5
- data/test/rbbt/test_entity.rb +8 -5
- data/test/rbbt/test_knowledge_base.rb +107 -30
- data/test/rbbt/test_packed_index.rb +5 -5
- data/test/rbbt/test_tsv.rb +1 -1
- data/test/rbbt/test_workflow.rb +83 -78
- data/test/rbbt/tsv/test_accessor.rb +4 -0
- data/test/rbbt/tsv/test_attach.rb +2 -5
- data/test/rbbt/tsv/test_index.rb +8 -2
- data/test/rbbt/tsv/test_manipulate.rb +0 -3
- data/test/rbbt/util/R/test_model.rb +0 -2
- data/test/rbbt/util/misc/test_bgzf.rb +3 -3
- data/test/rbbt/util/misc/test_communication.rb +5 -3
- data/test/rbbt/util/test_migrate.rb +1 -3
- data/test/rbbt/util/test_misc.rb +7 -2
- data/test/rbbt/util/test_procpath.rb +1 -1
- data/test/rbbt/util/test_python.rb +12 -8
- data/test/rbbt/workflow/step/test_dependencies.rb +0 -4
- data/test/rbbt/workflow/test_remote_workflow.rb +6 -5
- data/test/rbbt/workflow/util/test_archive.rb +4 -8
- data/test/rbbt/workflow/util/test_orchestrator.rb +5 -15
- data/test/test_helper.rb +25 -1
- metadata +5 -5
- data/test/rbbt/test_hpc.rb +0 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d1bbf2cefca44eecad7465a8d4dc1b88f53f18d1e63e310353061b43ad46bed9
|
4
|
+
data.tar.gz: f54f659da9eca312d116430d88019f067ce5ccf519724a2a448e3b4d31dfb767
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: eaad34515129ffd123064e9af920b5f83f3897c957de45e3bef04a1101d961d419863729f21665773b81f535f3fdba45d490ce41870f9db87f0e0e5d5f987233
|
7
|
+
data.tar.gz: d48552f30e6434b0e27e173eb62f41a3126ecb9065cba58f2f5737f8e3aef0e2183503076a40cc6586832d90458ae5e572a55171758452f3699a4225704bc25e
|
@@ -93,7 +93,7 @@ module Association
|
|
93
93
|
tmp_data = Persist.open_database(tmpfile, true, :double, "HDB")
|
94
94
|
|
95
95
|
tsv.with_monitor(options[:monitor]) do
|
96
|
-
tsv = tsv.reorder source_field,
|
96
|
+
tsv = tsv.reorder source_field, tsv.all_fields.values_at(*field_pos), :persist => persist, :persist_data => tmp_data if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
|
97
97
|
end
|
98
98
|
|
99
99
|
tsv.key_field = source_header
|
@@ -216,6 +216,7 @@ module Association
|
|
216
216
|
open_stream(stream, options.dup)
|
217
217
|
end
|
218
218
|
|
219
|
+
database.filename = file if Path === file && file.identifier_files.any?
|
219
220
|
database.entity_options = options[:entity_options] if options[:entity_options]
|
220
221
|
|
221
222
|
database
|
@@ -115,8 +115,14 @@ module Association
|
|
115
115
|
source_field = specs[:source][0]
|
116
116
|
target_field = specs[:target][0]
|
117
117
|
|
118
|
-
source_pos = all_fields.index source_field
|
119
|
-
target_pos = all_fields.index target_field
|
118
|
+
#source_pos = all_fields.index source_field
|
119
|
+
#target_pos = all_fields.index target_field
|
120
|
+
|
121
|
+
source_pos = TSV.identify_field all_fields.first, all_fields[1..-1], source_field
|
122
|
+
target_pos = TSV.identify_field all_fields.first, all_fields[1..-1], target_field
|
123
|
+
|
124
|
+
source_pos = source_pos == :key ? 0 : source_pos + 1
|
125
|
+
target_pos = target_pos == :key ? 0 : target_pos + 1
|
120
126
|
|
121
127
|
source_header = specs[:source][1] || specs[:source][0]
|
122
128
|
target_header = specs[:target][1] || specs[:target][0]
|
@@ -140,7 +146,11 @@ module Association
|
|
140
146
|
field_headers << header
|
141
147
|
end
|
142
148
|
|
143
|
-
field_pos = info_fields.collect
|
149
|
+
field_pos = info_fields.collect do |f|
|
150
|
+
p = TSV.identify_field all_fields.first, all_fields[1..-1], f
|
151
|
+
p == :key ? 0 : p + 1
|
152
|
+
end
|
153
|
+
|
144
154
|
field_pos.delete source_pos
|
145
155
|
|
146
156
|
source_format = specs[:source][2]
|
data/lib/rbbt/entity.rb
CHANGED
@@ -289,16 +289,17 @@ module Entity
|
|
289
289
|
end
|
290
290
|
|
291
291
|
def self.unpersist(method_name)
|
292
|
-
return unless persisted?
|
293
|
-
orig_name = UNPERSISTED_PREFIX + method_name.to_s
|
294
|
-
|
292
|
+
return unless orig_name = persisted?(method_name)
|
295
293
|
alias_method method_name, orig_name
|
296
294
|
remove_method orig_name
|
297
295
|
end
|
298
296
|
|
299
297
|
def self.persisted?(method_name)
|
300
|
-
|
301
|
-
|
298
|
+
["", "_multiple_", "_single_"].each do |type|
|
299
|
+
orig_name = (UNPERSISTED_PREFIX + type + method_name.to_s).to_sym
|
300
|
+
return orig_name if instance_methods.include?(orig_name)
|
301
|
+
end
|
302
|
+
return false
|
302
303
|
end
|
303
304
|
|
304
305
|
def self.with_persisted(method_name)
|
@@ -39,9 +39,8 @@ module HPC
|
|
39
39
|
(job.dependencies + job.input_dependencies).uniq.select{|d| ! d.done? || d.dirty? }
|
40
40
|
end
|
41
41
|
|
42
|
-
def self.job_chains(rules, job)
|
43
|
-
|
44
|
-
@@job_chains[Misc.digest([rules, job.path].inspect)] ||=
|
42
|
+
def self.job_chains(rules, job, computed = {})
|
43
|
+
computed[Misc.fingerprint([rules, job.path, job.object_id])] ||=
|
45
44
|
begin
|
46
45
|
chains = self.parse_chains(rules)
|
47
46
|
|
@@ -55,7 +54,7 @@ module HPC
|
|
55
54
|
dep_matches = check_chains(chains, dep)
|
56
55
|
common = matches & dep_matches
|
57
56
|
|
58
|
-
dep_chains = job_chains(rules, dep)
|
57
|
+
dep_chains = job_chains(rules, dep, computed)
|
59
58
|
found = []
|
60
59
|
dep_chains.each do |match,info|
|
61
60
|
if common.include?(match)
|
@@ -146,7 +146,7 @@ class KnowledgeBase
|
|
146
146
|
if conditions
|
147
147
|
Misc.tokenize(conditions).each do |condition|
|
148
148
|
if condition.index "="
|
149
|
-
key, value =
|
149
|
+
key, value = condition.split("=")
|
150
150
|
matches = matches.select{|m| Misc.match_value(m.info[key.strip], value)}
|
151
151
|
else
|
152
152
|
matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
|
@@ -158,17 +158,19 @@ class KnowledgeBase
|
|
158
158
|
end
|
159
159
|
|
160
160
|
def id_dbs(db)
|
161
|
+
# ToDo: Revise this, I'm not sure what id does anymore
|
162
|
+
# I think it deals with syndication
|
161
163
|
if db.include? '?'
|
162
|
-
all_dbs = kb.registry.keys
|
164
|
+
all_dbs = kb.registry.keys.collect{|k| k.to_s }
|
163
165
|
_name, _sep, _kb = db.partition("@")
|
164
166
|
case
|
165
167
|
when _name[0] == '?'
|
166
168
|
dbs = all_dbs.select{|_db|
|
167
169
|
n,_s,d=_db.partition("@");
|
168
|
-
d.nil? or d.empty? or (d ==
|
170
|
+
d.nil? or d.empty? or (d == _kb and assignments[_name].include?(n))
|
169
171
|
}
|
170
172
|
when _kb[0] == '?'
|
171
|
-
dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d)}
|
173
|
+
dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d) }
|
172
174
|
end
|
173
175
|
else
|
174
176
|
dbs = [db]
|
@@ -181,11 +183,12 @@ class KnowledgeBase
|
|
181
183
|
all_matches = []
|
182
184
|
path_rules = []
|
183
185
|
acc_var = nil
|
186
|
+
pre_acc_var_assignments = nil
|
184
187
|
rules.each do |rule|
|
185
188
|
rule = rule.strip
|
186
189
|
next if rule.empty?
|
187
190
|
|
188
|
-
if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+(
|
191
|
+
if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+(.*))?/)
|
189
192
|
Log.debug "Traverse rule: #{rule}"
|
190
193
|
path_rules << rule
|
191
194
|
|
@@ -199,17 +202,19 @@ class KnowledgeBase
|
|
199
202
|
|
200
203
|
next if matches.nil? or matches.empty?
|
201
204
|
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
205
|
+
# ToDo: Revise this, I'm not sure what id does anymore
|
206
|
+
#
|
207
|
+
#if db.include? '?'
|
208
|
+
# _name, _sep, _kb = db.partition("@")
|
209
|
+
# case
|
210
|
+
# when _kb[0] == '?'
|
211
|
+
# assignments[_kb] ||= []
|
212
|
+
# assignments[_kb] << _db.partition("@").reject{|p| p.empty?}.last
|
213
|
+
# when _name[0] == '?'
|
214
|
+
# assignments[_name] ||= []
|
215
|
+
# assignments[_name] << _db.partition("@").first
|
216
|
+
# end
|
217
|
+
#end
|
213
218
|
|
214
219
|
matches.each do |m|
|
215
220
|
rule_matches << m
|
@@ -244,11 +249,14 @@ class KnowledgeBase
|
|
244
249
|
|
245
250
|
elsif m = rule.match(/(\?[^\s{]+)\s*{/)
|
246
251
|
acc_var = m.captures.first
|
252
|
+
pre_acc_var_assignments = assignments.dup
|
247
253
|
Log.debug "Start assign block: #{acc_var}"
|
248
254
|
elsif m = rule.match(/^\s*}\s*$/)
|
249
255
|
Log.debug "Close assign block: #{acc_var}"
|
250
256
|
saved_assign = assignments[acc_var]
|
251
257
|
assignments.clear
|
258
|
+
assignments.merge!(pre_acc_var_assignments)
|
259
|
+
pre_acc_var_assignments = nil
|
252
260
|
assignments[acc_var] = saved_assign
|
253
261
|
all_matches = []
|
254
262
|
path_rules = []
|
@@ -21,7 +21,7 @@ module Persist
|
|
21
21
|
tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
|
22
22
|
tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
|
23
23
|
|
24
|
-
database = CONNECTIONS[path] ||= tokyocabinet_class.new
|
24
|
+
database = CONNECTIONS[path] ||= Log.ignore_stderr do tokyocabinet_class.new end
|
25
25
|
|
26
26
|
if big and not Open.exists?(path)
|
27
27
|
database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE)
|
data/lib/rbbt/resource/path.rb
CHANGED
@@ -161,8 +161,13 @@ module Path
|
|
161
161
|
def find(where = nil, caller_lib = nil, paths = nil)
|
162
162
|
|
163
163
|
if located?
|
164
|
-
|
165
|
-
|
164
|
+
path = self
|
165
|
+
path = path + '.gz' if File.exist?(path + '.gz')
|
166
|
+
path = path + '.bgz' if File.exist?(path + '.bgz')
|
167
|
+
self.annotate(path)
|
168
|
+
|
169
|
+
path.original = self
|
170
|
+
return path
|
166
171
|
end
|
167
172
|
|
168
173
|
if where == :all || where == 'all'
|
data/lib/rbbt/resource.rb
CHANGED
@@ -374,7 +374,7 @@ url='#{url}'
|
|
374
374
|
if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
|
375
375
|
unlocated = ([m["TOPLEVEL"],m["SUBPATH"],m["REST"]] * "/")
|
376
376
|
unlocated.gsub!(/\/+/,'/')
|
377
|
-
if self.subdir && ! self.subdir.empty?
|
377
|
+
if self.subdir && ! self.subdir.empty? && unlocated.include?(subdir)
|
378
378
|
subdir = self.subdir
|
379
379
|
subdir += "/" unless subdir.end_with?("/")
|
380
380
|
unlocated[subdir] = ""
|
@@ -181,6 +181,48 @@ module TSV
|
|
181
181
|
end
|
182
182
|
end
|
183
183
|
|
184
|
+
def self.traverse_enumerable(enum, options = {}, &block)
|
185
|
+
callback, bar, join = Misc.process_options options, :callback, :bar, :join
|
186
|
+
|
187
|
+
begin
|
188
|
+
error = false
|
189
|
+
if callback
|
190
|
+
bar.init if bar
|
191
|
+
while enum.any?
|
192
|
+
e = enum.pop
|
193
|
+
begin
|
194
|
+
callback.call yield(e)
|
195
|
+
rescue Exception
|
196
|
+
Log.warn "Traverse exception on element: #{Misc.fingerprint(e)}"
|
197
|
+
raise $!
|
198
|
+
ensure
|
199
|
+
bar.tick if bar
|
200
|
+
end
|
201
|
+
end
|
202
|
+
else
|
203
|
+
bar.init if bar
|
204
|
+
while enum.any?
|
205
|
+
e = enum.pop
|
206
|
+
begin
|
207
|
+
yield e
|
208
|
+
rescue Exception
|
209
|
+
Log.warn "Traverse exception on element: #{Misc.fingerprint(e)}"
|
210
|
+
raise $!
|
211
|
+
ensure
|
212
|
+
bar.tick if bar
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
rescue
|
218
|
+
error = true
|
219
|
+
raise $!
|
220
|
+
ensure
|
221
|
+
join.call(error) if join
|
222
|
+
Log::ProgressBar.remove_bar(bar, error) if bar
|
223
|
+
end
|
224
|
+
end
|
225
|
+
|
184
226
|
def self.traverse_priority_queue(queue, options = {}, &block)
|
185
227
|
callback, bar, join = Misc.process_options options, :callback, :bar, :join
|
186
228
|
|
@@ -396,6 +438,8 @@ module TSV
|
|
396
438
|
else
|
397
439
|
raise "Can not open obj for traversal #{Misc.fingerprint obj}"
|
398
440
|
end
|
441
|
+
when Enumerable
|
442
|
+
traverse_enumerable(obj, options, &block)
|
399
443
|
when nil
|
400
444
|
raise "Can not traverse nil object into #{stream_name(options[:into])}"
|
401
445
|
else
|
data/lib/rbbt/tsv/parser.rb
CHANGED
@@ -281,19 +281,20 @@ module TSV
|
|
281
281
|
end
|
282
282
|
|
283
283
|
values = values.collect{|v| v.length != num ? [v.first] * num : v}
|
284
|
+
|
284
285
|
all = values
|
285
286
|
all.unshift keys
|
286
|
-
Misc.zip_fields(all).each do |
|
287
|
-
key =
|
287
|
+
Misc.zip_fields(all).each do |vs|
|
288
|
+
key = vs.shift
|
288
289
|
if data.include? key
|
289
|
-
data[key] = data[key].zip(
|
290
|
-
old
|
291
|
-
old
|
290
|
+
data[key] = data[key].zip(vs).collect do |old, new|
|
291
|
+
old + [new]
|
292
292
|
end
|
293
293
|
else
|
294
|
-
data[key] =
|
294
|
+
data[key] = vs.collect{|v| [v] }
|
295
295
|
end
|
296
296
|
end
|
297
|
+
|
297
298
|
nil
|
298
299
|
end
|
299
300
|
|
data/lib/rbbt/tsv/serializers.rb
CHANGED
@@ -63,7 +63,7 @@ module TSV
|
|
63
63
|
class StringDoubleArraySerializer
|
64
64
|
def self.dump(array)
|
65
65
|
begin
|
66
|
-
|
66
|
+
array.collect{|a| a.collect{|a| a.to_s } * "|"} * "\t"
|
67
67
|
rescue Encoding::CompatibilityError
|
68
68
|
array.collect{|a| a.collect{|a| a.to_s.force_encoding('UTF-8')} * "|"} * "\t"
|
69
69
|
end
|
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -168,6 +168,8 @@ module TSV
|
|
168
168
|
end
|
169
169
|
when Set
|
170
170
|
get_stream(file.to_a, open_options)
|
171
|
+
when Enumerable
|
172
|
+
file
|
171
173
|
else
|
172
174
|
raise "Cannot get stream from: #{file.inspect}"
|
173
175
|
end
|
@@ -189,6 +191,16 @@ module TSV
|
|
189
191
|
pos = fields.index field
|
190
192
|
return pos if pos
|
191
193
|
return identify_field(key_field, fields, field.to_i) if field =~ /^\d+$/
|
194
|
+
if fields.select{|f| f.include?("(") }.any?
|
195
|
+
simplify_fields = fields.collect do |f|
|
196
|
+
if m = f.match(/(.*)\s+\(.*\)/)
|
197
|
+
m[1]
|
198
|
+
else
|
199
|
+
f
|
200
|
+
end
|
201
|
+
end
|
202
|
+
return identify_field(key_field, simplify_fields, field)
|
203
|
+
end
|
192
204
|
raise "Field '#{ field }' was not found. Options: (#{key_field || "NO_KEY_FIELD"}), #{(fields || ["NO_FIELDS"]) * ", "}" if pos.nil?
|
193
205
|
else
|
194
206
|
raise "Field '#{ field }' was not found. Options: (#{key_field || "NO_KEY_FIELD"}), #{(fields || ["NO_FIELDS"]) * ", "}"
|
data/lib/rbbt/util/misc/bgzf.rb
CHANGED
@@ -29,8 +29,8 @@ module Bgzf
|
|
29
29
|
|
30
30
|
def filename
|
31
31
|
@filename ||= begin
|
32
|
-
|
33
|
-
|
32
|
+
compressed_stream.respond_to?(:filename) ? compressed_stream.filename : rand(1000000000).to_s
|
33
|
+
end
|
34
34
|
end
|
35
35
|
|
36
36
|
def closed?
|
@@ -49,7 +49,8 @@ module Bgzf
|
|
49
49
|
|
50
50
|
def _index
|
51
51
|
@_index ||= begin
|
52
|
-
|
52
|
+
prefix_code = "BGZF index" + (filename || "").sub(/.bgz$/,'')
|
53
|
+
index = Persist.persist(prefix_code, :marshal) do
|
53
54
|
index = []
|
54
55
|
pos = 0
|
55
56
|
while true do
|
@@ -66,7 +67,7 @@ module Bgzf
|
|
66
67
|
end
|
67
68
|
index
|
68
69
|
end
|
69
|
-
@block_cache_size = Math.log(index.length).to_i + 1
|
70
|
+
@block_cache_size = Math.log(index.length + 1).to_i + 1
|
70
71
|
index
|
71
72
|
end
|
72
73
|
end
|
@@ -39,13 +39,13 @@ module Misc
|
|
39
39
|
end
|
40
40
|
end
|
41
41
|
if message
|
42
|
-
|
42
|
+
STDERR.pust "#{message }: #{ repeats } repeats"
|
43
43
|
else
|
44
|
-
puts "Benchmark for #{ repeats } repeats"
|
44
|
+
STDERR.puts "Benchmark for #{ repeats } repeats"
|
45
45
|
end
|
46
|
-
puts measure
|
46
|
+
STDERR.puts measure
|
47
47
|
rescue Exception
|
48
|
-
puts "Benchmark aborted"
|
48
|
+
STDERR.puts "Benchmark aborted"
|
49
49
|
raise $!
|
50
50
|
end
|
51
51
|
res
|
data/lib/rbbt/util/misc.rb
CHANGED
@@ -20,10 +20,15 @@ class RemoteWorkflow
|
|
20
20
|
|
21
21
|
def self.escape_url(url)
|
22
22
|
base, _sep, query = url.partition("?")
|
23
|
-
protocol,
|
24
|
-
|
23
|
+
protocol, _sep, host_and_path = base.partition("://")
|
24
|
+
host_and_path = protocol if host_and_path.nil?
|
25
|
+
|
26
|
+
host, _sep, path = host_and_path.partition("/")
|
27
|
+
|
25
28
|
path = path.split("/").collect{|p| CGI.escape(p) }* "/"
|
26
|
-
|
29
|
+
|
30
|
+
host_and_path = [host, path] * "/"
|
31
|
+
base = protocol ? [protocol, host_and_path] * "://" : host_and_path
|
27
32
|
|
28
33
|
if query && ! query.empty?
|
29
34
|
query = query.split("&").collect{|e| e.split("=").collect{|pe| CGI.escape(pe) } * "=" } * "&"
|
@@ -354,6 +354,10 @@ class RemoteStep < Step
|
|
354
354
|
i = {:status => :waiting, :pid => Process.pid, :path => path}
|
355
355
|
i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
|
356
356
|
end
|
357
|
+
|
358
|
+
def inspect
|
359
|
+
"RemoteWorkflow #{self.path}"
|
360
|
+
end
|
357
361
|
end
|
358
362
|
|
359
363
|
require 'rbbt/workflow/remote_workflow/remote_step/rest'
|
@@ -158,11 +158,11 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
|
|
158
158
|
info[key] = dep_info[key]
|
159
159
|
end
|
160
160
|
|
161
|
-
dep.info[:config_keys].
|
161
|
+
dep.info[:config_keys].each do |kinfo|
|
162
162
|
key, value, tokens = kinfo
|
163
163
|
|
164
164
|
info[key.to_s] = value if report_keys.include? key.to_s
|
165
|
-
end
|
165
|
+
end if dep.info[:config_keys]
|
166
166
|
end
|
167
167
|
|
168
168
|
summary = TSV.setup({}, "Task~Calls,Avg. Time,Total Time#:type=:list")
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -496,7 +496,7 @@ module Workflow
|
|
496
496
|
end
|
497
497
|
|
498
498
|
#overriden = true if dependencies.select{|d| d.overriden && d.clean_name != d.name }.any?
|
499
|
-
overriden = true if not_overriden && dependencies.select{|d| Symbol === d.overriden }.any?
|
499
|
+
overriden = true if ! not_overriden && dependencies.select{|d| Symbol === d.overriden }.any?
|
500
500
|
|
501
501
|
input_values = task.take_input_values(inputs)
|
502
502
|
if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden #&& ! dependencies.select{|d| d.overriden && d.clean_name != d.name }.any?
|
@@ -507,7 +507,7 @@ module Workflow
|
|
507
507
|
|
508
508
|
job = get_job_step step_path, task, input_values, dependencies
|
509
509
|
job.workflow = self
|
510
|
-
job.overriden
|
510
|
+
job.overriden = overriden if job.overriden.nil?
|
511
511
|
job.clean_name = jobname
|
512
512
|
|
513
513
|
#iif [job, not_overriden]
|
@@ -23,7 +23,6 @@ MDM2 TP53 inhibition false
|
|
23
23
|
TP53 NFKB1|GLI1 activation|activation true|true
|
24
24
|
END
|
25
25
|
|
26
|
-
|
27
26
|
effect_tsv = TSV.open effect, effect_options.dup
|
28
27
|
end
|
29
28
|
|
@@ -35,7 +34,6 @@ TP53 NFKB1|GLI1 activation|activation true|true
|
|
35
34
|
assert_equal %w(NFKB1 GLI1), effect_tsv["TP53"]["TG"]
|
36
35
|
end
|
37
36
|
|
38
|
-
|
39
37
|
def test_open_new_tsv
|
40
38
|
tsv = Association.database effect_tsv, :fields => ["Effect"], :target => "TG=~Associated Gene Name=>Ensembl Gene ID"
|
41
39
|
assert_equal ["Ensembl Gene ID", "Effect"], tsv.fields
|
@@ -30,7 +30,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
|
|
30
30
|
|
31
31
|
|
32
32
|
def test_open_persist
|
33
|
-
tsv = Association.open(
|
33
|
+
tsv = Association.open(EFFECT, EFFECT_OPTIONS, :persist => true, :update => true)
|
34
34
|
assert_equal "ENSG00000141510", tsv["MDM2"]["Ensembl Gene ID"].first
|
35
35
|
assert_equal ["false"], tsv["MDM2"]["directed?"]
|
36
36
|
end
|
@@ -8,67 +8,27 @@ module Gene
|
|
8
8
|
end
|
9
9
|
|
10
10
|
class TestEntityIdentifiers < Test::Unit::TestCase
|
11
|
-
def setup
|
12
|
-
require 'rbbt/sources/kegg'
|
13
|
-
require 'rbbt/sources/organism'
|
14
|
-
Gene.add_identifiers Organism.identifiers("NAMESPACE"), "Ensembl Gene ID", "Associated Gene Name"
|
15
|
-
Gene.add_identifiers KEGG.identifiers
|
16
|
-
Gene.add_identifiers Organism.identifiers("NAMESPACE"), "Ensembl Gene ID", "Associated Gene Name"
|
17
|
-
Gene.add_identifiers KEGG.identifiers
|
18
11
|
|
19
|
-
end
|
20
12
|
def test_name
|
21
13
|
Gene.add_identifiers datafile_test('identifiers'), "Ensembl Gene ID", "Associated Gene Name"
|
22
14
|
assert_equal "TP53", Gene.setup("ENSG00000141510").name
|
23
15
|
end
|
24
16
|
|
25
17
|
def test_name_organism
|
26
|
-
|
18
|
+
keyword_test :organism do
|
19
|
+
require 'rbbt/sources/organism'
|
20
|
+
Gene.add_identifiers Organism.identifiers("NAMESPACE"), "Ensembl Gene ID", "Associated Gene Name"
|
21
|
+
assert_equal "TP53", Gene.setup("ENSG00000141510", "Ensembl Gene ID", "Hsa/feb2014").name
|
22
|
+
end
|
27
23
|
end
|
28
24
|
|
29
25
|
def test_identifier_files
|
26
|
+
Gene.add_identifiers datafile_test('identifiers'), "Ensembl Gene ID", "Associated Gene Name"
|
30
27
|
assert Gene.identifier_files.any?
|
31
28
|
end
|
32
29
|
|
33
30
|
def test_Entity_identifier_files
|
31
|
+
Gene.add_identifiers datafile_test('identifiers'), "Ensembl Gene ID", "Associated Gene Name"
|
34
32
|
assert Entity.identifier_files("Ensembl Gene ID").any?
|
35
33
|
end
|
36
|
-
|
37
|
-
def test_translate_kegg
|
38
|
-
assert_match "hsa", Gene.setup("ENSG00000141510", "Ensembl Gene ID", "Hsa/feb2014").to("KEGG Gene ID")
|
39
|
-
assert_match "TP53", Gene.setup("ENSG00000141510", "Ensembl Gene ID", "Hsa/feb2014").to("KEGG Gene ID").to(:name)
|
40
|
-
end
|
41
|
-
|
42
|
-
def ___test_complex
|
43
|
-
|
44
|
-
file = Path.setup('/home/mvazquezg/git/workflows/PanCancer/share/network/gene_sets/CORUM_protein_complexes.gmt')
|
45
|
-
name = File.basename(file).sub(/\.gmt$/,'')
|
46
|
-
organism = "Hsa/feb2014"
|
47
|
-
key_field = "#{ name } Pathway ID"
|
48
|
-
description_field = "#{name} Pathway Description"
|
49
|
-
description_file = file.find + '.identifiers'
|
50
|
-
tsv = TSV.open(file, :fix => Proc.new{|l| p=l.split"\t"; [p[0], p[1], p[2..-1]*"|"]*"\t"})
|
51
|
-
tsv.namespace = organism
|
52
|
-
tsv.unnamed = true
|
53
|
-
gene_field, count = Organism.guess_id(organism, tsv.values.collect{|l| l.last}.flatten.uniq )
|
54
|
-
tsv.key_field = key_field
|
55
|
-
tsv.fields = [description_field, gene_field]
|
56
|
-
descriptions = tsv.slice(description_field)
|
57
|
-
Open.write(description_file, descriptions.to_single.to_s) #unless File.exist? description_file
|
58
|
-
values = tsv.slice(gene_field)
|
59
|
-
values.identifiers = description_file
|
60
|
-
|
61
|
-
mod = Module.new
|
62
|
-
mod_name = Misc.camel_case(key_field.gsub(/\s+/,'_').sub(/_ID$/,''))
|
63
|
-
Object.const_set(mod_name, mod)
|
64
|
-
mod.instance_eval do
|
65
|
-
extend Entity
|
66
|
-
add_identifiers Path.setup(description_file), key_field, description_field
|
67
|
-
|
68
|
-
annotation :format
|
69
|
-
end
|
70
|
-
entity = "CORUM:6052"
|
71
|
-
mod.setup(entity, :format => key_field)
|
72
|
-
puts entity.name
|
73
|
-
end
|
74
34
|
end
|