rbbt-util 5.42.0 → 5.43.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. checksums.yaml +4 -4
  2. data/lib/rbbt/annotations/util.rb +1 -1
  3. data/lib/rbbt/association/database.rb +2 -1
  4. data/lib/rbbt/association/index.rb +1 -0
  5. data/lib/rbbt/association/util.rb +13 -3
  6. data/lib/rbbt/entity.rb +6 -5
  7. data/lib/rbbt/hpc/orchestrate/batches.rb +1 -1
  8. data/lib/rbbt/hpc/orchestrate/chains.rb +3 -4
  9. data/lib/rbbt/knowledge_base/traverse.rb +24 -16
  10. data/lib/rbbt/persist/tsv/tokyocabinet.rb +1 -1
  11. data/lib/rbbt/resource/path.rb +7 -2
  12. data/lib/rbbt/resource.rb +1 -1
  13. data/lib/rbbt/tsv/parallel/traverse.rb +44 -0
  14. data/lib/rbbt/tsv/parser.rb +7 -6
  15. data/lib/rbbt/tsv/serializers.rb +1 -1
  16. data/lib/rbbt/tsv/util.rb +12 -0
  17. data/lib/rbbt/util/log/progress.rb +1 -1
  18. data/lib/rbbt/util/misc/bgzf.rb +5 -4
  19. data/lib/rbbt/util/misc/development.rb +4 -4
  20. data/lib/rbbt/util/misc.rb +1 -1
  21. data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +8 -3
  22. data/lib/rbbt/workflow/remote_workflow/remote_step.rb +4 -0
  23. data/lib/rbbt/workflow/util/trace.rb +2 -2
  24. data/lib/rbbt/workflow.rb +2 -2
  25. data/test/rbbt/association/test_database.rb +0 -2
  26. data/test/rbbt/association/test_open.rb +1 -1
  27. data/test/rbbt/entity/test_identifiers.rb +7 -47
  28. data/test/rbbt/hpc/orchestrate/test_batches.rb +2 -45
  29. data/test/rbbt/hpc/orchestrate/test_chains.rb +13 -44
  30. data/test/rbbt/hpc/orchestrate/test_rules.rb +2 -35
  31. data/test/rbbt/hpc/test_batch.rb +18 -22
  32. data/test/rbbt/hpc/test_hpc_test_workflows.rb +0 -0
  33. data/test/rbbt/hpc/test_pbs.rb +1 -1
  34. data/test/rbbt/hpc/test_slurm.rb +6 -10
  35. data/test/rbbt/knowledge_base/test_enrichment.rb +5 -3
  36. data/test/rbbt/knowledge_base/test_entity.rb +26 -6
  37. data/test/rbbt/knowledge_base/test_query.rb +1 -1
  38. data/test/rbbt/knowledge_base/test_registry.rb +34 -41
  39. data/test/rbbt/knowledge_base/test_traverse.rb +106 -58
  40. data/test/rbbt/persist/test_tsv.rb +4 -4
  41. data/test/rbbt/persist/tsv/test_lmdb.rb +7 -5
  42. data/test/rbbt/test_entity.rb +8 -5
  43. data/test/rbbt/test_knowledge_base.rb +107 -30
  44. data/test/rbbt/test_packed_index.rb +5 -5
  45. data/test/rbbt/test_tsv.rb +1 -1
  46. data/test/rbbt/test_workflow.rb +83 -78
  47. data/test/rbbt/tsv/test_accessor.rb +4 -0
  48. data/test/rbbt/tsv/test_attach.rb +2 -5
  49. data/test/rbbt/tsv/test_index.rb +8 -2
  50. data/test/rbbt/tsv/test_manipulate.rb +0 -3
  51. data/test/rbbt/util/R/test_model.rb +0 -2
  52. data/test/rbbt/util/misc/test_bgzf.rb +3 -3
  53. data/test/rbbt/util/misc/test_communication.rb +5 -3
  54. data/test/rbbt/util/test_migrate.rb +1 -3
  55. data/test/rbbt/util/test_misc.rb +7 -2
  56. data/test/rbbt/util/test_procpath.rb +1 -1
  57. data/test/rbbt/util/test_python.rb +12 -8
  58. data/test/rbbt/workflow/step/test_dependencies.rb +0 -4
  59. data/test/rbbt/workflow/test_remote_workflow.rb +6 -5
  60. data/test/rbbt/workflow/util/test_archive.rb +4 -8
  61. data/test/rbbt/workflow/util/test_orchestrator.rb +5 -15
  62. data/test/test_helper.rb +25 -1
  63. metadata +5 -5
  64. data/test/rbbt/test_hpc.rb +0 -16
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8b23de77e3900d1a81e8ffa349ab9f88a4283f30bb2cec62c04bff4539243170
4
- data.tar.gz: 494b18d5e2ba4802225617a03b2f69255ec9deab173e8d18e06654a7b415325c
3
+ metadata.gz: d1bbf2cefca44eecad7465a8d4dc1b88f53f18d1e63e310353061b43ad46bed9
4
+ data.tar.gz: f54f659da9eca312d116430d88019f067ce5ccf519724a2a448e3b4d31dfb767
5
5
  SHA512:
6
- metadata.gz: d57bd3bc7ff84a4e8101dd907f98ef20923dbcab95c3187523e6b84eba6774d900f269ed2457fd6e1a81821edf3e2d1f11fcbdd5863f791282d549bcf83a48a5
7
- data.tar.gz: a4857d82292ab4234cf10fbf2fcb242b466e6400f9476cb4de4bc625ff808542a690f27fbd9dede4ff692a82644561ebfc086ccdf953102cb4f7a0020b094849
6
+ metadata.gz: eaad34515129ffd123064e9af920b5f83f3897c957de45e3bef04a1101d961d419863729f21665773b81f535f3fdba45d490ce41870f9db87f0e0e5d5f987233
7
+ data.tar.gz: d48552f30e6434b0e27e173eb62f41a3126ecb9065cba58f2f5737f8e3aef0e2183503076a40cc6586832d90458ae5e572a55171758452f3699a4225704bc25e
@@ -43,7 +43,7 @@ module Annotated
43
43
  end
44
44
 
45
45
  def self.resolve_array(entry)
46
- if entry =~ /^Array:/
46
+ if String === entry && entry =~ /^Array:/
47
47
  entry["Array:".length..-1].split("|")
48
48
  else
49
49
  entry
@@ -93,7 +93,7 @@ module Association
93
93
  tmp_data = Persist.open_database(tmpfile, true, :double, "HDB")
94
94
 
95
95
  tsv.with_monitor(options[:monitor]) do
96
- tsv = tsv.reorder source_field, fields, :persist => persist, :persist_data => tmp_data if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
96
+ tsv = tsv.reorder source_field, tsv.all_fields.values_at(*field_pos), :persist => persist, :persist_data => tmp_data if true or source_field != tsv.key_field or (fields and tsv.fields != fields)
97
97
  end
98
98
 
99
99
  tsv.key_field = source_header
@@ -216,6 +216,7 @@ module Association
216
216
  open_stream(stream, options.dup)
217
217
  end
218
218
 
219
+ database.filename = file if Path === file && file.identifier_files.any?
219
220
  database.entity_options = options[:entity_options] if options[:entity_options]
220
221
 
221
222
  database
@@ -38,6 +38,7 @@ module Association
38
38
  data.fields = fields[1..-1]
39
39
  data.type = :list
40
40
  data.serializer ||= serializer
41
+ data.filename ||= file if String === file
41
42
 
42
43
  database.with_unnamed do
43
44
  database.with_monitor(options[:monitor]) do
@@ -115,8 +115,14 @@ module Association
115
115
  source_field = specs[:source][0]
116
116
  target_field = specs[:target][0]
117
117
 
118
- source_pos = all_fields.index source_field
119
- target_pos = all_fields.index target_field
118
+ #source_pos = all_fields.index source_field
119
+ #target_pos = all_fields.index target_field
120
+
121
+ source_pos = TSV.identify_field all_fields.first, all_fields[1..-1], source_field
122
+ target_pos = TSV.identify_field all_fields.first, all_fields[1..-1], target_field
123
+
124
+ source_pos = source_pos == :key ? 0 : source_pos + 1
125
+ target_pos = target_pos == :key ? 0 : target_pos + 1
120
126
 
121
127
  source_header = specs[:source][1] || specs[:source][0]
122
128
  target_header = specs[:target][1] || specs[:target][0]
@@ -140,7 +146,11 @@ module Association
140
146
  field_headers << header
141
147
  end
142
148
 
143
- field_pos = info_fields.collect{|f| raise "Field #{f} not found. Options: #{all_fields * ", "}" unless all_fields.include?(f); f == :key ? 0 : all_fields.index(f); }
149
+ field_pos = info_fields.collect do |f|
150
+ p = TSV.identify_field all_fields.first, all_fields[1..-1], f
151
+ p == :key ? 0 : p + 1
152
+ end
153
+
144
154
  field_pos.delete source_pos
145
155
 
146
156
  source_format = specs[:source][2]
data/lib/rbbt/entity.rb CHANGED
@@ -289,16 +289,17 @@ module Entity
289
289
  end
290
290
 
291
291
  def self.unpersist(method_name)
292
- return unless persisted? method_name
293
- orig_name = UNPERSISTED_PREFIX + method_name.to_s
294
-
292
+ return unless orig_name = persisted?(method_name)
295
293
  alias_method method_name, orig_name
296
294
  remove_method orig_name
297
295
  end
298
296
 
299
297
  def self.persisted?(method_name)
300
- orig_name = UNPERSISTED_PREFIX + method_name.to_s
301
- instance_methods.include? orig_name.to_sym
298
+ ["", "_multiple_", "_single_"].each do |type|
299
+ orig_name = (UNPERSISTED_PREFIX + type + method_name.to_s).to_sym
300
+ return orig_name if instance_methods.include?(orig_name)
301
+ end
302
+ return false
302
303
  end
303
304
 
304
305
  def self.with_persisted(method_name)
@@ -152,7 +152,7 @@ module HPC
152
152
  end
153
153
 
154
154
  def self.job_batches(rules, job)
155
- job_chains = self.job_chains(rules, job)
155
+ job_chains = self.job_chains(rules, job).dup
156
156
 
157
157
  workload = job_workload(job).uniq
158
158
 
@@ -39,9 +39,8 @@ module HPC
39
39
  (job.dependencies + job.input_dependencies).uniq.select{|d| ! d.done? || d.dirty? }
40
40
  end
41
41
 
42
- def self.job_chains(rules, job)
43
- @@job_chains ||= {}
44
- @@job_chains[Misc.digest([rules, job.path].inspect)] ||=
42
+ def self.job_chains(rules, job, computed = {})
43
+ computed[Misc.fingerprint([rules, job.path, job.object_id])] ||=
45
44
  begin
46
45
  chains = self.parse_chains(rules)
47
46
 
@@ -55,7 +54,7 @@ module HPC
55
54
  dep_matches = check_chains(chains, dep)
56
55
  common = matches & dep_matches
57
56
 
58
- dep_chains = job_chains(rules, dep)
57
+ dep_chains = job_chains(rules, dep, computed)
59
58
  found = []
60
59
  dep_chains.each do |match,info|
61
60
  if common.include?(match)
@@ -146,7 +146,7 @@ class KnowledgeBase
146
146
  if conditions
147
147
  Misc.tokenize(conditions).each do |condition|
148
148
  if condition.index "="
149
- key, value = conditions.split("=")
149
+ key, value = condition.split("=")
150
150
  matches = matches.select{|m| Misc.match_value(m.info[key.strip], value)}
151
151
  else
152
152
  matches = matches.select{|m| m.info[condition.strip].to_s =~ /\btrue\b/}
@@ -158,17 +158,19 @@ class KnowledgeBase
158
158
  end
159
159
 
160
160
  def id_dbs(db)
161
+ # ToDo: Revise this, I'm not sure what id does anymore
162
+ # I think it deals with syndication
161
163
  if db.include? '?'
162
- all_dbs = kb.registry.keys
164
+ all_dbs = kb.registry.keys.collect{|k| k.to_s }
163
165
  _name, _sep, _kb = db.partition("@")
164
166
  case
165
167
  when _name[0] == '?'
166
168
  dbs = all_dbs.select{|_db|
167
169
  n,_s,d=_db.partition("@");
168
- d.nil? or d.empty? or (d == _kd and assignments[_name].include?(n))
170
+ d.nil? or d.empty? or (d == _kb and assignments[_name].include?(n))
169
171
  }
170
172
  when _kb[0] == '?'
171
- dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d)}
173
+ dbs = all_dbs.select{|_db| n,_s,d=_db.partition("@"); n == _name and assignments[_kb].include?(d) }
172
174
  end
173
175
  else
174
176
  dbs = [db]
@@ -181,11 +183,12 @@ class KnowledgeBase
181
183
  all_matches = []
182
184
  path_rules = []
183
185
  acc_var = nil
186
+ pre_acc_var_assignments = nil
184
187
  rules.each do |rule|
185
188
  rule = rule.strip
186
189
  next if rule.empty?
187
190
 
188
- if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+([^\s]+))?/)
191
+ if m = rule.match(/([^\s]+)\s+([^\s=]+)\s+([^\s]+)(?:\s+-\s+(.*))?/)
189
192
  Log.debug "Traverse rule: #{rule}"
190
193
  path_rules << rule
191
194
 
@@ -199,17 +202,19 @@ class KnowledgeBase
199
202
 
200
203
  next if matches.nil? or matches.empty?
201
204
 
202
- if db.include? '?'
203
- _name, _sep, _kb = db.partition("@")
204
- case
205
- when _kb[0] == '?'
206
- assignments[_kb] ||= []
207
- assignments[_kb] << _db.partition("@").reject{|p| p.empty?}.last
208
- when _name[0] == '?'
209
- assignments[_name] ||= []
210
- assignments[_name] << _db.partition("@").first
211
- end
212
- end
205
+ # ToDo: Revise this, I'm not sure what id does anymore
206
+ #
207
+ #if db.include? '?'
208
+ # _name, _sep, _kb = db.partition("@")
209
+ # case
210
+ # when _kb[0] == '?'
211
+ # assignments[_kb] ||= []
212
+ # assignments[_kb] << _db.partition("@").reject{|p| p.empty?}.last
213
+ # when _name[0] == '?'
214
+ # assignments[_name] ||= []
215
+ # assignments[_name] << _db.partition("@").first
216
+ # end
217
+ #end
213
218
 
214
219
  matches.each do |m|
215
220
  rule_matches << m
@@ -244,11 +249,14 @@ class KnowledgeBase
244
249
 
245
250
  elsif m = rule.match(/(\?[^\s{]+)\s*{/)
246
251
  acc_var = m.captures.first
252
+ pre_acc_var_assignments = assignments.dup
247
253
  Log.debug "Start assign block: #{acc_var}"
248
254
  elsif m = rule.match(/^\s*}\s*$/)
249
255
  Log.debug "Close assign block: #{acc_var}"
250
256
  saved_assign = assignments[acc_var]
251
257
  assignments.clear
258
+ assignments.merge!(pre_acc_var_assignments)
259
+ pre_acc_var_assignments = nil
252
260
  assignments[acc_var] = saved_assign
253
261
  all_matches = []
254
262
  path_rules = []
@@ -21,7 +21,7 @@ module Persist
21
21
  tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
22
22
  tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
23
23
 
24
- database = CONNECTIONS[path] ||= tokyocabinet_class.new
24
+ database = CONNECTIONS[path] ||= Log.ignore_stderr do tokyocabinet_class.new end
25
25
 
26
26
  if big and not Open.exists?(path)
27
27
  database.tune(nil,nil,nil,tokyocabinet_class::TLARGE | tokyocabinet_class::TDEFLATE)
@@ -161,8 +161,13 @@ module Path
161
161
  def find(where = nil, caller_lib = nil, paths = nil)
162
162
 
163
163
  if located?
164
- self.original ||= self
165
- return self
164
+ path = self
165
+ path = path + '.gz' if File.exist?(path + '.gz')
166
+ path = path + '.bgz' if File.exist?(path + '.bgz')
167
+ self.annotate(path)
168
+
169
+ path.original = self
170
+ return path
166
171
  end
167
172
 
168
173
  if where == :all || where == 'all'
data/lib/rbbt/resource.rb CHANGED
@@ -374,7 +374,7 @@ url='#{url}'
374
374
  if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
375
375
  unlocated = ([m["TOPLEVEL"],m["SUBPATH"],m["REST"]] * "/")
376
376
  unlocated.gsub!(/\/+/,'/')
377
- if self.subdir && ! self.subdir.empty?
377
+ if self.subdir && ! self.subdir.empty? && unlocated.include?(subdir)
378
378
  subdir = self.subdir
379
379
  subdir += "/" unless subdir.end_with?("/")
380
380
  unlocated[subdir] = ""
@@ -181,6 +181,48 @@ module TSV
181
181
  end
182
182
  end
183
183
 
184
+ def self.traverse_enumerable(enum, options = {}, &block)
185
+ callback, bar, join = Misc.process_options options, :callback, :bar, :join
186
+
187
+ begin
188
+ error = false
189
+ if callback
190
+ bar.init if bar
191
+ while enum.any?
192
+ e = enum.pop
193
+ begin
194
+ callback.call yield(e)
195
+ rescue Exception
196
+ Log.warn "Traverse exception on element: #{Misc.fingerprint(e)}"
197
+ raise $!
198
+ ensure
199
+ bar.tick if bar
200
+ end
201
+ end
202
+ else
203
+ bar.init if bar
204
+ while enum.any?
205
+ e = enum.pop
206
+ begin
207
+ yield e
208
+ rescue Exception
209
+ Log.warn "Traverse exception on element: #{Misc.fingerprint(e)}"
210
+ raise $!
211
+ ensure
212
+ bar.tick if bar
213
+ end
214
+ end
215
+ end
216
+
217
+ rescue
218
+ error = true
219
+ raise $!
220
+ ensure
221
+ join.call(error) if join
222
+ Log::ProgressBar.remove_bar(bar, error) if bar
223
+ end
224
+ end
225
+
184
226
  def self.traverse_priority_queue(queue, options = {}, &block)
185
227
  callback, bar, join = Misc.process_options options, :callback, :bar, :join
186
228
 
@@ -396,6 +438,8 @@ module TSV
396
438
  else
397
439
  raise "Can not open obj for traversal #{Misc.fingerprint obj}"
398
440
  end
441
+ when Enumerable
442
+ traverse_enumerable(obj, options, &block)
399
443
  when nil
400
444
  raise "Can not traverse nil object into #{stream_name(options[:into])}"
401
445
  else
@@ -281,19 +281,20 @@ module TSV
281
281
  end
282
282
 
283
283
  values = values.collect{|v| v.length != num ? [v.first] * num : v}
284
+
284
285
  all = values
285
286
  all.unshift keys
286
- Misc.zip_fields(all).each do |values|
287
- key = values.shift
287
+ Misc.zip_fields(all).each do |vs|
288
+ key = vs.shift
288
289
  if data.include? key
289
- data[key] = data[key].zip(values).collect do |old, new|
290
- old.push new
291
- old
290
+ data[key] = data[key].zip(vs).collect do |old, new|
291
+ old + [new]
292
292
  end
293
293
  else
294
- data[key] = values.collect{|v| [v]}
294
+ data[key] = vs.collect{|v| [v] }
295
295
  end
296
296
  end
297
+
297
298
  nil
298
299
  end
299
300
 
@@ -63,7 +63,7 @@ module TSV
63
63
  class StringDoubleArraySerializer
64
64
  def self.dump(array)
65
65
  begin
66
- array.collect{|a| a.collect{|a| a.to_s } * "|"} * "\t"
66
+ array.collect{|a| a.collect{|a| a.to_s } * "|"} * "\t"
67
67
  rescue Encoding::CompatibilityError
68
68
  array.collect{|a| a.collect{|a| a.to_s.force_encoding('UTF-8')} * "|"} * "\t"
69
69
  end
data/lib/rbbt/tsv/util.rb CHANGED
@@ -168,6 +168,8 @@ module TSV
168
168
  end
169
169
  when Set
170
170
  get_stream(file.to_a, open_options)
171
+ when Enumerable
172
+ file
171
173
  else
172
174
  raise "Cannot get stream from: #{file.inspect}"
173
175
  end
@@ -189,6 +191,16 @@ module TSV
189
191
  pos = fields.index field
190
192
  return pos if pos
191
193
  return identify_field(key_field, fields, field.to_i) if field =~ /^\d+$/
194
+ if fields.select{|f| f.include?("(") }.any?
195
+ simplify_fields = fields.collect do |f|
196
+ if m = f.match(/(.*)\s+\(.*\)/)
197
+ m[1]
198
+ else
199
+ f
200
+ end
201
+ end
202
+ return identify_field(key_field, simplify_fields, field)
203
+ end
192
204
  raise "Field '#{ field }' was not found. Options: (#{key_field || "NO_KEY_FIELD"}), #{(fields || ["NO_FIELDS"]) * ", "}" if pos.nil?
193
205
  else
194
206
  raise "Field '#{ field }' was not found. Options: (#{key_field || "NO_KEY_FIELD"}), #{(fields || ["NO_FIELDS"]) * ", "}"
@@ -9,7 +9,7 @@ module Log
9
9
 
10
10
  def self.no_bar
11
11
  @@no_bar = false unless defined?(@@no_bar)
12
- @@no_bar || ENV["RBBT_NO_PROGRESS"] == "true"
12
+ (@@no_bar || ENV["RBBT_NO_PROGRESS"]).to_s == "true"
13
13
  end
14
14
 
15
15
  class ProgressBar
@@ -29,8 +29,8 @@ module Bgzf
29
29
 
30
30
  def filename
31
31
  @filename ||= begin
32
- compressed_stream.respond_to?(:filename) ? compressed_stream.filename : nil
33
- end
32
+ compressed_stream.respond_to?(:filename) ? compressed_stream.filename : rand(1000000000).to_s
33
+ end
34
34
  end
35
35
 
36
36
  def closed?
@@ -49,7 +49,8 @@ module Bgzf
49
49
 
50
50
  def _index
51
51
  @_index ||= begin
52
- index = Persist.persist("BGZF index" + (filename || "").sub(/.bgz$/,''), :marshal, :dir => Rbbt.var.bgzf_index) do
52
+ prefix_code = "BGZF index" + (filename || "").sub(/.bgz$/,'')
53
+ index = Persist.persist(prefix_code, :marshal) do
53
54
  index = []
54
55
  pos = 0
55
56
  while true do
@@ -66,7 +67,7 @@ module Bgzf
66
67
  end
67
68
  index
68
69
  end
69
- @block_cache_size = Math.log(index.length).to_i + 1
70
+ @block_cache_size = Math.log(index.length + 1).to_i + 1
70
71
  index
71
72
  end
72
73
  end
@@ -39,13 +39,13 @@ module Misc
39
39
  end
40
40
  end
41
41
  if message
42
- puts "#{message }: #{ repeats } repeats"
42
+ STDERR.pust "#{message }: #{ repeats } repeats"
43
43
  else
44
- puts "Benchmark for #{ repeats } repeats"
44
+ STDERR.puts "Benchmark for #{ repeats } repeats"
45
45
  end
46
- puts measure
46
+ STDERR.puts measure
47
47
  rescue Exception
48
- puts "Benchmark aborted"
48
+ STDERR.puts "Benchmark aborted"
49
49
  raise $!
50
50
  end
51
51
  res
@@ -69,7 +69,7 @@ module Misc
69
69
  end
70
70
 
71
71
  def self.tokenize(str)
72
- str.scan(/"[^"]*"|'[^']*'|[^"'\s]+/)
72
+ str.scan(/"([^"]*)"|'([^']*)'|([^"'\s]+)/).flatten.compact
73
73
  end
74
74
 
75
75
  def self.timespan(str, default = "s")
@@ -20,10 +20,15 @@ class RemoteWorkflow
20
20
 
21
21
  def self.escape_url(url)
22
22
  base, _sep, query = url.partition("?")
23
- protocol, path = base.split("://")
24
- path = protocol if path.nil?
23
+ protocol, _sep, host_and_path = base.partition("://")
24
+ host_and_path = protocol if host_and_path.nil?
25
+
26
+ host, _sep, path = host_and_path.partition("/")
27
+
25
28
  path = path.split("/").collect{|p| CGI.escape(p) }* "/"
26
- base = protocol ? [protocol, path] * "://" : path
29
+
30
+ host_and_path = [host, path] * "/"
31
+ base = protocol ? [protocol, host_and_path] * "://" : host_and_path
27
32
 
28
33
  if query && ! query.empty?
29
34
  query = query.split("&").collect{|e| e.split("=").collect{|pe| CGI.escape(pe) } * "=" } * "&"
@@ -354,6 +354,10 @@ class RemoteStep < Step
354
354
  i = {:status => :waiting, :pid => Process.pid, :path => path}
355
355
  i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
356
356
  end
357
+
358
+ def inspect
359
+ "RemoteWorkflow #{self.path}"
360
+ end
357
361
  end
358
362
 
359
363
  require 'rbbt/workflow/remote_workflow/remote_step/rest'
@@ -158,11 +158,11 @@ rbbt.png_plot('#{plot}', 'plot(timeline)', width=#{width}, height=#{height}, poi
158
158
  info[key] = dep_info[key]
159
159
  end
160
160
 
161
- dep.info[:config_keys].select do |kinfo|
161
+ dep.info[:config_keys].each do |kinfo|
162
162
  key, value, tokens = kinfo
163
163
 
164
164
  info[key.to_s] = value if report_keys.include? key.to_s
165
- end
165
+ end if dep.info[:config_keys]
166
166
  end
167
167
 
168
168
  summary = TSV.setup({}, "Task~Calls,Avg. Time,Total Time#:type=:list")
data/lib/rbbt/workflow.rb CHANGED
@@ -496,7 +496,7 @@ module Workflow
496
496
  end
497
497
 
498
498
  #overriden = true if dependencies.select{|d| d.overriden && d.clean_name != d.name }.any?
499
- overriden = true if not_overriden && dependencies.select{|d| Symbol === d.overriden }.any?
499
+ overriden = true if ! not_overriden && dependencies.select{|d| Symbol === d.overriden }.any?
500
500
 
501
501
  input_values = task.take_input_values(inputs)
502
502
  if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden #&& ! dependencies.select{|d| d.overriden && d.clean_name != d.name }.any?
@@ -507,7 +507,7 @@ module Workflow
507
507
 
508
508
  job = get_job_step step_path, task, input_values, dependencies
509
509
  job.workflow = self
510
- job.overriden ||= overriden
510
+ job.overriden = overriden if job.overriden.nil?
511
511
  job.clean_name = jobname
512
512
 
513
513
  #iif [job, not_overriden]
@@ -23,7 +23,6 @@ MDM2 TP53 inhibition false
23
23
  TP53 NFKB1|GLI1 activation|activation true|true
24
24
  END
25
25
 
26
-
27
26
  effect_tsv = TSV.open effect, effect_options.dup
28
27
  end
29
28
 
@@ -35,7 +34,6 @@ TP53 NFKB1|GLI1 activation|activation true|true
35
34
  assert_equal %w(NFKB1 GLI1), effect_tsv["TP53"]["TG"]
36
35
  end
37
36
 
38
-
39
37
  def test_open_new_tsv
40
38
  tsv = Association.database effect_tsv, :fields => ["Effect"], :target => "TG=~Associated Gene Name=>Ensembl Gene ID"
41
39
  assert_equal ["Ensembl Gene ID", "Effect"], tsv.fields
@@ -30,7 +30,7 @@ TP53 NFKB1|GLI1 activation|activation true|true
30
30
 
31
31
 
32
32
  def test_open_persist
33
- tsv = Association.open(EFFECT_TSV, EFFECT_OPTIONS, :persist => true, :update => true)
33
+ tsv = Association.open(EFFECT, EFFECT_OPTIONS, :persist => true, :update => true)
34
34
  assert_equal "ENSG00000141510", tsv["MDM2"]["Ensembl Gene ID"].first
35
35
  assert_equal ["false"], tsv["MDM2"]["directed?"]
36
36
  end
@@ -8,67 +8,27 @@ module Gene
8
8
  end
9
9
 
10
10
  class TestEntityIdentifiers < Test::Unit::TestCase
11
- def setup
12
- require 'rbbt/sources/kegg'
13
- require 'rbbt/sources/organism'
14
- Gene.add_identifiers Organism.identifiers("NAMESPACE"), "Ensembl Gene ID", "Associated Gene Name"
15
- Gene.add_identifiers KEGG.identifiers
16
- Gene.add_identifiers Organism.identifiers("NAMESPACE"), "Ensembl Gene ID", "Associated Gene Name"
17
- Gene.add_identifiers KEGG.identifiers
18
11
 
19
- end
20
12
  def test_name
21
13
  Gene.add_identifiers datafile_test('identifiers'), "Ensembl Gene ID", "Associated Gene Name"
22
14
  assert_equal "TP53", Gene.setup("ENSG00000141510").name
23
15
  end
24
16
 
25
17
  def test_name_organism
26
- assert_equal "TP53", Gene.setup("ENSG00000141510", "Ensembl Gene ID", "Hsa/feb2014").name
18
+ keyword_test :organism do
19
+ require 'rbbt/sources/organism'
20
+ Gene.add_identifiers Organism.identifiers("NAMESPACE"), "Ensembl Gene ID", "Associated Gene Name"
21
+ assert_equal "TP53", Gene.setup("ENSG00000141510", "Ensembl Gene ID", "Hsa/feb2014").name
22
+ end
27
23
  end
28
24
 
29
25
  def test_identifier_files
26
+ Gene.add_identifiers datafile_test('identifiers'), "Ensembl Gene ID", "Associated Gene Name"
30
27
  assert Gene.identifier_files.any?
31
28
  end
32
29
 
33
30
  def test_Entity_identifier_files
31
+ Gene.add_identifiers datafile_test('identifiers'), "Ensembl Gene ID", "Associated Gene Name"
34
32
  assert Entity.identifier_files("Ensembl Gene ID").any?
35
33
  end
36
-
37
- def test_translate_kegg
38
- assert_match "hsa", Gene.setup("ENSG00000141510", "Ensembl Gene ID", "Hsa/feb2014").to("KEGG Gene ID")
39
- assert_match "TP53", Gene.setup("ENSG00000141510", "Ensembl Gene ID", "Hsa/feb2014").to("KEGG Gene ID").to(:name)
40
- end
41
-
42
- def ___test_complex
43
-
44
- file = Path.setup('/home/mvazquezg/git/workflows/PanCancer/share/network/gene_sets/CORUM_protein_complexes.gmt')
45
- name = File.basename(file).sub(/\.gmt$/,'')
46
- organism = "Hsa/feb2014"
47
- key_field = "#{ name } Pathway ID"
48
- description_field = "#{name} Pathway Description"
49
- description_file = file.find + '.identifiers'
50
- tsv = TSV.open(file, :fix => Proc.new{|l| p=l.split"\t"; [p[0], p[1], p[2..-1]*"|"]*"\t"})
51
- tsv.namespace = organism
52
- tsv.unnamed = true
53
- gene_field, count = Organism.guess_id(organism, tsv.values.collect{|l| l.last}.flatten.uniq )
54
- tsv.key_field = key_field
55
- tsv.fields = [description_field, gene_field]
56
- descriptions = tsv.slice(description_field)
57
- Open.write(description_file, descriptions.to_single.to_s) #unless File.exist? description_file
58
- values = tsv.slice(gene_field)
59
- values.identifiers = description_file
60
-
61
- mod = Module.new
62
- mod_name = Misc.camel_case(key_field.gsub(/\s+/,'_').sub(/_ID$/,''))
63
- Object.const_set(mod_name, mod)
64
- mod.instance_eval do
65
- extend Entity
66
- add_identifiers Path.setup(description_file), key_field, description_field
67
-
68
- annotation :format
69
- end
70
- entity = "CORUM:6052"
71
- mod.setup(entity, :format => key_field)
72
- puts entity.name
73
- end
74
34
  end