rbbt-util 5.35.3 → 5.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 8be0e512e0b2cded02b65df8105bb3504297b703792f86b664d06904443660cd
4
- data.tar.gz: 568516ecd269492e81fb571a8fff377d2d486d30fb37883081f2d2aa8fbe811c
3
+ metadata.gz: d105d6c21ad018f28acf6a234feae8d81f327f3f94f228c01f118abed6c386f6
4
+ data.tar.gz: 5c883e0611f02203d394ec02e448664efdc42361be04e9d873e9ba3f5e6cc92e
5
5
  SHA512:
6
- metadata.gz: '09a9d4a2508ce0bc3cf9dc7b84071ec35f0fdf9e9ebf7566edb4d302b5af87b393b958475f83969a931a5da23aa82d00ca97ab2c6e63667730049558cf553a75'
7
- data.tar.gz: ea069e4c4c50bf0f71800d45156945485053afe0c2f91378fa58ce9442f222c99ea3f92ef30e6ae38e8ccce1afbef13e1a5b77d67a8ea75743353587be578fb6
6
+ metadata.gz: 20a4bccdeb0c587041b7f20f09bda19949713fcb362911a249b08112399c09178f8e50b5ae90f390bd6cd1389b3c9a47b7374c4fc05a83b073f1d43bdfdc6611
7
+ data.tar.gz: 1b3b112106b4ac1e261bcf1630ecaeea0ed27f1975aeead9694708241e9268fdac36da4f5b4be25c12288cb7f97b8b4163815b8d9010aba40f6f2434e654f99c
@@ -46,7 +46,9 @@ module Persist
46
46
  end
47
47
 
48
48
  def delete(key)
49
- out(key)
49
+ self.write_lock do
50
+ out(key)
51
+ end
50
52
  end
51
53
 
52
54
  def lock
data/lib/rbbt/persist.rb CHANGED
@@ -44,6 +44,8 @@ module Persist
44
44
  end
45
45
 
46
46
  def self.is_persisted?(path, persist_options = {})
47
+ return true if Open.remote?(path)
48
+ return true if Open.ssh?(path)
47
49
  return false if not Open.exists? path
48
50
  return false if TrueClass === persist_options[:update]
49
51
 
@@ -319,7 +319,8 @@ module Path
319
319
 
320
320
 
321
321
  def open(options = {}, &block)
322
- Open.open(self.produce.find, options, &block)
322
+ file = Open.remote?(self) || Open.ssh?(self) ? self : self.produce.find
323
+ Open.open(file, options, &block)
323
324
  end
324
325
 
325
326
  def to_s
@@ -331,7 +332,16 @@ module Path
331
332
  end
332
333
 
333
334
  def tsv(*args)
334
- TSV.open(self.produce, *args)
335
+ begin
336
+ path = self.produce
337
+ rescue Resource::ResourceNotFound => e
338
+ begin
339
+ path = self.set_extension('tsv').produce
340
+ rescue Resource::ResourceNotFound
341
+ raise e
342
+ end
343
+ end
344
+ TSV.open(path, *args)
335
345
  end
336
346
 
337
347
  def tsv_options(options = {})
@@ -58,6 +58,16 @@ module RbbtPython
58
58
  RbbtPython.import_method(module_name, method_name).call(*args)
59
59
  end
60
60
 
61
+ def self.get_class(module_name, class_name)
62
+ save_module_name = module_name.to_s.gsub(".", "_")
63
+ RbbtPython.pyimport(module_name, as: save_module_name)
64
+ RbbtPython.send(save_module_name).send(class_name)
65
+ end
66
+
67
+ def self.class_new_obj(module_name, class_name, args={})
68
+ RbbtPython.get_class(module_name, class_name).new(**args)
69
+ end
70
+
61
71
  def self.exec(script)
62
72
  PyCall.exec(script)
63
73
  end
@@ -175,6 +175,8 @@ module Workflow
175
175
 
176
176
  tasks[name] = task
177
177
  task_dependencies[name] = consume_dependencies
178
+
179
+ task
178
180
  end
179
181
 
180
182
  def unexport(*names)
@@ -37,6 +37,7 @@ module Workflow
37
37
  def documentation_markdown
38
38
  return "" if @libdir.nil?
39
39
  file = @libdir['workflow.md'].find
40
+ file = @libdir['README.md'].find unless file.exists?
40
41
  if file.exists?
41
42
  file.read
42
43
  else
@@ -1,4 +1,6 @@
1
1
  class RemoteWorkflow
2
+ RBBT_DEBUG_REMOTE_JSON = ENV["RBBT_DEBUG_REMOTE_JSON"] == 'true'
3
+
2
4
  module SSH
3
5
  #def self.run(server, script)
4
6
  # Log.debug "Run ssh script in #{server}:\n#{script}"
@@ -21,9 +23,16 @@ class RemoteWorkflow
21
23
 
22
24
  workflow, task, job, *rest = path.split("/")
23
25
 
26
+ workflow_name = begin
27
+ workflow = Kernel.const_get(workflow) if String === workflow
28
+ workflow.respond_to?(:complete_name) ? workflow.complete_name : workflow
29
+ rescue
30
+ workflow
31
+ end
32
+
24
33
  script =<<-EOF
25
34
  require 'rbbt/workflow'
26
- wf = Workflow.require_workflow "#{workflow}"
35
+ wf = Workflow.require_workflow "#{workflow_name}"
27
36
  EOF
28
37
 
29
38
  case task
@@ -86,7 +95,7 @@ STDOUT.write res.to_json
86
95
  EOF
87
96
 
88
97
  json = Misc.ssh_run(server, script)
89
- Log.debug "JSON (#{ url }): #{json}"
98
+ Log.debug "JSON (#{ url }): #{json}" if RBBT_DEBUG_REMOTE_JSON
90
99
  JSON.parse(json)
91
100
  end
92
101
 
@@ -172,11 +181,17 @@ job.clean
172
181
  def self.upload_inputs(server, inputs, input_types, input_id)
173
182
  TmpFile.with_file do |dir|
174
183
  if Step.save_inputs(inputs, input_types, dir)
175
- Dir.glob(File.join(dir, "*.as_step")).each do |file|
176
- path = Open.read(file).strip
177
- new = Step.migrate(path, :user, :target => server)
178
- Open.write(file, new)
179
- end
184
+ # Dir.glob(File.join(dir, "*.as_step")).each do |file|
185
+ # Log.medium "Migrating Step input #{file} #{ server }"
186
+ # path = Open.read(file).strip
187
+ # new = Step.migrate(path, :user, :target => server)
188
+ # Open.write(file, new)
189
+ # end
190
+
191
+ paths = Dir.glob(File.join(dir, "*.as_step")).collect{|f| Open.read(f).strip }
192
+ new = Step.migrate(paths, :user, :target => server)
193
+ paths.zip(new).each{|file,new| Open.write(file, new) }
194
+
180
195
  CMD.cmd_log("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
181
196
  end
182
197
  end
@@ -204,31 +219,56 @@ job.clean
204
219
  # rjob.run
205
220
  #end
206
221
 
222
+ def self.upload_dependencies(job, server, search_path = 'user', produce_dependencies = false)
223
+ server, path = parse_url(server) if server =~ /^ssh:\/\//
224
+ job.dependencies.each do |dep|
225
+ Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
226
+ dep.produce
227
+ end if produce_dependencies
228
+
229
+ job.input_dependencies.each do |dep|
230
+ Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
231
+ dep.produce
232
+ end if produce_dependencies
233
+
234
+ migrate_dependencies = job.rec_dependencies.select{|d| d.done? }.collect{|d| d.path }
235
+ Log.medium "Migrating #{migrate_dependencies.length} dependencies to #{ server }"
236
+ Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
237
+ end
238
+
239
+ def self.missing_dep_inputs(job)
240
+ inputs = job.inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
241
+ job.dependencies.each do |dep|
242
+ next if dep.done?
243
+ iif [dep, dep.inputs, dep.real_inputs]
244
+ inputs = dep.inputs.to_hash.slice(*dep.real_inputs.map{|i| i.to_s}).merge(inputs)
245
+ inputs = missing_dep_inputs(dep).merge(inputs)
246
+ end
247
+ inputs
248
+ end
249
+
207
250
  def self.relay_job(job, server, options = {})
208
- migrate, produce, produce_dependencies, search_path = Misc.process_options options.dup,
209
- :migrate, :produce, :produce_dependencies, :search_path
251
+ migrate, produce, produce_dependencies, search_path, run_type, slurm_options = Misc.process_options options.dup,
252
+ :migrate, :produce, :produce_dependencies, :search_path, :run_type, :slurm_options
210
253
 
211
254
  search_path ||= 'user'
212
255
 
213
256
  produce = true if migrate
214
257
 
215
258
  workflow_name = job.workflow.to_s
216
- inputs = job.recursive_inputs.to_hash
217
-
218
- job.dependencies.each do |dep|
219
- dep.produce
220
- end if options[:produce_dependencies]
221
-
222
- job.rec_dependencies.each do |dep|
223
- Step.migrate(dep.path, search_path, :target => server) if dep.done?
224
- end
259
+ remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{workflow_name}", "#{workflow_name}")
260
+ inputs = job.recursive_inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
261
+ Log.medium "Relaying dependency #{job.workflow}:#{job.short_path} to #{server} (#{inputs.keys * ", "})"
225
262
 
226
- remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{job.workflow.to_s}", "#{job.workflow.to_s}")
263
+ upload_dependencies(job, server, search_path, options[:produce_dependencies])
227
264
  rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
228
265
 
229
266
  override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
230
267
  rjob.override_dependencies = override_dependencies
231
268
 
269
+ rjob.run_type = run_type
270
+ rjob.slurm_options = slurm_options || {}
271
+
232
272
  if options[:migrate]
233
273
  rjob.produce
234
274
  Step.migrate(Rbbt.identify(job.path), 'user', :source => server)
@@ -255,7 +295,7 @@ job.clean
255
295
  @task_info ||= IndiferentHash.setup({})
256
296
 
257
297
  if @task_info[task].nil?
258
- task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url, task.to_s))
298
+ task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url || @url, task.to_s))
259
299
  task_info = RemoteWorkflow::SSH.fix_hash(task_info)
260
300
 
261
301
  task_info[:result_type] = task_info[:result_type].to_sym if task_info[:result_type]
@@ -1,6 +1,6 @@
1
1
  class RemoteStep
2
2
  module SSH
3
- attr_accessor :override_dependencies
3
+ attr_accessor :override_dependencies, :run_type, :slurm_options
4
4
 
5
5
  def init_job(cache_type = nil, other_params = {})
6
6
  return self if @url
@@ -55,12 +55,27 @@ class RemoteStep
55
55
  RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
56
56
  end
57
57
 
58
+ def _run_slurm
59
+ RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
60
+ end
61
+
62
+ def _orchestrate_slurm
63
+ RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
64
+ end
65
+
58
66
  def produce(*args)
59
67
  input_types = {}
60
68
  init_job
61
- @remote_path = _run
69
+ @remote_path = case @run_type
70
+ when 'run', :run, nil
71
+ _run
72
+ when 'slurm', :slurm
73
+ _run_slurm
74
+ when 'orchestrate', :orchestrate
75
+ _orchestrate_slurm
76
+ end
62
77
  @started = true
63
- while ! (done? || error?)
78
+ while ! (done? || error? || aborted?)
64
79
  sleep 1
65
80
  end
66
81
  raise self.get_exception if error?
@@ -163,6 +163,7 @@ class RemoteStep < Step
163
163
  begin
164
164
  status = info[:status]
165
165
  @done = true if status and status.to_sym == :done
166
+ Log.low "RemoteStep status '#{status}' #{self.url}"
166
167
  status
167
168
  rescue
168
169
  Log.exception $!
@@ -1,4 +1,12 @@
1
1
  class Step
2
+ attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
3
+ attr_accessor :task_name, :overriden
4
+ attr_accessor :pid
5
+ attr_accessor :exec
6
+ attr_accessor :relocated
7
+ attr_accessor :result, :mutex, :seen
8
+ attr_accessor :real_inputs, :original_task_name, :original_workflow
9
+
2
10
 
3
11
  INFO_SERIALIZER = begin
4
12
  if ENV["RBBT_INFO_SERIALIZER"]
@@ -86,12 +94,45 @@ class Step
86
94
  end
87
95
  end
88
96
 
97
+ def task_name
98
+ @task_name ||= begin
99
+ if @task.respond_to?(:name)
100
+
101
+ @task.name
102
+ else
103
+ @path.split("/")[-2]
104
+ end
105
+ end
106
+ end
107
+
108
+ def result_type
109
+ @result_type ||= if @task.respond_to?(:result_type)
110
+ @task.result_type || info[:result_type]
111
+ else
112
+ info[:result_type]
113
+ end
114
+ end
115
+
116
+ def result_type=(type)
117
+ @result_type = type
118
+ end
119
+
120
+ def result_description
121
+ @result_description ||= if @task.respond_to?(:result_description)
122
+ @task.result_description
123
+ else
124
+ nil
125
+ end
126
+ end
127
+
128
+ def result_description=(description)
129
+ @result_description = description
130
+ end
89
131
 
90
132
  def name
91
133
  @name ||= path.sub(/.*\/#{Regexp.quote task_name.to_s}\/(.*)/, '\1')
92
134
  end
93
135
 
94
-
95
136
  def short_path
96
137
  [task_name, name] * "/"
97
138
  end
@@ -105,10 +146,6 @@ class Step
105
146
  workflow.to_s + "#" + short_path
106
147
  end
107
148
 
108
- def task_name
109
- @task_name ||= task.name
110
- end
111
-
112
149
  def task_signature
113
150
  [workflow.to_s, task_name] * "#"
114
151
  end
@@ -3,7 +3,7 @@ class Step
3
3
  STREAM_CACHE = {}
4
4
  STREAM_CACHE_MUTEX = Mutex.new
5
5
  def self.purge_stream_cache
6
- Log.debug "Purging dup. stream cache"
6
+ # Log.debug "Purging dup. stream cache"
7
7
  STREAM_CACHE_MUTEX.synchronize do
8
8
  STREAM_CACHE.clear
9
9
  end
@@ -209,7 +209,7 @@ class Step
209
209
  end
210
210
 
211
211
  rescue TryAgain
212
- Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
212
+ #Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
213
213
  retry
214
214
  rescue Aborted, Interrupt
215
215
  Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
@@ -430,7 +430,6 @@ class Step
430
430
 
431
431
  required_dep_paths = []
432
432
  dep_step.each do |path,list|
433
- #required_dep_paths << path if list.length > 1
434
433
  required_dep_paths << path if (list & dependencies).any?
435
434
  end
436
435
 
@@ -442,42 +441,46 @@ class Step
442
441
  [dep.path] + dep.input_dependencies
443
442
  end.flatten)
444
443
 
445
- log :dependencies, "Dependencies for step #{Log.color :yellow, task.name.to_s || ""}"
446
444
 
447
445
  pre_deps = []
448
- compute_pre_deps = {}
449
- last_deps = []
446
+ simple_dependencies = []
447
+ compute_simple_dependencies = {}
450
448
  compute_last_deps = {}
451
449
  seen_paths = Set.new
452
450
  rec_dependencies.uniq.each do |step|
453
451
  next if seen_paths.include? step.path
454
452
  seen_paths << step.path
455
453
  next unless required_dep_paths.include? step.path
456
- if step.inputs.flatten.select{|i| Step === i}.any?
457
- if ComputeDependency === step
458
- next if produced.include? step.path
454
+ required_seen_paths = seen_paths & required_dep_paths
455
+
456
+ internal = step.inputs.select{|i| Step == i && required_paths.include?(i.path) && seen_paths.include?(i.path) }
457
+
458
+ if ComputeDependency === step
459
+ next if produced.include? step.path
460
+ if internal
459
461
  compute_last_deps[step.compute] ||= []
460
462
  compute_last_deps[step.compute] << step
461
463
  else
462
- last_deps << step
464
+ compute_simple_dependencies[step.compute] ||= []
465
+ compute_simple_dependencies[step.compute] << step
463
466
  end
464
467
  else
465
- if ComputeDependency === step
466
- next if produced.include? step.path
467
- compute_pre_deps[step.compute] ||= []
468
- compute_pre_deps[step.compute] << step
468
+ if internal
469
+ simple_dependencies.prepend(step)
469
470
  else
470
- pre_deps << step #if dependencies.include?(step)
471
+ simple_dependencies << step
471
472
  end
472
473
  end
473
474
  end
474
475
 
475
- Log.medium "Computing pre dependencies: #{Misc.fingerprint(compute_pre_deps)} - #{Log.color :blue, self.path}" if compute_pre_deps.any?
476
- compute_pre_deps.each do |type,list|
476
+ log :dependencies, "Processing dependencies for #{Log.color :yellow, task_name.to_s || ""}" if compute_simple_dependencies.any? || simple_dependencies.any? || compute_last_deps.any?
477
+
478
+ Log.debug "compute_simple_dependencies: #{Misc.fingerprint(compute_simple_dependencies)} - #{Log.color :blue, self.path}" if compute_simple_dependencies.any?
479
+ compute_simple_dependencies.each do |type,list|
477
480
  run_compute_dependencies(type, list, dep_step)
478
481
  end
479
482
 
480
- Log.medium "Processing pre dependencies: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
483
+ Log.low "pre_deps: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
481
484
  pre_deps.each do |step|
482
485
  next if compute_deps.include? step
483
486
  begin
@@ -487,8 +490,8 @@ class Step
487
490
  end
488
491
  end
489
492
 
490
- Log.medium "Processing last dependencies: #{Misc.fingerprint(last_deps)} - #{Log.color :blue, self.path}" if last_deps.any?
491
- last_deps.each do |step|
493
+ Log.debug "simple_dependencies: #{Misc.fingerprint(simple_dependencies)} - #{Log.color :blue, self.path}" if simple_dependencies.any?
494
+ simple_dependencies.each do |step|
492
495
  next if compute_deps.include? step
493
496
  begin Exception
494
497
  execute_and_dup(step, dep_step)
@@ -497,8 +500,8 @@ class Step
497
500
  end
498
501
  end
499
502
 
500
- Log.medium "Computing last dependencies: #{Misc.fingerprint(compute_last_deps)} - #{Log.color :blue, self.path}" if compute_last_deps.any?
501
- compute_last_deps.each do |type,list|
503
+ Log.low "compute_last_deps: #{Misc.fingerprint(compute_simple_dependencies)} - #{Log.color :blue, self.path}" if compute_simple_dependencies.any?
504
+ compute_simple_dependencies.each do |type,list|
502
505
  run_compute_dependencies(type, list, dep_step)
503
506
  end
504
507
 
@@ -1,5 +1,5 @@
1
1
  class Step
2
- def self.prepare_dependencies(jobs, tasks, cpus)
2
+ def self.produce_dependencies(jobs, tasks, cpus)
3
3
  deps = []
4
4
 
5
5
  jobs = [jobs] unless Array === jobs
@@ -96,7 +96,11 @@ class Step
96
96
  @exec = true if @exec.nil?
97
97
  begin
98
98
  old = Signal.trap("INT"){ Thread.current.raise Aborted }
99
- @task.exec_in((bindings ? bindings : self), *@inputs)
99
+ if @task.respond_to?(:exec_in)
100
+ @task.exec_in((bindings || self), *@inputs)
101
+ else
102
+ (bindings || self).instance_exec *@inputs, &@task
103
+ end
100
104
  ensure
101
105
  Signal.trap("INT", old)
102
106
  end
@@ -223,14 +227,15 @@ class Step
223
227
  end
224
228
 
225
229
  begin
226
- time_elapsed = total_time_elapsed = nil
227
- res = @mutex.synchronize do
228
- no_load = :stream if no_load
230
+ no_load = :stream if no_load
231
+ result_type = self.result_type || info[:result_type]
229
232
 
233
+ res = @mutex.synchronize do
234
+ time_elapsed = total_time_elapsed = nil
230
235
  Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
231
- result_type = @task.result_type if @task
232
- result_type = info[:result_type] if result_type.nil?
236
+
233
237
  result = Persist.persist "Job", result_type, :file => path, :check => persist_checks, :no_load => no_load do
238
+
234
239
  if Step === Step.log_relay_step and not self == Step.log_relay_step
235
240
  relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
236
241
  end
@@ -240,26 +245,30 @@ class Step
240
245
  @exec = false
241
246
  init_info(true)
242
247
 
243
- log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task.name.to_s || ""}"
248
+ workflow = @workflow || @task.respond_to?(:workflow) ? @task.workflow : nil
249
+ result_type = @task.respond_to?(:result_type) ? @task.result_type : nil
250
+ result_description = @task.respond_to?(:result_description) ? @task.result_description : nil
251
+
252
+ log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task_name}"
244
253
 
245
254
  merge_info({
246
- :issued => (issue_time = Time.now),
247
- :name => name,
248
- :pid => Process.pid.to_s,
249
- :pid_hostname => Socket.gethostname,
250
- :clean_name => clean_name,
251
- :workflow => (@workflow || @task.workflow).to_s,
252
- :task_name => @task.name,
253
- :result_type => @task.result_type,
254
- :result_description => @task.result_description,
255
- :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
256
- :versions => Rbbt.versions
255
+ :issued => (issue_time = Time.now),
256
+ :name => name,
257
+ :pid => Process.pid.to_s,
258
+ :pid_hostname => Socket.gethostname,
259
+ :clean_name => clean_name,
260
+ :workflow => workflow.to_s,
261
+ :task_name => task_name,
262
+ :result_type => result_type,
263
+ :result_description => result_description,
264
+ :dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
265
+ :versions => Rbbt.versions
257
266
  })
258
267
 
259
268
  new_inputs = []
260
269
  @inputs.each_with_index do |input,i|
261
- name = @task.inputs[i]
262
- type = @task.input_types[name]
270
+ name = @task.respond_to?(:inputs) ? @task.inputs[i] : nil
271
+ type = @task.respond_to?(:input_types) ? @task.input_types[i] : nil
263
272
 
264
273
  if type == :directory
265
274
  directory_inputs = file('directory_inputs')
@@ -303,7 +312,7 @@ class Step
303
312
 
304
313
  @inputs = new_inputs if @inputs
305
314
 
306
- if @inputs && ! task.inputs.nil?
315
+ if @inputs && task.respond_to?(:inputs) && ! task.inputs.nil?
307
316
  info_inputs = @inputs.collect do |i|
308
317
  if Path === i
309
318
  i.to_s
@@ -323,7 +332,7 @@ class Step
323
332
  end
324
333
 
325
334
  set_info :started, (start_time = Time.now)
326
- log :started, "Starting step #{Log.color :yellow, task.name.to_s || ""}"
335
+ log :started, "Starting step #{Log.color :yellow, task_name}"
327
336
 
328
337
  config_keys_pre = Rbbt::Config::GOT_KEYS.dup
329
338
  begin
@@ -357,7 +366,7 @@ class Step
357
366
  end
358
367
 
359
368
  if stream
360
- log :streaming, "Streaming step #{Log.color :yellow, task.name.to_s || ""}"
369
+ log :streaming, "Streaming step #{Log.color :yellow, task_name.to_s || ""}"
361
370
 
362
371
  callback = Proc.new do
363
372
  if AbortedStream === stream
@@ -377,7 +386,7 @@ class Step
377
386
  :time_elapsed => (time_elapsed = done_time - start_time),
378
387
  :versions => Rbbt.versions
379
388
  })
380
- log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
389
+ log :done, "Completed step #{Log.color :yellow, task_name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
381
390
  end
382
391
  end
383
392
  rescue
@@ -393,7 +402,7 @@ class Step
393
402
  if exception
394
403
  self.exception exception
395
404
  else
396
- log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
405
+ log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task_name.to_s || ""}" if status == :streaming
397
406
  end
398
407
  _clean_finished
399
408
  rescue
@@ -419,7 +428,6 @@ class Step
419
428
  })
420
429
  log :ending
421
430
  Step.purge_stream_cache
422
- Open.rm pid_file if Open.exist?(pid_file)
423
431
  end
424
432
 
425
433
  set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
@@ -430,16 +438,17 @@ class Step
430
438
  if result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
431
439
  Open.mv self.tmp_path, self.path
432
440
  end
441
+ Open.rm pid_file if Open.exist?(pid_file) unless stream
433
442
  result
434
443
  end # END PERSIST
435
- log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
444
+ log :done, "Completed step #{Log.color :yellow, task_name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
436
445
 
437
446
  if no_load
438
447
  @result ||= result
439
448
  self
440
449
  else
441
450
  Step.purge_stream_cache
442
- @result = prepare_result result, @task.result_description
451
+ @result = prepare_result result, result_description
443
452
  end
444
453
  end # END SYNC
445
454
  res
@@ -98,7 +98,7 @@ class Step
98
98
  end
99
99
 
100
100
  def done?
101
- @done ||= path and Open.exists? path
101
+ @done ||= path and Open.exists?(path)
102
102
  end
103
103
 
104
104
  def streaming?
@@ -3,20 +3,12 @@ require 'rbbt/persist/tsv'
3
3
  require 'rbbt/util/log'
4
4
  require 'rbbt/util/semaphore'
5
5
  require 'rbbt/workflow/step/accessor'
6
- require 'rbbt/workflow/step/prepare'
6
+ require 'rbbt/workflow/step/produce'
7
7
  require 'rbbt/workflow/step/status'
8
8
  require 'rbbt/workflow/step/info'
9
9
  require 'rbbt/workflow/step/save_load_inputs'
10
10
 
11
11
  class Step
12
- attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
13
- attr_accessor :task_name, :overriden
14
- attr_accessor :pid
15
- attr_accessor :exec
16
- attr_accessor :relocated
17
- attr_accessor :result, :mutex, :seen
18
- attr_accessor :real_inputs, :original_task_name, :original_workflow
19
-
20
12
  RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
21
13
 
22
14
  class << self
@@ -38,10 +30,12 @@ class Step
38
30
  end
39
31
 
40
32
 
41
- def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
33
+ def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil, &block)
42
34
  path = Path.setup(Misc.sanitize_filename(path)) if String === path
43
35
  path = path.call if Proc === path
44
36
 
37
+ task = block if block_given?
38
+
45
39
  @path = path
46
40
  @task = task
47
41
  @bindings = bindings
@@ -73,7 +67,8 @@ class Step
73
67
 
74
68
  load_inputs_from_info if @inputs.nil?
75
69
 
76
- NamedArray.setup(@inputs, task.inputs) if task && task.inputs && !(NamedArray === @inputs)
70
+ task_inputs = task.respond_to?(:inputs) ? task.inputs : nil
71
+ NamedArray.setup(@inputs, task_inputs) unless NamedArray === @inputs
77
72
 
78
73
  @inputs || []
79
74
  end
@@ -133,15 +128,6 @@ class Step
133
128
  v
134
129
  end
135
130
 
136
- def task_name
137
- @task_name ||= begin
138
- if @task.nil?
139
- @path.split("/")[-2]
140
- else
141
- @task.name
142
- end
143
- end
144
- end
145
131
 
146
132
  def path
147
133
  if Proc === @path
@@ -155,8 +141,11 @@ class Step
155
141
  attr_accessor :log_relay_step
156
142
  end
157
143
 
144
+
145
+ # May be deprecated: This is no loger used
158
146
  def relay_log(step)
159
- return self unless Task === self.task and not self.task.name.nil?
147
+ return self if self.task_name.nil?
148
+
160
149
  if not self.respond_to? :original_log
161
150
  class << self
162
151
  attr_accessor :relay_step
@@ -164,77 +153,58 @@ class Step
164
153
  def log(status, message = nil)
165
154
  self.status = status
166
155
  message Log.uncolor message
167
- relay_step.log([task.name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
156
+ relay_step.log([self.task_name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
168
157
  end
169
158
  end
170
159
  end
171
160
  @relay_step = step
172
- self
173
- end
174
161
 
175
- def result_type
176
- @result_type ||= if @task.nil?
177
- info[:result_type] || :binary
178
- else
179
- @task.result_type || info[:result_type] || :string
180
- end
181
- end
182
-
183
- def result_type=(type)
184
- @result_type = type
185
- end
186
-
187
- def result_description
188
- @result_description ||= if @task.nil?
189
- info[:result_description]
190
- else
191
- @task.result_description
192
- end
162
+ self
193
163
  end
194
164
 
195
165
  def prepare_result(value, description = nil, entity_info = nil)
196
166
  res = case
197
- when IO === value
198
- begin
199
- res = case result_type
200
- when :array
201
- array = []
202
- while line = value.gets
203
- array << line.chomp
204
- end
205
- array
206
- when :tsv
207
- begin
208
- TSV.open(value)
209
- rescue IOError
210
- TSV.setup({})
211
- end
212
- else
213
- value.read
214
- end
215
- value.join if value.respond_to? :join
216
- res
217
- rescue Exception
218
- value.abort if value.respond_to? :abort
219
- self.abort
220
- raise $!
221
- end
222
- when (not defined? Entity or description.nil? or not Entity.formats.include? description)
223
- value
224
- when (Annotated === value and info.empty?)
225
- value
226
- when Annotated === value
227
- annotations = value.annotations
228
- entity_info ||= begin
229
- entity_info = info.dup
230
- entity_info.merge! info[:inputs] if info[:inputs]
231
- entity_info
167
+ when IO === value
168
+ begin
169
+ res = case result_type
170
+ when :array
171
+ array = []
172
+ while line = value.gets
173
+ array << line.chomp
232
174
  end
233
- entity_info.each do |k,v|
234
- value.send("#{h}=", v) if annotations.include? k
235
- end
236
-
237
- value
175
+ array
176
+ when :tsv
177
+ begin
178
+ TSV.open(value)
179
+ rescue IOError
180
+ TSV.setup({})
181
+ end
182
+ else
183
+ value.read
184
+ end
185
+ value.join if value.respond_to? :join
186
+ res
187
+ rescue Exception
188
+ value.abort if value.respond_to? :abort
189
+ self.abort
190
+ raise $!
191
+ end
192
+ when (not defined? Entity or description.nil? or not Entity.formats.include? description)
193
+ value
194
+ when (Annotated === value and info.empty?)
195
+ value
196
+ when Annotated === value
197
+ annotations = value.annotations
198
+ entity_info ||= begin
199
+ entity_info = info.dup
200
+ entity_info.merge! info[:inputs] if info[:inputs]
201
+ entity_info
202
+ end
203
+ entity_info.each do |k,v|
204
+ value.send("#{h}=", v) if annotations.include? k
205
+ end
206
+
207
+ value
238
208
  else
239
209
  entity_info ||= begin
240
210
  entity_info = info.dup
@@ -371,8 +341,9 @@ class Step
371
341
  def step(name)
372
342
  @steps ||= {}
373
343
  @steps[name] ||= begin
344
+ name = name.to_sym
374
345
  deps = rec_dependencies.select{|step|
375
- step.task_name.to_sym == name.to_sym
346
+ step.task_name && step.task_name.to_sym == name
376
347
  }
377
348
  raise "Dependency step not found: #{ name }" if deps.empty?
378
349
  if (deps & self.dependencies).any?
@@ -385,3 +356,4 @@ class Step
385
356
  end
386
357
 
387
358
  require 'rbbt/workflow/step/run'
359
+ require 'rbbt/workflow/step/accessor'
@@ -88,13 +88,15 @@ module Task
88
88
  next
89
89
  end
90
90
 
91
+ task_name ||= task.name
92
+
91
93
  maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
92
94
  raise "Dependency task not found: #{dep}" if task.nil?
93
- next if seen.include? [wf, task.name, maps]
95
+ next if seen.include? [wf, task_name, maps]
94
96
 
95
97
  task.workflow = wf if wf
96
98
 
97
- seen << [wf, task.name, maps]
99
+ seen << [wf, task_name, maps]
98
100
  new_inputs = task.inputs - maps
99
101
  next unless new_inputs.any?
100
102
  if task_inputs[task].nil?
@@ -160,44 +160,50 @@ puts files * "\n"
160
160
  end
161
161
  end
162
162
 
163
- def self.migrate(path, search_path, options = {})
164
- if Step === path
165
- if options[:source]
166
- path = Rbbt.identify(path.path)
167
- else
168
- path = path.path
163
+ def self.migrate(paths, search_path, options = {})
164
+ subpath_files = {}
165
+ target_paths = []
166
+ resource = options[:resource] || Rbbt
167
+
168
+ target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
169
+
170
+ (Array === paths ? paths : [paths]).each do |path|
171
+ if Step === path
172
+ if options[:source]
173
+ path = Rbbt.identify(path.path)
174
+ else
175
+ path = path.path
176
+ end
169
177
  end
170
- end
171
- search_path = 'user' if search_path.nil?
178
+ search_path = 'user' if search_path.nil?
172
179
 
173
- resource = Rbbt
174
180
 
175
- path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
181
+ path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
176
182
 
177
- subpath_files = {}
178
- real_paths.sort.each do |path|
179
- parts = path.split("/")
180
- subpath = parts[0..-4] * "/" + "/"
183
+ real_paths.sort.each do |path|
184
+ parts = path.split("/")
185
+ subpath = parts[0..-4] * "/" + "/"
186
+
187
+ if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
188
+ subpath = subpath_files.keys.last
189
+ end
181
190
 
182
- if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
183
- subpath = subpath_files.keys.last
191
+ source = path.chars[subpath.length..-1] * ""
192
+
193
+ subpath_files[subpath] ||= []
194
+ subpath_files[subpath] << source
184
195
  end
185
196
 
186
- source = path.chars[subpath.length..-1] * ""
187
197
 
188
- subpath_files[subpath] ||= []
189
- subpath_files[subpath] << source
198
+ target_paths << File.join(target, *path.split("/")[-3..-1])
190
199
  end
191
200
 
192
- target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
193
-
194
- target_path = File.join(target, *path.split("/")[-3..-1])
195
201
 
196
202
  subpath_files.each do |subpath, files|
197
203
  Rbbt.migrate_files([subpath], target, options.merge(:files => files))
198
204
  end
199
205
 
200
- target_path
206
+ target_paths
201
207
  end
202
208
 
203
209
  def self.purge(path, recursive = false, skip_overriden = true)
data/lib/rbbt/workflow.rb CHANGED
@@ -212,11 +212,13 @@ module Workflow
212
212
  end
213
213
  workflow.load_documentation
214
214
 
215
+
215
216
  first ||= workflow
216
217
  end
217
- return first
218
218
 
219
- workflow
219
+ first.complete_name = wf_name
220
+
221
+ return first
220
222
  end
221
223
 
222
224
  attr_accessor :description
@@ -230,6 +232,12 @@ module Workflow
230
232
  attr_accessor :relay_tasks
231
233
 
232
234
  #{{{ ATTR DEFAULTS
235
+ #
236
+ attr_accessor :complete_name
237
+
238
+ def self.complete_name
239
+ @complete_name ||= self.to_s
240
+ end
233
241
 
234
242
  def self.workdir=(path)
235
243
  path = Path.setup path.dup unless Path === path
data/python/rbbt.py CHANGED
@@ -1,7 +1,93 @@
1
1
  import warnings
2
2
  import sys
3
3
  import os
4
- os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
5
4
 
6
5
  def rbbt():
7
6
  print("Rbbt")
7
+
8
+ def path(subdir = None, base_dir = None):
9
+ from pathlib import Path
10
+ import os
11
+
12
+ if (base_dir == None):
13
+ base_dir = os.path.join(Path.home(), ".rbbt")
14
+ if (subdir == None):
15
+ return base_dir
16
+ else:
17
+ return os.path.join(base_dir, subdir)
18
+
19
+ def inspect(obj):
20
+ print(dir(obj))
21
+
22
+ def log_tsv(tsv):
23
+ print(tsv)
24
+ print(tsv.keys())
25
+
26
+ def tsv_preamble(line, comment_char="#"):
27
+ import re
28
+ header = dict()
29
+ entries = re.sub(f"^{comment_char}:", '', line)
30
+ entries = re.sub(f"^{comment_char}:", '', line).split("#")
31
+ for entry in entries:
32
+ key, value = entry.split("=")
33
+ key = re.sub("^:","",key)
34
+ value = re.sub("^:","",value)
35
+ header[key] = value
36
+
37
+ return header
38
+
39
+
40
+ def tsv_header(filename, sep="\t", comment_char="#"):
41
+ import re
42
+
43
+ f = open(filename)
44
+ line = f.readline().strip()
45
+
46
+ if (not line.startswith(comment_char)):
47
+ header = {"fields":None, "type":"list", "start": 0}
48
+ else:
49
+ header = dict()
50
+ start = 0
51
+ if (line.startswith(f"{comment_char}:")):
52
+ header["preamble"]=tsv_preamble(line, comment_char)
53
+ if ("type" in header["preamble"]):
54
+ header["type"] = header["preamble"]["type"]
55
+ line = f.readline().strip()
56
+ start = 1
57
+
58
+ if (line.startswith(comment_char)):
59
+ header["all_fields"] = re.sub(f"^{comment_char}", "", line).split(sep)
60
+ header["key_field"] = header["all_fields"][0]
61
+ header["fields"] = header["all_fields"][1:]
62
+
63
+ header["start"] = start
64
+
65
+ f.close()
66
+ return header
67
+
68
+
69
+ def tsv_pandas(filename, sep="\t", comment_char="#", index_col=0, **kwargs):
70
+ import pandas
71
+
72
+ if (comment_char == ""):
73
+ tsv = pandas.read_table(filename, sep=sep, index_col=index_col, **kwargs)
74
+ else:
75
+ header = tsv_header(filename, sep=sep, comment_char="#")
76
+
77
+ if ("type" in header and header["type"] == "flat"):
78
+ return None
79
+ else:
80
+ if ("sep" in header):
81
+ sep=header["sep"]
82
+
83
+ tsv = pandas.read_table(filename, sep=sep, index_col=index_col, header=header["start"], **kwargs)
84
+
85
+ if ("fields" in header):
86
+ tsv.columns = header["fields"]
87
+ tsv.index.name = header["key_field"]
88
+
89
+ return tsv
90
+
91
+ def tsv(*args, **kwargs):
92
+ return tsv_pandas(*args, **kwargs)
93
+
@@ -54,8 +54,9 @@ tsvfile, excelfile = ARGV
54
54
 
55
55
  excelfile, tsvfile = tsvfile, nil if excelfile.nil? and tsvfile =~ /\.(xls|xlsx)$/
56
56
 
57
- raise ParameterException, "No files given" if tsvfile.nil? and excelfile.nil?
58
- raise ParameterException, "No excel file given" if excelfile.nil?
57
+ raise ParameterException, "No TSV file given" if tsvfile.nil?
58
+
59
+ excelfile = tsvfile.sub(/\.tsv(\.gz)?$/, '.xlsx') if excelfile.nil?
59
60
 
60
61
  require 'rbbt/tsv/excel'
61
62
  TSV.open(tsvfile, options).excel(excelfile, options)
@@ -213,11 +213,15 @@ The `recursive_clean` cleans all the job dependency steps recursively.
213
213
  -prov--provenance Report the jobs provenance
214
214
  -W--workflows* Load a list of workflows
215
215
  -R--requires* Require a list of files
216
- -pre--prepare* Prepare dependencies
217
- -prec--prepare_cpus* Number of dependencies prepared in parallel
216
+ -pro--produce* Prepare dependencies
217
+ -proc--produce_cpus* Number of dependencies prepared in parallel
218
218
  -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
219
219
  -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
220
220
  -PERF--procpath_performance* Measure performance using procpath
221
+ -r--relay* Relay job to SSH server
222
+ -sr--slurm_relay* Relay job to SSH SLURM server
223
+ -rdep--relay_dependencies* Relay dependencies instead of main job
224
+ -pdr--produce_dependencies_for_relay Prepare dependencies previous to relay jobs
221
225
  EOF
222
226
 
223
227
  workflow = ARGV.shift
@@ -415,13 +419,53 @@ begin
415
419
  exit 0
416
420
  end
417
421
 
418
- if tasks = options.delete(:prepare)
422
+ if tasks = options.delete(:produce)
419
423
  tasks = tasks.split(",")
420
- prepare_cpus = (options[:prepare_cpus] || 1)
421
- puts Step.prepare_dependencies(job, tasks, prepare_cpus)
424
+ produce_cpus = (options[:produce_cpus] || 1)
425
+ puts Step.produce_dependencies(job, tasks, prepare_cpus)
422
426
  exit 0
423
427
  end
424
428
 
429
+ def match_dependencies(queries, dependencies)
430
+ queries = queries.collect{|q| q.include?("#") ? q.split("#") : q }
431
+ dependencies.select do |dep|
432
+ queries.select do |q|
433
+ if Array === q
434
+ q.first == dep.workflow.to_s && q.last == dep.task_name.to_s
435
+ else
436
+ q.to_s == dep.task_name.to_s
437
+ end
438
+ end.any?
439
+ end
440
+ end
441
+
442
+ if server = options.delete(:relay)
443
+ require 'rbbt/workflow/remote_workflow'
444
+ relay_dependencies = options.delete(:relay_dependencies).split(",")
445
+ produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
446
+ if relay_dependencies
447
+ match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
448
+ RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
449
+ end
450
+ else
451
+ RemoteWorkflow::SSH.relay_job(job, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
452
+ end
453
+ end
454
+
455
+ if server = options.delete(:slurm_relay)
456
+ require 'rbbt/workflow/remote_workflow'
457
+ relay_dependencies = options.delete(:relay_dependencies).split(",")
458
+ produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
459
+ if relay_dependencies
460
+ match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
461
+ RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
462
+ end
463
+ else
464
+ RemoteWorkflow::SSH.relay_job(job, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
465
+ end
466
+ end
467
+
468
+
425
469
  if options[:procpath_performance]
426
470
  require 'rbbt/util/procpath'
427
471
  current_pid = job.info[:pid]
@@ -513,4 +513,92 @@ class TestWorkflow < Test::Unit::TestCase
513
513
  assert_equal "TLA", job.run
514
514
  assert job.overriden
515
515
  end
516
+
517
+ def test_anonymous_workflow
518
+ workflow = Module.new
519
+ workflow.extend Workflow
520
+ workflow.define_singleton_method(:to_s){"TestWorkflow"}
521
+
522
+ assert_equal "TestWorkflow", workflow.to_s
523
+
524
+ t1 = workflow.task :s1 => :string do
525
+ task_name.to_s
526
+ end
527
+
528
+ workflow.dep :s1
529
+ t2 = workflow.task :s2 => :string do
530
+ task_name.to_s
531
+ end
532
+
533
+ workflow.dep :s2
534
+ t3 = workflow.task :s3 => :string do
535
+ task_name.to_s
536
+ end
537
+
538
+ assert_equal "s3", workflow.job(:s3).run
539
+ end
540
+
541
+ def test_low_level_step
542
+ Log.with_severity 0 do
543
+ TmpFile.with_file do |tmpdir|
544
+ Path.setup tmpdir
545
+
546
+ s1 = Step.new tmpdir.s1 do "s1" end
547
+ s1.task_name = "s1"
548
+
549
+ # run clean run
550
+ assert_equal "s1", s1.run
551
+ s1.clean
552
+ assert_equal "s1", s1.run
553
+ assert_equal "s1", s1.run
554
+
555
+ # is persisted because it raises RbbtException
556
+ s1.task = proc do raise RbbtException end
557
+ assert_equal "s1", s1.run
558
+ s1.clean
559
+ assert_raises RbbtException do s1.run end
560
+ assert RbbtException === s1.get_exception
561
+ assert s1.error?
562
+
563
+ s1.recursive_clean
564
+ s1.task = proc do "s1" end
565
+
566
+ # add dependencies
567
+ s2 = Step.new tmpdir.s2 do
568
+ step(:s1).load + " -> s2"
569
+ end
570
+ s2.task_name = "s2"
571
+ s2.dependencies << s1
572
+
573
+ s3 = Step.new tmpdir.s3 do
574
+ step(:s2).load + " -> s3"
575
+ end
576
+ s3.task_name = "s3"
577
+ s3.dependencies << s2
578
+
579
+ assert_equal "s1 -> s2", s2.run
580
+ assert_equal "s1 -> s2 -> s3", s3.run
581
+
582
+ # Test recusive behaviour
583
+ s1.task = proc do raise RbbtException end
584
+ ComputeDependency.setup(s1, :produce)
585
+ s1.clean
586
+ s3.clean
587
+ s3.run
588
+ assert_equal "s1 -> s2 -> s3", s3.run
589
+
590
+ s1.task = proc do raise RbbtException end
591
+ ComputeDependency.setup(s1, :produce)
592
+ s3.recursive_clean
593
+ assert_raises RbbtException do s3.run end
594
+
595
+ s1.task = proc{|v| v }
596
+ s1.inputs = ["test"]
597
+ assert_equal "test", s1.run
598
+ s3.recursive_clean
599
+ assert_equal "test -> s2 -> s3", s3.run
600
+ end
601
+ end
602
+ end
603
+
516
604
  end
@@ -131,5 +131,10 @@ def python_print():
131
131
  random = RbbtPython.import_method :torch, :rand, :random
132
132
  assert random.call(1).numpy.to_f > 0
133
133
  end
134
+
135
+ def test_class_new_obj
136
+ obj = RbbtPython.class_new_obj("torch.nn", "Module")
137
+ assert_equal "Module()", obj.to_s
138
+ end
134
139
  end
135
140
 
@@ -294,5 +294,6 @@ class TestWorkflowDependency < Test::Unit::TestCase
294
294
  assert job.dependencies.first.file('foo').exists?
295
295
  assert_equal 'done'.reverse, job.run
296
296
  end
297
+
297
298
  end
298
299
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.35.3
4
+ version: 5.36.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-02-08 00:00:00.000000000 Z
11
+ date: 2023-03-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -366,7 +366,7 @@ files:
366
366
  - lib/rbbt/workflow/step/accessor.rb
367
367
  - lib/rbbt/workflow/step/dependencies.rb
368
368
  - lib/rbbt/workflow/step/info.rb
369
- - lib/rbbt/workflow/step/prepare.rb
369
+ - lib/rbbt/workflow/step/produce.rb
370
370
  - lib/rbbt/workflow/step/run.rb
371
371
  - lib/rbbt/workflow/step/save_load_inputs.rb
372
372
  - lib/rbbt/workflow/step/status.rb