rbbt-util 5.35.3 → 5.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/persist/tsv/adapter.rb +3 -1
- data/lib/rbbt/persist.rb +2 -0
- data/lib/rbbt/resource/path.rb +12 -2
- data/lib/rbbt/util/python.rb +10 -0
- data/lib/rbbt/workflow/definition.rb +2 -0
- data/lib/rbbt/workflow/doc.rb +1 -0
- data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +60 -20
- data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +18 -3
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +1 -0
- data/lib/rbbt/workflow/step/accessor.rb +42 -5
- data/lib/rbbt/workflow/step/dependencies.rb +25 -22
- data/lib/rbbt/workflow/step/{prepare.rb → produce.rb} +1 -1
- data/lib/rbbt/workflow/step/run.rb +37 -28
- data/lib/rbbt/workflow/step/status.rb +1 -1
- data/lib/rbbt/workflow/step.rb +55 -83
- data/lib/rbbt/workflow/task.rb +4 -2
- data/lib/rbbt/workflow/util/archive.rb +29 -23
- data/lib/rbbt/workflow.rb +10 -2
- data/python/rbbt.py +87 -1
- data/share/rbbt_commands/tsv/write_excel +3 -2
- data/share/rbbt_commands/workflow/task +49 -5
- data/test/rbbt/test_workflow.rb +88 -0
- data/test/rbbt/util/test_python.rb +5 -0
- data/test/rbbt/workflow/step/test_dependencies.rb +1 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d105d6c21ad018f28acf6a234feae8d81f327f3f94f228c01f118abed6c386f6
|
4
|
+
data.tar.gz: 5c883e0611f02203d394ec02e448664efdc42361be04e9d873e9ba3f5e6cc92e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 20a4bccdeb0c587041b7f20f09bda19949713fcb362911a249b08112399c09178f8e50b5ae90f390bd6cd1389b3c9a47b7374c4fc05a83b073f1d43bdfdc6611
|
7
|
+
data.tar.gz: 1b3b112106b4ac1e261bcf1630ecaeea0ed27f1975aeead9694708241e9268fdac36da4f5b4be25c12288cb7f97b8b4163815b8d9010aba40f6f2434e654f99c
|
data/lib/rbbt/persist.rb
CHANGED
data/lib/rbbt/resource/path.rb
CHANGED
@@ -319,7 +319,8 @@ module Path
|
|
319
319
|
|
320
320
|
|
321
321
|
def open(options = {}, &block)
|
322
|
-
Open.
|
322
|
+
file = Open.remote?(self) || Open.ssh?(self) ? self : self.produce.find
|
323
|
+
Open.open(file, options, &block)
|
323
324
|
end
|
324
325
|
|
325
326
|
def to_s
|
@@ -331,7 +332,16 @@ module Path
|
|
331
332
|
end
|
332
333
|
|
333
334
|
def tsv(*args)
|
334
|
-
|
335
|
+
begin
|
336
|
+
path = self.produce
|
337
|
+
rescue Resource::ResourceNotFound => e
|
338
|
+
begin
|
339
|
+
path = self.set_extension('tsv').produce
|
340
|
+
rescue Resource::ResourceNotFound
|
341
|
+
raise e
|
342
|
+
end
|
343
|
+
end
|
344
|
+
TSV.open(path, *args)
|
335
345
|
end
|
336
346
|
|
337
347
|
def tsv_options(options = {})
|
data/lib/rbbt/util/python.rb
CHANGED
@@ -58,6 +58,16 @@ module RbbtPython
|
|
58
58
|
RbbtPython.import_method(module_name, method_name).call(*args)
|
59
59
|
end
|
60
60
|
|
61
|
+
def self.get_class(module_name, class_name)
|
62
|
+
save_module_name = module_name.to_s.gsub(".", "_")
|
63
|
+
RbbtPython.pyimport(module_name, as: save_module_name)
|
64
|
+
RbbtPython.send(save_module_name).send(class_name)
|
65
|
+
end
|
66
|
+
|
67
|
+
def self.class_new_obj(module_name, class_name, args={})
|
68
|
+
RbbtPython.get_class(module_name, class_name).new(**args)
|
69
|
+
end
|
70
|
+
|
61
71
|
def self.exec(script)
|
62
72
|
PyCall.exec(script)
|
63
73
|
end
|
data/lib/rbbt/workflow/doc.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
class RemoteWorkflow
|
2
|
+
RBBT_DEBUG_REMOTE_JSON = ENV["RBBT_DEBUG_REMOTE_JSON"] == 'true'
|
3
|
+
|
2
4
|
module SSH
|
3
5
|
#def self.run(server, script)
|
4
6
|
# Log.debug "Run ssh script in #{server}:\n#{script}"
|
@@ -21,9 +23,16 @@ class RemoteWorkflow
|
|
21
23
|
|
22
24
|
workflow, task, job, *rest = path.split("/")
|
23
25
|
|
26
|
+
workflow_name = begin
|
27
|
+
workflow = Kernel.const_get(workflow) if String === workflow
|
28
|
+
workflow.respond_to?(:complete_name) ? workflow.complete_name : workflow
|
29
|
+
rescue
|
30
|
+
workflow
|
31
|
+
end
|
32
|
+
|
24
33
|
script =<<-EOF
|
25
34
|
require 'rbbt/workflow'
|
26
|
-
wf = Workflow.require_workflow "#{
|
35
|
+
wf = Workflow.require_workflow "#{workflow_name}"
|
27
36
|
EOF
|
28
37
|
|
29
38
|
case task
|
@@ -86,7 +95,7 @@ STDOUT.write res.to_json
|
|
86
95
|
EOF
|
87
96
|
|
88
97
|
json = Misc.ssh_run(server, script)
|
89
|
-
Log.debug "JSON (#{ url }): #{json}"
|
98
|
+
Log.debug "JSON (#{ url }): #{json}" if RBBT_DEBUG_REMOTE_JSON
|
90
99
|
JSON.parse(json)
|
91
100
|
end
|
92
101
|
|
@@ -172,11 +181,17 @@ job.clean
|
|
172
181
|
def self.upload_inputs(server, inputs, input_types, input_id)
|
173
182
|
TmpFile.with_file do |dir|
|
174
183
|
if Step.save_inputs(inputs, input_types, dir)
|
175
|
-
Dir.glob(File.join(dir, "*.as_step")).each do |file|
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
184
|
+
# Dir.glob(File.join(dir, "*.as_step")).each do |file|
|
185
|
+
# Log.medium "Migrating Step input #{file} #{ server }"
|
186
|
+
# path = Open.read(file).strip
|
187
|
+
# new = Step.migrate(path, :user, :target => server)
|
188
|
+
# Open.write(file, new)
|
189
|
+
# end
|
190
|
+
|
191
|
+
paths = Dir.glob(File.join(dir, "*.as_step")).collect{|f| Open.read(f).strip }
|
192
|
+
new = Step.migrate(paths, :user, :target => server)
|
193
|
+
paths.zip(new).each{|file,new| Open.write(file, new) }
|
194
|
+
|
180
195
|
CMD.cmd_log("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
|
181
196
|
end
|
182
197
|
end
|
@@ -204,31 +219,56 @@ job.clean
|
|
204
219
|
# rjob.run
|
205
220
|
#end
|
206
221
|
|
222
|
+
def self.upload_dependencies(job, server, search_path = 'user', produce_dependencies = false)
|
223
|
+
server, path = parse_url(server) if server =~ /^ssh:\/\//
|
224
|
+
job.dependencies.each do |dep|
|
225
|
+
Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
|
226
|
+
dep.produce
|
227
|
+
end if produce_dependencies
|
228
|
+
|
229
|
+
job.input_dependencies.each do |dep|
|
230
|
+
Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
|
231
|
+
dep.produce
|
232
|
+
end if produce_dependencies
|
233
|
+
|
234
|
+
migrate_dependencies = job.rec_dependencies.select{|d| d.done? }.collect{|d| d.path }
|
235
|
+
Log.medium "Migrating #{migrate_dependencies.length} dependencies to #{ server }"
|
236
|
+
Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
|
237
|
+
end
|
238
|
+
|
239
|
+
def self.missing_dep_inputs(job)
|
240
|
+
inputs = job.inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
|
241
|
+
job.dependencies.each do |dep|
|
242
|
+
next if dep.done?
|
243
|
+
iif [dep, dep.inputs, dep.real_inputs]
|
244
|
+
inputs = dep.inputs.to_hash.slice(*dep.real_inputs.map{|i| i.to_s}).merge(inputs)
|
245
|
+
inputs = missing_dep_inputs(dep).merge(inputs)
|
246
|
+
end
|
247
|
+
inputs
|
248
|
+
end
|
249
|
+
|
207
250
|
def self.relay_job(job, server, options = {})
|
208
|
-
migrate, produce, produce_dependencies, search_path = Misc.process_options options.dup,
|
209
|
-
:migrate, :produce, :produce_dependencies, :search_path
|
251
|
+
migrate, produce, produce_dependencies, search_path, run_type, slurm_options = Misc.process_options options.dup,
|
252
|
+
:migrate, :produce, :produce_dependencies, :search_path, :run_type, :slurm_options
|
210
253
|
|
211
254
|
search_path ||= 'user'
|
212
255
|
|
213
256
|
produce = true if migrate
|
214
257
|
|
215
258
|
workflow_name = job.workflow.to_s
|
216
|
-
|
217
|
-
|
218
|
-
job.
|
219
|
-
dep.produce
|
220
|
-
end if options[:produce_dependencies]
|
221
|
-
|
222
|
-
job.rec_dependencies.each do |dep|
|
223
|
-
Step.migrate(dep.path, search_path, :target => server) if dep.done?
|
224
|
-
end
|
259
|
+
remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{workflow_name}", "#{workflow_name}")
|
260
|
+
inputs = job.recursive_inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
|
261
|
+
Log.medium "Relaying dependency #{job.workflow}:#{job.short_path} to #{server} (#{inputs.keys * ", "})"
|
225
262
|
|
226
|
-
|
263
|
+
upload_dependencies(job, server, search_path, options[:produce_dependencies])
|
227
264
|
rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
|
228
265
|
|
229
266
|
override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
|
230
267
|
rjob.override_dependencies = override_dependencies
|
231
268
|
|
269
|
+
rjob.run_type = run_type
|
270
|
+
rjob.slurm_options = slurm_options || {}
|
271
|
+
|
232
272
|
if options[:migrate]
|
233
273
|
rjob.produce
|
234
274
|
Step.migrate(Rbbt.identify(job.path), 'user', :source => server)
|
@@ -255,7 +295,7 @@ job.clean
|
|
255
295
|
@task_info ||= IndiferentHash.setup({})
|
256
296
|
|
257
297
|
if @task_info[task].nil?
|
258
|
-
task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url, task.to_s))
|
298
|
+
task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url || @url, task.to_s))
|
259
299
|
task_info = RemoteWorkflow::SSH.fix_hash(task_info)
|
260
300
|
|
261
301
|
task_info[:result_type] = task_info[:result_type].to_sym if task_info[:result_type]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
class RemoteStep
|
2
2
|
module SSH
|
3
|
-
attr_accessor :override_dependencies
|
3
|
+
attr_accessor :override_dependencies, :run_type, :slurm_options
|
4
4
|
|
5
5
|
def init_job(cache_type = nil, other_params = {})
|
6
6
|
return self if @url
|
@@ -55,12 +55,27 @@ class RemoteStep
|
|
55
55
|
RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
|
56
56
|
end
|
57
57
|
|
58
|
+
def _run_slurm
|
59
|
+
RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
|
60
|
+
end
|
61
|
+
|
62
|
+
def _orchestrate_slurm
|
63
|
+
RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
|
64
|
+
end
|
65
|
+
|
58
66
|
def produce(*args)
|
59
67
|
input_types = {}
|
60
68
|
init_job
|
61
|
-
@remote_path =
|
69
|
+
@remote_path = case @run_type
|
70
|
+
when 'run', :run, nil
|
71
|
+
_run
|
72
|
+
when 'slurm', :slurm
|
73
|
+
_run_slurm
|
74
|
+
when 'orchestrate', :orchestrate
|
75
|
+
_orchestrate_slurm
|
76
|
+
end
|
62
77
|
@started = true
|
63
|
-
while ! (done? || error?)
|
78
|
+
while ! (done? || error? || aborted?)
|
64
79
|
sleep 1
|
65
80
|
end
|
66
81
|
raise self.get_exception if error?
|
@@ -1,4 +1,12 @@
|
|
1
1
|
class Step
|
2
|
+
attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
|
3
|
+
attr_accessor :task_name, :overriden
|
4
|
+
attr_accessor :pid
|
5
|
+
attr_accessor :exec
|
6
|
+
attr_accessor :relocated
|
7
|
+
attr_accessor :result, :mutex, :seen
|
8
|
+
attr_accessor :real_inputs, :original_task_name, :original_workflow
|
9
|
+
|
2
10
|
|
3
11
|
INFO_SERIALIZER = begin
|
4
12
|
if ENV["RBBT_INFO_SERIALIZER"]
|
@@ -86,12 +94,45 @@ class Step
|
|
86
94
|
end
|
87
95
|
end
|
88
96
|
|
97
|
+
def task_name
|
98
|
+
@task_name ||= begin
|
99
|
+
if @task.respond_to?(:name)
|
100
|
+
|
101
|
+
@task.name
|
102
|
+
else
|
103
|
+
@path.split("/")[-2]
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def result_type
|
109
|
+
@result_type ||= if @task.respond_to?(:result_type)
|
110
|
+
@task.result_type || info[:result_type]
|
111
|
+
else
|
112
|
+
info[:result_type]
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def result_type=(type)
|
117
|
+
@result_type = type
|
118
|
+
end
|
119
|
+
|
120
|
+
def result_description
|
121
|
+
@result_description ||= if @task.respond_to?(:result_description)
|
122
|
+
@task.result_description
|
123
|
+
else
|
124
|
+
nil
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def result_description=(description)
|
129
|
+
@result_description = description
|
130
|
+
end
|
89
131
|
|
90
132
|
def name
|
91
133
|
@name ||= path.sub(/.*\/#{Regexp.quote task_name.to_s}\/(.*)/, '\1')
|
92
134
|
end
|
93
135
|
|
94
|
-
|
95
136
|
def short_path
|
96
137
|
[task_name, name] * "/"
|
97
138
|
end
|
@@ -105,10 +146,6 @@ class Step
|
|
105
146
|
workflow.to_s + "#" + short_path
|
106
147
|
end
|
107
148
|
|
108
|
-
def task_name
|
109
|
-
@task_name ||= task.name
|
110
|
-
end
|
111
|
-
|
112
149
|
def task_signature
|
113
150
|
[workflow.to_s, task_name] * "#"
|
114
151
|
end
|
@@ -3,7 +3,7 @@ class Step
|
|
3
3
|
STREAM_CACHE = {}
|
4
4
|
STREAM_CACHE_MUTEX = Mutex.new
|
5
5
|
def self.purge_stream_cache
|
6
|
-
Log.debug "Purging dup. stream cache"
|
6
|
+
# Log.debug "Purging dup. stream cache"
|
7
7
|
STREAM_CACHE_MUTEX.synchronize do
|
8
8
|
STREAM_CACHE.clear
|
9
9
|
end
|
@@ -209,7 +209,7 @@ class Step
|
|
209
209
|
end
|
210
210
|
|
211
211
|
rescue TryAgain
|
212
|
-
Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
|
212
|
+
#Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
|
213
213
|
retry
|
214
214
|
rescue Aborted, Interrupt
|
215
215
|
Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
|
@@ -430,7 +430,6 @@ class Step
|
|
430
430
|
|
431
431
|
required_dep_paths = []
|
432
432
|
dep_step.each do |path,list|
|
433
|
-
#required_dep_paths << path if list.length > 1
|
434
433
|
required_dep_paths << path if (list & dependencies).any?
|
435
434
|
end
|
436
435
|
|
@@ -442,42 +441,46 @@ class Step
|
|
442
441
|
[dep.path] + dep.input_dependencies
|
443
442
|
end.flatten)
|
444
443
|
|
445
|
-
log :dependencies, "Dependencies for step #{Log.color :yellow, task.name.to_s || ""}"
|
446
444
|
|
447
445
|
pre_deps = []
|
448
|
-
|
449
|
-
|
446
|
+
simple_dependencies = []
|
447
|
+
compute_simple_dependencies = {}
|
450
448
|
compute_last_deps = {}
|
451
449
|
seen_paths = Set.new
|
452
450
|
rec_dependencies.uniq.each do |step|
|
453
451
|
next if seen_paths.include? step.path
|
454
452
|
seen_paths << step.path
|
455
453
|
next unless required_dep_paths.include? step.path
|
456
|
-
|
457
|
-
|
458
|
-
|
454
|
+
required_seen_paths = seen_paths & required_dep_paths
|
455
|
+
|
456
|
+
internal = step.inputs.select{|i| Step == i && required_paths.include?(i.path) && seen_paths.include?(i.path) }
|
457
|
+
|
458
|
+
if ComputeDependency === step
|
459
|
+
next if produced.include? step.path
|
460
|
+
if internal
|
459
461
|
compute_last_deps[step.compute] ||= []
|
460
462
|
compute_last_deps[step.compute] << step
|
461
463
|
else
|
462
|
-
|
464
|
+
compute_simple_dependencies[step.compute] ||= []
|
465
|
+
compute_simple_dependencies[step.compute] << step
|
463
466
|
end
|
464
467
|
else
|
465
|
-
if
|
466
|
-
|
467
|
-
compute_pre_deps[step.compute] ||= []
|
468
|
-
compute_pre_deps[step.compute] << step
|
468
|
+
if internal
|
469
|
+
simple_dependencies.prepend(step)
|
469
470
|
else
|
470
|
-
|
471
|
+
simple_dependencies << step
|
471
472
|
end
|
472
473
|
end
|
473
474
|
end
|
474
475
|
|
475
|
-
|
476
|
-
|
476
|
+
log :dependencies, "Processing dependencies for #{Log.color :yellow, task_name.to_s || ""}" if compute_simple_dependencies.any? || simple_dependencies.any? || compute_last_deps.any?
|
477
|
+
|
478
|
+
Log.debug "compute_simple_dependencies: #{Misc.fingerprint(compute_simple_dependencies)} - #{Log.color :blue, self.path}" if compute_simple_dependencies.any?
|
479
|
+
compute_simple_dependencies.each do |type,list|
|
477
480
|
run_compute_dependencies(type, list, dep_step)
|
478
481
|
end
|
479
482
|
|
480
|
-
Log.
|
483
|
+
Log.low "pre_deps: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
|
481
484
|
pre_deps.each do |step|
|
482
485
|
next if compute_deps.include? step
|
483
486
|
begin
|
@@ -487,8 +490,8 @@ class Step
|
|
487
490
|
end
|
488
491
|
end
|
489
492
|
|
490
|
-
Log.
|
491
|
-
|
493
|
+
Log.debug "simple_dependencies: #{Misc.fingerprint(simple_dependencies)} - #{Log.color :blue, self.path}" if simple_dependencies.any?
|
494
|
+
simple_dependencies.each do |step|
|
492
495
|
next if compute_deps.include? step
|
493
496
|
begin Exception
|
494
497
|
execute_and_dup(step, dep_step)
|
@@ -497,8 +500,8 @@ class Step
|
|
497
500
|
end
|
498
501
|
end
|
499
502
|
|
500
|
-
Log.
|
501
|
-
|
503
|
+
Log.low "compute_last_deps: #{Misc.fingerprint(compute_simple_dependencies)} - #{Log.color :blue, self.path}" if compute_simple_dependencies.any?
|
504
|
+
compute_simple_dependencies.each do |type,list|
|
502
505
|
run_compute_dependencies(type, list, dep_step)
|
503
506
|
end
|
504
507
|
|
@@ -96,7 +96,11 @@ class Step
|
|
96
96
|
@exec = true if @exec.nil?
|
97
97
|
begin
|
98
98
|
old = Signal.trap("INT"){ Thread.current.raise Aborted }
|
99
|
-
@task.
|
99
|
+
if @task.respond_to?(:exec_in)
|
100
|
+
@task.exec_in((bindings || self), *@inputs)
|
101
|
+
else
|
102
|
+
(bindings || self).instance_exec *@inputs, &@task
|
103
|
+
end
|
100
104
|
ensure
|
101
105
|
Signal.trap("INT", old)
|
102
106
|
end
|
@@ -223,14 +227,15 @@ class Step
|
|
223
227
|
end
|
224
228
|
|
225
229
|
begin
|
226
|
-
|
227
|
-
|
228
|
-
no_load = :stream if no_load
|
230
|
+
no_load = :stream if no_load
|
231
|
+
result_type = self.result_type || info[:result_type]
|
229
232
|
|
233
|
+
res = @mutex.synchronize do
|
234
|
+
time_elapsed = total_time_elapsed = nil
|
230
235
|
Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
|
231
|
-
|
232
|
-
result_type = info[:result_type] if result_type.nil?
|
236
|
+
|
233
237
|
result = Persist.persist "Job", result_type, :file => path, :check => persist_checks, :no_load => no_load do
|
238
|
+
|
234
239
|
if Step === Step.log_relay_step and not self == Step.log_relay_step
|
235
240
|
relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
|
236
241
|
end
|
@@ -240,26 +245,30 @@ class Step
|
|
240
245
|
@exec = false
|
241
246
|
init_info(true)
|
242
247
|
|
243
|
-
|
248
|
+
workflow = @workflow || @task.respond_to?(:workflow) ? @task.workflow : nil
|
249
|
+
result_type = @task.respond_to?(:result_type) ? @task.result_type : nil
|
250
|
+
result_description = @task.respond_to?(:result_description) ? @task.result_description : nil
|
251
|
+
|
252
|
+
log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task_name}"
|
244
253
|
|
245
254
|
merge_info({
|
246
|
-
:issued
|
247
|
-
:name
|
248
|
-
:pid
|
249
|
-
:pid_hostname
|
250
|
-
:clean_name
|
251
|
-
:workflow
|
252
|
-
:task_name
|
253
|
-
:result_type
|
254
|
-
:result_description =>
|
255
|
-
:dependencies
|
256
|
-
:versions
|
255
|
+
:issued => (issue_time = Time.now),
|
256
|
+
:name => name,
|
257
|
+
:pid => Process.pid.to_s,
|
258
|
+
:pid_hostname => Socket.gethostname,
|
259
|
+
:clean_name => clean_name,
|
260
|
+
:workflow => workflow.to_s,
|
261
|
+
:task_name => task_name,
|
262
|
+
:result_type => result_type,
|
263
|
+
:result_description => result_description,
|
264
|
+
:dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
|
265
|
+
:versions => Rbbt.versions
|
257
266
|
})
|
258
267
|
|
259
268
|
new_inputs = []
|
260
269
|
@inputs.each_with_index do |input,i|
|
261
|
-
name = @task.inputs[i]
|
262
|
-
type = @task.input_types[
|
270
|
+
name = @task.respond_to?(:inputs) ? @task.inputs[i] : nil
|
271
|
+
type = @task.respond_to?(:input_types) ? @task.input_types[i] : nil
|
263
272
|
|
264
273
|
if type == :directory
|
265
274
|
directory_inputs = file('directory_inputs')
|
@@ -303,7 +312,7 @@ class Step
|
|
303
312
|
|
304
313
|
@inputs = new_inputs if @inputs
|
305
314
|
|
306
|
-
if @inputs && ! task.inputs.nil?
|
315
|
+
if @inputs && task.respond_to?(:inputs) && ! task.inputs.nil?
|
307
316
|
info_inputs = @inputs.collect do |i|
|
308
317
|
if Path === i
|
309
318
|
i.to_s
|
@@ -323,7 +332,7 @@ class Step
|
|
323
332
|
end
|
324
333
|
|
325
334
|
set_info :started, (start_time = Time.now)
|
326
|
-
log :started, "Starting step #{Log.color :yellow,
|
335
|
+
log :started, "Starting step #{Log.color :yellow, task_name}"
|
327
336
|
|
328
337
|
config_keys_pre = Rbbt::Config::GOT_KEYS.dup
|
329
338
|
begin
|
@@ -357,7 +366,7 @@ class Step
|
|
357
366
|
end
|
358
367
|
|
359
368
|
if stream
|
360
|
-
log :streaming, "Streaming step #{Log.color :yellow,
|
369
|
+
log :streaming, "Streaming step #{Log.color :yellow, task_name.to_s || ""}"
|
361
370
|
|
362
371
|
callback = Proc.new do
|
363
372
|
if AbortedStream === stream
|
@@ -377,7 +386,7 @@ class Step
|
|
377
386
|
:time_elapsed => (time_elapsed = done_time - start_time),
|
378
387
|
:versions => Rbbt.versions
|
379
388
|
})
|
380
|
-
log :done, "Completed step #{Log.color :yellow,
|
389
|
+
log :done, "Completed step #{Log.color :yellow, task_name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
|
381
390
|
end
|
382
391
|
end
|
383
392
|
rescue
|
@@ -393,7 +402,7 @@ class Step
|
|
393
402
|
if exception
|
394
403
|
self.exception exception
|
395
404
|
else
|
396
|
-
log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow,
|
405
|
+
log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task_name.to_s || ""}" if status == :streaming
|
397
406
|
end
|
398
407
|
_clean_finished
|
399
408
|
rescue
|
@@ -419,7 +428,6 @@ class Step
|
|
419
428
|
})
|
420
429
|
log :ending
|
421
430
|
Step.purge_stream_cache
|
422
|
-
Open.rm pid_file if Open.exist?(pid_file)
|
423
431
|
end
|
424
432
|
|
425
433
|
set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
|
@@ -430,16 +438,17 @@ class Step
|
|
430
438
|
if result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
|
431
439
|
Open.mv self.tmp_path, self.path
|
432
440
|
end
|
441
|
+
Open.rm pid_file if Open.exist?(pid_file) unless stream
|
433
442
|
result
|
434
443
|
end # END PERSIST
|
435
|
-
log :done, "Completed step #{Log.color :yellow,
|
444
|
+
log :done, "Completed step #{Log.color :yellow, task_name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
|
436
445
|
|
437
446
|
if no_load
|
438
447
|
@result ||= result
|
439
448
|
self
|
440
449
|
else
|
441
450
|
Step.purge_stream_cache
|
442
|
-
@result = prepare_result result,
|
451
|
+
@result = prepare_result result, result_description
|
443
452
|
end
|
444
453
|
end # END SYNC
|
445
454
|
res
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -3,20 +3,12 @@ require 'rbbt/persist/tsv'
|
|
3
3
|
require 'rbbt/util/log'
|
4
4
|
require 'rbbt/util/semaphore'
|
5
5
|
require 'rbbt/workflow/step/accessor'
|
6
|
-
require 'rbbt/workflow/step/
|
6
|
+
require 'rbbt/workflow/step/produce'
|
7
7
|
require 'rbbt/workflow/step/status'
|
8
8
|
require 'rbbt/workflow/step/info'
|
9
9
|
require 'rbbt/workflow/step/save_load_inputs'
|
10
10
|
|
11
11
|
class Step
|
12
|
-
attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
|
13
|
-
attr_accessor :task_name, :overriden
|
14
|
-
attr_accessor :pid
|
15
|
-
attr_accessor :exec
|
16
|
-
attr_accessor :relocated
|
17
|
-
attr_accessor :result, :mutex, :seen
|
18
|
-
attr_accessor :real_inputs, :original_task_name, :original_workflow
|
19
|
-
|
20
12
|
RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
|
21
13
|
|
22
14
|
class << self
|
@@ -38,10 +30,12 @@ class Step
|
|
38
30
|
end
|
39
31
|
|
40
32
|
|
41
|
-
def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
|
33
|
+
def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil, &block)
|
42
34
|
path = Path.setup(Misc.sanitize_filename(path)) if String === path
|
43
35
|
path = path.call if Proc === path
|
44
36
|
|
37
|
+
task = block if block_given?
|
38
|
+
|
45
39
|
@path = path
|
46
40
|
@task = task
|
47
41
|
@bindings = bindings
|
@@ -73,7 +67,8 @@ class Step
|
|
73
67
|
|
74
68
|
load_inputs_from_info if @inputs.nil?
|
75
69
|
|
76
|
-
|
70
|
+
task_inputs = task.respond_to?(:inputs) ? task.inputs : nil
|
71
|
+
NamedArray.setup(@inputs, task_inputs) unless NamedArray === @inputs
|
77
72
|
|
78
73
|
@inputs || []
|
79
74
|
end
|
@@ -133,15 +128,6 @@ class Step
|
|
133
128
|
v
|
134
129
|
end
|
135
130
|
|
136
|
-
def task_name
|
137
|
-
@task_name ||= begin
|
138
|
-
if @task.nil?
|
139
|
-
@path.split("/")[-2]
|
140
|
-
else
|
141
|
-
@task.name
|
142
|
-
end
|
143
|
-
end
|
144
|
-
end
|
145
131
|
|
146
132
|
def path
|
147
133
|
if Proc === @path
|
@@ -155,8 +141,11 @@ class Step
|
|
155
141
|
attr_accessor :log_relay_step
|
156
142
|
end
|
157
143
|
|
144
|
+
|
145
|
+
# May be deprecated: This is no loger used
|
158
146
|
def relay_log(step)
|
159
|
-
return self
|
147
|
+
return self if self.task_name.nil?
|
148
|
+
|
160
149
|
if not self.respond_to? :original_log
|
161
150
|
class << self
|
162
151
|
attr_accessor :relay_step
|
@@ -164,77 +153,58 @@ class Step
|
|
164
153
|
def log(status, message = nil)
|
165
154
|
self.status = status
|
166
155
|
message Log.uncolor message
|
167
|
-
relay_step.log([
|
156
|
+
relay_step.log([self.task_name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
|
168
157
|
end
|
169
158
|
end
|
170
159
|
end
|
171
160
|
@relay_step = step
|
172
|
-
self
|
173
|
-
end
|
174
161
|
|
175
|
-
|
176
|
-
@result_type ||= if @task.nil?
|
177
|
-
info[:result_type] || :binary
|
178
|
-
else
|
179
|
-
@task.result_type || info[:result_type] || :string
|
180
|
-
end
|
181
|
-
end
|
182
|
-
|
183
|
-
def result_type=(type)
|
184
|
-
@result_type = type
|
185
|
-
end
|
186
|
-
|
187
|
-
def result_description
|
188
|
-
@result_description ||= if @task.nil?
|
189
|
-
info[:result_description]
|
190
|
-
else
|
191
|
-
@task.result_description
|
192
|
-
end
|
162
|
+
self
|
193
163
|
end
|
194
164
|
|
195
165
|
def prepare_result(value, description = nil, entity_info = nil)
|
196
166
|
res = case
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
end
|
205
|
-
array
|
206
|
-
when :tsv
|
207
|
-
begin
|
208
|
-
TSV.open(value)
|
209
|
-
rescue IOError
|
210
|
-
TSV.setup({})
|
211
|
-
end
|
212
|
-
else
|
213
|
-
value.read
|
214
|
-
end
|
215
|
-
value.join if value.respond_to? :join
|
216
|
-
res
|
217
|
-
rescue Exception
|
218
|
-
value.abort if value.respond_to? :abort
|
219
|
-
self.abort
|
220
|
-
raise $!
|
221
|
-
end
|
222
|
-
when (not defined? Entity or description.nil? or not Entity.formats.include? description)
|
223
|
-
value
|
224
|
-
when (Annotated === value and info.empty?)
|
225
|
-
value
|
226
|
-
when Annotated === value
|
227
|
-
annotations = value.annotations
|
228
|
-
entity_info ||= begin
|
229
|
-
entity_info = info.dup
|
230
|
-
entity_info.merge! info[:inputs] if info[:inputs]
|
231
|
-
entity_info
|
167
|
+
when IO === value
|
168
|
+
begin
|
169
|
+
res = case result_type
|
170
|
+
when :array
|
171
|
+
array = []
|
172
|
+
while line = value.gets
|
173
|
+
array << line.chomp
|
232
174
|
end
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
175
|
+
array
|
176
|
+
when :tsv
|
177
|
+
begin
|
178
|
+
TSV.open(value)
|
179
|
+
rescue IOError
|
180
|
+
TSV.setup({})
|
181
|
+
end
|
182
|
+
else
|
183
|
+
value.read
|
184
|
+
end
|
185
|
+
value.join if value.respond_to? :join
|
186
|
+
res
|
187
|
+
rescue Exception
|
188
|
+
value.abort if value.respond_to? :abort
|
189
|
+
self.abort
|
190
|
+
raise $!
|
191
|
+
end
|
192
|
+
when (not defined? Entity or description.nil? or not Entity.formats.include? description)
|
193
|
+
value
|
194
|
+
when (Annotated === value and info.empty?)
|
195
|
+
value
|
196
|
+
when Annotated === value
|
197
|
+
annotations = value.annotations
|
198
|
+
entity_info ||= begin
|
199
|
+
entity_info = info.dup
|
200
|
+
entity_info.merge! info[:inputs] if info[:inputs]
|
201
|
+
entity_info
|
202
|
+
end
|
203
|
+
entity_info.each do |k,v|
|
204
|
+
value.send("#{h}=", v) if annotations.include? k
|
205
|
+
end
|
206
|
+
|
207
|
+
value
|
238
208
|
else
|
239
209
|
entity_info ||= begin
|
240
210
|
entity_info = info.dup
|
@@ -371,8 +341,9 @@ class Step
|
|
371
341
|
def step(name)
|
372
342
|
@steps ||= {}
|
373
343
|
@steps[name] ||= begin
|
344
|
+
name = name.to_sym
|
374
345
|
deps = rec_dependencies.select{|step|
|
375
|
-
step.task_name.to_sym == name
|
346
|
+
step.task_name && step.task_name.to_sym == name
|
376
347
|
}
|
377
348
|
raise "Dependency step not found: #{ name }" if deps.empty?
|
378
349
|
if (deps & self.dependencies).any?
|
@@ -385,3 +356,4 @@ class Step
|
|
385
356
|
end
|
386
357
|
|
387
358
|
require 'rbbt/workflow/step/run'
|
359
|
+
require 'rbbt/workflow/step/accessor'
|
data/lib/rbbt/workflow/task.rb
CHANGED
@@ -88,13 +88,15 @@ module Task
|
|
88
88
|
next
|
89
89
|
end
|
90
90
|
|
91
|
+
task_name ||= task.name
|
92
|
+
|
91
93
|
maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
|
92
94
|
raise "Dependency task not found: #{dep}" if task.nil?
|
93
|
-
next if seen.include? [wf,
|
95
|
+
next if seen.include? [wf, task_name, maps]
|
94
96
|
|
95
97
|
task.workflow = wf if wf
|
96
98
|
|
97
|
-
seen << [wf,
|
99
|
+
seen << [wf, task_name, maps]
|
98
100
|
new_inputs = task.inputs - maps
|
99
101
|
next unless new_inputs.any?
|
100
102
|
if task_inputs[task].nil?
|
@@ -160,44 +160,50 @@ puts files * "\n"
|
|
160
160
|
end
|
161
161
|
end
|
162
162
|
|
163
|
-
def self.migrate(
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
163
|
+
def self.migrate(paths, search_path, options = {})
|
164
|
+
subpath_files = {}
|
165
|
+
target_paths = []
|
166
|
+
resource = options[:resource] || Rbbt
|
167
|
+
|
168
|
+
target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
|
169
|
+
|
170
|
+
(Array === paths ? paths : [paths]).each do |path|
|
171
|
+
if Step === path
|
172
|
+
if options[:source]
|
173
|
+
path = Rbbt.identify(path.path)
|
174
|
+
else
|
175
|
+
path = path.path
|
176
|
+
end
|
169
177
|
end
|
170
|
-
|
171
|
-
search_path = 'user' if search_path.nil?
|
178
|
+
search_path = 'user' if search_path.nil?
|
172
179
|
|
173
|
-
resource = Rbbt
|
174
180
|
|
175
|
-
|
181
|
+
path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
|
176
182
|
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
183
|
+
real_paths.sort.each do |path|
|
184
|
+
parts = path.split("/")
|
185
|
+
subpath = parts[0..-4] * "/" + "/"
|
186
|
+
|
187
|
+
if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
|
188
|
+
subpath = subpath_files.keys.last
|
189
|
+
end
|
181
190
|
|
182
|
-
|
183
|
-
|
191
|
+
source = path.chars[subpath.length..-1] * ""
|
192
|
+
|
193
|
+
subpath_files[subpath] ||= []
|
194
|
+
subpath_files[subpath] << source
|
184
195
|
end
|
185
196
|
|
186
|
-
source = path.chars[subpath.length..-1] * ""
|
187
197
|
|
188
|
-
|
189
|
-
subpath_files[subpath] << source
|
198
|
+
target_paths << File.join(target, *path.split("/")[-3..-1])
|
190
199
|
end
|
191
200
|
|
192
|
-
target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
|
193
|
-
|
194
|
-
target_path = File.join(target, *path.split("/")[-3..-1])
|
195
201
|
|
196
202
|
subpath_files.each do |subpath, files|
|
197
203
|
Rbbt.migrate_files([subpath], target, options.merge(:files => files))
|
198
204
|
end
|
199
205
|
|
200
|
-
|
206
|
+
target_paths
|
201
207
|
end
|
202
208
|
|
203
209
|
def self.purge(path, recursive = false, skip_overriden = true)
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -212,11 +212,13 @@ module Workflow
|
|
212
212
|
end
|
213
213
|
workflow.load_documentation
|
214
214
|
|
215
|
+
|
215
216
|
first ||= workflow
|
216
217
|
end
|
217
|
-
return first
|
218
218
|
|
219
|
-
|
219
|
+
first.complete_name = wf_name
|
220
|
+
|
221
|
+
return first
|
220
222
|
end
|
221
223
|
|
222
224
|
attr_accessor :description
|
@@ -230,6 +232,12 @@ module Workflow
|
|
230
232
|
attr_accessor :relay_tasks
|
231
233
|
|
232
234
|
#{{{ ATTR DEFAULTS
|
235
|
+
#
|
236
|
+
attr_accessor :complete_name
|
237
|
+
|
238
|
+
def self.complete_name
|
239
|
+
@complete_name ||= self.to_s
|
240
|
+
end
|
233
241
|
|
234
242
|
def self.workdir=(path)
|
235
243
|
path = Path.setup path.dup unless Path === path
|
data/python/rbbt.py
CHANGED
@@ -1,7 +1,93 @@
|
|
1
1
|
import warnings
|
2
2
|
import sys
|
3
3
|
import os
|
4
|
-
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
|
5
4
|
|
6
5
|
def rbbt():
|
7
6
|
print("Rbbt")
|
7
|
+
|
8
|
+
def path(subdir = None, base_dir = None):
|
9
|
+
from pathlib import Path
|
10
|
+
import os
|
11
|
+
|
12
|
+
if (base_dir == None):
|
13
|
+
base_dir = os.path.join(Path.home(), ".rbbt")
|
14
|
+
if (subdir == None):
|
15
|
+
return base_dir
|
16
|
+
else:
|
17
|
+
return os.path.join(base_dir, subdir)
|
18
|
+
|
19
|
+
def inspect(obj):
|
20
|
+
print(dir(obj))
|
21
|
+
|
22
|
+
def log_tsv(tsv):
|
23
|
+
print(tsv)
|
24
|
+
print(tsv.keys())
|
25
|
+
|
26
|
+
def tsv_preamble(line, comment_char="#"):
|
27
|
+
import re
|
28
|
+
header = dict()
|
29
|
+
entries = re.sub(f"^{comment_char}:", '', line)
|
30
|
+
entries = re.sub(f"^{comment_char}:", '', line).split("#")
|
31
|
+
for entry in entries:
|
32
|
+
key, value = entry.split("=")
|
33
|
+
key = re.sub("^:","",key)
|
34
|
+
value = re.sub("^:","",value)
|
35
|
+
header[key] = value
|
36
|
+
|
37
|
+
return header
|
38
|
+
|
39
|
+
|
40
|
+
def tsv_header(filename, sep="\t", comment_char="#"):
|
41
|
+
import re
|
42
|
+
|
43
|
+
f = open(filename)
|
44
|
+
line = f.readline().strip()
|
45
|
+
|
46
|
+
if (not line.startswith(comment_char)):
|
47
|
+
header = {"fields":None, "type":"list", "start": 0}
|
48
|
+
else:
|
49
|
+
header = dict()
|
50
|
+
start = 0
|
51
|
+
if (line.startswith(f"{comment_char}:")):
|
52
|
+
header["preamble"]=tsv_preamble(line, comment_char)
|
53
|
+
if ("type" in header["preamble"]):
|
54
|
+
header["type"] = header["preamble"]["type"]
|
55
|
+
line = f.readline().strip()
|
56
|
+
start = 1
|
57
|
+
|
58
|
+
if (line.startswith(comment_char)):
|
59
|
+
header["all_fields"] = re.sub(f"^{comment_char}", "", line).split(sep)
|
60
|
+
header["key_field"] = header["all_fields"][0]
|
61
|
+
header["fields"] = header["all_fields"][1:]
|
62
|
+
|
63
|
+
header["start"] = start
|
64
|
+
|
65
|
+
f.close()
|
66
|
+
return header
|
67
|
+
|
68
|
+
|
69
|
+
def tsv_pandas(filename, sep="\t", comment_char="#", index_col=0, **kwargs):
|
70
|
+
import pandas
|
71
|
+
|
72
|
+
if (comment_char == ""):
|
73
|
+
tsv = pandas.read_table(filename, sep=sep, index_col=index_col, **kwargs)
|
74
|
+
else:
|
75
|
+
header = tsv_header(filename, sep=sep, comment_char="#")
|
76
|
+
|
77
|
+
if ("type" in header and header["type"] == "flat"):
|
78
|
+
return None
|
79
|
+
else:
|
80
|
+
if ("sep" in header):
|
81
|
+
sep=header["sep"]
|
82
|
+
|
83
|
+
tsv = pandas.read_table(filename, sep=sep, index_col=index_col, header=header["start"], **kwargs)
|
84
|
+
|
85
|
+
if ("fields" in header):
|
86
|
+
tsv.columns = header["fields"]
|
87
|
+
tsv.index.name = header["key_field"]
|
88
|
+
|
89
|
+
return tsv
|
90
|
+
|
91
|
+
def tsv(*args, **kwargs):
|
92
|
+
return tsv_pandas(*args, **kwargs)
|
93
|
+
|
@@ -54,8 +54,9 @@ tsvfile, excelfile = ARGV
|
|
54
54
|
|
55
55
|
excelfile, tsvfile = tsvfile, nil if excelfile.nil? and tsvfile =~ /\.(xls|xlsx)$/
|
56
56
|
|
57
|
-
raise ParameterException, "No
|
58
|
-
|
57
|
+
raise ParameterException, "No TSV file given" if tsvfile.nil?
|
58
|
+
|
59
|
+
excelfile = tsvfile.sub(/\.tsv(\.gz)?$/, '.xlsx') if excelfile.nil?
|
59
60
|
|
60
61
|
require 'rbbt/tsv/excel'
|
61
62
|
TSV.open(tsvfile, options).excel(excelfile, options)
|
@@ -213,11 +213,15 @@ The `recursive_clean` cleans all the job dependency steps recursively.
|
|
213
213
|
-prov--provenance Report the jobs provenance
|
214
214
|
-W--workflows* Load a list of workflows
|
215
215
|
-R--requires* Require a list of files
|
216
|
-
-
|
217
|
-
-
|
216
|
+
-pro--produce* Prepare dependencies
|
217
|
+
-proc--produce_cpus* Number of dependencies prepared in parallel
|
218
218
|
-rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
|
219
219
|
-od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
|
220
220
|
-PERF--procpath_performance* Measure performance using procpath
|
221
|
+
-r--relay* Relay job to SSH server
|
222
|
+
-sr--slurm_relay* Relay job to SSH SLURM server
|
223
|
+
-rdep--relay_dependencies* Relay dependencies instead of main job
|
224
|
+
-pdr--produce_dependencies_for_relay Prepare dependencies previous to relay jobs
|
221
225
|
EOF
|
222
226
|
|
223
227
|
workflow = ARGV.shift
|
@@ -415,13 +419,53 @@ begin
|
|
415
419
|
exit 0
|
416
420
|
end
|
417
421
|
|
418
|
-
if tasks = options.delete(:
|
422
|
+
if tasks = options.delete(:produce)
|
419
423
|
tasks = tasks.split(",")
|
420
|
-
|
421
|
-
puts Step.
|
424
|
+
produce_cpus = (options[:produce_cpus] || 1)
|
425
|
+
puts Step.produce_dependencies(job, tasks, prepare_cpus)
|
422
426
|
exit 0
|
423
427
|
end
|
424
428
|
|
429
|
+
def match_dependencies(queries, dependencies)
|
430
|
+
queries = queries.collect{|q| q.include?("#") ? q.split("#") : q }
|
431
|
+
dependencies.select do |dep|
|
432
|
+
queries.select do |q|
|
433
|
+
if Array === q
|
434
|
+
q.first == dep.workflow.to_s && q.last == dep.task_name.to_s
|
435
|
+
else
|
436
|
+
q.to_s == dep.task_name.to_s
|
437
|
+
end
|
438
|
+
end.any?
|
439
|
+
end
|
440
|
+
end
|
441
|
+
|
442
|
+
if server = options.delete(:relay)
|
443
|
+
require 'rbbt/workflow/remote_workflow'
|
444
|
+
relay_dependencies = options.delete(:relay_dependencies).split(",")
|
445
|
+
produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
|
446
|
+
if relay_dependencies
|
447
|
+
match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
|
448
|
+
RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
|
449
|
+
end
|
450
|
+
else
|
451
|
+
RemoteWorkflow::SSH.relay_job(job, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
|
452
|
+
end
|
453
|
+
end
|
454
|
+
|
455
|
+
if server = options.delete(:slurm_relay)
|
456
|
+
require 'rbbt/workflow/remote_workflow'
|
457
|
+
relay_dependencies = options.delete(:relay_dependencies).split(",")
|
458
|
+
produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
|
459
|
+
if relay_dependencies
|
460
|
+
match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
|
461
|
+
RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
|
462
|
+
end
|
463
|
+
else
|
464
|
+
RemoteWorkflow::SSH.relay_job(job, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
|
465
|
+
end
|
466
|
+
end
|
467
|
+
|
468
|
+
|
425
469
|
if options[:procpath_performance]
|
426
470
|
require 'rbbt/util/procpath'
|
427
471
|
current_pid = job.info[:pid]
|
data/test/rbbt/test_workflow.rb
CHANGED
@@ -513,4 +513,92 @@ class TestWorkflow < Test::Unit::TestCase
|
|
513
513
|
assert_equal "TLA", job.run
|
514
514
|
assert job.overriden
|
515
515
|
end
|
516
|
+
|
517
|
+
def test_anonymous_workflow
|
518
|
+
workflow = Module.new
|
519
|
+
workflow.extend Workflow
|
520
|
+
workflow.define_singleton_method(:to_s){"TestWorkflow"}
|
521
|
+
|
522
|
+
assert_equal "TestWorkflow", workflow.to_s
|
523
|
+
|
524
|
+
t1 = workflow.task :s1 => :string do
|
525
|
+
task_name.to_s
|
526
|
+
end
|
527
|
+
|
528
|
+
workflow.dep :s1
|
529
|
+
t2 = workflow.task :s2 => :string do
|
530
|
+
task_name.to_s
|
531
|
+
end
|
532
|
+
|
533
|
+
workflow.dep :s2
|
534
|
+
t3 = workflow.task :s3 => :string do
|
535
|
+
task_name.to_s
|
536
|
+
end
|
537
|
+
|
538
|
+
assert_equal "s3", workflow.job(:s3).run
|
539
|
+
end
|
540
|
+
|
541
|
+
def test_low_level_step
|
542
|
+
Log.with_severity 0 do
|
543
|
+
TmpFile.with_file do |tmpdir|
|
544
|
+
Path.setup tmpdir
|
545
|
+
|
546
|
+
s1 = Step.new tmpdir.s1 do "s1" end
|
547
|
+
s1.task_name = "s1"
|
548
|
+
|
549
|
+
# run clean run
|
550
|
+
assert_equal "s1", s1.run
|
551
|
+
s1.clean
|
552
|
+
assert_equal "s1", s1.run
|
553
|
+
assert_equal "s1", s1.run
|
554
|
+
|
555
|
+
# is persisted because it raises RbbtException
|
556
|
+
s1.task = proc do raise RbbtException end
|
557
|
+
assert_equal "s1", s1.run
|
558
|
+
s1.clean
|
559
|
+
assert_raises RbbtException do s1.run end
|
560
|
+
assert RbbtException === s1.get_exception
|
561
|
+
assert s1.error?
|
562
|
+
|
563
|
+
s1.recursive_clean
|
564
|
+
s1.task = proc do "s1" end
|
565
|
+
|
566
|
+
# add dependencies
|
567
|
+
s2 = Step.new tmpdir.s2 do
|
568
|
+
step(:s1).load + " -> s2"
|
569
|
+
end
|
570
|
+
s2.task_name = "s2"
|
571
|
+
s2.dependencies << s1
|
572
|
+
|
573
|
+
s3 = Step.new tmpdir.s3 do
|
574
|
+
step(:s2).load + " -> s3"
|
575
|
+
end
|
576
|
+
s3.task_name = "s3"
|
577
|
+
s3.dependencies << s2
|
578
|
+
|
579
|
+
assert_equal "s1 -> s2", s2.run
|
580
|
+
assert_equal "s1 -> s2 -> s3", s3.run
|
581
|
+
|
582
|
+
# Test recusive behaviour
|
583
|
+
s1.task = proc do raise RbbtException end
|
584
|
+
ComputeDependency.setup(s1, :produce)
|
585
|
+
s1.clean
|
586
|
+
s3.clean
|
587
|
+
s3.run
|
588
|
+
assert_equal "s1 -> s2 -> s3", s3.run
|
589
|
+
|
590
|
+
s1.task = proc do raise RbbtException end
|
591
|
+
ComputeDependency.setup(s1, :produce)
|
592
|
+
s3.recursive_clean
|
593
|
+
assert_raises RbbtException do s3.run end
|
594
|
+
|
595
|
+
s1.task = proc{|v| v }
|
596
|
+
s1.inputs = ["test"]
|
597
|
+
assert_equal "test", s1.run
|
598
|
+
s3.recursive_clean
|
599
|
+
assert_equal "test -> s2 -> s3", s3.run
|
600
|
+
end
|
601
|
+
end
|
602
|
+
end
|
603
|
+
|
516
604
|
end
|
@@ -131,5 +131,10 @@ def python_print():
|
|
131
131
|
random = RbbtPython.import_method :torch, :rand, :random
|
132
132
|
assert random.call(1).numpy.to_f > 0
|
133
133
|
end
|
134
|
+
|
135
|
+
def test_class_new_obj
|
136
|
+
obj = RbbtPython.class_new_obj("torch.nn", "Module")
|
137
|
+
assert_equal "Module()", obj.to_s
|
138
|
+
end
|
134
139
|
end
|
135
140
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.36.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-02
|
11
|
+
date: 2023-03-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -366,7 +366,7 @@ files:
|
|
366
366
|
- lib/rbbt/workflow/step/accessor.rb
|
367
367
|
- lib/rbbt/workflow/step/dependencies.rb
|
368
368
|
- lib/rbbt/workflow/step/info.rb
|
369
|
-
- lib/rbbt/workflow/step/
|
369
|
+
- lib/rbbt/workflow/step/produce.rb
|
370
370
|
- lib/rbbt/workflow/step/run.rb
|
371
371
|
- lib/rbbt/workflow/step/save_load_inputs.rb
|
372
372
|
- lib/rbbt/workflow/step/status.rb
|