rbbt-util 5.35.3 → 5.36.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/persist/tsv/adapter.rb +3 -1
- data/lib/rbbt/persist.rb +2 -0
- data/lib/rbbt/resource/path.rb +12 -2
- data/lib/rbbt/util/python.rb +10 -0
- data/lib/rbbt/workflow/definition.rb +2 -0
- data/lib/rbbt/workflow/doc.rb +1 -0
- data/lib/rbbt/workflow/remote_workflow/driver/ssh.rb +60 -20
- data/lib/rbbt/workflow/remote_workflow/remote_step/ssh.rb +18 -3
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +1 -0
- data/lib/rbbt/workflow/step/accessor.rb +42 -5
- data/lib/rbbt/workflow/step/dependencies.rb +25 -22
- data/lib/rbbt/workflow/step/{prepare.rb → produce.rb} +1 -1
- data/lib/rbbt/workflow/step/run.rb +37 -28
- data/lib/rbbt/workflow/step/status.rb +1 -1
- data/lib/rbbt/workflow/step.rb +55 -83
- data/lib/rbbt/workflow/task.rb +4 -2
- data/lib/rbbt/workflow/util/archive.rb +29 -23
- data/lib/rbbt/workflow.rb +10 -2
- data/python/rbbt.py +87 -1
- data/share/rbbt_commands/tsv/write_excel +3 -2
- data/share/rbbt_commands/workflow/task +49 -5
- data/test/rbbt/test_workflow.rb +88 -0
- data/test/rbbt/util/test_python.rb +5 -0
- data/test/rbbt/workflow/step/test_dependencies.rb +1 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d105d6c21ad018f28acf6a234feae8d81f327f3f94f228c01f118abed6c386f6
|
4
|
+
data.tar.gz: 5c883e0611f02203d394ec02e448664efdc42361be04e9d873e9ba3f5e6cc92e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 20a4bccdeb0c587041b7f20f09bda19949713fcb362911a249b08112399c09178f8e50b5ae90f390bd6cd1389b3c9a47b7374c4fc05a83b073f1d43bdfdc6611
|
7
|
+
data.tar.gz: 1b3b112106b4ac1e261bcf1630ecaeea0ed27f1975aeead9694708241e9268fdac36da4f5b4be25c12288cb7f97b8b4163815b8d9010aba40f6f2434e654f99c
|
data/lib/rbbt/persist.rb
CHANGED
data/lib/rbbt/resource/path.rb
CHANGED
@@ -319,7 +319,8 @@ module Path
|
|
319
319
|
|
320
320
|
|
321
321
|
def open(options = {}, &block)
|
322
|
-
Open.
|
322
|
+
file = Open.remote?(self) || Open.ssh?(self) ? self : self.produce.find
|
323
|
+
Open.open(file, options, &block)
|
323
324
|
end
|
324
325
|
|
325
326
|
def to_s
|
@@ -331,7 +332,16 @@ module Path
|
|
331
332
|
end
|
332
333
|
|
333
334
|
def tsv(*args)
|
334
|
-
|
335
|
+
begin
|
336
|
+
path = self.produce
|
337
|
+
rescue Resource::ResourceNotFound => e
|
338
|
+
begin
|
339
|
+
path = self.set_extension('tsv').produce
|
340
|
+
rescue Resource::ResourceNotFound
|
341
|
+
raise e
|
342
|
+
end
|
343
|
+
end
|
344
|
+
TSV.open(path, *args)
|
335
345
|
end
|
336
346
|
|
337
347
|
def tsv_options(options = {})
|
data/lib/rbbt/util/python.rb
CHANGED
@@ -58,6 +58,16 @@ module RbbtPython
|
|
58
58
|
RbbtPython.import_method(module_name, method_name).call(*args)
|
59
59
|
end
|
60
60
|
|
61
|
+
def self.get_class(module_name, class_name)
|
62
|
+
save_module_name = module_name.to_s.gsub(".", "_")
|
63
|
+
RbbtPython.pyimport(module_name, as: save_module_name)
|
64
|
+
RbbtPython.send(save_module_name).send(class_name)
|
65
|
+
end
|
66
|
+
|
67
|
+
def self.class_new_obj(module_name, class_name, args={})
|
68
|
+
RbbtPython.get_class(module_name, class_name).new(**args)
|
69
|
+
end
|
70
|
+
|
61
71
|
def self.exec(script)
|
62
72
|
PyCall.exec(script)
|
63
73
|
end
|
data/lib/rbbt/workflow/doc.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
class RemoteWorkflow
|
2
|
+
RBBT_DEBUG_REMOTE_JSON = ENV["RBBT_DEBUG_REMOTE_JSON"] == 'true'
|
3
|
+
|
2
4
|
module SSH
|
3
5
|
#def self.run(server, script)
|
4
6
|
# Log.debug "Run ssh script in #{server}:\n#{script}"
|
@@ -21,9 +23,16 @@ class RemoteWorkflow
|
|
21
23
|
|
22
24
|
workflow, task, job, *rest = path.split("/")
|
23
25
|
|
26
|
+
workflow_name = begin
|
27
|
+
workflow = Kernel.const_get(workflow) if String === workflow
|
28
|
+
workflow.respond_to?(:complete_name) ? workflow.complete_name : workflow
|
29
|
+
rescue
|
30
|
+
workflow
|
31
|
+
end
|
32
|
+
|
24
33
|
script =<<-EOF
|
25
34
|
require 'rbbt/workflow'
|
26
|
-
wf = Workflow.require_workflow "#{
|
35
|
+
wf = Workflow.require_workflow "#{workflow_name}"
|
27
36
|
EOF
|
28
37
|
|
29
38
|
case task
|
@@ -86,7 +95,7 @@ STDOUT.write res.to_json
|
|
86
95
|
EOF
|
87
96
|
|
88
97
|
json = Misc.ssh_run(server, script)
|
89
|
-
Log.debug "JSON (#{ url }): #{json}"
|
98
|
+
Log.debug "JSON (#{ url }): #{json}" if RBBT_DEBUG_REMOTE_JSON
|
90
99
|
JSON.parse(json)
|
91
100
|
end
|
92
101
|
|
@@ -172,11 +181,17 @@ job.clean
|
|
172
181
|
def self.upload_inputs(server, inputs, input_types, input_id)
|
173
182
|
TmpFile.with_file do |dir|
|
174
183
|
if Step.save_inputs(inputs, input_types, dir)
|
175
|
-
Dir.glob(File.join(dir, "*.as_step")).each do |file|
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
184
|
+
# Dir.glob(File.join(dir, "*.as_step")).each do |file|
|
185
|
+
# Log.medium "Migrating Step input #{file} #{ server }"
|
186
|
+
# path = Open.read(file).strip
|
187
|
+
# new = Step.migrate(path, :user, :target => server)
|
188
|
+
# Open.write(file, new)
|
189
|
+
# end
|
190
|
+
|
191
|
+
paths = Dir.glob(File.join(dir, "*.as_step")).collect{|f| Open.read(f).strip }
|
192
|
+
new = Step.migrate(paths, :user, :target => server)
|
193
|
+
paths.zip(new).each{|file,new| Open.write(file, new) }
|
194
|
+
|
180
195
|
CMD.cmd_log("ssh '#{server}' mkdir -p .rbbt/tmp/tmp-ssh_job_inputs/; scp -r '#{dir}' #{server}:.rbbt/tmp/tmp-ssh_job_inputs/#{input_id}")
|
181
196
|
end
|
182
197
|
end
|
@@ -204,31 +219,56 @@ job.clean
|
|
204
219
|
# rjob.run
|
205
220
|
#end
|
206
221
|
|
222
|
+
def self.upload_dependencies(job, server, search_path = 'user', produce_dependencies = false)
|
223
|
+
server, path = parse_url(server) if server =~ /^ssh:\/\//
|
224
|
+
job.dependencies.each do |dep|
|
225
|
+
Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
|
226
|
+
dep.produce
|
227
|
+
end if produce_dependencies
|
228
|
+
|
229
|
+
job.input_dependencies.each do |dep|
|
230
|
+
Log.medium "Producing #{dep.workflow}:#{dep.short_path} dependency for #{job.workflow}:#{job.short_path}"
|
231
|
+
dep.produce
|
232
|
+
end if produce_dependencies
|
233
|
+
|
234
|
+
migrate_dependencies = job.rec_dependencies.select{|d| d.done? }.collect{|d| d.path }
|
235
|
+
Log.medium "Migrating #{migrate_dependencies.length} dependencies to #{ server }"
|
236
|
+
Step.migrate(migrate_dependencies, search_path, :target => server) if migrate_dependencies.any?
|
237
|
+
end
|
238
|
+
|
239
|
+
def self.missing_dep_inputs(job)
|
240
|
+
inputs = job.inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
|
241
|
+
job.dependencies.each do |dep|
|
242
|
+
next if dep.done?
|
243
|
+
iif [dep, dep.inputs, dep.real_inputs]
|
244
|
+
inputs = dep.inputs.to_hash.slice(*dep.real_inputs.map{|i| i.to_s}).merge(inputs)
|
245
|
+
inputs = missing_dep_inputs(dep).merge(inputs)
|
246
|
+
end
|
247
|
+
inputs
|
248
|
+
end
|
249
|
+
|
207
250
|
def self.relay_job(job, server, options = {})
|
208
|
-
migrate, produce, produce_dependencies, search_path = Misc.process_options options.dup,
|
209
|
-
:migrate, :produce, :produce_dependencies, :search_path
|
251
|
+
migrate, produce, produce_dependencies, search_path, run_type, slurm_options = Misc.process_options options.dup,
|
252
|
+
:migrate, :produce, :produce_dependencies, :search_path, :run_type, :slurm_options
|
210
253
|
|
211
254
|
search_path ||= 'user'
|
212
255
|
|
213
256
|
produce = true if migrate
|
214
257
|
|
215
258
|
workflow_name = job.workflow.to_s
|
216
|
-
|
217
|
-
|
218
|
-
job.
|
219
|
-
dep.produce
|
220
|
-
end if options[:produce_dependencies]
|
221
|
-
|
222
|
-
job.rec_dependencies.each do |dep|
|
223
|
-
Step.migrate(dep.path, search_path, :target => server) if dep.done?
|
224
|
-
end
|
259
|
+
remote_workflow = RemoteWorkflow.new("ssh://#{server}:#{workflow_name}", "#{workflow_name}")
|
260
|
+
inputs = job.recursive_inputs.to_hash.slice(*job.real_inputs.map{|i| i.to_s})
|
261
|
+
Log.medium "Relaying dependency #{job.workflow}:#{job.short_path} to #{server} (#{inputs.keys * ", "})"
|
225
262
|
|
226
|
-
|
263
|
+
upload_dependencies(job, server, search_path, options[:produce_dependencies])
|
227
264
|
rjob = remote_workflow.job(job.task_name.to_s, job.clean_name, inputs)
|
228
265
|
|
229
266
|
override_dependencies = job.rec_dependencies.select{|dep| dep.done? }.collect{|dep| [dep.workflow.to_s, dep.task_name.to_s] * "#" << "=" << Rbbt.identify(dep.path)}
|
230
267
|
rjob.override_dependencies = override_dependencies
|
231
268
|
|
269
|
+
rjob.run_type = run_type
|
270
|
+
rjob.slurm_options = slurm_options || {}
|
271
|
+
|
232
272
|
if options[:migrate]
|
233
273
|
rjob.produce
|
234
274
|
Step.migrate(Rbbt.identify(job.path), 'user', :source => server)
|
@@ -255,7 +295,7 @@ job.clean
|
|
255
295
|
@task_info ||= IndiferentHash.setup({})
|
256
296
|
|
257
297
|
if @task_info[task].nil?
|
258
|
-
task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url, task.to_s))
|
298
|
+
task_info = RemoteWorkflow::SSH.get_json(File.join(@base_url || @url, task.to_s))
|
259
299
|
task_info = RemoteWorkflow::SSH.fix_hash(task_info)
|
260
300
|
|
261
301
|
task_info[:result_type] = task_info[:result_type].to_sym if task_info[:result_type]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
class RemoteStep
|
2
2
|
module SSH
|
3
|
-
attr_accessor :override_dependencies
|
3
|
+
attr_accessor :override_dependencies, :run_type, :slurm_options
|
4
4
|
|
5
5
|
def init_job(cache_type = nil, other_params = {})
|
6
6
|
return self if @url
|
@@ -55,12 +55,27 @@ class RemoteStep
|
|
55
55
|
RemoteWorkflow::SSH.run_job(File.join(base_url, task.to_s), @input_id, @base_name)
|
56
56
|
end
|
57
57
|
|
58
|
+
def _run_slurm
|
59
|
+
RemoteWorkflow::SSH.run_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
|
60
|
+
end
|
61
|
+
|
62
|
+
def _orchestrate_slurm
|
63
|
+
RemoteWorkflow::SSH.orchestrate_slurm_job(File.join(base_url, task.to_s), @input_id, @base_name, @slurm_options || {})
|
64
|
+
end
|
65
|
+
|
58
66
|
def produce(*args)
|
59
67
|
input_types = {}
|
60
68
|
init_job
|
61
|
-
@remote_path =
|
69
|
+
@remote_path = case @run_type
|
70
|
+
when 'run', :run, nil
|
71
|
+
_run
|
72
|
+
when 'slurm', :slurm
|
73
|
+
_run_slurm
|
74
|
+
when 'orchestrate', :orchestrate
|
75
|
+
_orchestrate_slurm
|
76
|
+
end
|
62
77
|
@started = true
|
63
|
-
while ! (done? || error?)
|
78
|
+
while ! (done? || error? || aborted?)
|
64
79
|
sleep 1
|
65
80
|
end
|
66
81
|
raise self.get_exception if error?
|
@@ -1,4 +1,12 @@
|
|
1
1
|
class Step
|
2
|
+
attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
|
3
|
+
attr_accessor :task_name, :overriden
|
4
|
+
attr_accessor :pid
|
5
|
+
attr_accessor :exec
|
6
|
+
attr_accessor :relocated
|
7
|
+
attr_accessor :result, :mutex, :seen
|
8
|
+
attr_accessor :real_inputs, :original_task_name, :original_workflow
|
9
|
+
|
2
10
|
|
3
11
|
INFO_SERIALIZER = begin
|
4
12
|
if ENV["RBBT_INFO_SERIALIZER"]
|
@@ -86,12 +94,45 @@ class Step
|
|
86
94
|
end
|
87
95
|
end
|
88
96
|
|
97
|
+
def task_name
|
98
|
+
@task_name ||= begin
|
99
|
+
if @task.respond_to?(:name)
|
100
|
+
|
101
|
+
@task.name
|
102
|
+
else
|
103
|
+
@path.split("/")[-2]
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def result_type
|
109
|
+
@result_type ||= if @task.respond_to?(:result_type)
|
110
|
+
@task.result_type || info[:result_type]
|
111
|
+
else
|
112
|
+
info[:result_type]
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def result_type=(type)
|
117
|
+
@result_type = type
|
118
|
+
end
|
119
|
+
|
120
|
+
def result_description
|
121
|
+
@result_description ||= if @task.respond_to?(:result_description)
|
122
|
+
@task.result_description
|
123
|
+
else
|
124
|
+
nil
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def result_description=(description)
|
129
|
+
@result_description = description
|
130
|
+
end
|
89
131
|
|
90
132
|
def name
|
91
133
|
@name ||= path.sub(/.*\/#{Regexp.quote task_name.to_s}\/(.*)/, '\1')
|
92
134
|
end
|
93
135
|
|
94
|
-
|
95
136
|
def short_path
|
96
137
|
[task_name, name] * "/"
|
97
138
|
end
|
@@ -105,10 +146,6 @@ class Step
|
|
105
146
|
workflow.to_s + "#" + short_path
|
106
147
|
end
|
107
148
|
|
108
|
-
def task_name
|
109
|
-
@task_name ||= task.name
|
110
|
-
end
|
111
|
-
|
112
149
|
def task_signature
|
113
150
|
[workflow.to_s, task_name] * "#"
|
114
151
|
end
|
@@ -3,7 +3,7 @@ class Step
|
|
3
3
|
STREAM_CACHE = {}
|
4
4
|
STREAM_CACHE_MUTEX = Mutex.new
|
5
5
|
def self.purge_stream_cache
|
6
|
-
Log.debug "Purging dup. stream cache"
|
6
|
+
# Log.debug "Purging dup. stream cache"
|
7
7
|
STREAM_CACHE_MUTEX.synchronize do
|
8
8
|
STREAM_CACHE.clear
|
9
9
|
end
|
@@ -209,7 +209,7 @@ class Step
|
|
209
209
|
end
|
210
210
|
|
211
211
|
rescue TryAgain
|
212
|
-
Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
|
212
|
+
#Log.low "Retrying dep. #{Log.color :yellow, dependency.task_name.to_s} -- [#{dependency.status}] #{(dependency.messages || ["No message"]).last}"
|
213
213
|
retry
|
214
214
|
rescue Aborted, Interrupt
|
215
215
|
Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
|
@@ -430,7 +430,6 @@ class Step
|
|
430
430
|
|
431
431
|
required_dep_paths = []
|
432
432
|
dep_step.each do |path,list|
|
433
|
-
#required_dep_paths << path if list.length > 1
|
434
433
|
required_dep_paths << path if (list & dependencies).any?
|
435
434
|
end
|
436
435
|
|
@@ -442,42 +441,46 @@ class Step
|
|
442
441
|
[dep.path] + dep.input_dependencies
|
443
442
|
end.flatten)
|
444
443
|
|
445
|
-
log :dependencies, "Dependencies for step #{Log.color :yellow, task.name.to_s || ""}"
|
446
444
|
|
447
445
|
pre_deps = []
|
448
|
-
|
449
|
-
|
446
|
+
simple_dependencies = []
|
447
|
+
compute_simple_dependencies = {}
|
450
448
|
compute_last_deps = {}
|
451
449
|
seen_paths = Set.new
|
452
450
|
rec_dependencies.uniq.each do |step|
|
453
451
|
next if seen_paths.include? step.path
|
454
452
|
seen_paths << step.path
|
455
453
|
next unless required_dep_paths.include? step.path
|
456
|
-
|
457
|
-
|
458
|
-
|
454
|
+
required_seen_paths = seen_paths & required_dep_paths
|
455
|
+
|
456
|
+
internal = step.inputs.select{|i| Step == i && required_paths.include?(i.path) && seen_paths.include?(i.path) }
|
457
|
+
|
458
|
+
if ComputeDependency === step
|
459
|
+
next if produced.include? step.path
|
460
|
+
if internal
|
459
461
|
compute_last_deps[step.compute] ||= []
|
460
462
|
compute_last_deps[step.compute] << step
|
461
463
|
else
|
462
|
-
|
464
|
+
compute_simple_dependencies[step.compute] ||= []
|
465
|
+
compute_simple_dependencies[step.compute] << step
|
463
466
|
end
|
464
467
|
else
|
465
|
-
if
|
466
|
-
|
467
|
-
compute_pre_deps[step.compute] ||= []
|
468
|
-
compute_pre_deps[step.compute] << step
|
468
|
+
if internal
|
469
|
+
simple_dependencies.prepend(step)
|
469
470
|
else
|
470
|
-
|
471
|
+
simple_dependencies << step
|
471
472
|
end
|
472
473
|
end
|
473
474
|
end
|
474
475
|
|
475
|
-
|
476
|
-
|
476
|
+
log :dependencies, "Processing dependencies for #{Log.color :yellow, task_name.to_s || ""}" if compute_simple_dependencies.any? || simple_dependencies.any? || compute_last_deps.any?
|
477
|
+
|
478
|
+
Log.debug "compute_simple_dependencies: #{Misc.fingerprint(compute_simple_dependencies)} - #{Log.color :blue, self.path}" if compute_simple_dependencies.any?
|
479
|
+
compute_simple_dependencies.each do |type,list|
|
477
480
|
run_compute_dependencies(type, list, dep_step)
|
478
481
|
end
|
479
482
|
|
480
|
-
Log.
|
483
|
+
Log.low "pre_deps: #{Misc.fingerprint(pre_deps)} - #{Log.color :blue, self.path}" if pre_deps.any?
|
481
484
|
pre_deps.each do |step|
|
482
485
|
next if compute_deps.include? step
|
483
486
|
begin
|
@@ -487,8 +490,8 @@ class Step
|
|
487
490
|
end
|
488
491
|
end
|
489
492
|
|
490
|
-
Log.
|
491
|
-
|
493
|
+
Log.debug "simple_dependencies: #{Misc.fingerprint(simple_dependencies)} - #{Log.color :blue, self.path}" if simple_dependencies.any?
|
494
|
+
simple_dependencies.each do |step|
|
492
495
|
next if compute_deps.include? step
|
493
496
|
begin Exception
|
494
497
|
execute_and_dup(step, dep_step)
|
@@ -497,8 +500,8 @@ class Step
|
|
497
500
|
end
|
498
501
|
end
|
499
502
|
|
500
|
-
Log.
|
501
|
-
|
503
|
+
Log.low "compute_last_deps: #{Misc.fingerprint(compute_simple_dependencies)} - #{Log.color :blue, self.path}" if compute_simple_dependencies.any?
|
504
|
+
compute_simple_dependencies.each do |type,list|
|
502
505
|
run_compute_dependencies(type, list, dep_step)
|
503
506
|
end
|
504
507
|
|
@@ -96,7 +96,11 @@ class Step
|
|
96
96
|
@exec = true if @exec.nil?
|
97
97
|
begin
|
98
98
|
old = Signal.trap("INT"){ Thread.current.raise Aborted }
|
99
|
-
@task.
|
99
|
+
if @task.respond_to?(:exec_in)
|
100
|
+
@task.exec_in((bindings || self), *@inputs)
|
101
|
+
else
|
102
|
+
(bindings || self).instance_exec *@inputs, &@task
|
103
|
+
end
|
100
104
|
ensure
|
101
105
|
Signal.trap("INT", old)
|
102
106
|
end
|
@@ -223,14 +227,15 @@ class Step
|
|
223
227
|
end
|
224
228
|
|
225
229
|
begin
|
226
|
-
|
227
|
-
|
228
|
-
no_load = :stream if no_load
|
230
|
+
no_load = :stream if no_load
|
231
|
+
result_type = self.result_type || info[:result_type]
|
229
232
|
|
233
|
+
res = @mutex.synchronize do
|
234
|
+
time_elapsed = total_time_elapsed = nil
|
230
235
|
Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
|
231
|
-
|
232
|
-
result_type = info[:result_type] if result_type.nil?
|
236
|
+
|
233
237
|
result = Persist.persist "Job", result_type, :file => path, :check => persist_checks, :no_load => no_load do
|
238
|
+
|
234
239
|
if Step === Step.log_relay_step and not self == Step.log_relay_step
|
235
240
|
relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
|
236
241
|
end
|
@@ -240,26 +245,30 @@ class Step
|
|
240
245
|
@exec = false
|
241
246
|
init_info(true)
|
242
247
|
|
243
|
-
|
248
|
+
workflow = @workflow || @task.respond_to?(:workflow) ? @task.workflow : nil
|
249
|
+
result_type = @task.respond_to?(:result_type) ? @task.result_type : nil
|
250
|
+
result_description = @task.respond_to?(:result_description) ? @task.result_description : nil
|
251
|
+
|
252
|
+
log :setup, "#{Log.color :green, "Setup"} step #{Log.color :yellow, task_name}"
|
244
253
|
|
245
254
|
merge_info({
|
246
|
-
:issued
|
247
|
-
:name
|
248
|
-
:pid
|
249
|
-
:pid_hostname
|
250
|
-
:clean_name
|
251
|
-
:workflow
|
252
|
-
:task_name
|
253
|
-
:result_type
|
254
|
-
:result_description =>
|
255
|
-
:dependencies
|
256
|
-
:versions
|
255
|
+
:issued => (issue_time = Time.now),
|
256
|
+
:name => name,
|
257
|
+
:pid => Process.pid.to_s,
|
258
|
+
:pid_hostname => Socket.gethostname,
|
259
|
+
:clean_name => clean_name,
|
260
|
+
:workflow => workflow.to_s,
|
261
|
+
:task_name => task_name,
|
262
|
+
:result_type => result_type,
|
263
|
+
:result_description => result_description,
|
264
|
+
:dependencies => dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]},
|
265
|
+
:versions => Rbbt.versions
|
257
266
|
})
|
258
267
|
|
259
268
|
new_inputs = []
|
260
269
|
@inputs.each_with_index do |input,i|
|
261
|
-
name = @task.inputs[i]
|
262
|
-
type = @task.input_types[
|
270
|
+
name = @task.respond_to?(:inputs) ? @task.inputs[i] : nil
|
271
|
+
type = @task.respond_to?(:input_types) ? @task.input_types[i] : nil
|
263
272
|
|
264
273
|
if type == :directory
|
265
274
|
directory_inputs = file('directory_inputs')
|
@@ -303,7 +312,7 @@ class Step
|
|
303
312
|
|
304
313
|
@inputs = new_inputs if @inputs
|
305
314
|
|
306
|
-
if @inputs && ! task.inputs.nil?
|
315
|
+
if @inputs && task.respond_to?(:inputs) && ! task.inputs.nil?
|
307
316
|
info_inputs = @inputs.collect do |i|
|
308
317
|
if Path === i
|
309
318
|
i.to_s
|
@@ -323,7 +332,7 @@ class Step
|
|
323
332
|
end
|
324
333
|
|
325
334
|
set_info :started, (start_time = Time.now)
|
326
|
-
log :started, "Starting step #{Log.color :yellow,
|
335
|
+
log :started, "Starting step #{Log.color :yellow, task_name}"
|
327
336
|
|
328
337
|
config_keys_pre = Rbbt::Config::GOT_KEYS.dup
|
329
338
|
begin
|
@@ -357,7 +366,7 @@ class Step
|
|
357
366
|
end
|
358
367
|
|
359
368
|
if stream
|
360
|
-
log :streaming, "Streaming step #{Log.color :yellow,
|
369
|
+
log :streaming, "Streaming step #{Log.color :yellow, task_name.to_s || ""}"
|
361
370
|
|
362
371
|
callback = Proc.new do
|
363
372
|
if AbortedStream === stream
|
@@ -377,7 +386,7 @@ class Step
|
|
377
386
|
:time_elapsed => (time_elapsed = done_time - start_time),
|
378
387
|
:versions => Rbbt.versions
|
379
388
|
})
|
380
|
-
log :done, "Completed step #{Log.color :yellow,
|
389
|
+
log :done, "Completed step #{Log.color :yellow, task_name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
|
381
390
|
end
|
382
391
|
end
|
383
392
|
rescue
|
@@ -393,7 +402,7 @@ class Step
|
|
393
402
|
if exception
|
394
403
|
self.exception exception
|
395
404
|
else
|
396
|
-
log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow,
|
405
|
+
log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task_name.to_s || ""}" if status == :streaming
|
397
406
|
end
|
398
407
|
_clean_finished
|
399
408
|
rescue
|
@@ -419,7 +428,6 @@ class Step
|
|
419
428
|
})
|
420
429
|
log :ending
|
421
430
|
Step.purge_stream_cache
|
422
|
-
Open.rm pid_file if Open.exist?(pid_file)
|
423
431
|
end
|
424
432
|
|
425
433
|
set_info :dependencies, dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]}
|
@@ -430,16 +438,17 @@ class Step
|
|
430
438
|
if result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
|
431
439
|
Open.mv self.tmp_path, self.path
|
432
440
|
end
|
441
|
+
Open.rm pid_file if Open.exist?(pid_file) unless stream
|
433
442
|
result
|
434
443
|
end # END PERSIST
|
435
|
-
log :done, "Completed step #{Log.color :yellow,
|
444
|
+
log :done, "Completed step #{Log.color :yellow, task_name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec." unless stream or time_elapsed.nil?
|
436
445
|
|
437
446
|
if no_load
|
438
447
|
@result ||= result
|
439
448
|
self
|
440
449
|
else
|
441
450
|
Step.purge_stream_cache
|
442
|
-
@result = prepare_result result,
|
451
|
+
@result = prepare_result result, result_description
|
443
452
|
end
|
444
453
|
end # END SYNC
|
445
454
|
res
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -3,20 +3,12 @@ require 'rbbt/persist/tsv'
|
|
3
3
|
require 'rbbt/util/log'
|
4
4
|
require 'rbbt/util/semaphore'
|
5
5
|
require 'rbbt/workflow/step/accessor'
|
6
|
-
require 'rbbt/workflow/step/
|
6
|
+
require 'rbbt/workflow/step/produce'
|
7
7
|
require 'rbbt/workflow/step/status'
|
8
8
|
require 'rbbt/workflow/step/info'
|
9
9
|
require 'rbbt/workflow/step/save_load_inputs'
|
10
10
|
|
11
11
|
class Step
|
12
|
-
attr_accessor :clean_name, :path, :task, :workflow, :inputs, :dependencies, :bindings
|
13
|
-
attr_accessor :task_name, :overriden
|
14
|
-
attr_accessor :pid
|
15
|
-
attr_accessor :exec
|
16
|
-
attr_accessor :relocated
|
17
|
-
attr_accessor :result, :mutex, :seen
|
18
|
-
attr_accessor :real_inputs, :original_task_name, :original_workflow
|
19
|
-
|
20
12
|
RBBT_DEBUG_CLEAN = ENV["RBBT_DEBUG_CLEAN"] == 'true'
|
21
13
|
|
22
14
|
class << self
|
@@ -38,10 +30,12 @@ class Step
|
|
38
30
|
end
|
39
31
|
|
40
32
|
|
41
|
-
def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil)
|
33
|
+
def initialize(path, task = nil, inputs = nil, dependencies = nil, bindings = nil, clean_name = nil, &block)
|
42
34
|
path = Path.setup(Misc.sanitize_filename(path)) if String === path
|
43
35
|
path = path.call if Proc === path
|
44
36
|
|
37
|
+
task = block if block_given?
|
38
|
+
|
45
39
|
@path = path
|
46
40
|
@task = task
|
47
41
|
@bindings = bindings
|
@@ -73,7 +67,8 @@ class Step
|
|
73
67
|
|
74
68
|
load_inputs_from_info if @inputs.nil?
|
75
69
|
|
76
|
-
|
70
|
+
task_inputs = task.respond_to?(:inputs) ? task.inputs : nil
|
71
|
+
NamedArray.setup(@inputs, task_inputs) unless NamedArray === @inputs
|
77
72
|
|
78
73
|
@inputs || []
|
79
74
|
end
|
@@ -133,15 +128,6 @@ class Step
|
|
133
128
|
v
|
134
129
|
end
|
135
130
|
|
136
|
-
def task_name
|
137
|
-
@task_name ||= begin
|
138
|
-
if @task.nil?
|
139
|
-
@path.split("/")[-2]
|
140
|
-
else
|
141
|
-
@task.name
|
142
|
-
end
|
143
|
-
end
|
144
|
-
end
|
145
131
|
|
146
132
|
def path
|
147
133
|
if Proc === @path
|
@@ -155,8 +141,11 @@ class Step
|
|
155
141
|
attr_accessor :log_relay_step
|
156
142
|
end
|
157
143
|
|
144
|
+
|
145
|
+
# May be deprecated: This is no loger used
|
158
146
|
def relay_log(step)
|
159
|
-
return self
|
147
|
+
return self if self.task_name.nil?
|
148
|
+
|
160
149
|
if not self.respond_to? :original_log
|
161
150
|
class << self
|
162
151
|
attr_accessor :relay_step
|
@@ -164,77 +153,58 @@ class Step
|
|
164
153
|
def log(status, message = nil)
|
165
154
|
self.status = status
|
166
155
|
message Log.uncolor message
|
167
|
-
relay_step.log([
|
156
|
+
relay_step.log([self.task_name.to_s, status.to_s] * ">", message.nil? ? nil : message ) unless (relay_step.done? or relay_step.error? or relay_step.aborted?)
|
168
157
|
end
|
169
158
|
end
|
170
159
|
end
|
171
160
|
@relay_step = step
|
172
|
-
self
|
173
|
-
end
|
174
161
|
|
175
|
-
|
176
|
-
@result_type ||= if @task.nil?
|
177
|
-
info[:result_type] || :binary
|
178
|
-
else
|
179
|
-
@task.result_type || info[:result_type] || :string
|
180
|
-
end
|
181
|
-
end
|
182
|
-
|
183
|
-
def result_type=(type)
|
184
|
-
@result_type = type
|
185
|
-
end
|
186
|
-
|
187
|
-
def result_description
|
188
|
-
@result_description ||= if @task.nil?
|
189
|
-
info[:result_description]
|
190
|
-
else
|
191
|
-
@task.result_description
|
192
|
-
end
|
162
|
+
self
|
193
163
|
end
|
194
164
|
|
195
165
|
def prepare_result(value, description = nil, entity_info = nil)
|
196
166
|
res = case
|
197
|
-
|
198
|
-
|
199
|
-
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
end
|
205
|
-
array
|
206
|
-
when :tsv
|
207
|
-
begin
|
208
|
-
TSV.open(value)
|
209
|
-
rescue IOError
|
210
|
-
TSV.setup({})
|
211
|
-
end
|
212
|
-
else
|
213
|
-
value.read
|
214
|
-
end
|
215
|
-
value.join if value.respond_to? :join
|
216
|
-
res
|
217
|
-
rescue Exception
|
218
|
-
value.abort if value.respond_to? :abort
|
219
|
-
self.abort
|
220
|
-
raise $!
|
221
|
-
end
|
222
|
-
when (not defined? Entity or description.nil? or not Entity.formats.include? description)
|
223
|
-
value
|
224
|
-
when (Annotated === value and info.empty?)
|
225
|
-
value
|
226
|
-
when Annotated === value
|
227
|
-
annotations = value.annotations
|
228
|
-
entity_info ||= begin
|
229
|
-
entity_info = info.dup
|
230
|
-
entity_info.merge! info[:inputs] if info[:inputs]
|
231
|
-
entity_info
|
167
|
+
when IO === value
|
168
|
+
begin
|
169
|
+
res = case result_type
|
170
|
+
when :array
|
171
|
+
array = []
|
172
|
+
while line = value.gets
|
173
|
+
array << line.chomp
|
232
174
|
end
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
175
|
+
array
|
176
|
+
when :tsv
|
177
|
+
begin
|
178
|
+
TSV.open(value)
|
179
|
+
rescue IOError
|
180
|
+
TSV.setup({})
|
181
|
+
end
|
182
|
+
else
|
183
|
+
value.read
|
184
|
+
end
|
185
|
+
value.join if value.respond_to? :join
|
186
|
+
res
|
187
|
+
rescue Exception
|
188
|
+
value.abort if value.respond_to? :abort
|
189
|
+
self.abort
|
190
|
+
raise $!
|
191
|
+
end
|
192
|
+
when (not defined? Entity or description.nil? or not Entity.formats.include? description)
|
193
|
+
value
|
194
|
+
when (Annotated === value and info.empty?)
|
195
|
+
value
|
196
|
+
when Annotated === value
|
197
|
+
annotations = value.annotations
|
198
|
+
entity_info ||= begin
|
199
|
+
entity_info = info.dup
|
200
|
+
entity_info.merge! info[:inputs] if info[:inputs]
|
201
|
+
entity_info
|
202
|
+
end
|
203
|
+
entity_info.each do |k,v|
|
204
|
+
value.send("#{h}=", v) if annotations.include? k
|
205
|
+
end
|
206
|
+
|
207
|
+
value
|
238
208
|
else
|
239
209
|
entity_info ||= begin
|
240
210
|
entity_info = info.dup
|
@@ -371,8 +341,9 @@ class Step
|
|
371
341
|
def step(name)
|
372
342
|
@steps ||= {}
|
373
343
|
@steps[name] ||= begin
|
344
|
+
name = name.to_sym
|
374
345
|
deps = rec_dependencies.select{|step|
|
375
|
-
step.task_name.to_sym == name
|
346
|
+
step.task_name && step.task_name.to_sym == name
|
376
347
|
}
|
377
348
|
raise "Dependency step not found: #{ name }" if deps.empty?
|
378
349
|
if (deps & self.dependencies).any?
|
@@ -385,3 +356,4 @@ class Step
|
|
385
356
|
end
|
386
357
|
|
387
358
|
require 'rbbt/workflow/step/run'
|
359
|
+
require 'rbbt/workflow/step/accessor'
|
data/lib/rbbt/workflow/task.rb
CHANGED
@@ -88,13 +88,15 @@ module Task
|
|
88
88
|
next
|
89
89
|
end
|
90
90
|
|
91
|
+
task_name ||= task.name
|
92
|
+
|
91
93
|
maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
|
92
94
|
raise "Dependency task not found: #{dep}" if task.nil?
|
93
|
-
next if seen.include? [wf,
|
95
|
+
next if seen.include? [wf, task_name, maps]
|
94
96
|
|
95
97
|
task.workflow = wf if wf
|
96
98
|
|
97
|
-
seen << [wf,
|
99
|
+
seen << [wf, task_name, maps]
|
98
100
|
new_inputs = task.inputs - maps
|
99
101
|
next unless new_inputs.any?
|
100
102
|
if task_inputs[task].nil?
|
@@ -160,44 +160,50 @@ puts files * "\n"
|
|
160
160
|
end
|
161
161
|
end
|
162
162
|
|
163
|
-
def self.migrate(
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
163
|
+
def self.migrate(paths, search_path, options = {})
|
164
|
+
subpath_files = {}
|
165
|
+
target_paths = []
|
166
|
+
resource = options[:resource] || Rbbt
|
167
|
+
|
168
|
+
target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
|
169
|
+
|
170
|
+
(Array === paths ? paths : [paths]).each do |path|
|
171
|
+
if Step === path
|
172
|
+
if options[:source]
|
173
|
+
path = Rbbt.identify(path.path)
|
174
|
+
else
|
175
|
+
path = path.path
|
176
|
+
end
|
169
177
|
end
|
170
|
-
|
171
|
-
search_path = 'user' if search_path.nil?
|
178
|
+
search_path = 'user' if search_path.nil?
|
172
179
|
|
173
|
-
resource = Rbbt
|
174
180
|
|
175
|
-
|
181
|
+
path, real_paths, lpath = self.migrate_source_paths(path, resource, options[:source], options[:recursive])
|
176
182
|
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
183
|
+
real_paths.sort.each do |path|
|
184
|
+
parts = path.split("/")
|
185
|
+
subpath = parts[0..-4] * "/" + "/"
|
186
|
+
|
187
|
+
if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
|
188
|
+
subpath = subpath_files.keys.last
|
189
|
+
end
|
181
190
|
|
182
|
-
|
183
|
-
|
191
|
+
source = path.chars[subpath.length..-1] * ""
|
192
|
+
|
193
|
+
subpath_files[subpath] ||= []
|
194
|
+
subpath_files[subpath] << source
|
184
195
|
end
|
185
196
|
|
186
|
-
source = path.chars[subpath.length..-1] * ""
|
187
197
|
|
188
|
-
|
189
|
-
subpath_files[subpath] << source
|
198
|
+
target_paths << File.join(target, *path.split("/")[-3..-1])
|
190
199
|
end
|
191
200
|
|
192
|
-
target = Rbbt.migrate_target_path('var/jobs', search_path, resource, options[:target])
|
193
|
-
|
194
|
-
target_path = File.join(target, *path.split("/")[-3..-1])
|
195
201
|
|
196
202
|
subpath_files.each do |subpath, files|
|
197
203
|
Rbbt.migrate_files([subpath], target, options.merge(:files => files))
|
198
204
|
end
|
199
205
|
|
200
|
-
|
206
|
+
target_paths
|
201
207
|
end
|
202
208
|
|
203
209
|
def self.purge(path, recursive = false, skip_overriden = true)
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -212,11 +212,13 @@ module Workflow
|
|
212
212
|
end
|
213
213
|
workflow.load_documentation
|
214
214
|
|
215
|
+
|
215
216
|
first ||= workflow
|
216
217
|
end
|
217
|
-
return first
|
218
218
|
|
219
|
-
|
219
|
+
first.complete_name = wf_name
|
220
|
+
|
221
|
+
return first
|
220
222
|
end
|
221
223
|
|
222
224
|
attr_accessor :description
|
@@ -230,6 +232,12 @@ module Workflow
|
|
230
232
|
attr_accessor :relay_tasks
|
231
233
|
|
232
234
|
#{{{ ATTR DEFAULTS
|
235
|
+
#
|
236
|
+
attr_accessor :complete_name
|
237
|
+
|
238
|
+
def self.complete_name
|
239
|
+
@complete_name ||= self.to_s
|
240
|
+
end
|
233
241
|
|
234
242
|
def self.workdir=(path)
|
235
243
|
path = Path.setup path.dup unless Path === path
|
data/python/rbbt.py
CHANGED
@@ -1,7 +1,93 @@
|
|
1
1
|
import warnings
|
2
2
|
import sys
|
3
3
|
import os
|
4
|
-
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
|
5
4
|
|
6
5
|
def rbbt():
|
7
6
|
print("Rbbt")
|
7
|
+
|
8
|
+
def path(subdir = None, base_dir = None):
|
9
|
+
from pathlib import Path
|
10
|
+
import os
|
11
|
+
|
12
|
+
if (base_dir == None):
|
13
|
+
base_dir = os.path.join(Path.home(), ".rbbt")
|
14
|
+
if (subdir == None):
|
15
|
+
return base_dir
|
16
|
+
else:
|
17
|
+
return os.path.join(base_dir, subdir)
|
18
|
+
|
19
|
+
def inspect(obj):
|
20
|
+
print(dir(obj))
|
21
|
+
|
22
|
+
def log_tsv(tsv):
|
23
|
+
print(tsv)
|
24
|
+
print(tsv.keys())
|
25
|
+
|
26
|
+
def tsv_preamble(line, comment_char="#"):
|
27
|
+
import re
|
28
|
+
header = dict()
|
29
|
+
entries = re.sub(f"^{comment_char}:", '', line)
|
30
|
+
entries = re.sub(f"^{comment_char}:", '', line).split("#")
|
31
|
+
for entry in entries:
|
32
|
+
key, value = entry.split("=")
|
33
|
+
key = re.sub("^:","",key)
|
34
|
+
value = re.sub("^:","",value)
|
35
|
+
header[key] = value
|
36
|
+
|
37
|
+
return header
|
38
|
+
|
39
|
+
|
40
|
+
def tsv_header(filename, sep="\t", comment_char="#"):
|
41
|
+
import re
|
42
|
+
|
43
|
+
f = open(filename)
|
44
|
+
line = f.readline().strip()
|
45
|
+
|
46
|
+
if (not line.startswith(comment_char)):
|
47
|
+
header = {"fields":None, "type":"list", "start": 0}
|
48
|
+
else:
|
49
|
+
header = dict()
|
50
|
+
start = 0
|
51
|
+
if (line.startswith(f"{comment_char}:")):
|
52
|
+
header["preamble"]=tsv_preamble(line, comment_char)
|
53
|
+
if ("type" in header["preamble"]):
|
54
|
+
header["type"] = header["preamble"]["type"]
|
55
|
+
line = f.readline().strip()
|
56
|
+
start = 1
|
57
|
+
|
58
|
+
if (line.startswith(comment_char)):
|
59
|
+
header["all_fields"] = re.sub(f"^{comment_char}", "", line).split(sep)
|
60
|
+
header["key_field"] = header["all_fields"][0]
|
61
|
+
header["fields"] = header["all_fields"][1:]
|
62
|
+
|
63
|
+
header["start"] = start
|
64
|
+
|
65
|
+
f.close()
|
66
|
+
return header
|
67
|
+
|
68
|
+
|
69
|
+
def tsv_pandas(filename, sep="\t", comment_char="#", index_col=0, **kwargs):
|
70
|
+
import pandas
|
71
|
+
|
72
|
+
if (comment_char == ""):
|
73
|
+
tsv = pandas.read_table(filename, sep=sep, index_col=index_col, **kwargs)
|
74
|
+
else:
|
75
|
+
header = tsv_header(filename, sep=sep, comment_char="#")
|
76
|
+
|
77
|
+
if ("type" in header and header["type"] == "flat"):
|
78
|
+
return None
|
79
|
+
else:
|
80
|
+
if ("sep" in header):
|
81
|
+
sep=header["sep"]
|
82
|
+
|
83
|
+
tsv = pandas.read_table(filename, sep=sep, index_col=index_col, header=header["start"], **kwargs)
|
84
|
+
|
85
|
+
if ("fields" in header):
|
86
|
+
tsv.columns = header["fields"]
|
87
|
+
tsv.index.name = header["key_field"]
|
88
|
+
|
89
|
+
return tsv
|
90
|
+
|
91
|
+
def tsv(*args, **kwargs):
|
92
|
+
return tsv_pandas(*args, **kwargs)
|
93
|
+
|
@@ -54,8 +54,9 @@ tsvfile, excelfile = ARGV
|
|
54
54
|
|
55
55
|
excelfile, tsvfile = tsvfile, nil if excelfile.nil? and tsvfile =~ /\.(xls|xlsx)$/
|
56
56
|
|
57
|
-
raise ParameterException, "No
|
58
|
-
|
57
|
+
raise ParameterException, "No TSV file given" if tsvfile.nil?
|
58
|
+
|
59
|
+
excelfile = tsvfile.sub(/\.tsv(\.gz)?$/, '.xlsx') if excelfile.nil?
|
59
60
|
|
60
61
|
require 'rbbt/tsv/excel'
|
61
62
|
TSV.open(tsvfile, options).excel(excelfile, options)
|
@@ -213,11 +213,15 @@ The `recursive_clean` cleans all the job dependency steps recursively.
|
|
213
213
|
-prov--provenance Report the jobs provenance
|
214
214
|
-W--workflows* Load a list of workflows
|
215
215
|
-R--requires* Require a list of files
|
216
|
-
-
|
217
|
-
-
|
216
|
+
-pro--produce* Prepare dependencies
|
217
|
+
-proc--produce_cpus* Number of dependencies prepared in parallel
|
218
218
|
-rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
|
219
219
|
-od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
|
220
220
|
-PERF--procpath_performance* Measure performance using procpath
|
221
|
+
-r--relay* Relay job to SSH server
|
222
|
+
-sr--slurm_relay* Relay job to SSH SLURM server
|
223
|
+
-rdep--relay_dependencies* Relay dependencies instead of main job
|
224
|
+
-pdr--produce_dependencies_for_relay Prepare dependencies previous to relay jobs
|
221
225
|
EOF
|
222
226
|
|
223
227
|
workflow = ARGV.shift
|
@@ -415,13 +419,53 @@ begin
|
|
415
419
|
exit 0
|
416
420
|
end
|
417
421
|
|
418
|
-
if tasks = options.delete(:
|
422
|
+
if tasks = options.delete(:produce)
|
419
423
|
tasks = tasks.split(",")
|
420
|
-
|
421
|
-
puts Step.
|
424
|
+
produce_cpus = (options[:produce_cpus] || 1)
|
425
|
+
puts Step.produce_dependencies(job, tasks, prepare_cpus)
|
422
426
|
exit 0
|
423
427
|
end
|
424
428
|
|
429
|
+
def match_dependencies(queries, dependencies)
|
430
|
+
queries = queries.collect{|q| q.include?("#") ? q.split("#") : q }
|
431
|
+
dependencies.select do |dep|
|
432
|
+
queries.select do |q|
|
433
|
+
if Array === q
|
434
|
+
q.first == dep.workflow.to_s && q.last == dep.task_name.to_s
|
435
|
+
else
|
436
|
+
q.to_s == dep.task_name.to_s
|
437
|
+
end
|
438
|
+
end.any?
|
439
|
+
end
|
440
|
+
end
|
441
|
+
|
442
|
+
if server = options.delete(:relay)
|
443
|
+
require 'rbbt/workflow/remote_workflow'
|
444
|
+
relay_dependencies = options.delete(:relay_dependencies).split(",")
|
445
|
+
produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
|
446
|
+
if relay_dependencies
|
447
|
+
match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
|
448
|
+
RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
|
449
|
+
end
|
450
|
+
else
|
451
|
+
RemoteWorkflow::SSH.relay_job(job, server, :run_type => :run, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
|
452
|
+
end
|
453
|
+
end
|
454
|
+
|
455
|
+
if server = options.delete(:slurm_relay)
|
456
|
+
require 'rbbt/workflow/remote_workflow'
|
457
|
+
relay_dependencies = options.delete(:relay_dependencies).split(",")
|
458
|
+
produce_dependencies_for_relay = options.delete(:produce_dependencies_for_relay)
|
459
|
+
if relay_dependencies
|
460
|
+
match_dependencies(relay_dependencies, job.rec_dependencies).each do |dep|
|
461
|
+
RemoteWorkflow::SSH.relay_job(dep, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
|
462
|
+
end
|
463
|
+
else
|
464
|
+
RemoteWorkflow::SSH.relay_job(job, server, :run_type => :orchestrate, :migrate => true, :produce_dependencies => produce_dependencies_for_relay)
|
465
|
+
end
|
466
|
+
end
|
467
|
+
|
468
|
+
|
425
469
|
if options[:procpath_performance]
|
426
470
|
require 'rbbt/util/procpath'
|
427
471
|
current_pid = job.info[:pid]
|
data/test/rbbt/test_workflow.rb
CHANGED
@@ -513,4 +513,92 @@ class TestWorkflow < Test::Unit::TestCase
|
|
513
513
|
assert_equal "TLA", job.run
|
514
514
|
assert job.overriden
|
515
515
|
end
|
516
|
+
|
517
|
+
def test_anonymous_workflow
|
518
|
+
workflow = Module.new
|
519
|
+
workflow.extend Workflow
|
520
|
+
workflow.define_singleton_method(:to_s){"TestWorkflow"}
|
521
|
+
|
522
|
+
assert_equal "TestWorkflow", workflow.to_s
|
523
|
+
|
524
|
+
t1 = workflow.task :s1 => :string do
|
525
|
+
task_name.to_s
|
526
|
+
end
|
527
|
+
|
528
|
+
workflow.dep :s1
|
529
|
+
t2 = workflow.task :s2 => :string do
|
530
|
+
task_name.to_s
|
531
|
+
end
|
532
|
+
|
533
|
+
workflow.dep :s2
|
534
|
+
t3 = workflow.task :s3 => :string do
|
535
|
+
task_name.to_s
|
536
|
+
end
|
537
|
+
|
538
|
+
assert_equal "s3", workflow.job(:s3).run
|
539
|
+
end
|
540
|
+
|
541
|
+
def test_low_level_step
|
542
|
+
Log.with_severity 0 do
|
543
|
+
TmpFile.with_file do |tmpdir|
|
544
|
+
Path.setup tmpdir
|
545
|
+
|
546
|
+
s1 = Step.new tmpdir.s1 do "s1" end
|
547
|
+
s1.task_name = "s1"
|
548
|
+
|
549
|
+
# run clean run
|
550
|
+
assert_equal "s1", s1.run
|
551
|
+
s1.clean
|
552
|
+
assert_equal "s1", s1.run
|
553
|
+
assert_equal "s1", s1.run
|
554
|
+
|
555
|
+
# is persisted because it raises RbbtException
|
556
|
+
s1.task = proc do raise RbbtException end
|
557
|
+
assert_equal "s1", s1.run
|
558
|
+
s1.clean
|
559
|
+
assert_raises RbbtException do s1.run end
|
560
|
+
assert RbbtException === s1.get_exception
|
561
|
+
assert s1.error?
|
562
|
+
|
563
|
+
s1.recursive_clean
|
564
|
+
s1.task = proc do "s1" end
|
565
|
+
|
566
|
+
# add dependencies
|
567
|
+
s2 = Step.new tmpdir.s2 do
|
568
|
+
step(:s1).load + " -> s2"
|
569
|
+
end
|
570
|
+
s2.task_name = "s2"
|
571
|
+
s2.dependencies << s1
|
572
|
+
|
573
|
+
s3 = Step.new tmpdir.s3 do
|
574
|
+
step(:s2).load + " -> s3"
|
575
|
+
end
|
576
|
+
s3.task_name = "s3"
|
577
|
+
s3.dependencies << s2
|
578
|
+
|
579
|
+
assert_equal "s1 -> s2", s2.run
|
580
|
+
assert_equal "s1 -> s2 -> s3", s3.run
|
581
|
+
|
582
|
+
# Test recusive behaviour
|
583
|
+
s1.task = proc do raise RbbtException end
|
584
|
+
ComputeDependency.setup(s1, :produce)
|
585
|
+
s1.clean
|
586
|
+
s3.clean
|
587
|
+
s3.run
|
588
|
+
assert_equal "s1 -> s2 -> s3", s3.run
|
589
|
+
|
590
|
+
s1.task = proc do raise RbbtException end
|
591
|
+
ComputeDependency.setup(s1, :produce)
|
592
|
+
s3.recursive_clean
|
593
|
+
assert_raises RbbtException do s3.run end
|
594
|
+
|
595
|
+
s1.task = proc{|v| v }
|
596
|
+
s1.inputs = ["test"]
|
597
|
+
assert_equal "test", s1.run
|
598
|
+
s3.recursive_clean
|
599
|
+
assert_equal "test -> s2 -> s3", s3.run
|
600
|
+
end
|
601
|
+
end
|
602
|
+
end
|
603
|
+
|
516
604
|
end
|
@@ -131,5 +131,10 @@ def python_print():
|
|
131
131
|
random = RbbtPython.import_method :torch, :rand, :random
|
132
132
|
assert random.call(1).numpy.to_f > 0
|
133
133
|
end
|
134
|
+
|
135
|
+
def test_class_new_obj
|
136
|
+
obj = RbbtPython.class_new_obj("torch.nn", "Module")
|
137
|
+
assert_equal "Module()", obj.to_s
|
138
|
+
end
|
134
139
|
end
|
135
140
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.36.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2023-02
|
11
|
+
date: 2023-03-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -366,7 +366,7 @@ files:
|
|
366
366
|
- lib/rbbt/workflow/step/accessor.rb
|
367
367
|
- lib/rbbt/workflow/step/dependencies.rb
|
368
368
|
- lib/rbbt/workflow/step/info.rb
|
369
|
-
- lib/rbbt/workflow/step/
|
369
|
+
- lib/rbbt/workflow/step/produce.rb
|
370
370
|
- lib/rbbt/workflow/step/run.rb
|
371
371
|
- lib/rbbt/workflow/step/save_load_inputs.rb
|
372
372
|
- lib/rbbt/workflow/step/status.rb
|