rbbt-util 5.29.0 → 5.29.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc/orchestrate.rb +1 -0
- data/lib/rbbt/hpc/slurm.rb +1 -1
- data/lib/rbbt/util/misc/inspect.rb +13 -3
- data/lib/rbbt/workflow/accessor.rb +6 -1
- data/lib/rbbt/workflow/step/accessor.rb +17 -9
- data/lib/rbbt/workflow/step/dependencies.rb +1 -2
- data/lib/rbbt/workflow/step/run.rb +0 -1
- data/lib/rbbt/workflow/util/provenance.rb +5 -2
- data/test/rbbt/test_workflow.rb +7 -7
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 9b88fc549c1c1dc5cd56f06d933776d56540e3d0f4bacf77f04a449abcda974f
|
4
|
+
data.tar.gz: caf80ab624418c6c0a744038d98569c3a32d18dc8d9de162b42363e1c281e8c2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8944f1d996afa5610f70046f4e61cf326461050f06f347529cd7e403440e0d9b47d0c33b862bc83e89d4479ce889e773aef0bbabfaf0dae797c3a2164f0e8a8c
|
7
|
+
data.tar.gz: fa0051835f35e23873bde6a075d81805d2878643e35e96e20c5d7423f0c7c48eaf25b2c5afad48215481b0ddd17b9d251555eae5b5c4a7fa420d9796dba9130b
|
data/lib/rbbt/hpc/orchestrate.rb
CHANGED
@@ -3,6 +3,7 @@ module HPC
|
|
3
3
|
module SLURM
|
4
4
|
def self.orchestrate_job(job, options, seen = {})
|
5
5
|
return if job.done?
|
6
|
+
return unless job.path.split("/")[-4] == "jobs"
|
6
7
|
options.delete "recursive_clean"
|
7
8
|
options.delete "tail"
|
8
9
|
rules = YAML.load(Open.read(options[:rules])) if options[:rules]
|
data/lib/rbbt/hpc/slurm.rb
CHANGED
@@ -494,6 +494,7 @@ EOF
|
|
494
494
|
dry_run = options.delete :dry_run
|
495
495
|
tail = options.delete :tail
|
496
496
|
dependencies = options.delete :slurm_dependencies
|
497
|
+
options[:jobname] = job.clean_name
|
497
498
|
|
498
499
|
workflow = job.workflow
|
499
500
|
|
@@ -518,7 +519,6 @@ EOF
|
|
518
519
|
inputs_dir = File.join(tmp_directory, 'inputs_dir')
|
519
520
|
saved = Step.save_job_inputs(job, inputs_dir)
|
520
521
|
|
521
|
-
|
522
522
|
if saved && saved.any?
|
523
523
|
options[:inputs_dir] = inputs_dir
|
524
524
|
cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
|
@@ -287,7 +287,9 @@ module Misc
|
|
287
287
|
when Symbol
|
288
288
|
obj.to_s
|
289
289
|
when (defined?(Path) and Path)
|
290
|
-
if Step
|
290
|
+
if defined?(Step) && Open.exists?(Step.info_file(obj))
|
291
|
+
obj2str(Workflow.load_step(obj))
|
292
|
+
elsif defined?(Step) && Step === obj.resource
|
291
293
|
"Step file: " + obj
|
292
294
|
else
|
293
295
|
if obj.exists?
|
@@ -322,7 +324,11 @@ module Misc
|
|
322
324
|
remove_long_items(obj)
|
323
325
|
when File
|
324
326
|
if obj.respond_to? :filename and obj.filename
|
325
|
-
|
327
|
+
if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
|
328
|
+
obj2str(Workflow.load_step(obj.filename))
|
329
|
+
else
|
330
|
+
"<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
|
331
|
+
end
|
326
332
|
else
|
327
333
|
"<IO:" << obj.path << "--" << mtime_str(obj.path) << ">"
|
328
334
|
end
|
@@ -330,7 +336,11 @@ module Misc
|
|
330
336
|
"<IO:" << obj.short_path << ">"
|
331
337
|
when IO
|
332
338
|
if obj.respond_to? :filename and obj.filename
|
333
|
-
|
339
|
+
if defined?(Step) && Open.exists?(Step.info_file(obj.filename))
|
340
|
+
obj2str(Workflow.load_step(obj.filename))
|
341
|
+
else
|
342
|
+
"<IO:" << obj.filename << "--" << mtime_str(obj.filename) << ">"
|
343
|
+
end
|
334
344
|
else
|
335
345
|
|
336
346
|
if obj.respond_to? :obj2str
|
@@ -16,6 +16,10 @@ end
|
|
16
16
|
|
17
17
|
module Workflow
|
18
18
|
|
19
|
+
def self.job_path?(path)
|
20
|
+
path.split("/")[-4] == "jobs"
|
21
|
+
end
|
22
|
+
|
19
23
|
def log(status, message = nil, &block)
|
20
24
|
Step.log(status, message, nil, &block)
|
21
25
|
end
|
@@ -301,8 +305,9 @@ module Workflow
|
|
301
305
|
|
302
306
|
def setup_override_dependency(dep, workflow, task_name)
|
303
307
|
dep = Step === dep ? dep : Workflow.load_step(dep)
|
308
|
+
dep.workflow = workflow
|
304
309
|
dep.info[:name] = dep.name
|
305
|
-
dep.original_task_name ||= dep.task_name
|
310
|
+
dep.original_task_name ||= dep.task_name if dep.workflow
|
306
311
|
begin
|
307
312
|
workflow = Kernel.const_get workflow if String === workflow
|
308
313
|
dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
|
@@ -94,17 +94,21 @@ class Step
|
|
94
94
|
Log.debug "Saving job input #{name} (#{type}) into #{path}"
|
95
95
|
case
|
96
96
|
when Step === value
|
97
|
-
Open.
|
97
|
+
Open.ln_s(value.path, path)
|
98
98
|
when type.to_s == "file"
|
99
99
|
if String === value && File.exists?(value)
|
100
|
-
Open.
|
100
|
+
Open.ln_s(value, path)
|
101
101
|
else
|
102
102
|
Open.write(path + '.yaml', value.to_yaml)
|
103
103
|
end
|
104
104
|
when Array === value
|
105
|
-
Open.write(path, value * "\n")
|
105
|
+
Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
|
106
106
|
when IO === value
|
107
|
-
|
107
|
+
if value.filename && String === value.filename && File.exists?(value.filename)
|
108
|
+
Open.ln_s(value.filename, path)
|
109
|
+
else
|
110
|
+
Open.write(path, value)
|
111
|
+
end
|
108
112
|
else
|
109
113
|
Open.write(path, value.to_s)
|
110
114
|
end
|
@@ -117,13 +121,17 @@ class Step
|
|
117
121
|
task_name = Symbol === job.overriden ? job.overriden : job.task_name
|
118
122
|
workflow = job.workflow
|
119
123
|
workflow = Kernel.const_get workflow if String === workflow
|
120
|
-
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
+
if workflow
|
125
|
+
task_info = workflow.task_info(task_name)
|
126
|
+
input_types = task_info[:input_types]
|
127
|
+
task_inputs = task_info[:inputs]
|
128
|
+
input_defaults = task_info[:input_defaults]
|
129
|
+
else
|
130
|
+
task_info = input_types = task_inputs = input_defaults = {}
|
131
|
+
end
|
124
132
|
|
125
133
|
inputs = {}
|
126
|
-
real_inputs = job.real_inputs ||
|
134
|
+
real_inputs = job.real_inputs || job.info[:real_inputs]
|
127
135
|
job.recursive_inputs.zip(job.recursive_inputs.fields).each do |value,name|
|
128
136
|
next unless task_inputs.include? name.to_sym
|
129
137
|
next unless real_inputs.include? name.to_sym
|
@@ -103,7 +103,7 @@ class Step
|
|
103
103
|
end
|
104
104
|
|
105
105
|
job.dup_inputs unless status == 'done' or job.started?
|
106
|
-
job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started?
|
106
|
+
job.init_info(status == 'noinfo') unless status == 'waiting' || status == 'done' || job.started? || ! Workflow.job_path?(job.path)
|
107
107
|
|
108
108
|
canfail = ComputeDependency === job && job.canfail?
|
109
109
|
end
|
@@ -130,7 +130,6 @@ class Step
|
|
130
130
|
(inputs.flatten.select{|i| Step === i} + inputs.flatten.select{|dep| Path === dep && Step === dep.resource}.collect{|dep| dep.resource})
|
131
131
|
end
|
132
132
|
|
133
|
-
|
134
133
|
def execute_dependency(dependency, log = true)
|
135
134
|
task_name = self.task_name
|
136
135
|
canfail_paths = self.canfail_paths
|
@@ -122,7 +122,6 @@ class Step
|
|
122
122
|
reject{|dependency| (defined?(WorkflowRemoteClient) && WorkflowRemoteClient::RemoteStep === dependency) || Open.remote?(dependency.path) }.
|
123
123
|
reject{|dependency| dependency.error? }.
|
124
124
|
#select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
|
125
|
-
#select{|dependency| Open.exists?(dependency.path) || ((Open.exists?(dependency.info_file) && (dependency.status == :cleaned) || dependency.status == :waiting)) }.
|
126
125
|
select{|dependency| dependency.updatable? }.
|
127
126
|
collect{|dependency| Workflow.relocate_dependency(self, dependency)}
|
128
127
|
end
|
@@ -22,11 +22,14 @@ class Step
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def self.prov_report_msg(status, name, path, info = nil)
|
25
|
-
parts = path.sub(/\{.*/,'').
|
25
|
+
parts = path.sub(/\{.*/,'').split "/"
|
26
26
|
|
27
|
+
parts.pop
|
28
|
+
|
27
29
|
task = Log.color(:yellow, parts.pop)
|
28
30
|
workflow = Log.color(:magenta, parts.pop)
|
29
|
-
if status.to_s == 'noinfo'
|
31
|
+
#if status.to_s == 'noinfo' && parts.last != 'jobs'
|
32
|
+
if ! Workflow.job_path?(path)
|
30
33
|
task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:green, "file"), Log.color(:magenta, "-")
|
31
34
|
end
|
32
35
|
|
data/test/rbbt/test_workflow.rb
CHANGED
@@ -386,9 +386,9 @@ class TestWorkflow < Test::Unit::TestCase
|
|
386
386
|
job.run
|
387
387
|
Misc.with_env "RBBT_UPDATE", 'true' do
|
388
388
|
assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
|
389
|
-
job = TestWF.job(:t3)
|
390
|
-
job.step(:t1).clean
|
391
|
-
assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
|
389
|
+
#job = TestWF.job(:t3)
|
390
|
+
#job.step(:t1).clean
|
391
|
+
#assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
|
392
392
|
job = TestWF.job(:t3).recursive_clean
|
393
393
|
job.run
|
394
394
|
assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
|
@@ -427,7 +427,7 @@ class TestWorkflow < Test::Unit::TestCase
|
|
427
427
|
TmpFile.with_file do |dir|
|
428
428
|
Path.setup(dir)
|
429
429
|
Step.save_job_inputs(job, dir)
|
430
|
-
assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.
|
430
|
+
assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.yaml']
|
431
431
|
inputs = Workflow.load_inputs(dir, [:file], :file => :file)
|
432
432
|
assert_equal inputs, {:file => 'code'}
|
433
433
|
end
|
@@ -455,9 +455,9 @@ class TestWorkflow < Test::Unit::TestCase
|
|
455
455
|
job.run
|
456
456
|
Misc.with_env "RBBT_UPDATE", 'true' do
|
457
457
|
assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
|
458
|
-
job = TestWF.job(:t3)
|
459
|
-
job.step(:t1).clean
|
460
|
-
assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
|
458
|
+
#job = TestWF.job(:t3)
|
459
|
+
#job.step(:t1).clean
|
460
|
+
#assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
|
461
461
|
job = TestWF.job(:t3).recursive_clean
|
462
462
|
job.run
|
463
463
|
assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.29.
|
4
|
+
version: 5.29.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-01-
|
11
|
+
date: 2021-01-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|