rbbt-util 5.34.25 → 5.34.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc/batch.rb +28 -1
- data/lib/rbbt/hpc/lsf.rb +5 -1
- data/lib/rbbt/hpc/orchestrate.old.rb +1 -1
- data/lib/rbbt/hpc/orchestrate.rb +2 -2
- data/lib/rbbt/hpc/slurm.rb +10 -2
- data/lib/rbbt/monitor.rb +3 -3
- data/lib/rbbt/persist.rb +1 -3
- data/lib/rbbt/resource/path.rb +1 -1
- data/lib/rbbt/util/misc/serialize.rb +26 -0
- data/lib/rbbt/util/misc.rb +2 -0
- data/lib/rbbt/workflow/dependencies.rb +30 -13
- data/lib/rbbt/workflow/step/accessor.rb +5 -1
- data/lib/rbbt/workflow/step/dependencies.rb +1 -0
- data/lib/rbbt/workflow/step/run.rb +1 -1
- data/lib/rbbt/workflow/step/save_load_inputs.rb +1 -1
- data/lib/rbbt/workflow/step/status.rb +1 -1
- data/lib/rbbt/workflow.rb +16 -20
- data/share/rbbt_commands/system/clean +1 -1
- data/test/rbbt/test_workflow.rb +43 -2
- data/test/rbbt/util/misc/test_serialize.rb +24 -0
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 40a3d86f01180255239e05fc23309b2c798eb94ee4c213ab8699f7954c1f5346
|
|
4
|
+
data.tar.gz: 3357e23002f032213a6b4c01ef046527092c8e4bf9ebce96dfc714176c3cb239
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8454ec0e14dac9fbdb5fa00a2219248b5037c890289fb5b013d69d6dac63e869b5160620d1ac744392546bf58dc8acfb1afba5c9172f366ddae01fc6cc008810
|
|
7
|
+
data.tar.gz: c85e5b6defde582d6af2c08281dd659bd8f71ae9867d4258d2c96c9824b3d0fe7020472fc6ccec3e66f4f70228bdc3dea425756abd510ded61a7f59420b78b98
|
data/lib/rbbt/hpc/batch.rb
CHANGED
|
@@ -560,6 +560,16 @@ env > #{batch_options[:fenv]}
|
|
|
560
560
|
def run_job(job, options = {})
|
|
561
561
|
system = self.to_s.split("::").last
|
|
562
562
|
|
|
563
|
+
if (batch_job = job.info[:batch_job]) && job_queued(batch_job)
|
|
564
|
+
Log.info "Job #{job.short_path} already queued in #{batch_job}"
|
|
565
|
+
return batch_job
|
|
566
|
+
end
|
|
567
|
+
|
|
568
|
+
if job.running?
|
|
569
|
+
Log.info "Job #{job.short_path} already running in #{job.info[:pid]}"
|
|
570
|
+
return job.info[:batch_job]
|
|
571
|
+
end
|
|
572
|
+
|
|
563
573
|
batch_base_dir, clean_batch_job, remove_batch_dir, procpath, tail, batch_dependencies, dry_run = Misc.process_options options,
|
|
564
574
|
:batch_base_dir, :clean_batch_job, :remove_batch_dir, :batch_procpath, :tail, :batch_dependencies, :dry_run,
|
|
565
575
|
:batch_base_dir => File.expand_path(File.join('~/rbbt-batch'))
|
|
@@ -583,6 +593,8 @@ env > #{batch_options[:fenv]}
|
|
|
583
593
|
|
|
584
594
|
batch_job = run_template(batch_dir, dry_run)
|
|
585
595
|
|
|
596
|
+
hold_dependencies(job, batch_job) unless dry_run
|
|
597
|
+
|
|
586
598
|
return [batch_job, batch_dir] unless tail
|
|
587
599
|
|
|
588
600
|
t_monitor = Thread.new do
|
|
@@ -603,6 +615,17 @@ env > #{batch_options[:fenv]}
|
|
|
603
615
|
end
|
|
604
616
|
end
|
|
605
617
|
|
|
618
|
+
def hold_dependencies(job, batch_job)
|
|
619
|
+
job.set_info :batch_job, batch_job
|
|
620
|
+
job.set_info :batch_system, self.batch_system
|
|
621
|
+
job.dependencies.each do |dep|
|
|
622
|
+
next unless dep.waiting?
|
|
623
|
+
next if (dep_batch_job = dep.info[:batch_job]) && job_queued(dep_batch_job)
|
|
624
|
+
|
|
625
|
+
hold_dependencies(dep, batch_job)
|
|
626
|
+
end
|
|
627
|
+
end
|
|
628
|
+
|
|
606
629
|
def follow_job(batch_dir, tail = true)
|
|
607
630
|
fjob = File.join(batch_dir, 'job.id')
|
|
608
631
|
fout = File.join(batch_dir, 'std.out')
|
|
@@ -672,7 +695,7 @@ env > #{batch_options[:fenv]}
|
|
|
672
695
|
terr = Misc.consume_stream(err, true, STDERR) if err
|
|
673
696
|
tout = Misc.consume_stream(out, true, STDOUT) if out
|
|
674
697
|
|
|
675
|
-
sleep 3 while
|
|
698
|
+
sleep 3 while job_queued(job)
|
|
676
699
|
rescue Aborted
|
|
677
700
|
ensure
|
|
678
701
|
begin
|
|
@@ -692,6 +715,10 @@ env > #{batch_options[:fenv]}
|
|
|
692
715
|
end
|
|
693
716
|
end
|
|
694
717
|
|
|
718
|
+
def job_queued(job)
|
|
719
|
+
job_status(job).split(/\s+/).include?(job.to_s)
|
|
720
|
+
end
|
|
721
|
+
|
|
695
722
|
def wait_for_job(batch_dir, time = 1)
|
|
696
723
|
fexit = File.join(batch_dir, 'exit.status')
|
|
697
724
|
fjob = File.join(batch_dir, 'job.id')
|
data/lib/rbbt/hpc/lsf.rb
CHANGED
|
@@ -5,6 +5,10 @@ module HPC
|
|
|
5
5
|
extend HPC::TemplateGeneration
|
|
6
6
|
extend HPC::Orchestration
|
|
7
7
|
|
|
8
|
+
def self.batch_system
|
|
9
|
+
"LSF"
|
|
10
|
+
end
|
|
11
|
+
|
|
8
12
|
def self.batch_system_variables
|
|
9
13
|
<<-EOF
|
|
10
14
|
let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
|
|
@@ -13,7 +17,7 @@ let MAX_MEMORY_DEFAULT="$(grep MemTotal /proc/meminfo|grep -o "[[:digit:]]*") /
|
|
|
13
17
|
export MAX_MEMORY_DEFAULT
|
|
14
18
|
export MAX_MEMORY
|
|
15
19
|
export BATCH_JOB_ID=$LSF_JOBID
|
|
16
|
-
export BATCH_SYSTEM
|
|
20
|
+
export BATCH_SYSTEM=#{batch_system}
|
|
17
21
|
EOF
|
|
18
22
|
end
|
|
19
23
|
|
|
@@ -206,7 +206,7 @@ module HPC
|
|
|
206
206
|
options.delete "detach"
|
|
207
207
|
options.delete "jobname"
|
|
208
208
|
|
|
209
|
-
rules =
|
|
209
|
+
rules = Misc.load_yaml(options[:orchestration_rules]) if options[:orchestration_rules]
|
|
210
210
|
rules ||= {}
|
|
211
211
|
IndiferentHash.setup(rules)
|
|
212
212
|
|
data/lib/rbbt/hpc/orchestrate.rb
CHANGED
|
@@ -37,9 +37,9 @@ module HPC
|
|
|
37
37
|
prepare_for_execution(job)
|
|
38
38
|
|
|
39
39
|
if options[:orchestration_rules]
|
|
40
|
-
rules =
|
|
40
|
+
rules = Misc.load_yaml(options[:orchestration_rules])
|
|
41
41
|
elsif Rbbt.etc.slurm["default.yaml"].exists?
|
|
42
|
-
rules =
|
|
42
|
+
rules = Misc.load_yaml(Rbbt.etc.slurm["default.yaml"])
|
|
43
43
|
else
|
|
44
44
|
rules = {}
|
|
45
45
|
end
|
data/lib/rbbt/hpc/slurm.rb
CHANGED
|
@@ -6,6 +6,10 @@ module HPC
|
|
|
6
6
|
extend HPC::TemplateGeneration
|
|
7
7
|
extend HPC::Orchestration
|
|
8
8
|
|
|
9
|
+
def self.batch_system
|
|
10
|
+
"SLURM"
|
|
11
|
+
end
|
|
12
|
+
|
|
9
13
|
def self.batch_system_variables
|
|
10
14
|
<<-EOF
|
|
11
15
|
let TOTAL_PROCESORS="$(cat /proc/cpuinfo|grep ^processor |wc -l)"
|
|
@@ -16,7 +20,7 @@ MAX_MEMORY="$MAX_MEMORY_DEFAULT"
|
|
|
16
20
|
export MAX_MEMORY_DEFAULT
|
|
17
21
|
export MAX_MEMORY
|
|
18
22
|
export BATCH_JOB_ID=$SLURM_JOB_ID
|
|
19
|
-
export BATCH_SYSTEM
|
|
23
|
+
export BATCH_SYSTEM=#{batch_system}
|
|
20
24
|
EOF
|
|
21
25
|
end
|
|
22
26
|
|
|
@@ -145,7 +149,11 @@ export BATCH_SYSTEM=SLURM
|
|
|
145
149
|
if job.nil?
|
|
146
150
|
CMD.cmd("squeue").read
|
|
147
151
|
else
|
|
148
|
-
|
|
152
|
+
begin
|
|
153
|
+
CMD.cmd("squeue --job #{job}").read
|
|
154
|
+
rescue
|
|
155
|
+
""
|
|
156
|
+
end
|
|
149
157
|
end
|
|
150
158
|
end
|
|
151
159
|
|
data/lib/rbbt/monitor.rb
CHANGED
|
@@ -64,14 +64,14 @@ module Rbbt
|
|
|
64
64
|
lock_info[f].merge!(file_time(f))
|
|
65
65
|
if File.size(f) > 0
|
|
66
66
|
info = Open.open(f) do |s|
|
|
67
|
-
|
|
67
|
+
Misc.load_yaml(s)
|
|
68
68
|
end
|
|
69
69
|
IndiferentHash.setup(info)
|
|
70
70
|
lock_info[f][:pid] = info[:pid]
|
|
71
71
|
lock_info[f][:ppid] = info[:ppid]
|
|
72
72
|
end
|
|
73
73
|
rescue Exception
|
|
74
|
-
|
|
74
|
+
Log.exception $!
|
|
75
75
|
end
|
|
76
76
|
end
|
|
77
77
|
lock_info
|
|
@@ -272,7 +272,7 @@ module Rbbt
|
|
|
272
272
|
def self.load_lock(lock)
|
|
273
273
|
begin
|
|
274
274
|
info = Misc.insist 3 do
|
|
275
|
-
|
|
275
|
+
Misc.load_yaml(lock)
|
|
276
276
|
end
|
|
277
277
|
info.values_at "pid", "ppid", "time"
|
|
278
278
|
rescue Exception
|
data/lib/rbbt/persist.rb
CHANGED
data/lib/rbbt/resource/path.rb
CHANGED
|
@@ -150,7 +150,7 @@ module Path
|
|
|
150
150
|
search_path_file = File.join(ENV['HOME'], '.rbbt/etc/search_paths')
|
|
151
151
|
if File.exist?(search_path_file)
|
|
152
152
|
begin
|
|
153
|
-
|
|
153
|
+
Misc.load_yaml(search_path_file).each do |where, location|
|
|
154
154
|
SEARCH_PATHS[where.to_sym] = location
|
|
155
155
|
end
|
|
156
156
|
rescue
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
module Misc
|
|
2
|
+
def self.load_yaml(yaml)
|
|
3
|
+
case yaml
|
|
4
|
+
when IO, StringIO
|
|
5
|
+
if YAML.respond_to?(:unsafe_load)
|
|
6
|
+
YAML.unsafe_load(yaml)
|
|
7
|
+
else
|
|
8
|
+
YAML.load(yaml)
|
|
9
|
+
end
|
|
10
|
+
when (defined?(Path) && Path)
|
|
11
|
+
yaml.open do |io|
|
|
12
|
+
load_yaml(io)
|
|
13
|
+
end
|
|
14
|
+
when String
|
|
15
|
+
if Misc.is_filename?(yaml)
|
|
16
|
+
File.open(yaml) do |io|
|
|
17
|
+
load_yaml(io)
|
|
18
|
+
end
|
|
19
|
+
else
|
|
20
|
+
load_yaml(StringIO.new(yaml))
|
|
21
|
+
end
|
|
22
|
+
else
|
|
23
|
+
raise "Unkown YAML object: #{Misc.fingerprint yaml}"
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
data/lib/rbbt/util/misc.rb
CHANGED
|
@@ -72,12 +72,7 @@ module Workflow
|
|
|
72
72
|
def setup_override_dependency(dep, workflow, task_name)
|
|
73
73
|
return [] if dep == :skip || dep == 'skip'
|
|
74
74
|
|
|
75
|
-
if not Step === dep
|
|
76
|
-
located = Open.exists?(dep) || Open.exists?(dep + '.info')
|
|
77
|
-
dep = Workflow.load_step(dep)
|
|
78
|
-
else
|
|
79
|
-
located = true
|
|
80
|
-
end
|
|
75
|
+
dep = Workflow.load_step(dep) if not Step === dep
|
|
81
76
|
|
|
82
77
|
dep.original_workflow ||= dep.workflow if dep.workflow
|
|
83
78
|
dep.original_task_name ||= dep.task_name if dep.task_name
|
|
@@ -88,7 +83,6 @@ module Workflow
|
|
|
88
83
|
dep.workflow = workflow
|
|
89
84
|
dep.info[:name] = dep.name
|
|
90
85
|
|
|
91
|
-
|
|
92
86
|
begin
|
|
93
87
|
workflow = Kernel.const_get workflow if String === workflow
|
|
94
88
|
dep.task = workflow.tasks[task_name] if dep.task.nil? && workflow.tasks.include?(task_name)
|
|
@@ -97,13 +91,17 @@ module Workflow
|
|
|
97
91
|
end
|
|
98
92
|
|
|
99
93
|
dep.task_name = task_name
|
|
100
|
-
dep.overriden = dep.original_task_name.to_sym if dep.original_task_name
|
|
94
|
+
dep.overriden = dep.original_task_name.to_sym if dep.original_task_name
|
|
101
95
|
|
|
102
96
|
dep.extend step_module
|
|
103
97
|
|
|
104
98
|
dep
|
|
105
99
|
end
|
|
106
100
|
|
|
101
|
+
def unlocated_override?(dep)
|
|
102
|
+
String === dep && ! (Open.exists?(dep) || Open.exists?(dep + '.info'))
|
|
103
|
+
end
|
|
104
|
+
|
|
107
105
|
def assign_dep_inputs(_inputs, options, all_d, task_info)
|
|
108
106
|
IndiferentHash.setup(_inputs)
|
|
109
107
|
|
|
@@ -155,6 +153,7 @@ module Workflow
|
|
|
155
153
|
|
|
156
154
|
override_dependencies = override_dependencies(inputs)
|
|
157
155
|
|
|
156
|
+
overriden = false
|
|
158
157
|
dependencies.each do |dependency|
|
|
159
158
|
_inputs = IndiferentHash.setup(inputs.dup)
|
|
160
159
|
jobname = orig_jobname
|
|
@@ -165,6 +164,7 @@ module Workflow
|
|
|
165
164
|
workflow, dep_task, options = dependency
|
|
166
165
|
|
|
167
166
|
if override_dependencies[workflow.to_s] && value = override_dependencies[workflow.to_s][dep_task]
|
|
167
|
+
overriden = true if (options.nil? || ! options[:not_overriden]) && ! unlocated_override?(value)
|
|
168
168
|
setup_override_dependency(value, workflow, dep_task)
|
|
169
169
|
else
|
|
170
170
|
|
|
@@ -177,15 +177,23 @@ module Workflow
|
|
|
177
177
|
|
|
178
178
|
job = workflow._job(dep_task, jobname, _inputs)
|
|
179
179
|
ComputeDependency.setup(job, compute) if compute
|
|
180
|
+
|
|
181
|
+
overriden = true if TrueClass === job.overriden && (options.nil? || ! options[:not_overriden])
|
|
182
|
+
|
|
180
183
|
job
|
|
181
184
|
end
|
|
182
185
|
when Step
|
|
183
|
-
dependency
|
|
186
|
+
job = dependency
|
|
187
|
+
overriden = true if TrueClass === job.overriden && (options.nil? || ! options[:not_overriden])
|
|
188
|
+
job
|
|
184
189
|
when Symbol
|
|
185
190
|
if override_dependencies[self.to_s] && value = override_dependencies[self.to_s][dependency]
|
|
191
|
+
overriden = true if (options.nil? || ! options[:not_overriden]) && ! unlocated_override?(value)
|
|
186
192
|
setup_override_dependency(value, self, dependency)
|
|
187
193
|
else
|
|
188
|
-
_job(dependency, jobname, _inputs)
|
|
194
|
+
job = _job(dependency, jobname, _inputs)
|
|
195
|
+
overriden = true if TrueClass === job.overriden && (options.nil? || ! options[:not_overriden])
|
|
196
|
+
job
|
|
189
197
|
end
|
|
190
198
|
when Proc
|
|
191
199
|
if DependencyBlock === dependency
|
|
@@ -193,6 +201,7 @@ module Workflow
|
|
|
193
201
|
wf, task_name, options = orig_dep
|
|
194
202
|
|
|
195
203
|
if override_dependencies[wf.to_s] && value = override_dependencies[wf.to_s][task_name]
|
|
204
|
+
overriden = true if (options.nil? || ! options[:not_overriden]) && ! unlocated_override?(value)
|
|
196
205
|
dep = setup_override_dependency(value, wf, task_name)
|
|
197
206
|
else
|
|
198
207
|
|
|
@@ -208,16 +217,20 @@ module Workflow
|
|
|
208
217
|
dep.each{|d|
|
|
209
218
|
next if d.nil?
|
|
210
219
|
if Hash === d
|
|
220
|
+
d = d.merge(options)
|
|
211
221
|
d[:workflow] ||= wf
|
|
212
222
|
d[:task] ||= task_name
|
|
213
223
|
_override_dependencies = override_dependencies.merge(override_dependencies(d[:inputs] || {}))
|
|
214
224
|
d = if _override_dependencies[d[:workflow].to_s] && value = _override_dependencies[d[:workflow].to_s][d[:task]]
|
|
225
|
+
overriden = true if (options.nil? || ! options[:not_overriden]) && ! unlocated_override?(value)
|
|
215
226
|
setup_override_dependency(value, d[:workflow], d[:task])
|
|
216
227
|
else
|
|
217
228
|
task_info = d[:workflow].task_info(d[:task])
|
|
218
229
|
|
|
219
230
|
_inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info)
|
|
220
|
-
d[:workflow]._job(d[:task], d[:jobname], _inputs)
|
|
231
|
+
job = d[:workflow]._job(d[:task], d[:jobname], _inputs)
|
|
232
|
+
overriden = true if TrueClass === job.overriden && (d.nil? || ! d[:not_overriden])
|
|
233
|
+
job
|
|
221
234
|
end
|
|
222
235
|
end
|
|
223
236
|
ComputeDependency.setup(d, compute) if compute
|
|
@@ -232,11 +245,15 @@ module Workflow
|
|
|
232
245
|
dep[:workflow] ||= wf || self
|
|
233
246
|
_override_dependencies = override_dependencies.merge(override_dependencies(dep[:inputs] || {}))
|
|
234
247
|
if _override_dependencies[dep[:workflow].to_s] && value = _override_dependencies[dep[:workflow].to_s][dep[:task]]
|
|
248
|
+
overriden = true if (options.nil? || ! options[:not_overriden]) && ! unlocated_override?(value)
|
|
235
249
|
setup_override_dependency(value, dep[:workflow], dep[:task])
|
|
236
250
|
else
|
|
237
251
|
task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
|
|
238
252
|
_inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
|
|
239
|
-
|
|
253
|
+
job = dep[:workflow]._job(dep[:task], dep[:jobname], _inputs)
|
|
254
|
+
job = d[:workflow]._job(d[:task], d[:jobname], _inputs)
|
|
255
|
+
overriden = true if TrueClass === job.overriden && (d.nil? || ! d[:not_overriden])
|
|
256
|
+
job
|
|
240
257
|
end
|
|
241
258
|
end
|
|
242
259
|
end
|
|
@@ -248,6 +265,6 @@ module Workflow
|
|
|
248
265
|
|
|
249
266
|
real_dependencies << real_dep
|
|
250
267
|
end
|
|
251
|
-
real_dependencies.flatten.compact
|
|
268
|
+
[real_dependencies.flatten.compact, overriden]
|
|
252
269
|
end
|
|
253
270
|
end
|
|
@@ -143,8 +143,12 @@ class Step
|
|
|
143
143
|
|
|
144
144
|
def init_info(force = false)
|
|
145
145
|
return nil if @exec || info_file.nil? || (Open.exists?(info_file) && ! force)
|
|
146
|
+
batch_job = info[:batch_job] if Open.exists?(info_file)
|
|
147
|
+
batch_system = info[:batch_system] if Open.exists?(info_file)
|
|
146
148
|
Open.lock(info_file, :lock => info_lock) do
|
|
147
149
|
i = {:status => :waiting, :pid => Process.pid, :path => path, :real_inputs => real_inputs, :overriden => overriden}
|
|
150
|
+
i[:batch_job] = batch_job if batch_job
|
|
151
|
+
i[:batch_system] = batch_system if batch_system
|
|
148
152
|
i[:dependencies] = dependencies.collect{|dep| [dep.task_name, dep.name, dep.path]} if dependencies
|
|
149
153
|
Misc.sensiblewrite(info_file, Step.serialize_info(i), :force => true, :lock => false)
|
|
150
154
|
@info_cache = IndiferentHash.setup(i)
|
|
@@ -253,7 +257,7 @@ class Step
|
|
|
253
257
|
#Open.read(file(name)).split /\n|,\s*/
|
|
254
258
|
Open.read(file(name)).split "\n"
|
|
255
259
|
when :yaml
|
|
256
|
-
|
|
260
|
+
Misc.load_yaml(file(name))
|
|
257
261
|
when :marshal
|
|
258
262
|
Marshal.load(Open.open(file(name)))
|
|
259
263
|
else
|
|
@@ -82,7 +82,7 @@ module Workflow
|
|
|
82
82
|
when :file, :binary
|
|
83
83
|
Log.debug "Pointing #{ input } to #{file}"
|
|
84
84
|
if file =~ /\.yaml/
|
|
85
|
-
inputs[input.to_sym] =
|
|
85
|
+
inputs[input.to_sym] = Misc.load_yaml(file)
|
|
86
86
|
else
|
|
87
87
|
if File.symlink?(file)
|
|
88
88
|
link_target = File.expand_path(File.readlink(file), File.dirname(file))
|
|
@@ -15,7 +15,7 @@ class Step
|
|
|
15
15
|
raise "DO NOT CLEAN"
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
-
if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir)
|
|
18
|
+
if (Open.exists?(path) or Open.broken_link?(path)) or Open.exists?(pid_file) or Open.exists?(info_file) or Open.exists?(files_dir) or Open.broken_link?(files_dir) or Open.exists?(pid_file)
|
|
19
19
|
|
|
20
20
|
@result = nil
|
|
21
21
|
@pid = nil
|
data/lib/rbbt/workflow.rb
CHANGED
|
@@ -285,6 +285,7 @@ module Workflow
|
|
|
285
285
|
|
|
286
286
|
workdir[taskname][tagged_jobname].find
|
|
287
287
|
end
|
|
288
|
+
|
|
288
289
|
def import_task(workflow, orig, new)
|
|
289
290
|
orig_task = workflow.tasks[orig]
|
|
290
291
|
new_task = orig_task.dup
|
|
@@ -442,10 +443,8 @@ module Workflow
|
|
|
442
443
|
end
|
|
443
444
|
|
|
444
445
|
real_inputs = {}
|
|
445
|
-
has_overriden_inputs = false
|
|
446
446
|
|
|
447
447
|
inputs.each do |k,v|
|
|
448
|
-
#has_overriden_inputs = true if String === k and k.include? "#"
|
|
449
448
|
next unless (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s))
|
|
450
449
|
default = all_defaults[k]
|
|
451
450
|
next if default == v
|
|
@@ -461,12 +460,10 @@ module Workflow
|
|
|
461
460
|
|
|
462
461
|
jobname = DEFAULT_NAME if jobname.nil? or jobname.empty?
|
|
463
462
|
|
|
464
|
-
dependencies = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
|
|
465
|
-
|
|
466
|
-
overriden_deps = dependencies.select{|d| d.overriden }
|
|
467
|
-
true_overriden_deps = overriden_deps.select{|d| TrueClass === d.overriden }
|
|
463
|
+
dependencies, overriden = real_dependencies(task, jobname, defaults.merge(inputs), task_dependencies[taskname] || [])
|
|
468
464
|
|
|
469
|
-
|
|
465
|
+
#overriden_deps = dependencies.select{|d| Symbol === d.overriden }
|
|
466
|
+
#overriden_target_deps = overriden_deps.select{|d| TrueClass === d.overriden }
|
|
470
467
|
|
|
471
468
|
extension = task.extension
|
|
472
469
|
|
|
@@ -492,19 +489,20 @@ module Workflow
|
|
|
492
489
|
step_path = step_path taskname, jobname, input_values, dependencies, extension
|
|
493
490
|
end
|
|
494
491
|
|
|
495
|
-
|
|
496
492
|
job = get_job_step step_path, task, input_values, dependencies
|
|
497
493
|
job.workflow = self
|
|
494
|
+
job.overriden ||= overriden
|
|
498
495
|
job.clean_name = jobname
|
|
499
496
|
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
497
|
+
#iif [job, not_overriden]
|
|
498
|
+
#case not_overriden
|
|
499
|
+
#when TrueClass
|
|
500
|
+
# job.overriden = false
|
|
501
|
+
#when :not_overriden_dep
|
|
502
|
+
# job.overriden = true if overriden_target_deps.any?
|
|
503
|
+
#else
|
|
504
|
+
# job.overriden = true if overriden_deps.any?
|
|
505
|
+
#end
|
|
508
506
|
|
|
509
507
|
job.real_inputs = real_inputs.keys
|
|
510
508
|
job
|
|
@@ -836,8 +834,7 @@ module Workflow
|
|
|
836
834
|
end
|
|
837
835
|
|
|
838
836
|
def self.load_remote_tasks(filename)
|
|
839
|
-
|
|
840
|
-
remote_workflow_tasks = YAML.load(yaml_text)
|
|
837
|
+
remote_workflow_tasks = Misc.load_yaml(filename)
|
|
841
838
|
Workflow.process_remote_tasks(remote_workflow_tasks)
|
|
842
839
|
end
|
|
843
840
|
|
|
@@ -861,8 +858,7 @@ module Workflow
|
|
|
861
858
|
end
|
|
862
859
|
|
|
863
860
|
def self.load_relay_tasks(filename)
|
|
864
|
-
|
|
865
|
-
relay_workflow_tasks = YAML.load(yaml_text)
|
|
861
|
+
remote_workflow_tasks = Misc.load_yaml(filename)
|
|
866
862
|
Workflow.process_relay_tasks(relay_workflow_tasks)
|
|
867
863
|
end
|
|
868
864
|
end
|
|
@@ -46,7 +46,7 @@ if locks.any?
|
|
|
46
46
|
puts
|
|
47
47
|
puts Log.color(:magenta, "Locks:")
|
|
48
48
|
locks.each do |file,info|
|
|
49
|
-
if force or (info[:pid]
|
|
49
|
+
if force or (info[:pid] && ! Misc.pid_exists?(info[:pid]))
|
|
50
50
|
puts " Removing #{ file }"
|
|
51
51
|
File.unlink file
|
|
52
52
|
end
|
data/test/rbbt/test_workflow.rb
CHANGED
|
@@ -166,6 +166,33 @@ for this dependency
|
|
|
166
166
|
dep = dependencies.flatten.first
|
|
167
167
|
end
|
|
168
168
|
|
|
169
|
+
task :overr_orig => :string do
|
|
170
|
+
"over"
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
task :overr_alt => :string do
|
|
174
|
+
"alt"
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
dep :overr_orig
|
|
178
|
+
task :overr_target => :string do
|
|
179
|
+
step(:overr_orig).load.reverse
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
dep :overr_alt, :not_overriden => true
|
|
183
|
+
dep :overr_target, "TestWF#overr_orig" => :overr_alt, :not_overriden => true
|
|
184
|
+
task :overr_action => :string do
|
|
185
|
+
step(:overr_target).load.upcase
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
dep :overr_alt, :not_overriden => true
|
|
190
|
+
dep :overr_target, "TestWF#overr_orig" => :overr_alt
|
|
191
|
+
task :overr_action2 => :string do
|
|
192
|
+
step(:overr_target).load.upcase
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
|
|
169
196
|
end
|
|
170
197
|
|
|
171
198
|
TestWF.workdir = Rbbt.tmp.test.jobs.TestWF
|
|
@@ -419,7 +446,7 @@ class TestWorkflow < Test::Unit::TestCase
|
|
|
419
446
|
TmpFile.with_file do |dir|
|
|
420
447
|
Path.setup(dir)
|
|
421
448
|
Step.save_job_inputs(job, dir)
|
|
422
|
-
assert_equal Dir.glob(dir + "/*"), [dir.file.find]
|
|
449
|
+
assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.as_path']
|
|
423
450
|
end
|
|
424
451
|
end
|
|
425
452
|
|
|
@@ -427,7 +454,7 @@ class TestWorkflow < Test::Unit::TestCase
|
|
|
427
454
|
TmpFile.with_file do |dir|
|
|
428
455
|
Path.setup(dir)
|
|
429
456
|
Step.save_job_inputs(job, dir)
|
|
430
|
-
assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.
|
|
457
|
+
assert_equal Dir.glob(dir + "/*"), [dir.file.find + '.as_path']
|
|
431
458
|
inputs = Workflow.load_inputs(dir, [:file], :file => :file)
|
|
432
459
|
assert_equal inputs, {:file => 'code'}
|
|
433
460
|
end
|
|
@@ -472,4 +499,18 @@ class TestWorkflow < Test::Unit::TestCase
|
|
|
472
499
|
end
|
|
473
500
|
end
|
|
474
501
|
end
|
|
502
|
+
|
|
503
|
+
def test_overriden
|
|
504
|
+
job = TestWF.job(:overr_action)
|
|
505
|
+
job.recursive_clean
|
|
506
|
+
assert_equal "TLA", job.run
|
|
507
|
+
|
|
508
|
+
assert Symbol === job.step(:overr_orig).overriden
|
|
509
|
+
assert TrueClass === job.step(:overr_target).overriden
|
|
510
|
+
assert ! job.overriden
|
|
511
|
+
|
|
512
|
+
job = TestWF.job(:overr_action2)
|
|
513
|
+
assert_equal "TLA", job.run
|
|
514
|
+
assert job.overriden
|
|
515
|
+
end
|
|
475
516
|
end
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
|
|
2
|
+
require 'rbbt/util/misc/serialize'
|
|
3
|
+
|
|
4
|
+
class TestClass < Test::Unit::TestCase
|
|
5
|
+
def test_load_yaml
|
|
6
|
+
yaml_txt=<<-EOF
|
|
7
|
+
---
|
|
8
|
+
a: b
|
|
9
|
+
EOF
|
|
10
|
+
yaml_sio = StringIO.new yaml_txt
|
|
11
|
+
|
|
12
|
+
assert_equal "b", Misc.load_yaml(yaml_txt)["a"]
|
|
13
|
+
assert_equal "b", Misc.load_yaml(yaml_sio)["a"]
|
|
14
|
+
|
|
15
|
+
TmpFile.with_file yaml_txt do |yaml_file|
|
|
16
|
+
assert_equal "b", Misc.load_yaml(yaml_file)["a"]
|
|
17
|
+
Open.open(yaml_file) do |yaml_io|
|
|
18
|
+
assert_equal "b", Misc.load_yaml(yaml_io)["a"]
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: rbbt-util
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.34.
|
|
4
|
+
version: 5.34.27
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Miguel Vazquez
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2022-12-
|
|
11
|
+
date: 2022-12-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rake
|
|
@@ -323,6 +323,7 @@ files:
|
|
|
323
323
|
- lib/rbbt/util/misc/omics.rb
|
|
324
324
|
- lib/rbbt/util/misc/options.rb
|
|
325
325
|
- lib/rbbt/util/misc/pipes.rb
|
|
326
|
+
- lib/rbbt/util/misc/serialize.rb
|
|
326
327
|
- lib/rbbt/util/misc/system.rb
|
|
327
328
|
- lib/rbbt/util/named_array.rb
|
|
328
329
|
- lib/rbbt/util/open.rb
|
|
@@ -559,6 +560,7 @@ files:
|
|
|
559
560
|
- test/rbbt/util/misc/test_multipart_payload.rb
|
|
560
561
|
- test/rbbt/util/misc/test_omics.rb
|
|
561
562
|
- test/rbbt/util/misc/test_pipes.rb
|
|
563
|
+
- test/rbbt/util/misc/test_serialize.rb
|
|
562
564
|
- test/rbbt/util/python/test_util.rb
|
|
563
565
|
- test/rbbt/util/simpleopt/test_get.rb
|
|
564
566
|
- test/rbbt/util/simpleopt/test_parse.rb
|
|
@@ -634,6 +636,7 @@ test_files:
|
|
|
634
636
|
- test/rbbt/util/misc/test_development.rb
|
|
635
637
|
- test/rbbt/util/misc/test_omics.rb
|
|
636
638
|
- test/rbbt/util/misc/test_pipes.rb
|
|
639
|
+
- test/rbbt/util/misc/test_serialize.rb
|
|
637
640
|
- test/rbbt/util/misc/test_format.rb
|
|
638
641
|
- test/rbbt/util/misc/test_communication.rb
|
|
639
642
|
- test/rbbt/util/misc/test_lock.rb
|