rbbt-util 5.33.6 → 5.33.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/hpc/batch.rb +1 -1
- data/lib/rbbt/hpc/orchestrate/chains.rb +1 -89
- data/lib/rbbt/hpc/orchestrate.rb +23 -1
- data/lib/rbbt/hpc/slurm.rb +2 -2
- data/lib/rbbt/resource/path.rb +1 -2
- data/lib/rbbt/resource/util.rb +20 -7
- data/lib/rbbt/util/migrate.rb +6 -0
- data/lib/rbbt/util/misc/inspect.rb +8 -7
- data/lib/rbbt/workflow/step/run.rb +2 -2
- data/lib/rbbt/workflow/step/save_load_inputs.rb +183 -40
- data/lib/rbbt/workflow/step.rb +2 -2
- data/share/install/software/lib/install_helpers +4 -0
- data/share/rbbt_commands/hpc/list +53 -17
- data/share/rbbt_commands/lsf/list +53 -17
- data/share/rbbt_commands/slurm/list +53 -17
- data/share/rbbt_commands/workflow/task +12 -1
- data/test/rbbt/workflow/step/test_save_load_inputs.rb +91 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5c1ff60078b8ea97b0d68c7ec8beb1831889cd0e4ef841c781073dad938924be
|
4
|
+
data.tar.gz: 75dd8c9528cc2015c83b3d106027a5bdc29ba5d9eca53b8024ab19eb824c5273
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 60379d7fd62c52f115a198bab61ef24ba2dc4be5c0fba9d4974637afaea9481b99530d0cadbae07b03db92fec75dee1ea1bb23d3fa96906d6cc918794ab0973b
|
7
|
+
data.tar.gz: 7c7ffd9b0847083722e6976f98eb700fef719f7fba7b36914e690538a93de047c7c46d42c0715ac430edc9c392ad5b0f1b0e97de65f812ba2eed3835263dbbe9
|
data/lib/rbbt/hpc/batch.rb
CHANGED
@@ -544,7 +544,7 @@ env > #{batch_options[:fenv]}
|
|
544
544
|
Misc.add_defaults options,
|
545
545
|
:batch_dir => batch_dir,
|
546
546
|
:inputs_dir => File.join(batch_dir, "inputs_dir"),
|
547
|
-
:workflows => workflows_to_load * ","
|
547
|
+
:workflows => workflows_to_load.uniq * ","
|
548
548
|
|
549
549
|
options[:procpath_performance] ||= File.join(batch_dir, "procpath##{procpath.gsub(',', '#')}") if procpath
|
550
550
|
|
@@ -1,6 +1,7 @@
|
|
1
1
|
module HPC
|
2
2
|
module Orchestration
|
3
3
|
def self.check_chains(chains, job)
|
4
|
+
return [] if Symbol === job.overriden
|
4
5
|
matches = []
|
5
6
|
chains.each do |name, chain|
|
6
7
|
next unless chain[:tasks].include?(job.workflow.to_s)
|
@@ -36,95 +37,6 @@ module HPC
|
|
36
37
|
(job.dependencies + job.input_dependencies).uniq.select{|d| ! d.done? || d.dirty? }
|
37
38
|
end
|
38
39
|
|
39
|
-
#def self.job_workload(job)
|
40
|
-
# workload = []
|
41
|
-
# heap = []
|
42
|
-
# heap << job
|
43
|
-
# while job = heap.pop
|
44
|
-
# next if job.done?
|
45
|
-
# workload << job
|
46
|
-
# heap.concat job_dependencies(job)
|
47
|
-
# heap.uniq!
|
48
|
-
# end
|
49
|
-
# workload.uniq
|
50
|
-
#end
|
51
|
-
|
52
|
-
#def self.top_level_job(jobs)
|
53
|
-
# top = jobs.select do |job|
|
54
|
-
# (jobs - job_workload(job)).empty? &&
|
55
|
-
# (job_workload(job) - jobs).select{|j| (job_workload(j) & jobs).any? }.empty?
|
56
|
-
# end
|
57
|
-
# return nil if top.length != 1
|
58
|
-
# top.first
|
59
|
-
#end
|
60
|
-
|
61
|
-
#def self.job_chains(rules, job)
|
62
|
-
# workload = job_workload(job)
|
63
|
-
# chains = parse_chains(rules)
|
64
|
-
|
65
|
-
# chain_jobs = {}
|
66
|
-
# workload.each do |job|
|
67
|
-
# check_chains(chains, job).each do |match|
|
68
|
-
# chain_jobs[match] ||= []
|
69
|
-
# chain_jobs[match] << job
|
70
|
-
# end
|
71
|
-
# end
|
72
|
-
|
73
|
-
# job_chains = []
|
74
|
-
|
75
|
-
# seen = []
|
76
|
-
# chain_jobs.sort_by{|name,jobs| jobs.length }.reverse.each do |name,jobs|
|
77
|
-
# remain = jobs - seen
|
78
|
-
# next unless remain.length > 1
|
79
|
-
# top_level_job = top_level_job(jobs)
|
80
|
-
# next if top_level_job.nil?
|
81
|
-
# job_chains << {:jobs => remain, :rules => chains[name][:rules], :top_level_job => top_level_job}
|
82
|
-
# seen.concat remain
|
83
|
-
# end
|
84
|
-
|
85
|
-
# job_chains
|
86
|
-
#end
|
87
|
-
|
88
|
-
#def self._job_chains(rules, job)
|
89
|
-
# workload = job_workload(job)
|
90
|
-
# chains = parse_chains(rules)
|
91
|
-
|
92
|
-
# matches = check_chains(chains, job)
|
93
|
-
|
94
|
-
# job_chains = {}
|
95
|
-
# job.dependencies.each do |dep|
|
96
|
-
# dep_chains = _job_chains(rules, dep)
|
97
|
-
# matches.each do |match|
|
98
|
-
# if dep_chains[match] && dep_chains[match].include?(dep)
|
99
|
-
# dep_chains[match].prepend job
|
100
|
-
# end
|
101
|
-
# end
|
102
|
-
# job_chains.merge!(dep_chains)
|
103
|
-
# end
|
104
|
-
|
105
|
-
# matches.each do |match|
|
106
|
-
# job_chains[match] ||= [job]
|
107
|
-
# end
|
108
|
-
|
109
|
-
# job_chains
|
110
|
-
#end
|
111
|
-
|
112
|
-
#def self.job_chains(rules, job)
|
113
|
-
# job_chains = self._job_chains(rules, job)
|
114
|
-
# iif job_chains
|
115
|
-
# chains = parse_chains(rules)
|
116
|
-
|
117
|
-
# seen = []
|
118
|
-
# job_chains.collect do |name,jobs|
|
119
|
-
# remain = jobs - seen
|
120
|
-
# next unless remain.length > 1
|
121
|
-
# top_level_job = top_level_job(jobs)
|
122
|
-
# next if top_level_job.nil?
|
123
|
-
# seen.concat remain
|
124
|
-
# {:jobs => remain, :rules => chains[name][:rules], :top_level_job => top_level_job}
|
125
|
-
# end.compact
|
126
|
-
#end
|
127
|
-
|
128
40
|
def self.job_chains(rules, job)
|
129
41
|
chains = self.parse_chains(rules)
|
130
42
|
|
data/lib/rbbt/hpc/orchestrate.rb
CHANGED
@@ -5,6 +5,23 @@ require 'rbbt/hpc/orchestrate/batches'
|
|
5
5
|
module HPC
|
6
6
|
module Orchestration
|
7
7
|
|
8
|
+
def prepare_for_execution(job)
|
9
|
+
rec_dependencies = job.rec_dependencies(true)
|
10
|
+
|
11
|
+
return if rec_dependencies.empty?
|
12
|
+
|
13
|
+
all_deps = rec_dependencies + [job]
|
14
|
+
|
15
|
+
all_deps.each do |dep|
|
16
|
+
begin
|
17
|
+
Step.prepare_for_execution(dep)
|
18
|
+
rescue RbbtException
|
19
|
+
next
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
|
8
25
|
def orchestrate_job(job, options)
|
9
26
|
options.delete "recursive_clean"
|
10
27
|
options.delete "clean_task"
|
@@ -13,6 +30,10 @@ module HPC
|
|
13
30
|
options.delete "printpath"
|
14
31
|
options.delete "detach"
|
15
32
|
options.delete "jobname"
|
33
|
+
options.delete "load_inputs"
|
34
|
+
|
35
|
+
Log.high "Prepare for exec"
|
36
|
+
prepare_for_execution(job)
|
16
37
|
|
17
38
|
if options[:orchestration_rules]
|
18
39
|
rules = YAML.load(Open.read(options[:orchestration_rules]))
|
@@ -25,6 +46,7 @@ module HPC
|
|
25
46
|
IndiferentHash.setup(rules)
|
26
47
|
|
27
48
|
batches = HPC::Orchestration.job_batches(rules, job)
|
49
|
+
Log.high "Compute #{batches.length} batches"
|
28
50
|
|
29
51
|
batch_ids = {}
|
30
52
|
while batches.any?
|
@@ -62,7 +84,7 @@ module HPC
|
|
62
84
|
puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
|
63
85
|
puts Log.color(:magenta, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
|
64
86
|
puts Log.color(:yellow, "Path: ") + top[:top_level].path
|
65
|
-
puts Log.color(:yellow, "Options: ") +
|
87
|
+
puts Log.color(:yellow, "Options: ") + job_options.inspect
|
66
88
|
batch_ids[top] = top[:top_level].task_signature
|
67
89
|
else
|
68
90
|
id = run_job(top[:top_level], job_options)
|
data/lib/rbbt/hpc/slurm.rb
CHANGED
@@ -94,8 +94,8 @@ export BATCH_SYSTEM=SLURM
|
|
94
94
|
|
95
95
|
return if Open.exists?(fexit)
|
96
96
|
|
97
|
-
|
98
|
-
|
97
|
+
Log.info "Issuing SLURM file: #{fcmd}"
|
98
|
+
Log.debug Open.read(fcmd)
|
99
99
|
|
100
100
|
if File.exists?(fjob)
|
101
101
|
job = Open.read(fjob).to_i
|
data/lib/rbbt/resource/path.rb
CHANGED
@@ -167,8 +167,7 @@ module Path
|
|
167
167
|
|
168
168
|
@path ||= {}
|
169
169
|
rsearch_paths = (resource and resource.respond_to?(:search_paths)) ? resource.search_paths : nil
|
170
|
-
|
171
|
-
key = Misc.obj2digest(key_elems.inspect)
|
170
|
+
key = [where, caller_lib, rsearch_paths, paths].inspect
|
172
171
|
self.sub!('~/', Etc.getpwuid.dir + '/') if self.include? "~"
|
173
172
|
|
174
173
|
return @path[key] if @path[key]
|
data/lib/rbbt/resource/util.rb
CHANGED
@@ -1,13 +1,26 @@
|
|
1
1
|
module Path
|
2
2
|
|
3
3
|
def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
|
4
|
-
file = caller.reject{|l|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
}.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
|
4
|
+
#file = caller.reject{|l|
|
5
|
+
# l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
|
6
|
+
# l =~ /rbbt\/resource\/path\.rb/ or
|
7
|
+
# l =~ /rbbt\/persist.rb/ or
|
8
|
+
# l =~ /rbbt\/util\/misc\.rb/ or
|
9
|
+
# l =~ /progress-monitor\.rb/
|
10
|
+
#}.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
|
11
|
+
|
12
|
+
|
13
|
+
if file.nil?
|
14
|
+
caller_dup = caller.dup
|
15
|
+
while file = caller_dup.shift
|
16
|
+
break unless file =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
|
17
|
+
file =~ /rbbt\/resource\/path\.rb/ or
|
18
|
+
file =~ /rbbt\/persist.rb/ or
|
19
|
+
file =~ /rbbt\/util\/misc\.rb/ or
|
20
|
+
file =~ /progress-monitor\.rb/
|
21
|
+
end
|
22
|
+
file = file.sub(/\.rb[^\w].*/,'.rb')
|
23
|
+
end
|
11
24
|
|
12
25
|
relative_to = [relative_to] unless Array === relative_to
|
13
26
|
file = File.expand_path(file)
|
data/lib/rbbt/util/migrate.rb
CHANGED
@@ -1,4 +1,6 @@
|
|
1
1
|
module Rbbt
|
2
|
+
|
3
|
+
prepare_for_execution(job)
|
2
4
|
def self.migrate_source_paths(path, resource = Rbbt, source = nil)
|
3
5
|
if source
|
4
6
|
lpath, *paths = Misc.ssh_run(source, <<-EOF).split("\n")
|
@@ -44,6 +46,8 @@ puts resource[path].find(search_path)
|
|
44
46
|
excludes += (options[:exclude] || "").split(/,\s*/)
|
45
47
|
excludes_str = excludes.collect{|s| "--exclude '#{s}'" } * " "
|
46
48
|
|
49
|
+
hard_link = options[:hard_link]
|
50
|
+
|
47
51
|
other = options[:other] || []
|
48
52
|
|
49
53
|
test_str = options[:test] ? '-nv' : ''
|
@@ -82,6 +86,8 @@ puts resource[path].find(search_path)
|
|
82
86
|
# rsync_args = "-avztAXHP --copy-unsafe-links"
|
83
87
|
rsync_args = "-avztAHP --copy-unsafe-links"
|
84
88
|
|
89
|
+
rsync_args << " --link-dest '#{source_path}'" if hard_link && ! options[:source]
|
90
|
+
|
85
91
|
cmd = "rsync #{rsync_args} #{test_str} #{files_from_str} #{excludes_str} '#{source_path}' #{target_path} #{other * " "}"
|
86
92
|
|
87
93
|
cmd << " && rm -Rf #{source_path}" if options[:delete] && ! options[:files]
|
@@ -80,7 +80,7 @@ module Misc
|
|
80
80
|
end
|
81
81
|
when (defined? TSV and TSV)
|
82
82
|
obj.with_unnamed do
|
83
|
-
"TSV:{"<< fingerprint(obj.all_fields|| [])
|
83
|
+
"TSV:{"<< fingerprint(obj.all_fields|| []) << ";" << fingerprint(obj.keys) << "}"
|
84
84
|
end
|
85
85
|
when Hash
|
86
86
|
if obj.length > 10
|
@@ -297,14 +297,14 @@ module Misc
|
|
297
297
|
str = case obj
|
298
298
|
when nil
|
299
299
|
'nil'
|
300
|
+
when Symbol
|
301
|
+
obj.to_s
|
300
302
|
when TrueClass
|
301
303
|
'true'
|
302
304
|
when FalseClass
|
303
305
|
'false'
|
304
306
|
when Hash
|
305
307
|
"{"<< obj.collect{|k,v| obj2str(k) + '=>' << obj2str(v)}*"," << "}"
|
306
|
-
when Symbol
|
307
|
-
obj.to_s
|
308
308
|
when (defined?(Path) and Path)
|
309
309
|
if defined?(Step) && Open.exists?(Step.info_file(obj))
|
310
310
|
obj2str(Workflow.load_step(obj))
|
@@ -324,9 +324,10 @@ module Misc
|
|
324
324
|
end
|
325
325
|
when String
|
326
326
|
good_filename = Misc.is_filename?(obj, false) && ! %w(. ..).include?(obj) && %w(. /).include?(obj[0])
|
327
|
-
|
328
|
-
|
329
|
-
|
327
|
+
if good_filename
|
328
|
+
obj = obj.dup
|
329
|
+
obj.extend Path
|
330
|
+
obj2str obj
|
330
331
|
else
|
331
332
|
obj = obj.chomp if String === obj
|
332
333
|
if obj.length > HASH2MD5_MAX_STRING_LENGTH
|
@@ -339,7 +340,7 @@ module Misc
|
|
339
340
|
if obj.length > HASH2MD5_MAX_ARRAY_LENGTH
|
340
341
|
"[" << sample_large_obj(obj, HASH2MD5_MAX_ARRAY_LENGTH).collect{|v| obj2str(v)} * "," << "]"
|
341
342
|
else
|
342
|
-
"[" << obj.collect{|v| obj2str(v)} * "," << "]"
|
343
|
+
"[" << obj.collect{|v| obj2str(v) } * "," << "]"
|
343
344
|
end
|
344
345
|
when TSV::Parser
|
345
346
|
remove_long_items(obj)
|
@@ -181,8 +181,8 @@ class Step
|
|
181
181
|
# end
|
182
182
|
#end
|
183
183
|
|
184
|
-
Log.
|
185
|
-
Log.
|
184
|
+
Log.medium "Some newer files found: #{Misc.fingerprint outdated_time}" if outdated_time.any?
|
185
|
+
Log.medium "Some outdated files found: #{Misc.fingerprint outdated_dep}" if outdated_dep.any?
|
186
186
|
|
187
187
|
outdated_time + outdated_dep
|
188
188
|
end
|
@@ -1,4 +1,5 @@
|
|
1
1
|
module Workflow
|
2
|
+
|
2
3
|
def self.load_inputs(dir, input_names, input_types)
|
3
4
|
inputs = {}
|
4
5
|
if File.exists?(dir) && ! File.directory?(dir)
|
@@ -24,17 +25,31 @@ module Workflow
|
|
24
25
|
|
25
26
|
type = :io if file.split(".").last == 'as_io'
|
26
27
|
|
28
|
+
type = :step if file.split(".").last == 'as_step'
|
29
|
+
|
30
|
+
type = :step_file if file.split(".").last == 'as_step_file'
|
31
|
+
|
27
32
|
type = :path if file.split(".").last == 'as_path'
|
28
33
|
|
34
|
+
type = :filename if file.split(".").last == 'as_filename'
|
35
|
+
|
29
36
|
type = :nofile if file.split(".").last == 'nofile'
|
30
37
|
|
31
38
|
case type
|
32
39
|
when :nofile
|
33
40
|
inputs[input.to_sym] = Open.realpath(file)
|
34
41
|
when :path
|
35
|
-
inputs[input.to_sym] = Open.
|
42
|
+
inputs[input.to_sym] = Open.read(file).strip
|
36
43
|
when :io
|
37
44
|
inputs[input.to_sym] = Open.open(Open.realpath(file))
|
45
|
+
when :step
|
46
|
+
inputs[input.to_sym] = Workflow.load_step(Open.read(file).strip)
|
47
|
+
when :step_file
|
48
|
+
path = Open.read(file).strip
|
49
|
+
path.extend Path
|
50
|
+
step_path = path.match(/(.*)\.files/)[1]
|
51
|
+
path.resource = Step.new step_path
|
52
|
+
inputs[input.to_sym] = path
|
38
53
|
when :file, :binary
|
39
54
|
Log.debug "Pointing #{ input } to #{file}"
|
40
55
|
if file =~ /\.yaml/
|
@@ -87,52 +102,124 @@ module Workflow
|
|
87
102
|
job(task_name, jobname, inputs)
|
88
103
|
end
|
89
104
|
|
105
|
+
#def self.load_inputs_old(dir, input_names, input_types)
|
106
|
+
# inputs = {}
|
107
|
+
# if File.exists?(dir) && ! File.directory?(dir)
|
108
|
+
# Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
|
109
|
+
# tarfile = dir
|
110
|
+
# digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
|
111
|
+
# tmpdir = Rbbt.tmp.input_bundle[digest].find
|
112
|
+
# Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
|
113
|
+
# files = tmpdir.glob("*")
|
114
|
+
# if files.length == 1 && File.directory?(files.first)
|
115
|
+
# tmpdir = files.first
|
116
|
+
# end
|
117
|
+
# load_inputs(tmpdir, input_names, input_types)
|
118
|
+
# else
|
119
|
+
# dir = Path.setup(dir.dup)
|
120
|
+
# input_names.each do |input|
|
121
|
+
# file = dir[input].find
|
122
|
+
# file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
|
123
|
+
# Log.debug "Trying #{ input }: #{file}"
|
124
|
+
# next unless file and (File.symlink?(file) || file.exists?)
|
125
|
+
|
126
|
+
# type = input_types[input]
|
127
|
+
|
128
|
+
# type = :io if file.split(".").last == 'as_io'
|
129
|
+
|
130
|
+
# type = :path if file.split(".").last == 'as_path'
|
131
|
+
|
132
|
+
# type = :filename if file.split(".").last == 'as_filename'
|
133
|
+
|
134
|
+
# type = :nofile if file.split(".").last == 'nofile'
|
135
|
+
|
136
|
+
# case type
|
137
|
+
# when :nofile
|
138
|
+
# inputs[input.to_sym] = Open.realpath(file)
|
139
|
+
# when :path
|
140
|
+
# inputs[input.to_sym] = Open.realpath(Open.read(file).strip)
|
141
|
+
# when :io
|
142
|
+
# inputs[input.to_sym] = Open.open(Open.realpath(file))
|
143
|
+
# when :file, :binary
|
144
|
+
# Log.debug "Pointing #{ input } to #{file}"
|
145
|
+
# if file =~ /\.yaml/
|
146
|
+
# inputs[input.to_sym] = YAML.load(Open.read(file))
|
147
|
+
# else
|
148
|
+
# if File.symlink?(file)
|
149
|
+
# link_target = File.expand_path(File.readlink(file), File.dirname(file))
|
150
|
+
# inputs[input.to_sym] = link_target
|
151
|
+
# else
|
152
|
+
# inputs[input.to_sym] = Open.realpath(file)
|
153
|
+
# end
|
154
|
+
# end
|
155
|
+
# when :text
|
156
|
+
# Log.debug "Reading #{ input } from #{file}"
|
157
|
+
# inputs[input.to_sym] = Open.read(file)
|
158
|
+
# when :array
|
159
|
+
# Log.debug "Reading array #{ input } from #{file}"
|
160
|
+
# inputs[input.to_sym] = Open.read(file).split("\n")
|
161
|
+
# when :tsv
|
162
|
+
# Log.debug "Opening tsv #{ input } from #{file}"
|
163
|
+
# inputs[input.to_sym] = TSV.open(file)
|
164
|
+
# when :boolean
|
165
|
+
# inputs[input.to_sym] = (file.read.strip == 'true')
|
166
|
+
# else
|
167
|
+
# Log.debug "Loading #{ input } from #{file}"
|
168
|
+
# inputs[input.to_sym] = file.read.strip
|
169
|
+
# end
|
170
|
+
|
171
|
+
# end
|
172
|
+
# inputs = IndiferentHash.setup(inputs)
|
173
|
+
|
174
|
+
# dir.glob("*#*").each do |od|
|
175
|
+
# name = File.basename(od)
|
176
|
+
# value = Open.read(od)
|
177
|
+
# Log.debug "Loading override dependency #{ name } as #{value}"
|
178
|
+
# inputs[name] = value.chomp
|
179
|
+
# end
|
180
|
+
|
181
|
+
# inputs
|
182
|
+
# end
|
183
|
+
#end
|
90
184
|
end
|
91
185
|
|
92
186
|
class Step
|
187
|
+
def self.save_input(name, value, type, dir)
|
188
|
+
path = File.join(dir, name.to_s)
|
189
|
+
|
190
|
+
|
191
|
+
case value
|
192
|
+
when Path
|
193
|
+
if Step === value.resource
|
194
|
+
path = path + '.as_step_file'
|
195
|
+
else
|
196
|
+
path = path + '.as_path'
|
197
|
+
end
|
198
|
+
when String
|
199
|
+
if Misc.is_filename?(value, false)
|
200
|
+
value = value.dup
|
201
|
+
value.extend Path
|
202
|
+
return save_input(name, value, type, dir)
|
203
|
+
end
|
204
|
+
when IO
|
205
|
+
path = path + '.as_io'
|
206
|
+
when Step
|
207
|
+
value = value.path
|
208
|
+
path = path + '.as_step'
|
209
|
+
when Array
|
210
|
+
value = value * "\n"
|
211
|
+
end
|
212
|
+
|
213
|
+
Log.debug "Saving job input #{name} (#{type}) into #{path}"
|
214
|
+
Open.write(path, value.to_s)
|
215
|
+
end
|
216
|
+
|
93
217
|
def self.save_inputs(inputs, input_types, input_options, dir)
|
94
218
|
inputs.each do |name,value|
|
95
219
|
type = input_types[name]
|
96
220
|
type = type.to_s if type
|
97
|
-
|
98
|
-
|
99
|
-
path = path + '.as_io' if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
|
100
|
-
Log.debug "Saving job input #{name} (#{type}) into #{path}"
|
101
|
-
|
102
|
-
case
|
103
|
-
when IO === value
|
104
|
-
Open.write(path, value.to_s)
|
105
|
-
when Step === value
|
106
|
-
Open.ln_s(value.path, path)
|
107
|
-
when type.to_s == "binary"
|
108
|
-
if String === value && File.exists?(value)
|
109
|
-
value = File.expand_path(value)
|
110
|
-
Open.ln_s(value, path)
|
111
|
-
elsif String === value && Misc.is_filename?(value, false)
|
112
|
-
Open.write(path + '.as_path' , value)
|
113
|
-
else
|
114
|
-
Open.write(path, value, :mode => 'wb')
|
115
|
-
end
|
116
|
-
when type.to_s == "file"
|
117
|
-
if String === value && File.exists?(value)
|
118
|
-
value = File.expand_path(value)
|
119
|
-
Open.ln_s(value, path)
|
120
|
-
else
|
121
|
-
value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
|
122
|
-
value = "#{value}" if Path === value
|
123
|
-
Open.write(path + '.yaml', value.to_yaml)
|
124
|
-
end
|
125
|
-
when Array === value
|
126
|
-
Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
|
127
|
-
when IO === value
|
128
|
-
if value.filename && String === value.filename && File.exists?(value.filename)
|
129
|
-
Open.ln_s(value.filename, path)
|
130
|
-
else
|
131
|
-
Open.write(path, value)
|
132
|
-
end
|
133
|
-
else
|
134
|
-
Open.write(path, value.to_s)
|
135
|
-
end
|
221
|
+
|
222
|
+
save_input(name, value, type, dir)
|
136
223
|
end.any?
|
137
224
|
end
|
138
225
|
|
@@ -176,4 +263,60 @@ class Step
|
|
176
263
|
|
177
264
|
inputs.keys
|
178
265
|
end
|
266
|
+
|
267
|
+
#def self.save_inputs_old(inputs, input_types, input_options, dir)
|
268
|
+
# inputs.each do |name,value|
|
269
|
+
# type = input_types[name]
|
270
|
+
# type = type.to_s if type
|
271
|
+
# path = File.join(dir, name.to_s)
|
272
|
+
|
273
|
+
# if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
|
274
|
+
# path = path + '.as_io'
|
275
|
+
# elsif Misc.is_filename?(value, false)
|
276
|
+
# path = path + '.as_filename'
|
277
|
+
# end
|
278
|
+
|
279
|
+
# Log.debug "Saving job input #{name} (#{type}) into #{path}"
|
280
|
+
|
281
|
+
# case
|
282
|
+
# when IO === value
|
283
|
+
# Open.write(path, value.to_s)
|
284
|
+
# when Step === value
|
285
|
+
# Open.ln_s(value.path, path)
|
286
|
+
# when type.to_s == "binary"
|
287
|
+
# if String === value && File.exists?(value)
|
288
|
+
# value = File.expand_path(value)
|
289
|
+
# Open.ln_s(value, path)
|
290
|
+
# elsif String === value && Misc.is_filename?(value, false)
|
291
|
+
# Open.write(path + '.as_path' , value)
|
292
|
+
# else
|
293
|
+
# Open.write(path, value, :mode => 'wb')
|
294
|
+
# end
|
295
|
+
# when TSV === value
|
296
|
+
# Open.write(path, value.to_s)
|
297
|
+
# when Array === value
|
298
|
+
# Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
|
299
|
+
# when %w(file tsv array).include?(type.to_s)
|
300
|
+
# if String === value && File.exists?(value)
|
301
|
+
# value = File.expand_path(value)
|
302
|
+
# Open.ln_s(value, path)
|
303
|
+
# elsif String === value && Misc.is_filename?(value, false)
|
304
|
+
# Open.write(path + '.as_path' , value)
|
305
|
+
# else
|
306
|
+
# value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
|
307
|
+
# value = "#{value}" if Path === value
|
308
|
+
# Open.write(path + '.yaml', value.to_yaml)
|
309
|
+
# end
|
310
|
+
# when IO === value
|
311
|
+
# if value.filename && String === value.filename && File.exists?(value.filename)
|
312
|
+
# Open.ln_s(value.filename, path)
|
313
|
+
# else
|
314
|
+
# Open.write(path, value)
|
315
|
+
# end
|
316
|
+
# else
|
317
|
+
# Open.write(path, value.to_s)
|
318
|
+
# end
|
319
|
+
# end.any?
|
320
|
+
#end
|
321
|
+
|
179
322
|
end
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -376,9 +376,9 @@ class Step
|
|
376
376
|
}
|
377
377
|
raise "Dependency step not found: #{ name }" if deps.empty?
|
378
378
|
if (deps & self.dependencies).any?
|
379
|
-
(deps & self.dependencies).
|
379
|
+
(deps & self.dependencies).last
|
380
380
|
else
|
381
|
-
deps.
|
381
|
+
deps.last
|
382
382
|
end
|
383
383
|
end
|
384
384
|
end
|
@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
|
|
22
22
|
-s--search* Regular expression
|
23
23
|
-t--tail* Show the last lines of the STDERR
|
24
24
|
-l--long Show more entries
|
25
|
+
-c--compressed Show compressed information about entries
|
25
26
|
-p--progress Report progress of job and the dependencies
|
26
27
|
-BP--batch_parameters show batch parameters
|
27
28
|
-BPP--batch_procpath show Procpath performance summary
|
@@ -140,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
140
141
|
rescue
|
141
142
|
nodes = []
|
142
143
|
end
|
143
|
-
elsif job_nodes[id]
|
144
|
+
elsif job_nodes && job_nodes[id]
|
144
145
|
nodes = job_nodes[id].reject{|n| n.include? "("}
|
145
146
|
else
|
146
147
|
nodes = []
|
@@ -185,6 +186,40 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
185
186
|
end
|
186
187
|
|
187
188
|
|
189
|
+
count += 1
|
190
|
+
|
191
|
+
if options[:compressed]
|
192
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) : Log.color(:green, id)
|
193
|
+
if different_system
|
194
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id)
|
195
|
+
else
|
196
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
|
197
|
+
end
|
198
|
+
prog_rep = []
|
199
|
+
if options[:progress]
|
200
|
+
step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
|
201
|
+
if step_line
|
202
|
+
require 'rbbt/workflow'
|
203
|
+
step_path = step_line.split(": ").last.strip
|
204
|
+
step = Step.new step_path
|
205
|
+
step.load_dependencies_from_info
|
206
|
+
has_bar = false
|
207
|
+
(step.rec_dependencies + [step]).reverse.each do |j|
|
208
|
+
next if j.done?
|
209
|
+
if j.file(:progress).exists?
|
210
|
+
bar = Log::ProgressBar.new
|
211
|
+
bar.load(j.file(:progress).yaml)
|
212
|
+
prog_rep << bar.report_msg.split("·")[0..1]
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
workflow, task, name = step_path.split("/")[-3..-1]
|
218
|
+
job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
|
219
|
+
puts [job_str, status, prog_rep ].flatten * " "
|
220
|
+
next
|
221
|
+
end
|
222
|
+
|
188
223
|
puts Log.color :blue, dir
|
189
224
|
puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
|
190
225
|
puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
|
@@ -232,22 +267,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
232
267
|
cpu_average = {}
|
233
268
|
rss_average = {}
|
234
269
|
perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
270
|
+
time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
|
271
|
+
time = time.to_f
|
272
|
+
|
273
|
+
cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
|
274
|
+
cpu_average[stat_pid] ||= {}
|
275
|
+
cpu_average[stat_pid][time] ||= []
|
276
|
+
cpu_average[stat_pid][time] << cpu.to_f
|
277
|
+
rss_average[time] ||= []
|
278
|
+
rss_average[time] << rss.to_f * page_size
|
244
279
|
end
|
245
280
|
|
246
281
|
ticks = 0
|
247
282
|
cpu_average.each do |stat_pid, cpu_average_pid|
|
248
|
-
|
249
|
-
|
250
|
-
|
283
|
+
start = cpu_average_pid.keys.sort.first
|
284
|
+
eend = cpu_average_pid.keys.sort.last
|
285
|
+
ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
|
251
286
|
end
|
252
287
|
start = rss_average.keys.sort.first
|
253
288
|
eend = rss_average.keys.sort.last
|
@@ -284,7 +319,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
284
319
|
puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
|
285
320
|
else
|
286
321
|
puts Log.color(:magenta, "Log tail: ")
|
287
|
-
puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m' | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
|
322
|
+
puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m' | grep -v -e '^[[:space:]]*$' | grep -v \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
|
288
323
|
end
|
289
324
|
end
|
290
325
|
|
@@ -305,12 +340,13 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
305
340
|
has_bar = true
|
306
341
|
end
|
307
342
|
end
|
308
|
-
|
343
|
+
step_status = step.status
|
344
|
+
step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
|
345
|
+
step_status = Log.color :green, step_status if step_status.to_s == 'done'
|
346
|
+
puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
|
309
347
|
end
|
310
348
|
end
|
311
349
|
|
312
|
-
count += 1
|
313
|
-
|
314
350
|
end
|
315
351
|
|
316
352
|
puts
|
@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
|
|
22
22
|
-s--search* Regular expression
|
23
23
|
-t--tail* Show the last lines of the STDERR
|
24
24
|
-l--long Show more entries
|
25
|
+
-c--compressed Show compressed information about entries
|
25
26
|
-p--progress Report progress of job and the dependencies
|
26
27
|
-BP--batch_parameters show batch parameters
|
27
28
|
-BPP--batch_procpath show Procpath performance summary
|
@@ -140,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
140
141
|
rescue
|
141
142
|
nodes = []
|
142
143
|
end
|
143
|
-
elsif job_nodes[id]
|
144
|
+
elsif job_nodes && job_nodes[id]
|
144
145
|
nodes = job_nodes[id].reject{|n| n.include? "("}
|
145
146
|
else
|
146
147
|
nodes = []
|
@@ -185,6 +186,40 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
185
186
|
end
|
186
187
|
|
187
188
|
|
189
|
+
count += 1
|
190
|
+
|
191
|
+
if options[:compressed]
|
192
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) : Log.color(:green, id)
|
193
|
+
if different_system
|
194
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id)
|
195
|
+
else
|
196
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
|
197
|
+
end
|
198
|
+
prog_rep = []
|
199
|
+
if options[:progress]
|
200
|
+
step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
|
201
|
+
if step_line
|
202
|
+
require 'rbbt/workflow'
|
203
|
+
step_path = step_line.split(": ").last.strip
|
204
|
+
step = Step.new step_path
|
205
|
+
step.load_dependencies_from_info
|
206
|
+
has_bar = false
|
207
|
+
(step.rec_dependencies + [step]).reverse.each do |j|
|
208
|
+
next if j.done?
|
209
|
+
if j.file(:progress).exists?
|
210
|
+
bar = Log::ProgressBar.new
|
211
|
+
bar.load(j.file(:progress).yaml)
|
212
|
+
prog_rep << bar.report_msg.split("·")[0..1]
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
workflow, task, name = step_path.split("/")[-3..-1]
|
218
|
+
job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
|
219
|
+
puts [job_str, status, prog_rep ].flatten * " "
|
220
|
+
next
|
221
|
+
end
|
222
|
+
|
188
223
|
puts Log.color :blue, dir
|
189
224
|
puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
|
190
225
|
puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
|
@@ -232,22 +267,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
232
267
|
cpu_average = {}
|
233
268
|
rss_average = {}
|
234
269
|
perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
270
|
+
time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
|
271
|
+
time = time.to_f
|
272
|
+
|
273
|
+
cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
|
274
|
+
cpu_average[stat_pid] ||= {}
|
275
|
+
cpu_average[stat_pid][time] ||= []
|
276
|
+
cpu_average[stat_pid][time] << cpu.to_f
|
277
|
+
rss_average[time] ||= []
|
278
|
+
rss_average[time] << rss.to_f * page_size
|
244
279
|
end
|
245
280
|
|
246
281
|
ticks = 0
|
247
282
|
cpu_average.each do |stat_pid, cpu_average_pid|
|
248
|
-
|
249
|
-
|
250
|
-
|
283
|
+
start = cpu_average_pid.keys.sort.first
|
284
|
+
eend = cpu_average_pid.keys.sort.last
|
285
|
+
ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
|
251
286
|
end
|
252
287
|
start = rss_average.keys.sort.first
|
253
288
|
eend = rss_average.keys.sort.last
|
@@ -284,7 +319,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
284
319
|
puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
|
285
320
|
else
|
286
321
|
puts Log.color(:magenta, "Log tail: ")
|
287
|
-
puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m' | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
|
322
|
+
puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m' | grep -v -e '^[[:space:]]*$' | grep -v \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
|
288
323
|
end
|
289
324
|
end
|
290
325
|
|
@@ -305,12 +340,13 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
305
340
|
has_bar = true
|
306
341
|
end
|
307
342
|
end
|
308
|
-
|
343
|
+
step_status = step.status
|
344
|
+
step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
|
345
|
+
step_status = Log.color :green, step_status if step_status.to_s == 'done'
|
346
|
+
puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
|
309
347
|
end
|
310
348
|
end
|
311
349
|
|
312
|
-
count += 1
|
313
|
-
|
314
350
|
end
|
315
351
|
|
316
352
|
puts
|
@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
|
|
22
22
|
-s--search* Regular expression
|
23
23
|
-t--tail* Show the last lines of the STDERR
|
24
24
|
-l--long Show more entries
|
25
|
+
-c--compressed Show compressed information about entries
|
25
26
|
-p--progress Report progress of job and the dependencies
|
26
27
|
-BP--batch_parameters show batch parameters
|
27
28
|
-BPP--batch_procpath show Procpath performance summary
|
@@ -140,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
140
141
|
rescue
|
141
142
|
nodes = []
|
142
143
|
end
|
143
|
-
elsif job_nodes[id]
|
144
|
+
elsif job_nodes && job_nodes[id]
|
144
145
|
nodes = job_nodes[id].reject{|n| n.include? "("}
|
145
146
|
else
|
146
147
|
nodes = []
|
@@ -185,6 +186,40 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
185
186
|
end
|
186
187
|
|
187
188
|
|
189
|
+
count += 1
|
190
|
+
|
191
|
+
if options[:compressed]
|
192
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) : Log.color(:green, id)
|
193
|
+
if different_system
|
194
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id)
|
195
|
+
else
|
196
|
+
status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
|
197
|
+
end
|
198
|
+
prog_rep = []
|
199
|
+
if options[:progress]
|
200
|
+
step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
|
201
|
+
if step_line
|
202
|
+
require 'rbbt/workflow'
|
203
|
+
step_path = step_line.split(": ").last.strip
|
204
|
+
step = Step.new step_path
|
205
|
+
step.load_dependencies_from_info
|
206
|
+
has_bar = false
|
207
|
+
(step.rec_dependencies + [step]).reverse.each do |j|
|
208
|
+
next if j.done?
|
209
|
+
if j.file(:progress).exists?
|
210
|
+
bar = Log::ProgressBar.new
|
211
|
+
bar.load(j.file(:progress).yaml)
|
212
|
+
prog_rep << bar.report_msg.split("·")[0..1]
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
workflow, task, name = step_path.split("/")[-3..-1]
|
218
|
+
job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
|
219
|
+
puts [job_str, status, prog_rep ].flatten * " "
|
220
|
+
next
|
221
|
+
end
|
222
|
+
|
188
223
|
puts Log.color :blue, dir
|
189
224
|
puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
|
190
225
|
puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
|
@@ -232,22 +267,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
232
267
|
cpu_average = {}
|
233
268
|
rss_average = {}
|
234
269
|
perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
270
|
+
time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
|
271
|
+
time = time.to_f
|
272
|
+
|
273
|
+
cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
|
274
|
+
cpu_average[stat_pid] ||= {}
|
275
|
+
cpu_average[stat_pid][time] ||= []
|
276
|
+
cpu_average[stat_pid][time] << cpu.to_f
|
277
|
+
rss_average[time] ||= []
|
278
|
+
rss_average[time] << rss.to_f * page_size
|
244
279
|
end
|
245
280
|
|
246
281
|
ticks = 0
|
247
282
|
cpu_average.each do |stat_pid, cpu_average_pid|
|
248
|
-
|
249
|
-
|
250
|
-
|
283
|
+
start = cpu_average_pid.keys.sort.first
|
284
|
+
eend = cpu_average_pid.keys.sort.last
|
285
|
+
ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
|
251
286
|
end
|
252
287
|
start = rss_average.keys.sort.first
|
253
288
|
eend = rss_average.keys.sort.last
|
@@ -284,7 +319,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
284
319
|
puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
|
285
320
|
else
|
286
321
|
puts Log.color(:magenta, "Log tail: ")
|
287
|
-
puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m' | grep -v -e '^[[:space:]]*$' | tail -n #{tail.to_i} ").read
|
322
|
+
puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m' | grep -v -e '^[[:space:]]*$' | grep -v \"\\(STDOUT\\|STDERR\\):[[:space:]]*$\" | tail -n #{tail.to_i} ").read
|
288
323
|
end
|
289
324
|
end
|
290
325
|
|
@@ -305,12 +340,13 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
|
|
305
340
|
has_bar = true
|
306
341
|
end
|
307
342
|
end
|
308
|
-
|
343
|
+
step_status = step.status
|
344
|
+
step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
|
345
|
+
step_status = Log.color :green, step_status if step_status.to_s == 'done'
|
346
|
+
puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
|
309
347
|
end
|
310
348
|
end
|
311
349
|
|
312
|
-
count += 1
|
313
|
-
|
314
350
|
end
|
315
351
|
|
316
352
|
puts
|
@@ -76,6 +76,16 @@ def fix_options(workflow, task, job_options)
|
|
76
76
|
type = input_types[name]
|
77
77
|
type = type.to_sym if type
|
78
78
|
|
79
|
+
if Step === value
|
80
|
+
job_options_cleaned[name] = value
|
81
|
+
next
|
82
|
+
end
|
83
|
+
|
84
|
+
if Path === value && Step === value.resource
|
85
|
+
job_options_cleaned[name] = value
|
86
|
+
next
|
87
|
+
end
|
88
|
+
|
79
89
|
value = case type
|
80
90
|
when nil
|
81
91
|
value
|
@@ -148,6 +158,7 @@ def fix_options(workflow, task, job_options)
|
|
148
158
|
else
|
149
159
|
value
|
150
160
|
end
|
161
|
+
|
151
162
|
job_options_cleaned[name] = value
|
152
163
|
end
|
153
164
|
|
@@ -219,8 +230,8 @@ help = !!options.delete(:help)
|
|
219
230
|
do_fork = !!options.delete(:fork)
|
220
231
|
detach = !!options.delete(:detach)
|
221
232
|
do_exec = !!options.delete(:exec)
|
222
|
-
clean = !!options.delete(:clean)
|
223
233
|
clean_task = options.delete(:clean_task)
|
234
|
+
clean = !!options.delete(:clean) || clean_task
|
224
235
|
override_deps = options.delete(:override_deps)
|
225
236
|
recursive_clean = !!options.delete(:recursive_clean)
|
226
237
|
out = options.include?(:output) ? File.open(options[:output], 'wb') : STDOUT
|
@@ -26,11 +26,26 @@ module TestSaveLoadWF
|
|
26
26
|
task :prefix => :array do
|
27
27
|
step(:reverse).run.collect{|e| "A-#{e}" }
|
28
28
|
end
|
29
|
+
|
30
|
+
input :integer, :integer
|
31
|
+
input :float, :float
|
32
|
+
input :file, :file
|
33
|
+
input :file_no_file, :file, "", nil, :nofile => true
|
34
|
+
input :string, :string
|
35
|
+
input :select, :select
|
36
|
+
input :array, :array
|
37
|
+
input :array_no_file, :array, "", nil, :nofile => true
|
38
|
+
input :tsv, :tsv, "", nil, :nofile => true
|
39
|
+
input :tsv_no_file, :tsv, "", nil, :nofile => true
|
40
|
+
input :binary, :binary, "", nil, :nofile => true
|
41
|
+
input :binary_no_file, :tsv, "", nil, :nofile => true
|
42
|
+
task :save_test => :string do
|
43
|
+
"DONE"
|
44
|
+
end
|
29
45
|
end
|
30
46
|
|
31
47
|
class TestSaveLoad < Test::Unit::TestCase
|
32
48
|
def test_save
|
33
|
-
Log.with_severity 0 do
|
34
49
|
job = TestSaveLoadWF.job(:prefix)
|
35
50
|
job.recursive_clean
|
36
51
|
job = TestSaveLoadWF.job(:prefix)
|
@@ -40,6 +55,81 @@ class TestSaveLoad < Test::Unit::TestCase
|
|
40
55
|
newjob = TestSaveLoadWF.job_for_directory_inputs(:reverse, directory)
|
41
56
|
assert_equal job.rec_dependencies.last.path, newjob.path
|
42
57
|
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_save_all_normal
|
61
|
+
tsv = TSV.setup({}, "Key~Value#:type=:single")
|
62
|
+
tsv["key"] = "value"
|
63
|
+
options = {
|
64
|
+
:float => 0.5,
|
65
|
+
:integer => 15,
|
66
|
+
:string => "STRING",
|
67
|
+
:select => "option",
|
68
|
+
:array => [0,1,2,3],
|
69
|
+
:tsv => tsv,
|
70
|
+
}
|
71
|
+
Log.with_severity 0 do
|
72
|
+
Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
|
73
|
+
job = TestSaveLoadWF.job(:save_test, nil, options)
|
74
|
+
TmpFile.with_file do |directory|
|
75
|
+
Step.save_job_inputs(job, directory)
|
76
|
+
newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
|
77
|
+
assert_equal job.path, newjob.path
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def test_save_all_file
|
84
|
+
tsv = TSV.setup({}, "Key~Value#:type=:single")
|
85
|
+
tsv["key"] = "value"
|
86
|
+
Log.with_severity 0 do
|
87
|
+
Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
|
88
|
+
TmpFile.with_file do |input_file|
|
89
|
+
Path.setup(input_file)
|
90
|
+
options = {
|
91
|
+
:float => input_file.float,
|
92
|
+
:integer => input_file.integer,
|
93
|
+
:string => input_file.string,
|
94
|
+
:select => input_file.select,
|
95
|
+
:array => input_file.array,
|
96
|
+
:tsv => input_file.tsv_file,
|
97
|
+
}
|
98
|
+
job = TestSaveLoadWF.job(:save_test, nil, options)
|
99
|
+
TmpFile.with_file do |directory|
|
100
|
+
Step.save_job_inputs(job, directory)
|
101
|
+
newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
|
102
|
+
assert_equal job.path, newjob.path
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def test_save_all_job_file
|
110
|
+
tsv = TSV.setup({}, "Key~Value#:type=:single")
|
111
|
+
tsv["key"] = "value"
|
112
|
+
Log.with_severity 0 do
|
113
|
+
Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
|
114
|
+
TmpFile.with_file do |input_file|
|
115
|
+
Path.setup(input_file)
|
116
|
+
options = {
|
117
|
+
:float => input_file.float,
|
118
|
+
:integer => input_file.integer,
|
119
|
+
:string => input_file.string,
|
120
|
+
:select => input_file.select,
|
121
|
+
:array => TestSaveLoadWF.job(:list),
|
122
|
+
:binary => TestSaveLoadWF.job(:list).file('binary'),
|
123
|
+
:tsv => input_file.tsv_file,
|
124
|
+
}
|
125
|
+
job = TestSaveLoadWF.job(:save_test, nil, options)
|
126
|
+
TmpFile.with_file do |directory|
|
127
|
+
Step.save_job_inputs(job, directory)
|
128
|
+
newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
|
129
|
+
assert_equal job.path, newjob.path
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
43
133
|
end
|
44
134
|
end
|
45
135
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.33.
|
4
|
+
version: 5.33.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2022-03-
|
11
|
+
date: 2022-03-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|