rbbt-util 5.33.8 → 5.33.9

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: aaa014ffdad4e098212cc3f7cefc4d04ac62a7f7fcbf17bbbab9415c6253d9ef
4
- data.tar.gz: b8a2a233686ec2f229ed79bb19d87d8eb79ad945142305f7b61dcdb11f5f7912
3
+ metadata.gz: 5c1ff60078b8ea97b0d68c7ec8beb1831889cd0e4ef841c781073dad938924be
4
+ data.tar.gz: 75dd8c9528cc2015c83b3d106027a5bdc29ba5d9eca53b8024ab19eb824c5273
5
5
  SHA512:
6
- metadata.gz: 564242bef1dd92c2f53a5fe6c967624de26d7ae7eeb4da1ef2a1f682db9cf6d4b0bf0f8e76b3a600afe6af7bb443df28a8fe50e570c3c56265319eb607d6c1fc
7
- data.tar.gz: ac7c01fa7e8fccdff8dfc861b82f28889a68f77886c198ff55b09558b36857ec9c29a8bae979bc7d088f311ad81515676bc7a4cc30097fd6c4b7586e6838c0e3
6
+ metadata.gz: 60379d7fd62c52f115a198bab61ef24ba2dc4be5c0fba9d4974637afaea9481b99530d0cadbae07b03db92fec75dee1ea1bb23d3fa96906d6cc918794ab0973b
7
+ data.tar.gz: 7c7ffd9b0847083722e6976f98eb700fef719f7fba7b36914e690538a93de047c7c46d42c0715ac430edc9c392ad5b0f1b0e97de65f812ba2eed3835263dbbe9
@@ -30,6 +30,7 @@ module HPC
30
30
  options.delete "printpath"
31
31
  options.delete "detach"
32
32
  options.delete "jobname"
33
+ options.delete "load_inputs"
33
34
 
34
35
  Log.high "Prepare for exec"
35
36
  prepare_for_execution(job)
@@ -94,8 +94,8 @@ export BATCH_SYSTEM=SLURM
94
94
 
95
95
  return if Open.exists?(fexit)
96
96
 
97
- STDERR.puts Log.color(:magenta, "Issuing SLURM file: #{fcmd}")
98
- STDERR.puts Open.read(fcmd)
97
+ Log.info "Issuing SLURM file: #{fcmd}"
98
+ Log.debug Open.read(fcmd)
99
99
 
100
100
  if File.exists?(fjob)
101
101
  job = Open.read(fjob).to_i
@@ -1,4 +1,5 @@
1
1
  module Workflow
2
+
2
3
  def self.load_inputs(dir, input_names, input_types)
3
4
  inputs = {}
4
5
  if File.exists?(dir) && ! File.directory?(dir)
@@ -24,17 +25,31 @@ module Workflow
24
25
 
25
26
  type = :io if file.split(".").last == 'as_io'
26
27
 
28
+ type = :step if file.split(".").last == 'as_step'
29
+
30
+ type = :step_file if file.split(".").last == 'as_step_file'
31
+
27
32
  type = :path if file.split(".").last == 'as_path'
28
33
 
34
+ type = :filename if file.split(".").last == 'as_filename'
35
+
29
36
  type = :nofile if file.split(".").last == 'nofile'
30
37
 
31
38
  case type
32
39
  when :nofile
33
40
  inputs[input.to_sym] = Open.realpath(file)
34
41
  when :path
35
- inputs[input.to_sym] = Open.realpath(Open.read(file).strip)
42
+ inputs[input.to_sym] = Open.read(file).strip
36
43
  when :io
37
44
  inputs[input.to_sym] = Open.open(Open.realpath(file))
45
+ when :step
46
+ inputs[input.to_sym] = Workflow.load_step(Open.read(file).strip)
47
+ when :step_file
48
+ path = Open.read(file).strip
49
+ path.extend Path
50
+ step_path = path.match(/(.*)\.files/)[1]
51
+ path.resource = Step.new step_path
52
+ inputs[input.to_sym] = path
38
53
  when :file, :binary
39
54
  Log.debug "Pointing #{ input } to #{file}"
40
55
  if file =~ /\.yaml/
@@ -87,52 +102,124 @@ module Workflow
87
102
  job(task_name, jobname, inputs)
88
103
  end
89
104
 
105
+ #def self.load_inputs_old(dir, input_names, input_types)
106
+ # inputs = {}
107
+ # if File.exists?(dir) && ! File.directory?(dir)
108
+ # Log.debug "Loading inputs from #{dir}, not a directory trying as tar.gz"
109
+ # tarfile = dir
110
+ # digest = CMD.cmd("md5sum '#{tarfile}'").read.split(" ").first
111
+ # tmpdir = Rbbt.tmp.input_bundle[digest].find
112
+ # Misc.untar(tarfile, tmpdir) unless File.exists? tmpdir
113
+ # files = tmpdir.glob("*")
114
+ # if files.length == 1 && File.directory?(files.first)
115
+ # tmpdir = files.first
116
+ # end
117
+ # load_inputs(tmpdir, input_names, input_types)
118
+ # else
119
+ # dir = Path.setup(dir.dup)
120
+ # input_names.each do |input|
121
+ # file = dir[input].find
122
+ # file = dir.glob(input.to_s + ".*").reject{|f| f =~ /\.md5$/}.first if file.nil? or not (File.symlink?(file) || file.exists?)
123
+ # Log.debug "Trying #{ input }: #{file}"
124
+ # next unless file and (File.symlink?(file) || file.exists?)
125
+
126
+ # type = input_types[input]
127
+
128
+ # type = :io if file.split(".").last == 'as_io'
129
+
130
+ # type = :path if file.split(".").last == 'as_path'
131
+
132
+ # type = :filename if file.split(".").last == 'as_filename'
133
+
134
+ # type = :nofile if file.split(".").last == 'nofile'
135
+
136
+ # case type
137
+ # when :nofile
138
+ # inputs[input.to_sym] = Open.realpath(file)
139
+ # when :path
140
+ # inputs[input.to_sym] = Open.realpath(Open.read(file).strip)
141
+ # when :io
142
+ # inputs[input.to_sym] = Open.open(Open.realpath(file))
143
+ # when :file, :binary
144
+ # Log.debug "Pointing #{ input } to #{file}"
145
+ # if file =~ /\.yaml/
146
+ # inputs[input.to_sym] = YAML.load(Open.read(file))
147
+ # else
148
+ # if File.symlink?(file)
149
+ # link_target = File.expand_path(File.readlink(file), File.dirname(file))
150
+ # inputs[input.to_sym] = link_target
151
+ # else
152
+ # inputs[input.to_sym] = Open.realpath(file)
153
+ # end
154
+ # end
155
+ # when :text
156
+ # Log.debug "Reading #{ input } from #{file}"
157
+ # inputs[input.to_sym] = Open.read(file)
158
+ # when :array
159
+ # Log.debug "Reading array #{ input } from #{file}"
160
+ # inputs[input.to_sym] = Open.read(file).split("\n")
161
+ # when :tsv
162
+ # Log.debug "Opening tsv #{ input } from #{file}"
163
+ # inputs[input.to_sym] = TSV.open(file)
164
+ # when :boolean
165
+ # inputs[input.to_sym] = (file.read.strip == 'true')
166
+ # else
167
+ # Log.debug "Loading #{ input } from #{file}"
168
+ # inputs[input.to_sym] = file.read.strip
169
+ # end
170
+
171
+ # end
172
+ # inputs = IndiferentHash.setup(inputs)
173
+
174
+ # dir.glob("*#*").each do |od|
175
+ # name = File.basename(od)
176
+ # value = Open.read(od)
177
+ # Log.debug "Loading override dependency #{ name } as #{value}"
178
+ # inputs[name] = value.chomp
179
+ # end
180
+
181
+ # inputs
182
+ # end
183
+ #end
90
184
  end
91
185
 
92
186
  class Step
187
+ def self.save_input(name, value, type, dir)
188
+ path = File.join(dir, name.to_s)
189
+
190
+
191
+ case value
192
+ when Path
193
+ if Step === value.resource
194
+ path = path + '.as_step_file'
195
+ else
196
+ path = path + '.as_path'
197
+ end
198
+ when String
199
+ if Misc.is_filename?(value, false)
200
+ value = value.dup
201
+ value.extend Path
202
+ return save_input(name, value, type, dir)
203
+ end
204
+ when IO
205
+ path = path + '.as_io'
206
+ when Step
207
+ value = value.path
208
+ path = path + '.as_step'
209
+ when Array
210
+ value = value * "\n"
211
+ end
212
+
213
+ Log.debug "Saving job input #{name} (#{type}) into #{path}"
214
+ Open.write(path, value.to_s)
215
+ end
216
+
93
217
  def self.save_inputs(inputs, input_types, input_options, dir)
94
218
  inputs.each do |name,value|
95
219
  type = input_types[name]
96
220
  type = type.to_s if type
97
- path = File.join(dir, name.to_s)
98
-
99
- path = path + '.as_io' if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
100
- Log.debug "Saving job input #{name} (#{type}) into #{path}"
101
-
102
- case
103
- when IO === value
104
- Open.write(path, value.to_s)
105
- when Step === value
106
- Open.ln_s(value.path, path)
107
- when type.to_s == "binary"
108
- if String === value && File.exists?(value)
109
- value = File.expand_path(value)
110
- Open.ln_s(value, path)
111
- elsif String === value && Misc.is_filename?(value, false)
112
- Open.write(path + '.as_path' , value)
113
- else
114
- Open.write(path, value, :mode => 'wb')
115
- end
116
- when type.to_s == "file"
117
- if String === value && File.exists?(value)
118
- value = File.expand_path(value)
119
- Open.ln_s(value, path)
120
- else
121
- value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
122
- value = "#{value}" if Path === value
123
- Open.write(path + '.yaml', value.to_yaml)
124
- end
125
- when Array === value
126
- Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
127
- when IO === value
128
- if value.filename && String === value.filename && File.exists?(value.filename)
129
- Open.ln_s(value.filename, path)
130
- else
131
- Open.write(path, value)
132
- end
133
- else
134
- Open.write(path, value.to_s)
135
- end
221
+
222
+ save_input(name, value, type, dir)
136
223
  end.any?
137
224
  end
138
225
 
@@ -176,4 +263,60 @@ class Step
176
263
 
177
264
  inputs.keys
178
265
  end
266
+
267
+ #def self.save_inputs_old(inputs, input_types, input_options, dir)
268
+ # inputs.each do |name,value|
269
+ # type = input_types[name]
270
+ # type = type.to_s if type
271
+ # path = File.join(dir, name.to_s)
272
+
273
+ # if (IO === value || Step === value) && ! (input_options[name] && input_options[name][:nofile])
274
+ # path = path + '.as_io'
275
+ # elsif Misc.is_filename?(value, false)
276
+ # path = path + '.as_filename'
277
+ # end
278
+
279
+ # Log.debug "Saving job input #{name} (#{type}) into #{path}"
280
+
281
+ # case
282
+ # when IO === value
283
+ # Open.write(path, value.to_s)
284
+ # when Step === value
285
+ # Open.ln_s(value.path, path)
286
+ # when type.to_s == "binary"
287
+ # if String === value && File.exists?(value)
288
+ # value = File.expand_path(value)
289
+ # Open.ln_s(value, path)
290
+ # elsif String === value && Misc.is_filename?(value, false)
291
+ # Open.write(path + '.as_path' , value)
292
+ # else
293
+ # Open.write(path, value, :mode => 'wb')
294
+ # end
295
+ # when TSV === value
296
+ # Open.write(path, value.to_s)
297
+ # when Array === value
298
+ # Open.write(path, value.collect{|v| Step === v ? v.path : v.to_s} * "\n")
299
+ # when %w(file tsv array).include?(type.to_s)
300
+ # if String === value && File.exists?(value)
301
+ # value = File.expand_path(value)
302
+ # Open.ln_s(value, path)
303
+ # elsif String === value && Misc.is_filename?(value, false)
304
+ # Open.write(path + '.as_path' , value)
305
+ # else
306
+ # value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
307
+ # value = "#{value}" if Path === value
308
+ # Open.write(path + '.yaml', value.to_yaml)
309
+ # end
310
+ # when IO === value
311
+ # if value.filename && String === value.filename && File.exists?(value.filename)
312
+ # Open.ln_s(value.filename, path)
313
+ # else
314
+ # Open.write(path, value)
315
+ # end
316
+ # else
317
+ # Open.write(path, value.to_s)
318
+ # end
319
+ # end.any?
320
+ #end
321
+
179
322
  end
@@ -376,9 +376,9 @@ class Step
376
376
  }
377
377
  raise "Dependency step not found: #{ name }" if deps.empty?
378
378
  if (deps & self.dependencies).any?
379
- (deps & self.dependencies).first
379
+ (deps & self.dependencies).last
380
380
  else
381
- deps.first
381
+ deps.last
382
382
  end
383
383
  end
384
384
  end
@@ -141,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
141
141
  rescue
142
142
  nodes = []
143
143
  end
144
- elsif job_nodes[id]
144
+ elsif job_nodes && job_nodes[id]
145
145
  nodes = job_nodes[id].reject{|n| n.include? "("}
146
146
  else
147
147
  nodes = []
@@ -340,7 +340,10 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
340
340
  has_bar = true
341
341
  end
342
342
  end
343
- puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
343
+ step_status = step.status
344
+ step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
345
+ step_status = Log.color :green, step_status if step_status.to_s == 'done'
346
+ puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
344
347
  end
345
348
  end
346
349
 
@@ -141,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
141
141
  rescue
142
142
  nodes = []
143
143
  end
144
- elsif job_nodes[id]
144
+ elsif job_nodes && job_nodes[id]
145
145
  nodes = job_nodes[id].reject{|n| n.include? "("}
146
146
  else
147
147
  nodes = []
@@ -340,7 +340,10 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
340
340
  has_bar = true
341
341
  end
342
342
  end
343
- puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
343
+ step_status = step.status
344
+ step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
345
+ step_status = Log.color :green, step_status if step_status.to_s == 'done'
346
+ puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
344
347
  end
345
348
  end
346
349
 
@@ -141,7 +141,7 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
141
141
  rescue
142
142
  nodes = []
143
143
  end
144
- elsif job_nodes[id]
144
+ elsif job_nodes && job_nodes[id]
145
145
  nodes = job_nodes[id].reject{|n| n.include? "("}
146
146
  else
147
147
  nodes = []
@@ -340,7 +340,10 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
340
340
  has_bar = true
341
341
  end
342
342
  end
343
- puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step.status}" unless has_bar
343
+ step_status = step.status
344
+ step_status = Log.color :red, step_status if step_status.to_s == 'cleaned'
345
+ step_status = Log.color :green, step_status if step_status.to_s == 'done'
346
+ puts Log.color(:magenta, "Progress: ") + Log.color(:yellow, step.task_signature) + " #{step_status}" unless has_bar
344
347
  end
345
348
  end
346
349
 
@@ -76,6 +76,16 @@ def fix_options(workflow, task, job_options)
76
76
  type = input_types[name]
77
77
  type = type.to_sym if type
78
78
 
79
+ if Step === value
80
+ job_options_cleaned[name] = value
81
+ next
82
+ end
83
+
84
+ if Path === value && Step === value.resource
85
+ job_options_cleaned[name] = value
86
+ next
87
+ end
88
+
79
89
  value = case type
80
90
  when nil
81
91
  value
@@ -148,6 +158,7 @@ def fix_options(workflow, task, job_options)
148
158
  else
149
159
  value
150
160
  end
161
+
151
162
  job_options_cleaned[name] = value
152
163
  end
153
164
 
@@ -26,11 +26,26 @@ module TestSaveLoadWF
26
26
  task :prefix => :array do
27
27
  step(:reverse).run.collect{|e| "A-#{e}" }
28
28
  end
29
+
30
+ input :integer, :integer
31
+ input :float, :float
32
+ input :file, :file
33
+ input :file_no_file, :file, "", nil, :nofile => true
34
+ input :string, :string
35
+ input :select, :select
36
+ input :array, :array
37
+ input :array_no_file, :array, "", nil, :nofile => true
38
+ input :tsv, :tsv, "", nil, :nofile => true
39
+ input :tsv_no_file, :tsv, "", nil, :nofile => true
40
+ input :binary, :binary, "", nil, :nofile => true
41
+ input :binary_no_file, :tsv, "", nil, :nofile => true
42
+ task :save_test => :string do
43
+ "DONE"
44
+ end
29
45
  end
30
46
 
31
47
  class TestSaveLoad < Test::Unit::TestCase
32
48
  def test_save
33
- Log.with_severity 0 do
34
49
  job = TestSaveLoadWF.job(:prefix)
35
50
  job.recursive_clean
36
51
  job = TestSaveLoadWF.job(:prefix)
@@ -40,6 +55,81 @@ class TestSaveLoad < Test::Unit::TestCase
40
55
  newjob = TestSaveLoadWF.job_for_directory_inputs(:reverse, directory)
41
56
  assert_equal job.rec_dependencies.last.path, newjob.path
42
57
  end
58
+ end
59
+
60
+ def test_save_all_normal
61
+ tsv = TSV.setup({}, "Key~Value#:type=:single")
62
+ tsv["key"] = "value"
63
+ options = {
64
+ :float => 0.5,
65
+ :integer => 15,
66
+ :string => "STRING",
67
+ :select => "option",
68
+ :array => [0,1,2,3],
69
+ :tsv => tsv,
70
+ }
71
+ Log.with_severity 0 do
72
+ Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
73
+ job = TestSaveLoadWF.job(:save_test, nil, options)
74
+ TmpFile.with_file do |directory|
75
+ Step.save_job_inputs(job, directory)
76
+ newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
77
+ assert_equal job.path, newjob.path
78
+ end
79
+ end
80
+ end
81
+ end
82
+
83
+ def test_save_all_file
84
+ tsv = TSV.setup({}, "Key~Value#:type=:single")
85
+ tsv["key"] = "value"
86
+ Log.with_severity 0 do
87
+ Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
88
+ TmpFile.with_file do |input_file|
89
+ Path.setup(input_file)
90
+ options = {
91
+ :float => input_file.float,
92
+ :integer => input_file.integer,
93
+ :string => input_file.string,
94
+ :select => input_file.select,
95
+ :array => input_file.array,
96
+ :tsv => input_file.tsv_file,
97
+ }
98
+ job = TestSaveLoadWF.job(:save_test, nil, options)
99
+ TmpFile.with_file do |directory|
100
+ Step.save_job_inputs(job, directory)
101
+ newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
102
+ assert_equal job.path, newjob.path
103
+ end
104
+ end
105
+ end
106
+ end
107
+ end
108
+
109
+ def test_save_all_job_file
110
+ tsv = TSV.setup({}, "Key~Value#:type=:single")
111
+ tsv["key"] = "value"
112
+ Log.with_severity 0 do
113
+ Misc.with_env "RBBT_DEBUG_JOB_HASH", "true" do
114
+ TmpFile.with_file do |input_file|
115
+ Path.setup(input_file)
116
+ options = {
117
+ :float => input_file.float,
118
+ :integer => input_file.integer,
119
+ :string => input_file.string,
120
+ :select => input_file.select,
121
+ :array => TestSaveLoadWF.job(:list),
122
+ :binary => TestSaveLoadWF.job(:list).file('binary'),
123
+ :tsv => input_file.tsv_file,
124
+ }
125
+ job = TestSaveLoadWF.job(:save_test, nil, options)
126
+ TmpFile.with_file do |directory|
127
+ Step.save_job_inputs(job, directory)
128
+ newjob = TestSaveLoadWF.job_for_directory_inputs(:save_test, directory)
129
+ assert_equal job.path, newjob.path
130
+ end
131
+ end
132
+ end
43
133
  end
44
134
  end
45
135
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.33.8
4
+ version: 5.33.9
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-03-14 00:00:00.000000000 Z
11
+ date: 2022-03-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake