rbbt-util 5.33.7 → 5.33.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f4272048a5c86e74b051600f0db895ea42f4ad51509a0de9ce88452d27133746
4
- data.tar.gz: f10d19e028de390beb5e2649c7d2f1e0a7754d3235abe3eeba2790946b347f64
3
+ metadata.gz: aaa014ffdad4e098212cc3f7cefc4d04ac62a7f7fcbf17bbbab9415c6253d9ef
4
+ data.tar.gz: b8a2a233686ec2f229ed79bb19d87d8eb79ad945142305f7b61dcdb11f5f7912
5
5
  SHA512:
6
- metadata.gz: 0667eb9d5783d7722c33f72a66a1c406d330d086a37660aa7682006d10fe15860a9e4cbfcb461f17ed15b146fa294a6f6d8158d937884f14dc7952908f63c357
7
- data.tar.gz: f3ba25b347bc4a7f7b56ecdbd1a0bef932b646ac030b3d98fe6ddb76f36a6863073f4692747ceef32c42e4344538a7ee19b531a25b16cc2fa208b201887b441e
6
+ metadata.gz: 564242bef1dd92c2f53a5fe6c967624de26d7ae7eeb4da1ef2a1f682db9cf6d4b0bf0f8e76b3a600afe6af7bb443df28a8fe50e570c3c56265319eb607d6c1fc
7
+ data.tar.gz: ac7c01fa7e8fccdff8dfc861b82f28889a68f77886c198ff55b09558b36857ec9c29a8bae979bc7d088f311ad81515676bc7a4cc30097fd6c4b7586e6838c0e3
@@ -13,7 +13,11 @@ module HPC
13
13
  all_deps = rec_dependencies + [job]
14
14
 
15
15
  all_deps.each do |dep|
16
- Step.prepare_for_execution(dep)
16
+ begin
17
+ Step.prepare_for_execution(dep)
18
+ rescue RbbtException
19
+ next
20
+ end
17
21
  end
18
22
 
19
23
  end
@@ -40,8 +44,8 @@ module HPC
40
44
 
41
45
  IndiferentHash.setup(rules)
42
46
 
43
- Log.high "Compute batches"
44
47
  batches = HPC::Orchestration.job_batches(rules, job)
48
+ Log.high "Compute #{batches.length} batches"
45
49
 
46
50
  batch_ids = {}
47
51
  while batches.any?
@@ -79,7 +83,7 @@ module HPC
79
83
  puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
80
84
  puts Log.color(:magenta, "Deps: ") + Log.color(:blue, job_options[:batch_dependencies]*", ")
81
85
  puts Log.color(:yellow, "Path: ") + top[:top_level].path
82
- puts Log.color(:yellow, "Options: ") + Misc.fingerprint(job_options)
86
+ puts Log.color(:yellow, "Options: ") + job_options.inspect
83
87
  batch_ids[top] = top[:top_level].task_signature
84
88
  else
85
89
  id = run_job(top[:top_level], job_options)
@@ -167,8 +167,7 @@ module Path
167
167
 
168
168
  @path ||= {}
169
169
  rsearch_paths = (resource and resource.respond_to?(:search_paths)) ? resource.search_paths : nil
170
- key_elems = [where, caller_lib, rsearch_paths, paths]
171
- key = Misc.obj2digest(key_elems.inspect)
170
+ key = [where, caller_lib, rsearch_paths, paths].inspect
172
171
  self.sub!('~/', Etc.getpwuid.dir + '/') if self.include? "~"
173
172
 
174
173
  return @path[key] if @path[key]
@@ -1,13 +1,26 @@
1
1
  module Path
2
2
 
3
3
  def self.caller_lib_dir(file = nil, relative_to = ['lib', 'bin'])
4
- file = caller.reject{|l|
5
- l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
6
- l =~ /rbbt\/resource\/path\.rb/ or
7
- l =~ /rbbt\/persist.rb/ or
8
- l =~ /rbbt\/util\/misc\.rb/ or
9
- l =~ /progress-monitor\.rb/
10
- }.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
4
+ #file = caller.reject{|l|
5
+ # l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
6
+ # l =~ /rbbt\/resource\/path\.rb/ or
7
+ # l =~ /rbbt\/persist.rb/ or
8
+ # l =~ /rbbt\/util\/misc\.rb/ or
9
+ # l =~ /progress-monitor\.rb/
10
+ #}.first.sub(/\.rb[^\w].*/,'.rb') if file.nil?
11
+
12
+
13
+ if file.nil?
14
+ caller_dup = caller.dup
15
+ while file = caller_dup.shift
16
+ break unless file =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
17
+ file =~ /rbbt\/resource\/path\.rb/ or
18
+ file =~ /rbbt\/persist.rb/ or
19
+ file =~ /rbbt\/util\/misc\.rb/ or
20
+ file =~ /progress-monitor\.rb/
21
+ end
22
+ file = file.sub(/\.rb[^\w].*/,'.rb')
23
+ end
11
24
 
12
25
  relative_to = [relative_to] unless Array === relative_to
13
26
  file = File.expand_path(file)
@@ -80,7 +80,7 @@ module Misc
80
80
  end
81
81
  when (defined? TSV and TSV)
82
82
  obj.with_unnamed do
83
- "TSV:{"<< fingerprint(obj.all_fields|| []).inspect << ";" << fingerprint(obj.keys).inspect << "}"
83
+ "TSV:{"<< fingerprint(obj.all_fields|| []) << ";" << fingerprint(obj.keys) << "}"
84
84
  end
85
85
  when Hash
86
86
  if obj.length > 10
@@ -297,14 +297,14 @@ module Misc
297
297
  str = case obj
298
298
  when nil
299
299
  'nil'
300
+ when Symbol
301
+ obj.to_s
300
302
  when TrueClass
301
303
  'true'
302
304
  when FalseClass
303
305
  'false'
304
306
  when Hash
305
307
  "{"<< obj.collect{|k,v| obj2str(k) + '=>' << obj2str(v)}*"," << "}"
306
- when Symbol
307
- obj.to_s
308
308
  when (defined?(Path) and Path)
309
309
  if defined?(Step) && Open.exists?(Step.info_file(obj))
310
310
  obj2str(Workflow.load_step(obj))
@@ -324,9 +324,10 @@ module Misc
324
324
  end
325
325
  when String
326
326
  good_filename = Misc.is_filename?(obj, false) && ! %w(. ..).include?(obj) && %w(. /).include?(obj[0])
327
- is_path = Path === obj
328
- if good_filename && ! is_path
329
- obj2str Path.setup(obj.dup)
327
+ if good_filename
328
+ obj = obj.dup
329
+ obj.extend Path
330
+ obj2str obj
330
331
  else
331
332
  obj = obj.chomp if String === obj
332
333
  if obj.length > HASH2MD5_MAX_STRING_LENGTH
@@ -339,7 +340,7 @@ module Misc
339
340
  if obj.length > HASH2MD5_MAX_ARRAY_LENGTH
340
341
  "[" << sample_large_obj(obj, HASH2MD5_MAX_ARRAY_LENGTH).collect{|v| obj2str(v)} * "," << "]"
341
342
  else
342
- "[" << obj.collect{|v| obj2str(v)} * "," << "]"
343
+ "[" << obj.collect{|v| obj2str(v) } * "," << "]"
343
344
  end
344
345
  when TSV::Parser
345
346
  remove_long_items(obj)
@@ -185,6 +185,10 @@ compile(){
185
185
  local name=$1; shift
186
186
  local extra="$@"
187
187
 
188
+ if [ -f bootstrap ]; then
189
+ ./bootstrap
190
+ fi
191
+
188
192
  if [ -f Makefile -o -f makefile ]; then
189
193
  make -j 4 || exit -1
190
194
  make install || echo "No install"
@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
22
22
  -s--search* Regular expression
23
23
  -t--tail* Show the last lines of the STDERR
24
24
  -l--long Show more entries
25
+ -c--compressed Show compressed information about entries
25
26
  -p--progress Report progress of job and the dependencies
26
27
  -BP--batch_parameters show batch parameters
27
28
  -BPP--batch_procpath show Procpath performance summary
@@ -185,6 +186,40 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
185
186
  end
186
187
 
187
188
 
189
+ count += 1
190
+
191
+ if options[:compressed]
192
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) : Log.color(:green, id)
193
+ if different_system
194
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id)
195
+ else
196
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
197
+ end
198
+ prog_rep = []
199
+ if options[:progress]
200
+ step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
201
+ if step_line
202
+ require 'rbbt/workflow'
203
+ step_path = step_line.split(": ").last.strip
204
+ step = Step.new step_path
205
+ step.load_dependencies_from_info
206
+ has_bar = false
207
+ (step.rec_dependencies + [step]).reverse.each do |j|
208
+ next if j.done?
209
+ if j.file(:progress).exists?
210
+ bar = Log::ProgressBar.new
211
+ bar.load(j.file(:progress).yaml)
212
+ prog_rep << bar.report_msg.split("·")[0..1]
213
+ end
214
+ end
215
+ end
216
+ end
217
+ workflow, task, name = step_path.split("/")[-3..-1]
218
+ job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
219
+ puts [job_str, status, prog_rep ].flatten * " "
220
+ next
221
+ end
222
+
188
223
  puts Log.color :blue, dir
189
224
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
190
225
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
@@ -232,22 +267,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
232
267
  cpu_average = {}
233
268
  rss_average = {}
234
269
  perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
235
- time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
236
- time = time.to_f
237
-
238
- cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
239
- cpu_average[stat_pid] ||= {}
240
- cpu_average[stat_pid][time] ||= []
241
- cpu_average[stat_pid][time] << cpu.to_f
242
- rss_average[time] ||= []
243
- rss_average[time] << rss.to_f * page_size
270
+ time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
271
+ time = time.to_f
272
+
273
+ cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
274
+ cpu_average[stat_pid] ||= {}
275
+ cpu_average[stat_pid][time] ||= []
276
+ cpu_average[stat_pid][time] << cpu.to_f
277
+ rss_average[time] ||= []
278
+ rss_average[time] << rss.to_f * page_size
244
279
  end
245
280
 
246
281
  ticks = 0
247
282
  cpu_average.each do |stat_pid, cpu_average_pid|
248
- start = cpu_average_pid.keys.sort.first
249
- eend = cpu_average_pid.keys.sort.last
250
- ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
283
+ start = cpu_average_pid.keys.sort.first
284
+ eend = cpu_average_pid.keys.sort.last
285
+ ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
251
286
  end
252
287
  start = rss_average.keys.sort.first
253
288
  eend = rss_average.keys.sort.last
@@ -309,8 +344,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
309
344
  end
310
345
  end
311
346
 
312
- count += 1
313
-
314
347
  end
315
348
 
316
349
  puts
@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
22
22
  -s--search* Regular expression
23
23
  -t--tail* Show the last lines of the STDERR
24
24
  -l--long Show more entries
25
+ -c--compressed Show compressed information about entries
25
26
  -p--progress Report progress of job and the dependencies
26
27
  -BP--batch_parameters show batch parameters
27
28
  -BPP--batch_procpath show Procpath performance summary
@@ -185,6 +186,40 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
185
186
  end
186
187
 
187
188
 
189
+ count += 1
190
+
191
+ if options[:compressed]
192
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) : Log.color(:green, id)
193
+ if different_system
194
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id)
195
+ else
196
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
197
+ end
198
+ prog_rep = []
199
+ if options[:progress]
200
+ step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
201
+ if step_line
202
+ require 'rbbt/workflow'
203
+ step_path = step_line.split(": ").last.strip
204
+ step = Step.new step_path
205
+ step.load_dependencies_from_info
206
+ has_bar = false
207
+ (step.rec_dependencies + [step]).reverse.each do |j|
208
+ next if j.done?
209
+ if j.file(:progress).exists?
210
+ bar = Log::ProgressBar.new
211
+ bar.load(j.file(:progress).yaml)
212
+ prog_rep << bar.report_msg.split("·")[0..1]
213
+ end
214
+ end
215
+ end
216
+ end
217
+ workflow, task, name = step_path.split("/")[-3..-1]
218
+ job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
219
+ puts [job_str, status, prog_rep ].flatten * " "
220
+ next
221
+ end
222
+
188
223
  puts Log.color :blue, dir
189
224
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
190
225
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
@@ -232,22 +267,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
232
267
  cpu_average = {}
233
268
  rss_average = {}
234
269
  perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
235
- time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
236
- time = time.to_f
237
-
238
- cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
239
- cpu_average[stat_pid] ||= {}
240
- cpu_average[stat_pid][time] ||= []
241
- cpu_average[stat_pid][time] << cpu.to_f
242
- rss_average[time] ||= []
243
- rss_average[time] << rss.to_f * page_size
270
+ time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
271
+ time = time.to_f
272
+
273
+ cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
274
+ cpu_average[stat_pid] ||= {}
275
+ cpu_average[stat_pid][time] ||= []
276
+ cpu_average[stat_pid][time] << cpu.to_f
277
+ rss_average[time] ||= []
278
+ rss_average[time] << rss.to_f * page_size
244
279
  end
245
280
 
246
281
  ticks = 0
247
282
  cpu_average.each do |stat_pid, cpu_average_pid|
248
- start = cpu_average_pid.keys.sort.first
249
- eend = cpu_average_pid.keys.sort.last
250
- ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
283
+ start = cpu_average_pid.keys.sort.first
284
+ eend = cpu_average_pid.keys.sort.last
285
+ ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
251
286
  end
252
287
  start = rss_average.keys.sort.first
253
288
  eend = rss_average.keys.sort.last
@@ -309,8 +344,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
309
344
  end
310
345
  end
311
346
 
312
- count += 1
313
-
314
347
  end
315
348
 
316
349
  puts
@@ -22,6 +22,7 @@ $ rbbt slurm list [options]
22
22
  -s--search* Regular expression
23
23
  -t--tail* Show the last lines of the STDERR
24
24
  -l--long Show more entries
25
+ -c--compressed Show compressed information about entries
25
26
  -p--progress Report progress of job and the dependencies
26
27
  -BP--batch_parameters show batch parameters
27
28
  -BPP--batch_procpath show Procpath performance summary
@@ -185,6 +186,40 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
185
186
  end
186
187
 
187
188
 
189
+ count += 1
190
+
191
+ if options[:compressed]
192
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) : Log.color(:green, id)
193
+ if different_system
194
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : Log.color(:green, id)
195
+ else
196
+ status = exit_status ? (exit_status == 0 ? Log.color(:green, "Done") : Log.color(:red, "Error")) + " (#{ id })" : (running_jobs.include?(id) || $norunningjobs ? Log.color(:green, id) : Log.color(:red, id) )
197
+ end
198
+ prog_rep = []
199
+ if options[:progress]
200
+ step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
201
+ if step_line
202
+ require 'rbbt/workflow'
203
+ step_path = step_line.split(": ").last.strip
204
+ step = Step.new step_path
205
+ step.load_dependencies_from_info
206
+ has_bar = false
207
+ (step.rec_dependencies + [step]).reverse.each do |j|
208
+ next if j.done?
209
+ if j.file(:progress).exists?
210
+ bar = Log::ProgressBar.new
211
+ bar.load(j.file(:progress).yaml)
212
+ prog_rep << bar.report_msg.split("·")[0..1]
213
+ end
214
+ end
215
+ end
216
+ end
217
+ workflow, task, name = step_path.split("/")[-3..-1]
218
+ job_str = [Log.color(:yellow, workflow), Log.color(:magenta, task), name] * "/"
219
+ puts [job_str, status, prog_rep ].flatten * " "
220
+ next
221
+ end
222
+
188
223
  puts Log.color :blue, dir
189
224
  puts Log.color(:magenta, "Creation: ") << File.mtime(File.join(dir, 'command.batch')).to_s if long
190
225
  puts Log.color(:magenta, "Started: ") << File.ctime(File.join(dir, 'std.err')).to_s if File.exist?(File.join(dir, 'std.err')) && long
@@ -232,22 +267,22 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
232
267
  cpu_average = {}
233
268
  rss_average = {}
234
269
  perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
235
- time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
236
- time = time.to_f
237
-
238
- cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
239
- cpu_average[stat_pid] ||= {}
240
- cpu_average[stat_pid][time] ||= []
241
- cpu_average[stat_pid][time] << cpu.to_f
242
- rss_average[time] ||= []
243
- rss_average[time] << rss.to_f * page_size
270
+ time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
271
+ time = time.to_f
272
+
273
+ cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
274
+ cpu_average[stat_pid] ||= {}
275
+ cpu_average[stat_pid][time] ||= []
276
+ cpu_average[stat_pid][time] << cpu.to_f
277
+ rss_average[time] ||= []
278
+ rss_average[time] << rss.to_f * page_size
244
279
  end
245
280
 
246
281
  ticks = 0
247
282
  cpu_average.each do |stat_pid, cpu_average_pid|
248
- start = cpu_average_pid.keys.sort.first
249
- eend = cpu_average_pid.keys.sort.last
250
- ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
283
+ start = cpu_average_pid.keys.sort.first
284
+ eend = cpu_average_pid.keys.sort.last
285
+ ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
251
286
  end
252
287
  start = rss_average.keys.sort.first
253
288
  eend = rss_average.keys.sort.last
@@ -309,8 +344,6 @@ workdir.glob("**/command.batch").sort_by{|f| File.mtime(f)}.each do |fcmd|
309
344
  end
310
345
  end
311
346
 
312
- count += 1
313
-
314
347
  end
315
348
 
316
349
  puts
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.33.7
4
+ version: 5.33.8
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-03-10 00:00:00.000000000 Z
11
+ date: 2022-03-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake