rbbt-util 5.29.4 → 5.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3ec6302ccfe3f38a074f7f0d10511090c8f4db4186228ad93adb2888e0edbf5e
4
- data.tar.gz: fcaa50b654461f128b9539fc47ed00b008d3f713e89b8bbe963c2b898c3c168b
3
+ metadata.gz: b0e0e6d9165c16d73d1122a8d85e8038befd335215646bc3779400f5747b4b07
4
+ data.tar.gz: 7c0e92d49399111157fae43149c7ec8e598aa64510591fc2f74314bbefc89b2d
5
5
  SHA512:
6
- metadata.gz: 2a2537aef150df77142a593742399d28bb96334decc0e33b69e9cbac2853085487100aa90cd711b342bac7eda536d15c080be22891d5b9f38bfa4601282ae5de
7
- data.tar.gz: d79cc4afa294d63cebd79f73b759bc46c3c4e0285db7be7406a594aa49e5b572b0c4f91b98c73b5f64c31c908d58d8dee43853ec18b1ead9e814b4370de0d60d
6
+ metadata.gz: 6f1ecdb8ea4481b12138f6b63d2508f888ad60d42b56cc187ac74e3d705bb40db76d52f7e615bbd3b7fba74e49298325dcf7e5477f5527e55aa4a42e6744586d
7
+ data.tar.gz: 5b3761d1300940911383fc0659a4335b422db217a9118a0fd41ebd2374cec87884775564f003d5a0a14555eff68fcac7b70c68c3cea72c6d802908a48b8b514c
@@ -33,7 +33,8 @@ module HPC
33
33
  group = File.basename(File.dirname(ENV['HOME']))
34
34
 
35
35
  if contain_and_sync
36
- contain = "/scratch/tmp/rbbt-#{user}" if contain.nil?
36
+ random_file = TmpFile.random_name
37
+ contain = "/scratch/tmp/rbbt-#{user}/#{random_file}" if contain.nil?
37
38
  sync = "~/.rbbt/var/jobs" if sync.nil?
38
39
  wipe_container = "post" if wipe_container.nil?
39
40
  end
@@ -243,7 +244,7 @@ EOF
243
244
  end
244
245
 
245
246
  if contain
246
- rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}/.rbbt/var/jobs')
247
+ rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}/workdir')
247
248
  end
248
249
  end
249
250
 
@@ -251,6 +252,10 @@ EOF
251
252
  cmd =<<-EOF
252
253
  #{exec_cmd} \\
253
254
  #{rbbt_cmd}
255
+ EOF
256
+ annotate_cmd =<<-EOF
257
+ #{exec_cmd} \\
258
+ workflow write_info --recursive --force=false --check_pid "$step_path" slurm_job $SLURM_JOB_ID
254
259
  EOF
255
260
 
256
261
  header +=<<-EOF
@@ -260,11 +265,14 @@ EOF
260
265
  run +=<<-EOF
261
266
 
262
267
  # Run command
263
- #{cmd}
268
+ step_path=$(#{cmd})
264
269
 
265
270
  # Save exit status
266
271
  exit_status=$?
267
272
 
273
+ # Annotate info with SLURM job_info
274
+ #{annotate_cmd}
275
+
268
276
  EOF
269
277
 
270
278
  # CODA
@@ -286,7 +294,7 @@ EOF
286
294
  sync = sync.strip
287
295
  source = File.join(File.expand_path(contain), source)
288
296
  else
289
- source = File.join(File.expand_path(contain), '.rbbt/var/jobs')
297
+ source = File.join(File.expand_path(contain), 'workdir/var/jobs')
290
298
  end
291
299
 
292
300
  target = File.expand_path(sync)
@@ -516,6 +524,7 @@ EOF
516
524
  dry_run = options.delete :dry_run
517
525
  tail = options.delete :tail
518
526
  dependencies = options.delete :slurm_dependencies
527
+ procpath = options.delete :SLURM_procpath
519
528
  options[:jobname] = job.clean_name
520
529
 
521
530
  workflow = job.workflow
@@ -543,13 +552,15 @@ EOF
543
552
 
544
553
  if saved && saved.any?
545
554
  options[:inputs_dir] = inputs_dir
546
- cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
555
+ cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
547
556
  else
548
- cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--log', (options[:log] || Log.severity).to_s]
557
+ cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--log', (options[:log] || Log.severity).to_s]
549
558
  end
550
559
 
551
560
  cmd << "--override_deps='#{override_deps.gsub("'", '\'')}'" if override_deps and not override_deps.empty?
552
561
 
562
+ cmd << "--procpath_performance='#{tmp_directory}/procpath##{procpath.gsub(',', '#')}'" if procpath
563
+
553
564
  template = self.template(cmd, options)
554
565
  jobid = self.issue_template(template, options.merge(:slurm_basedir => slurm_basedir, :dry_run => dry_run, :slurm_dependencies => dependencies))
555
566
 
@@ -104,9 +104,6 @@ module Persist
104
104
  write(true) if closed? || ! write?
105
105
  res = begin
106
106
  yield
107
- rescue Exception
108
- Log.exception $!
109
- raise $!
110
107
  ensure
111
108
  close
112
109
  end
@@ -115,7 +112,6 @@ module Persist
115
112
  end
116
113
 
117
114
  def read_and_close
118
- #return yield if @locked
119
115
  if read? || write?
120
116
  begin
121
117
  return yield
@@ -134,6 +130,45 @@ module Persist
134
130
  end
135
131
  end
136
132
 
133
+ def read_lock
134
+ read if closed?
135
+ if read?
136
+ begin
137
+ return yield
138
+ ensure
139
+ close
140
+ end
141
+ end
142
+
143
+ lock do
144
+ close
145
+ read true
146
+ begin
147
+ yield
148
+ end
149
+ end
150
+ end
151
+
152
+ def write_lock
153
+ write if closed?
154
+ if write?
155
+ begin
156
+ return yield
157
+ ensure
158
+ close
159
+ end
160
+ end
161
+
162
+ lock do
163
+ close
164
+ write true
165
+ begin
166
+ yield
167
+ end
168
+ end
169
+ end
170
+
171
+
137
172
  def merge!(hash)
138
173
  hash.each do |key,values|
139
174
  self[key] = values
@@ -141,38 +176,38 @@ module Persist
141
176
  end
142
177
 
143
178
  def range(*args)
144
- self.read_and_close do
179
+ self.read_lock do
145
180
  super(*args)
146
181
  end
147
182
  end
148
183
 
149
184
  def include?(*args)
150
- self.read_and_close do
185
+ self.read_lock do
151
186
  super(*args) #- TSV::ENTRY_KEYS.to_a
152
187
  end
153
188
  end
154
189
 
155
190
  def [](*args)
156
- self.read_and_close do
191
+ self.read_lock do
157
192
  super(*args) #- TSV::ENTRY_KEYS.to_a
158
193
  end
159
194
  end
160
195
 
161
196
  def []=(*args)
162
- self.write_and_close do
197
+ self.write_lock do
163
198
  super(*args) #- TSV::ENTRY_KEYS.to_a
164
199
  end
165
200
  end
166
201
 
167
202
  def keys(*args)
168
- self.read_and_close do
203
+ self.read_lock do
169
204
  super(*args)
170
205
  end
171
206
  end
172
207
 
173
208
 
174
209
  def prefix(key)
175
- self.read_and_close do
210
+ self.read_lock do
176
211
  range(key, 1, key + MAX_CHAR, 1)
177
212
  end
178
213
  end
@@ -184,13 +219,13 @@ module Persist
184
219
 
185
220
 
186
221
  def size(*args)
187
- self.read_and_close do
222
+ self.read_lock do
188
223
  super(*args)
189
224
  end
190
225
  end
191
226
 
192
227
  def each(*args, &block)
193
- self.read_and_close do
228
+ self.read_lock do
194
229
  super(*args, &block)
195
230
  end
196
231
  end
@@ -208,7 +243,7 @@ module Persist
208
243
  end
209
244
 
210
245
  def values_at(*keys)
211
- self.read_and_close do
246
+ self.read_lock do
212
247
  keys.collect do |k|
213
248
  self[k]
214
249
  end
data/lib/rbbt/util/cmd.rb CHANGED
@@ -217,7 +217,7 @@ module CMD
217
217
  end
218
218
  end
219
219
 
220
- def self.cmd_log(*args)
220
+ def self.cmd_pid(*args)
221
221
  all_args = *args
222
222
 
223
223
  all_args << {} unless Hash === all_args.last
@@ -248,4 +248,9 @@ module CMD
248
248
  nil
249
249
  end
250
250
 
251
+ def self.cmd_log(*args)
252
+ cmd_pid(*args)
253
+ nil
254
+ end
255
+
251
256
  end
@@ -242,48 +242,6 @@ module Misc
242
242
 
243
243
  return options
244
244
 
245
- options = {}
246
- string.split(/#/).each do |str|
247
- if str.match(/(.*)=(.*)/)
248
- option, value = $1, $2
249
- else
250
- option, value = str, true
251
- end
252
-
253
- option = option.sub(":",'').to_sym if option.chars.first == ':'
254
- value = value.sub(":",'').to_sym if String === value and value.chars.first == ':'
255
-
256
- if value == true
257
- options[option] = option.to_s.chars.first != '!'
258
- else
259
- options[option] = Thread.start do
260
- $SAFE = 0;
261
- case
262
- when value =~ /^(?:true|T)$/i
263
- true
264
- when value =~ /^(?:false|F)$/i
265
- false
266
- when Symbol === value
267
- value
268
- when (String === value and value =~ /^\/(.*)\/$/)
269
- Regexp.new /#{$1}/
270
- else
271
- begin
272
- Kernel.const_get value
273
- rescue
274
- begin
275
- raise if value =~ /[a-z]/ and defined? value
276
- eval(value)
277
- rescue Exception
278
- value
279
- end
280
- end
281
- end
282
- end.value
283
- end
284
- end
285
-
286
- options
287
245
  end
288
246
 
289
247
  end
@@ -0,0 +1,49 @@
1
+ require 'rbbt/util/cmd'
2
+ module ProcPath
3
+ CMD.tool :procpath do
4
+ 'pip install procpath'
5
+ end
6
+
7
+ def self.record(pid, path, options = {})
8
+ IndiferentHash.setup(options)
9
+ options = Misc.add_defaults options, "interval" => 30
10
+
11
+ cmd_options = %w(interval recnum reevalnum).inject({}){|acc,k| acc[k] = options[k]; acc}
12
+
13
+ Log.debug "ProcPath recording #{pid} in #{path} (#{Misc.fingerprint options})"
14
+ procpath_thread = Thread.new do
15
+ begin
16
+ procpath_pid = CMD.cmd_pid(:procpath, "record --database-file '#{path}' '$..children[?(@.stat.pid == #{pid})]'", cmd_options.merge(:nofail => true, :add_option_dashes => true))
17
+ rescue Exception
18
+ Log.exceptions $!
19
+ Process.kill "INT", procpath_pid
20
+ end
21
+ end
22
+
23
+ procpath_thread.report_on_exception = false
24
+
25
+ Process.wait pid.to_i
26
+ procpath_thread.raise Interrupt
27
+ end
28
+
29
+ def self.plot(path, output, options = {})
30
+ IndiferentHash.setup(options)
31
+ options = Misc.add_defaults options, "query-name" => 'rss', 'epsilon' => 0.5, "moving-average-window" => 10
32
+
33
+ cmd_options = %w(query-name epsilon monitor-average-window title logarithmic after before custom-query-file custom-value-expr).inject({}){|acc,k| acc[k] = options[k]; acc}
34
+ CMD.cmd_log(:procpath, "plot --database-file '#{path}' --plot-file '#{output}' ", cmd_options.merge(:nofail => true, :add_option_dashes => true))
35
+ end
36
+
37
+ def self.monitor(pid, path)
38
+ database, options_str = path.split("#")
39
+ options = options_str.nil? ? {} : Misc.string2hash(options_str)
40
+
41
+ database = File.expand_path database
42
+ Log.low "ProcPath monitor #{pid} in #{database} (#{Misc.fingerprint options})"
43
+
44
+ ProcPath.record(pid, database + '.sqlite3', options)
45
+ ProcPath.plot(database + '.sqlite3', database + '.cpu.svg', options.merge("query-name" => 'cpu'))
46
+ ProcPath.plot(database + '.sqlite3', database + '.rss.svg', options.merge("query-name" => 'rss'))
47
+ end
48
+ end
49
+
@@ -505,8 +505,8 @@ class Step
505
505
 
506
506
  def running?
507
507
  return false if ! (started? || status == :ending)
508
- pid = info[:pid]
509
- return nil if pid.nil?
508
+ return nil unless Open.exist?(self.pid_file)
509
+ pid = Open.read(self.pid_file).to_i
510
510
 
511
511
  return false if done? or error? or aborted?
512
512
 
@@ -530,8 +530,7 @@ class Step
530
530
  end
531
531
 
532
532
  def nopid?
533
- pid = info[:pid] || Open.exists?(pid_file)
534
- ! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
533
+ ! Open.exists?(pid_file) && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
535
534
  end
536
535
 
537
536
  def aborted?
@@ -373,7 +373,6 @@ class Step
373
373
  Log.exception $!
374
374
  ensure
375
375
  Step.purge_stream_cache
376
- set_info :pid, nil
377
376
  Open.rm pid_file if Open.exist?(pid_file)
378
377
  end
379
378
  end
@@ -388,7 +387,6 @@ class Step
388
387
  _clean_finished
389
388
  rescue
390
389
  stop_dependencies
391
- set_info :pid, nil
392
390
  Open.rm pid_file if Open.exist?(pid_file)
393
391
  end
394
392
  end
@@ -449,7 +447,7 @@ class Step
449
447
  ensure
450
448
  no_load = false unless IO === result
451
449
  Open.rm pid_file if Open.exist?(pid_file) unless no_load
452
- set_info :pid, nil unless no_load
450
+ #set_info :pid, nil unless no_load
453
451
  end
454
452
  end
455
453
 
@@ -559,7 +557,7 @@ class Step
559
557
  RbbtSemaphore.post_semaphore(semaphore) if semaphore
560
558
  Kernel.exit! -1
561
559
  end
562
- set_info :pid, nil
560
+ #set_info :pid, nil
563
561
  ensure
564
562
  RbbtSemaphore.post_semaphore(semaphore) if semaphore
565
563
  end
@@ -57,7 +57,7 @@ module Task
57
57
  puts Log.color(:magenta, "Input select options")
58
58
  puts
59
59
  selects.collect{|p| p}.uniq.each do |input,options|
60
- puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| o.to_s} * ", ") << "\n"
60
+ puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
61
61
  puts unless Log.compact
62
62
  end
63
63
  puts
@@ -21,6 +21,8 @@ $ rbbt mnl [options]
21
21
  -s--search* Regular expression
22
22
  -t--tail* Show the last lines of the STDERR
23
23
  -SBP--sbatch_parameters show sbatch parameters
24
+ -PERF--procpath_performance show Procpath performance summary
25
+ -sacct--sacct_peformance show sacct performance summary
24
26
  EOF
25
27
 
26
28
  if options[:help]
@@ -140,9 +142,62 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
140
142
 
141
143
  if options[:sbatch_parameters]
142
144
  puts Log.color(:magenta, "SBATCH parameters: ")
143
- puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
145
+ text = CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
146
+ lines = text.split("\n").collect{|line| header, _sep, value = line.partition(/\s+/); Log.color(:yellow, header + ": ") + value}
147
+ puts Log.color :yellow, lines * "\n"
144
148
  end
145
149
 
150
+ fprocpath = File.join(dir, 'procpath.sqlite3')
151
+ if options[:procpath_performance] && Open.exists?(fprocpath)
152
+ puts Log.color(:magenta, "Procpath summary: ")
153
+ require 'rbbt/tsv/csv'
154
+ meta = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from meta;' "))
155
+ perf = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from record;' "))
156
+
157
+ page_size = meta["page_size"].first.to_f
158
+ clock_ticks = meta["clock_ticks"].first.to_f
159
+
160
+ cpu_average = {}
161
+ rss_average = {}
162
+ perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
163
+ time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
164
+ time = time.to_f
165
+
166
+ cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
167
+ cpu_average[stat_pid] ||= {}
168
+ cpu_average[stat_pid][time] ||= []
169
+ cpu_average[stat_pid][time] << cpu.to_f
170
+ rss_average[time] ||= []
171
+ rss_average[time] << rss.to_f * page_size
172
+ end
173
+
174
+ ticks = 0
175
+ cpu_average.each do |stat_pid, cpu_average_pid|
176
+ start = cpu_average_pid.keys.sort.first
177
+ eend = cpu_average_pid.keys.sort.last
178
+ ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
179
+ end
180
+ start = rss_average.keys.sort.first
181
+ eend = rss_average.keys.sort.last
182
+ time_elapsed = eend - start
183
+ puts Log.color(:yellow, "CPU average: ") + "%.2f" % ( ticks / clock_ticks / time_elapsed * 100).to_s
184
+ puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s
185
+
186
+ end
187
+
188
+ if options[:sacct_peformance]
189
+ begin
190
+ tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
191
+ values = tsv[tsv.keys.first]
192
+ if values.compact.any?
193
+ puts Log.color(:magenta, "SACCT performance: ")
194
+ puts values.zip(values.fields).collect{|v,t| Log.color(:yellow, t + ": ") + v.to_s } * "\n"
195
+ end
196
+ rescue
197
+ end
198
+ end
199
+
200
+
146
201
  if tail && File.exists?(File.join(dir, 'std.err'))
147
202
  if exit_status && exit_status != 0
148
203
  puts Log.color(:magenta, "First error or exception found: ")
@@ -21,6 +21,7 @@ $slurm_options = SOPT.get <<EOF
21
21
  -CS--contain_and_sync Contain and sync to default locations
22
22
  -ci--copy_image When using a container directory, copy image there
23
23
  -t--tail Tail the logs
24
+ -SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
24
25
  -q--queue* Queue
25
26
  -t--task_cpus* Tasks
26
27
  -W--workflows* Additional workflows
@@ -20,6 +20,7 @@ $slurm_options = SOPT.get <<EOF
20
20
  -CS--contain_and_sync Contain and sync to default locations
21
21
  -ci--copy_image When using a container directory, copy image there
22
22
  -t--tail Tail the logs
23
+ -SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
23
24
  -q--queue* Queue
24
25
  -t--task_cpus* Tasks
25
26
  -W--workflows* Additional workflows
@@ -35,7 +35,7 @@ file = case file
35
35
  fields = options[:fields]
36
36
  raise ParameterException, "Please specify the fields to slice" if fields.nil?
37
37
 
38
- options[:header_hash] = options["header_hash"]
38
+ options[:header_hash] ||= options["header_hash"]
39
39
 
40
40
  case
41
41
  when options[:tokyocabinet]
@@ -45,8 +45,8 @@ when options[:tokyocabinet_bd]
45
45
  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
46
46
  puts tsv.summary
47
47
  else
48
- stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :keys => fields, :unnamed => true) do |*p|
49
- p * "\t"
48
+ stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :fields => fields.split(","), :unnamed => true) do |k,fields,names|
49
+ [k,fields].flatten * "\t"
50
50
  end
51
51
  puts stream.read
52
52
  exit 0
@@ -86,7 +86,7 @@ messages = info[:messages]
86
86
  backtrace = info[:backtrace]
87
87
  pid = info[:pid]
88
88
  exception = info[:exception]
89
- rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid, :archived_info]
89
+ rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :archived_info]
90
90
 
91
91
 
92
92
  puts Log.color(:magenta, "File") << ": " << step.path
@@ -203,6 +203,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
203
203
  -prec--prepare_cpus* Number of dependencies prepared in parallel
204
204
  -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
205
205
  -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
206
+ -PERF--procpath_performance* Measure performance using procpath
206
207
  EOF
207
208
 
208
209
  workflow = ARGV.shift
@@ -407,6 +408,13 @@ begin
407
408
  exit 0
408
409
  end
409
410
 
411
+ if options[:procpath_performance]
412
+ require 'rbbt/util/procpath'
413
+ job.fork
414
+ job.soft_grace
415
+ pid = job.info[:pid]
416
+ ProcPath.monitor(pid, options[:procpath_performance])
417
+ end
410
418
 
411
419
  if do_fork
412
420
  ENV["RBBT_NO_PROGRESS"] = "true"
@@ -423,7 +431,6 @@ begin
423
431
  res = job
424
432
  end
425
433
 
426
-
427
434
  if options.delete(:printpath)
428
435
  job.join
429
436
  raise job.messages.last if (job.error? || job.aborted?) && job.messages
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rbbt/workflow'
4
+
5
+ require 'rbbt-util'
6
+ require 'rbbt-util'
7
+ require 'rbbt/util/simpleopt'
8
+
9
+ $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
10
+
11
+ options = SOPT.setup <<EOF
12
+ Examine the info of a job result
13
+
14
+ $ rbbt workflow info <job-result> <key> <value>
15
+
16
+ -h--help Help
17
+ -f--force Write info even if key is already present
18
+ -r--recursive Write info for all dependencies as well
19
+ -p--check_pid Check that recursive jobs where created by the same process
20
+ EOF
21
+
22
+ SOPT.usage if options[:help]
23
+
24
+ file, key, value = ARGV
25
+
26
+ force, recursive, check_pid = options.values_at :force, :recursive, :check_pid
27
+
28
+ def get_step(file)
29
+ file = file.sub(/\.(info|files)/,'')
30
+ step = Workflow.load_step file
31
+ step
32
+ end
33
+
34
+ raise ParameterException if key.nil? || value.nil?
35
+
36
+ if %w(DELETE nil).include? value
37
+ value = nil
38
+ force = true
39
+ end
40
+
41
+ step = get_step file
42
+
43
+ step.set_info key, value if force || ! step.info.include?(key)
44
+
45
+ pid = step.info[:pid]
46
+ host = step.info[:pid_hostname]
47
+
48
+ step.rec_dependencies.each do |dep|
49
+ dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
50
+ rescue
51
+ Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
52
+ end if recursive
@@ -0,0 +1,23 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt/util/procpath'
3
+
4
+ class TestProcPath < Test::Unit::TestCase
5
+ def test_record_and_plot
6
+ Log.with_severity 0 do
7
+ pid = Process.fork do
8
+ a = ""
9
+ (0..1000).each do
10
+ a << (0..rand(10000).to_i).to_a.collect{|i| "TEST #{i}" } * " "
11
+ sleep 0.1
12
+ end
13
+ end
14
+
15
+ TmpFile.with_file(nil, false) do |db|
16
+
17
+ ProcPath.record(pid, db, :interval => '1', "recnum" => 100)
18
+ ProcPath.plot(db, db + '.svg', "moving-average-window" => 1 )
19
+ end
20
+ end
21
+ end
22
+ end
23
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.29.4
4
+ version: 5.30.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-01-28 00:00:00.000000000 Z
11
+ date: 2021-02-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -300,6 +300,7 @@ files:
300
300
  - lib/rbbt/util/misc/system.rb
301
301
  - lib/rbbt/util/named_array.rb
302
302
  - lib/rbbt/util/open.rb
303
+ - lib/rbbt/util/procpath.rb
303
304
  - lib/rbbt/util/python.rb
304
305
  - lib/rbbt/util/semaphore.rb
305
306
  - lib/rbbt/util/simpleDSL.rb
@@ -438,6 +439,7 @@ files:
438
439
  - share/rbbt_commands/workflow/server
439
440
  - share/rbbt_commands/workflow/task
440
441
  - share/rbbt_commands/workflow/trace
442
+ - share/rbbt_commands/workflow/write_info
441
443
  - share/unicorn.rb
442
444
  - share/workflow_config.ru
443
445
  - test/rbbt/annotations/test_util.rb
@@ -516,6 +518,7 @@ files:
516
518
  - test/rbbt/util/test_log.rb
517
519
  - test/rbbt/util/test_misc.rb
518
520
  - test/rbbt/util/test_open.rb
521
+ - test/rbbt/util/test_procpath.rb
519
522
  - test/rbbt/util/test_python.rb
520
523
  - test/rbbt/util/test_semaphore.rb
521
524
  - test/rbbt/util/test_simpleDSL.rb
@@ -563,6 +566,7 @@ test_files:
563
566
  - test/rbbt/workflow/test_task.rb
564
567
  - test/rbbt/resource/test_path.rb
565
568
  - test/rbbt/util/test_colorize.rb
569
+ - test/rbbt/util/test_procpath.rb
566
570
  - test/rbbt/util/misc/test_omics.rb
567
571
  - test/rbbt/util/misc/test_pipes.rb
568
572
  - test/rbbt/util/misc/test_format.rb