rbbt-util 5.29.4 → 5.30.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3ec6302ccfe3f38a074f7f0d10511090c8f4db4186228ad93adb2888e0edbf5e
4
- data.tar.gz: fcaa50b654461f128b9539fc47ed00b008d3f713e89b8bbe963c2b898c3c168b
3
+ metadata.gz: b0e0e6d9165c16d73d1122a8d85e8038befd335215646bc3779400f5747b4b07
4
+ data.tar.gz: 7c0e92d49399111157fae43149c7ec8e598aa64510591fc2f74314bbefc89b2d
5
5
  SHA512:
6
- metadata.gz: 2a2537aef150df77142a593742399d28bb96334decc0e33b69e9cbac2853085487100aa90cd711b342bac7eda536d15c080be22891d5b9f38bfa4601282ae5de
7
- data.tar.gz: d79cc4afa294d63cebd79f73b759bc46c3c4e0285db7be7406a594aa49e5b572b0c4f91b98c73b5f64c31c908d58d8dee43853ec18b1ead9e814b4370de0d60d
6
+ metadata.gz: 6f1ecdb8ea4481b12138f6b63d2508f888ad60d42b56cc187ac74e3d705bb40db76d52f7e615bbd3b7fba74e49298325dcf7e5477f5527e55aa4a42e6744586d
7
+ data.tar.gz: 5b3761d1300940911383fc0659a4335b422db217a9118a0fd41ebd2374cec87884775564f003d5a0a14555eff68fcac7b70c68c3cea72c6d802908a48b8b514c
@@ -33,7 +33,8 @@ module HPC
33
33
  group = File.basename(File.dirname(ENV['HOME']))
34
34
 
35
35
  if contain_and_sync
36
- contain = "/scratch/tmp/rbbt-#{user}" if contain.nil?
36
+ random_file = TmpFile.random_name
37
+ contain = "/scratch/tmp/rbbt-#{user}/#{random_file}" if contain.nil?
37
38
  sync = "~/.rbbt/var/jobs" if sync.nil?
38
39
  wipe_container = "post" if wipe_container.nil?
39
40
  end
@@ -243,7 +244,7 @@ EOF
243
244
  end
244
245
 
245
246
  if contain
246
- rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}/.rbbt/var/jobs')
247
+ rbbt_cmd << " " << %(--workdir_all='#{contain.gsub("'", '\'')}/workdir')
247
248
  end
248
249
  end
249
250
 
@@ -251,6 +252,10 @@ EOF
251
252
  cmd =<<-EOF
252
253
  #{exec_cmd} \\
253
254
  #{rbbt_cmd}
255
+ EOF
256
+ annotate_cmd =<<-EOF
257
+ #{exec_cmd} \\
258
+ workflow write_info --recursive --force=false --check_pid "$step_path" slurm_job $SLURM_JOB_ID
254
259
  EOF
255
260
 
256
261
  header +=<<-EOF
@@ -260,11 +265,14 @@ EOF
260
265
  run +=<<-EOF
261
266
 
262
267
  # Run command
263
- #{cmd}
268
+ step_path=$(#{cmd})
264
269
 
265
270
  # Save exit status
266
271
  exit_status=$?
267
272
 
273
+ # Annotate info with SLURM job_info
274
+ #{annotate_cmd}
275
+
268
276
  EOF
269
277
 
270
278
  # CODA
@@ -286,7 +294,7 @@ EOF
286
294
  sync = sync.strip
287
295
  source = File.join(File.expand_path(contain), source)
288
296
  else
289
- source = File.join(File.expand_path(contain), '.rbbt/var/jobs')
297
+ source = File.join(File.expand_path(contain), 'workdir/var/jobs')
290
298
  end
291
299
 
292
300
  target = File.expand_path(sync)
@@ -516,6 +524,7 @@ EOF
516
524
  dry_run = options.delete :dry_run
517
525
  tail = options.delete :tail
518
526
  dependencies = options.delete :slurm_dependencies
527
+ procpath = options.delete :SLURM_procpath
519
528
  options[:jobname] = job.clean_name
520
529
 
521
530
  workflow = job.workflow
@@ -543,13 +552,15 @@ EOF
543
552
 
544
553
  if saved && saved.any?
545
554
  options[:inputs_dir] = inputs_dir
546
- cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
555
+ cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--load_inputs', inputs_dir, '--log', (options[:log] || Log.severity).to_s]
547
556
  else
548
- cmd = ['workflow', 'task', workflow.to_s, task.to_s, '-pf', '--log', (options[:log] || Log.severity).to_s]
557
+ cmd = ['workflow', 'task', workflow.to_s, task.to_s, '--printpath', '--log', (options[:log] || Log.severity).to_s]
549
558
  end
550
559
 
551
560
  cmd << "--override_deps='#{override_deps.gsub("'", '\'')}'" if override_deps and not override_deps.empty?
552
561
 
562
+ cmd << "--procpath_performance='#{tmp_directory}/procpath##{procpath.gsub(',', '#')}'" if procpath
563
+
553
564
  template = self.template(cmd, options)
554
565
  jobid = self.issue_template(template, options.merge(:slurm_basedir => slurm_basedir, :dry_run => dry_run, :slurm_dependencies => dependencies))
555
566
 
@@ -104,9 +104,6 @@ module Persist
104
104
  write(true) if closed? || ! write?
105
105
  res = begin
106
106
  yield
107
- rescue Exception
108
- Log.exception $!
109
- raise $!
110
107
  ensure
111
108
  close
112
109
  end
@@ -115,7 +112,6 @@ module Persist
115
112
  end
116
113
 
117
114
  def read_and_close
118
- #return yield if @locked
119
115
  if read? || write?
120
116
  begin
121
117
  return yield
@@ -134,6 +130,45 @@ module Persist
134
130
  end
135
131
  end
136
132
 
133
+ def read_lock
134
+ read if closed?
135
+ if read?
136
+ begin
137
+ return yield
138
+ ensure
139
+ close
140
+ end
141
+ end
142
+
143
+ lock do
144
+ close
145
+ read true
146
+ begin
147
+ yield
148
+ end
149
+ end
150
+ end
151
+
152
+ def write_lock
153
+ write if closed?
154
+ if write?
155
+ begin
156
+ return yield
157
+ ensure
158
+ close
159
+ end
160
+ end
161
+
162
+ lock do
163
+ close
164
+ write true
165
+ begin
166
+ yield
167
+ end
168
+ end
169
+ end
170
+
171
+
137
172
  def merge!(hash)
138
173
  hash.each do |key,values|
139
174
  self[key] = values
@@ -141,38 +176,38 @@ module Persist
141
176
  end
142
177
 
143
178
  def range(*args)
144
- self.read_and_close do
179
+ self.read_lock do
145
180
  super(*args)
146
181
  end
147
182
  end
148
183
 
149
184
  def include?(*args)
150
- self.read_and_close do
185
+ self.read_lock do
151
186
  super(*args) #- TSV::ENTRY_KEYS.to_a
152
187
  end
153
188
  end
154
189
 
155
190
  def [](*args)
156
- self.read_and_close do
191
+ self.read_lock do
157
192
  super(*args) #- TSV::ENTRY_KEYS.to_a
158
193
  end
159
194
  end
160
195
 
161
196
  def []=(*args)
162
- self.write_and_close do
197
+ self.write_lock do
163
198
  super(*args) #- TSV::ENTRY_KEYS.to_a
164
199
  end
165
200
  end
166
201
 
167
202
  def keys(*args)
168
- self.read_and_close do
203
+ self.read_lock do
169
204
  super(*args)
170
205
  end
171
206
  end
172
207
 
173
208
 
174
209
  def prefix(key)
175
- self.read_and_close do
210
+ self.read_lock do
176
211
  range(key, 1, key + MAX_CHAR, 1)
177
212
  end
178
213
  end
@@ -184,13 +219,13 @@ module Persist
184
219
 
185
220
 
186
221
  def size(*args)
187
- self.read_and_close do
222
+ self.read_lock do
188
223
  super(*args)
189
224
  end
190
225
  end
191
226
 
192
227
  def each(*args, &block)
193
- self.read_and_close do
228
+ self.read_lock do
194
229
  super(*args, &block)
195
230
  end
196
231
  end
@@ -208,7 +243,7 @@ module Persist
208
243
  end
209
244
 
210
245
  def values_at(*keys)
211
- self.read_and_close do
246
+ self.read_lock do
212
247
  keys.collect do |k|
213
248
  self[k]
214
249
  end
data/lib/rbbt/util/cmd.rb CHANGED
@@ -217,7 +217,7 @@ module CMD
217
217
  end
218
218
  end
219
219
 
220
- def self.cmd_log(*args)
220
+ def self.cmd_pid(*args)
221
221
  all_args = *args
222
222
 
223
223
  all_args << {} unless Hash === all_args.last
@@ -248,4 +248,9 @@ module CMD
248
248
  nil
249
249
  end
250
250
 
251
+ def self.cmd_log(*args)
252
+ cmd_pid(*args)
253
+ nil
254
+ end
255
+
251
256
  end
@@ -242,48 +242,6 @@ module Misc
242
242
 
243
243
  return options
244
244
 
245
- options = {}
246
- string.split(/#/).each do |str|
247
- if str.match(/(.*)=(.*)/)
248
- option, value = $1, $2
249
- else
250
- option, value = str, true
251
- end
252
-
253
- option = option.sub(":",'').to_sym if option.chars.first == ':'
254
- value = value.sub(":",'').to_sym if String === value and value.chars.first == ':'
255
-
256
- if value == true
257
- options[option] = option.to_s.chars.first != '!'
258
- else
259
- options[option] = Thread.start do
260
- $SAFE = 0;
261
- case
262
- when value =~ /^(?:true|T)$/i
263
- true
264
- when value =~ /^(?:false|F)$/i
265
- false
266
- when Symbol === value
267
- value
268
- when (String === value and value =~ /^\/(.*)\/$/)
269
- Regexp.new /#{$1}/
270
- else
271
- begin
272
- Kernel.const_get value
273
- rescue
274
- begin
275
- raise if value =~ /[a-z]/ and defined? value
276
- eval(value)
277
- rescue Exception
278
- value
279
- end
280
- end
281
- end
282
- end.value
283
- end
284
- end
285
-
286
- options
287
245
  end
288
246
 
289
247
  end
@@ -0,0 +1,49 @@
1
+ require 'rbbt/util/cmd'
2
+ module ProcPath
3
+ CMD.tool :procpath do
4
+ 'pip install procpath'
5
+ end
6
+
7
+ def self.record(pid, path, options = {})
8
+ IndiferentHash.setup(options)
9
+ options = Misc.add_defaults options, "interval" => 30
10
+
11
+ cmd_options = %w(interval recnum reevalnum).inject({}){|acc,k| acc[k] = options[k]; acc}
12
+
13
+ Log.debug "ProcPath recording #{pid} in #{path} (#{Misc.fingerprint options})"
14
+ procpath_thread = Thread.new do
15
+ begin
16
+ procpath_pid = CMD.cmd_pid(:procpath, "record --database-file '#{path}' '$..children[?(@.stat.pid == #{pid})]'", cmd_options.merge(:nofail => true, :add_option_dashes => true))
17
+ rescue Exception
18
+ Log.exceptions $!
19
+ Process.kill "INT", procpath_pid
20
+ end
21
+ end
22
+
23
+ procpath_thread.report_on_exception = false
24
+
25
+ Process.wait pid.to_i
26
+ procpath_thread.raise Interrupt
27
+ end
28
+
29
+ def self.plot(path, output, options = {})
30
+ IndiferentHash.setup(options)
31
+ options = Misc.add_defaults options, "query-name" => 'rss', 'epsilon' => 0.5, "moving-average-window" => 10
32
+
33
+ cmd_options = %w(query-name epsilon monitor-average-window title logarithmic after before custom-query-file custom-value-expr).inject({}){|acc,k| acc[k] = options[k]; acc}
34
+ CMD.cmd_log(:procpath, "plot --database-file '#{path}' --plot-file '#{output}' ", cmd_options.merge(:nofail => true, :add_option_dashes => true))
35
+ end
36
+
37
+ def self.monitor(pid, path)
38
+ database, options_str = path.split("#")
39
+ options = options_str.nil? ? {} : Misc.string2hash(options_str)
40
+
41
+ database = File.expand_path database
42
+ Log.low "ProcPath monitor #{pid} in #{database} (#{Misc.fingerprint options})"
43
+
44
+ ProcPath.record(pid, database + '.sqlite3', options)
45
+ ProcPath.plot(database + '.sqlite3', database + '.cpu.svg', options.merge("query-name" => 'cpu'))
46
+ ProcPath.plot(database + '.sqlite3', database + '.rss.svg', options.merge("query-name" => 'rss'))
47
+ end
48
+ end
49
+
@@ -505,8 +505,8 @@ class Step
505
505
 
506
506
  def running?
507
507
  return false if ! (started? || status == :ending)
508
- pid = info[:pid]
509
- return nil if pid.nil?
508
+ return nil unless Open.exist?(self.pid_file)
509
+ pid = Open.read(self.pid_file).to_i
510
510
 
511
511
  return false if done? or error? or aborted?
512
512
 
@@ -530,8 +530,7 @@ class Step
530
530
  end
531
531
 
532
532
  def nopid?
533
- pid = info[:pid] || Open.exists?(pid_file)
534
- ! pid && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
533
+ ! Open.exists?(pid_file) && ! (status.nil? || status == :aborted || status == :done || status == :error || status == :cleaned)
535
534
  end
536
535
 
537
536
  def aborted?
@@ -373,7 +373,6 @@ class Step
373
373
  Log.exception $!
374
374
  ensure
375
375
  Step.purge_stream_cache
376
- set_info :pid, nil
377
376
  Open.rm pid_file if Open.exist?(pid_file)
378
377
  end
379
378
  end
@@ -388,7 +387,6 @@ class Step
388
387
  _clean_finished
389
388
  rescue
390
389
  stop_dependencies
391
- set_info :pid, nil
392
390
  Open.rm pid_file if Open.exist?(pid_file)
393
391
  end
394
392
  end
@@ -449,7 +447,7 @@ class Step
449
447
  ensure
450
448
  no_load = false unless IO === result
451
449
  Open.rm pid_file if Open.exist?(pid_file) unless no_load
452
- set_info :pid, nil unless no_load
450
+ #set_info :pid, nil unless no_load
453
451
  end
454
452
  end
455
453
 
@@ -559,7 +557,7 @@ class Step
559
557
  RbbtSemaphore.post_semaphore(semaphore) if semaphore
560
558
  Kernel.exit! -1
561
559
  end
562
- set_info :pid, nil
560
+ #set_info :pid, nil
563
561
  ensure
564
562
  RbbtSemaphore.post_semaphore(semaphore) if semaphore
565
563
  end
@@ -57,7 +57,7 @@ module Task
57
57
  puts Log.color(:magenta, "Input select options")
58
58
  puts
59
59
  selects.collect{|p| p}.uniq.each do |input,options|
60
- puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| o.to_s} * ", ") << "\n"
60
+ puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
61
61
  puts unless Log.compact
62
62
  end
63
63
  puts
@@ -21,6 +21,8 @@ $ rbbt mnl [options]
21
21
  -s--search* Regular expression
22
22
  -t--tail* Show the last lines of the STDERR
23
23
  -SBP--sbatch_parameters show sbatch parameters
24
+ -PERF--procpath_performance show Procpath performance summary
25
+ -sacct--sacct_peformance show sacct performance summary
24
26
  EOF
25
27
 
26
28
  if options[:help]
@@ -140,9 +142,62 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
140
142
 
141
143
  if options[:sbatch_parameters]
142
144
  puts Log.color(:magenta, "SBATCH parameters: ")
143
- puts Log.color :blue, CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
145
+ text = CMD.cmd('grep "^#SBATCH" |tail -n +6', :in => Open.read(fcmd)).read.strip
146
+ lines = text.split("\n").collect{|line| header, _sep, value = line.partition(/\s+/); Log.color(:yellow, header + ": ") + value}
147
+ puts Log.color :yellow, lines * "\n"
144
148
  end
145
149
 
150
+ fprocpath = File.join(dir, 'procpath.sqlite3')
151
+ if options[:procpath_performance] && Open.exists?(fprocpath)
152
+ puts Log.color(:magenta, "Procpath summary: ")
153
+ require 'rbbt/tsv/csv'
154
+ meta = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from meta;' "))
155
+ perf = TSV.csv(CMD.cmd("sqlite3 -header -csv #{fprocpath} 'select * from record;' "))
156
+
157
+ page_size = meta["page_size"].first.to_f
158
+ clock_ticks = meta["clock_ticks"].first.to_f
159
+
160
+ cpu_average = {}
161
+ rss_average = {}
162
+ perf.through :key, ["ts", 'stat_pid', "stat_utime", "stat_stime", "stat_cutime", "stat_cstime", "stat_rss"] do |k, values|
163
+ time, stat_pid, ucpu, scpu, ccpu, cscpu, rss = values
164
+ time = time.to_f
165
+
166
+ cpu = Misc.sum([ucpu, scpu].collect{|v| v.to_f})
167
+ cpu_average[stat_pid] ||= {}
168
+ cpu_average[stat_pid][time] ||= []
169
+ cpu_average[stat_pid][time] << cpu.to_f
170
+ rss_average[time] ||= []
171
+ rss_average[time] << rss.to_f * page_size
172
+ end
173
+
174
+ ticks = 0
175
+ cpu_average.each do |stat_pid, cpu_average_pid|
176
+ start = cpu_average_pid.keys.sort.first
177
+ eend = cpu_average_pid.keys.sort.last
178
+ ticks += Misc.sum(cpu_average_pid[eend]) - Misc.sum(cpu_average_pid[start])
179
+ end
180
+ start = rss_average.keys.sort.first
181
+ eend = rss_average.keys.sort.last
182
+ time_elapsed = eend - start
183
+ puts Log.color(:yellow, "CPU average: ") + "%.2f" % ( ticks / clock_ticks / time_elapsed * 100).to_s
184
+ puts Log.color(:yellow, "RSS average: ") + "%.2f GB" % Misc.mean(rss_average.collect{|t,l| Misc.sum(l) / (1024 * 1024 * 1024)}).to_s
185
+
186
+ end
187
+
188
+ if options[:sacct_peformance]
189
+ begin
190
+ tsv = TSV.open(CMD.cmd("sacct -j #{id} -o 'jobid,AveRSS,MaxRSS,MaxDiskRead,MaxDiskWrite' -P|grep 'JobID\\|\.batch'"), :header_hash => '', :sep => "|", :type => :list)
191
+ values = tsv[tsv.keys.first]
192
+ if values.compact.any?
193
+ puts Log.color(:magenta, "SACCT performance: ")
194
+ puts values.zip(values.fields).collect{|v,t| Log.color(:yellow, t + ": ") + v.to_s } * "\n"
195
+ end
196
+ rescue
197
+ end
198
+ end
199
+
200
+
146
201
  if tail && File.exists?(File.join(dir, 'std.err'))
147
202
  if exit_status && exit_status != 0
148
203
  puts Log.color(:magenta, "First error or exception found: ")
@@ -21,6 +21,7 @@ $slurm_options = SOPT.get <<EOF
21
21
  -CS--contain_and_sync Contain and sync to default locations
22
22
  -ci--copy_image When using a container directory, copy image there
23
23
  -t--tail Tail the logs
24
+ -SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
24
25
  -q--queue* Queue
25
26
  -t--task_cpus* Tasks
26
27
  -W--workflows* Additional workflows
@@ -20,6 +20,7 @@ $slurm_options = SOPT.get <<EOF
20
20
  -CS--contain_and_sync Contain and sync to default locations
21
21
  -ci--copy_image When using a container directory, copy image there
22
22
  -t--tail Tail the logs
23
+ -SPERF--SLURM_procpath* Save Procpath performance for SLURM job; specify only options
23
24
  -q--queue* Queue
24
25
  -t--task_cpus* Tasks
25
26
  -W--workflows* Additional workflows
@@ -35,7 +35,7 @@ file = case file
35
35
  fields = options[:fields]
36
36
  raise ParameterException, "Please specify the fields to slice" if fields.nil?
37
37
 
38
- options[:header_hash] = options["header_hash"]
38
+ options[:header_hash] ||= options["header_hash"]
39
39
 
40
40
  case
41
41
  when options[:tokyocabinet]
@@ -45,8 +45,8 @@ when options[:tokyocabinet_bd]
45
45
  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
46
46
  puts tsv.summary
47
47
  else
48
- stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :keys => fields, :unnamed => true) do |*p|
49
- p * "\t"
48
+ stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :fields => fields.split(","), :unnamed => true) do |k,fields,names|
49
+ [k,fields].flatten * "\t"
50
50
  end
51
51
  puts stream.read
52
52
  exit 0
@@ -86,7 +86,7 @@ messages = info[:messages]
86
86
  backtrace = info[:backtrace]
87
87
  pid = info[:pid]
88
88
  exception = info[:exception]
89
- rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :pid, :archived_info]
89
+ rest = info.keys - [:inputs, :dependencies, :status, :time_elapsed, :messages, :backtrace, :exception, :archived_info]
90
90
 
91
91
 
92
92
  puts Log.color(:magenta, "File") << ": " << step.path
@@ -203,6 +203,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
203
203
  -prec--prepare_cpus* Number of dependencies prepared in parallel
204
204
  -rwt--remote_workflow_tasks* Load a yaml file describing remote workflow tasks
205
205
  -od--override_deps* Override deps using 'Workflow#task=<path>' array_separated
206
+ -PERF--procpath_performance* Measure performance using procpath
206
207
  EOF
207
208
 
208
209
  workflow = ARGV.shift
@@ -407,6 +408,13 @@ begin
407
408
  exit 0
408
409
  end
409
410
 
411
+ if options[:procpath_performance]
412
+ require 'rbbt/util/procpath'
413
+ job.fork
414
+ job.soft_grace
415
+ pid = job.info[:pid]
416
+ ProcPath.monitor(pid, options[:procpath_performance])
417
+ end
410
418
 
411
419
  if do_fork
412
420
  ENV["RBBT_NO_PROGRESS"] = "true"
@@ -423,7 +431,6 @@ begin
423
431
  res = job
424
432
  end
425
433
 
426
-
427
434
  if options.delete(:printpath)
428
435
  job.join
429
436
  raise job.messages.last if (job.error? || job.aborted?) && job.messages
@@ -0,0 +1,52 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'rbbt/workflow'
4
+
5
+ require 'rbbt-util'
6
+ require 'rbbt-util'
7
+ require 'rbbt/util/simpleopt'
8
+
9
+ $0 = "rbbt #{$previous_commands*""} #{ File.basename(__FILE__) }" if $previous_commands
10
+
11
+ options = SOPT.setup <<EOF
12
+ Examine the info of a job result
13
+
14
+ $ rbbt workflow info <job-result> <key> <value>
15
+
16
+ -h--help Help
17
+ -f--force Write info even if key is already present
18
+ -r--recursive Write info for all dependencies as well
19
+ -p--check_pid Check that recursive jobs where created by the same process
20
+ EOF
21
+
22
+ SOPT.usage if options[:help]
23
+
24
+ file, key, value = ARGV
25
+
26
+ force, recursive, check_pid = options.values_at :force, :recursive, :check_pid
27
+
28
+ def get_step(file)
29
+ file = file.sub(/\.(info|files)/,'')
30
+ step = Workflow.load_step file
31
+ step
32
+ end
33
+
34
+ raise ParameterException if key.nil? || value.nil?
35
+
36
+ if %w(DELETE nil).include? value
37
+ value = nil
38
+ force = true
39
+ end
40
+
41
+ step = get_step file
42
+
43
+ step.set_info key, value if force || ! step.info.include?(key)
44
+
45
+ pid = step.info[:pid]
46
+ host = step.info[:pid_hostname]
47
+
48
+ step.rec_dependencies.each do |dep|
49
+ dep.set_info key, value if (force || ! dep.info.include?(key)) && (!check_pid || dep.info[:pid].to_s == pid and dep.info[:pid_hostname] == host)
50
+ rescue
51
+ Log.warn "Could no set info #{key} for #{dep.path}: #{$!.message}"
52
+ end if recursive
@@ -0,0 +1,23 @@
1
+ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helper.rb')
2
+ require 'rbbt/util/procpath'
3
+
4
+ class TestProcPath < Test::Unit::TestCase
5
+ def test_record_and_plot
6
+ Log.with_severity 0 do
7
+ pid = Process.fork do
8
+ a = ""
9
+ (0..1000).each do
10
+ a << (0..rand(10000).to_i).to_a.collect{|i| "TEST #{i}" } * " "
11
+ sleep 0.1
12
+ end
13
+ end
14
+
15
+ TmpFile.with_file(nil, false) do |db|
16
+
17
+ ProcPath.record(pid, db, :interval => '1', "recnum" => 100)
18
+ ProcPath.plot(db, db + '.svg', "moving-average-window" => 1 )
19
+ end
20
+ end
21
+ end
22
+ end
23
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.29.4
4
+ version: 5.30.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-01-28 00:00:00.000000000 Z
11
+ date: 2021-02-01 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake
@@ -300,6 +300,7 @@ files:
300
300
  - lib/rbbt/util/misc/system.rb
301
301
  - lib/rbbt/util/named_array.rb
302
302
  - lib/rbbt/util/open.rb
303
+ - lib/rbbt/util/procpath.rb
303
304
  - lib/rbbt/util/python.rb
304
305
  - lib/rbbt/util/semaphore.rb
305
306
  - lib/rbbt/util/simpleDSL.rb
@@ -438,6 +439,7 @@ files:
438
439
  - share/rbbt_commands/workflow/server
439
440
  - share/rbbt_commands/workflow/task
440
441
  - share/rbbt_commands/workflow/trace
442
+ - share/rbbt_commands/workflow/write_info
441
443
  - share/unicorn.rb
442
444
  - share/workflow_config.ru
443
445
  - test/rbbt/annotations/test_util.rb
@@ -516,6 +518,7 @@ files:
516
518
  - test/rbbt/util/test_log.rb
517
519
  - test/rbbt/util/test_misc.rb
518
520
  - test/rbbt/util/test_open.rb
521
+ - test/rbbt/util/test_procpath.rb
519
522
  - test/rbbt/util/test_python.rb
520
523
  - test/rbbt/util/test_semaphore.rb
521
524
  - test/rbbt/util/test_simpleDSL.rb
@@ -563,6 +566,7 @@ test_files:
563
566
  - test/rbbt/workflow/test_task.rb
564
567
  - test/rbbt/resource/test_path.rb
565
568
  - test/rbbt/util/test_colorize.rb
569
+ - test/rbbt/util/test_procpath.rb
566
570
  - test/rbbt/util/misc/test_omics.rb
567
571
  - test/rbbt/util/misc/test_pipes.rb
568
572
  - test/rbbt/util/misc/test_format.rb