rbbt-util 5.30.7 → 5.30.12

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 3275cafa7bc74ca88f5d487287ff736113dad5ee6e607bb13f283bb4aa5e0562
4
- data.tar.gz: 1a7642c60ce3f3828790599c2522ad1e083bd877c53de10401b93735e952934d
3
+ metadata.gz: 6d407fc5c2e576419062f2f72e52a4c67d8c740447b54eb7838ecaf769ee1c33
4
+ data.tar.gz: a9f11e1db995f7ea7179964316b6c7347e01fcd7323b6237e8762320455ca8ce
5
5
  SHA512:
6
- metadata.gz: bc9717f6611646c099305587bdcf224243102a500ba68047e382df49e583a3c764489e09b7c9aa4993eab3953d72dc9ce206d5450213e1ae832b1622c96fd309
7
- data.tar.gz: 791cdba47b74e1f1fd47c439bb3f2b025e540002ec2b1e18d846bab08c6fd858cc25a94f9c84056383c5125cca22c19f00d43ae0c53a8c77ef4de423c19f0d8c
6
+ metadata.gz: 46718e67f59ab2b9ae5e3fd1d6fb0e7deffe5b1f5daf95669d7cc11022c846e829640cee45033aaced7409c291eacc29bd04253ff5f978e98e4860d777046188
7
+ data.tar.gz: b214c317054c28c81d742a0eb5af3c3a1bddcc3fb48223ba7d2ea7f3ab95e155e642d4100d5353bc04cbfc77bfcdd215df57c37be0a3fc0de0fb6757e5c3a269
@@ -67,6 +67,7 @@ module HPC
67
67
  def self.piggyback(job, job_rules, job_deps)
68
68
  return false unless job_rules["skip"]
69
69
  final_deps = job_deps - job_deps.collect{|dep| get_recursive_job_dependencies(dep)}.flatten.uniq
70
+ final_deps = final_deps.reject{|dep| dep.done? }
70
71
  return final_deps.first if final_deps.length == 1
71
72
  return false
72
73
  end
@@ -104,19 +105,19 @@ module HPC
104
105
  def self.workload(job, rules, chains, options, seen = nil)
105
106
  return [] if job.done?
106
107
  if seen.nil?
107
- seen = {}
108
+ seen = {}
108
109
  target_job = true
109
110
  end
110
111
 
111
112
  job_rules = self.job_rules(rules, job)
112
113
  job_deps = get_job_dependencies(job)
113
114
 
114
-
115
115
  chain = chains[job]
116
- chain -= seen.keys if chain
116
+ chain = chain.reject{|j| seen.include? j.path} if chain
117
+ chain = chain.reject{|dep| dep.done? } if chain
117
118
  piggyback = piggyback(job, job_rules, job_deps)
118
119
  dep_ids = job_deps.collect do |dep|
119
- seen[dep] = nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep)
120
+ seen[dep.path] ||= nil if chain && chain.include?(dep) #&& ! job.input_dependencies.include?(dep)
120
121
  next_options = IndiferentHash.setup(options.dup)
121
122
  if piggyback and piggyback == dep
122
123
  next_options[:piggyback] ||= []
@@ -129,32 +130,44 @@ module HPC
129
130
 
130
131
  ids = [ids].flatten.compact.collect{|id| ['canfail', id] * ":"} if job.canfail_paths.include? dep.path
131
132
 
132
- seen[dep] = ids
133
+ seen[dep.path] = ids
133
134
  ids
134
135
  end.compact.flatten.uniq
135
136
 
136
- return seen[job] || dep_ids if seen.include?(job)
137
- return seen[piggyback] if piggyback
137
+ return seen[job.path] || dep_ids if seen.include?(job.path)
138
+
139
+ if piggyback and seen[piggyback.path]
140
+ return seen[job.path] = seen[piggyback.path]
141
+ end
138
142
 
139
143
  job_rules.delete :chain_tasks
140
144
  job_rules.delete :tasks
141
145
  job_rules.delete :workflow
142
146
 
143
- config_keys = job_rules.delete(:config_keys)
144
147
 
145
148
  job_options = IndiferentHash.setup(options.merge(job_rules).merge(:slurm_dependencies => dep_ids))
146
149
  job_options.delete :orchestration_rules
150
+
151
+ config_keys = job_rules.delete(:config_keys)
147
152
  if config_keys
148
153
  config_keys.gsub!(/,\s+/,',')
149
154
  job_options[:config_keys] = job_options[:config_keys] ? config_keys + "," + job_options[:config_keys] : config_keys
150
155
  end
151
156
 
152
-
153
157
  if options[:piggyback]
154
158
  manifest = options[:piggyback].uniq
155
159
  manifest += [job]
156
160
  manifest.concat chain if chain
161
+
157
162
  job = options[:piggyback].first
163
+
164
+ job_rules = self.job_rules(rules, job)
165
+ new_config_keys = self.job_rules(rules, job)[:config_keys]
166
+ if new_config_keys
167
+ new_config_keys = new_config_keys.gsub(/,\s+/,',')
168
+ job_options[:config_keys] = job_options[:config_keys] ? new_config_keys + "," + job_options[:config_keys] : new_config_keys
169
+ end
170
+
158
171
  job_options.delete :piggyback
159
172
  else
160
173
  manifest = [job]
@@ -163,7 +176,9 @@ module HPC
163
176
 
164
177
  manifest.uniq!
165
178
 
166
- job_options[:manifest] = manifest.collect{|j| j.workflow_short_path }
179
+ job_options[:manifest] = manifest.collect{|j| j.task_signature }
180
+
181
+ job_options[:config_keys] = job_options[:config_keys].split(",").uniq * "," if job_options[:config_keys]
167
182
 
168
183
  if options[:dry_run]
169
184
  puts Log.color(:magenta, "Manifest: ") + Log.color(:blue, job_options[:manifest] * ", ") + " - tasks: #{job_options[:task_cpus] || 1} - time: #{job_options[:time]} - config: #{job_options[:config_keys]}"
@@ -21,6 +21,8 @@ module HPC
21
21
  exclusive = options.delete :exclusive
22
22
  highmem = options.delete :highmem
23
23
 
24
+ slurm_step_path = options.delete :slurm_step_path
25
+
24
26
  manifest = options.delete :manifest
25
27
 
26
28
  queue = options.delete(:queue) || Rbbt::Config.get('queue', :slurm_queue, :slurm, :SLURM, :default => 'bsc_ls')
@@ -264,6 +266,10 @@ EOF
264
266
  #MANIFEST: #{manifest * ", "}
265
267
  EOF
266
268
 
269
+ header +=<<-EOF if slurm_step_path
270
+ #STEP_PATH: #{slurm_step_path}
271
+ EOF
272
+
267
273
  header +=<<-EOF
268
274
  #CMD: #{rbbt_cmd}
269
275
  EOF
@@ -536,7 +542,9 @@ EOF
536
542
  dependencies = options.delete :slurm_dependencies
537
543
  procpath = options.delete :SLURM_procpath
538
544
 
539
- options[:jobname] = job.clean_name
545
+ options[:jobname] = job.clean_name
546
+ options[:slurm_step_path] = job.path
547
+
540
548
  log_level = options.delete :log
541
549
  log_level ||= Log.severity
542
550
 
data/lib/rbbt/util/cmd.rb CHANGED
@@ -101,6 +101,7 @@ module CMD
101
101
  no_fail = options.delete(:nofail) if no_fail.nil?
102
102
  no_wait = options.delete(:no_wait)
103
103
  xvfb = options.delete(:xvfb)
104
+ bar = options.delete(:progress_bar)
104
105
 
105
106
  dont_close_in = options.delete(:dont_close_in)
106
107
 
@@ -183,6 +184,7 @@ module CMD
183
184
 
184
185
  err_thread = Thread.new do
185
186
  while line = serr.gets
187
+ bar.process(line) if bar
186
188
  sout.log = line
187
189
  Log.log "STDERR [#{pid}]: " + line, stderr
188
190
  end if Integer === stderr and log
@@ -220,6 +222,8 @@ module CMD
220
222
  def self.cmd_pid(*args)
221
223
  all_args = *args
222
224
 
225
+ bar = all_args.last[:progress_bar] if Hash === all_args.last
226
+
223
227
  all_args << {} unless Hash === all_args.last
224
228
 
225
229
  level = all_args.last[:log] || 0
@@ -233,17 +237,27 @@ module CMD
233
237
  io = cmd(*all_args)
234
238
  pid = io.pids.first
235
239
 
240
+ line = "" if bar
236
241
  while c = io.getc
237
242
  STDERR << c if Log.severity <= level
243
+ line << c if bar
238
244
  if c == "\n"
245
+ bar.process(line) if bar
239
246
  if pid
240
247
  Log.logn "STDOUT [#{pid}]: ", level
241
248
  else
242
249
  Log.logn "STDOUT: ", level
243
250
  end
251
+ line = "" if bar
244
252
  end
245
253
  end
246
- io.join
254
+ begin
255
+ io.join
256
+ bar.remove if bar
257
+ rescue
258
+ bar.remove(true) if bar
259
+ raise $!
260
+ end
247
261
 
248
262
  nil
249
263
  end
@@ -90,6 +90,8 @@ module Rbbt::Config
90
90
  options = tokens.pop if Hash === tokens.last
91
91
  default = options.nil? ? nil : options[:default]
92
92
 
93
+ tokens = ["key:" + key] if tokens.empty?
94
+
93
95
  tokens = tokens.flatten
94
96
  file, _sep, line = caller.reject{|l|
95
97
  l =~ /rbbt\/(?:resource\.rb|workflow\.rb)/ or
@@ -106,7 +108,6 @@ module Rbbt::Config
106
108
 
107
109
  entries = CACHE[key.to_s]
108
110
  priorities = {}
109
- tokens = tokens + ["key:" << key.to_s]
110
111
  tokens.each do |token|
111
112
  token_prio = match entries, token.to_s
112
113
  token_prio.each do |prio, values|
@@ -145,7 +146,6 @@ module Rbbt::Config
145
146
  Rbbt::Config.load_file(Rbbt.etc.config_profile[config].find)
146
147
  else
147
148
  key, value, *tokens = config.split(/\s/)
148
- tokens = ['key:' << key << '::0'] if tokens.empty?
149
149
  tokens = tokens.collect do |tok|
150
150
  tok, _sep, prio = tok.partition("::")
151
151
  prio = "0" if prio.nil? or prio.empty?
@@ -18,11 +18,11 @@ module Log
18
18
  attr_accessor :default_file
19
19
  end
20
20
 
21
- attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes
21
+ attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes, :process, :callback
22
22
 
23
23
  def initialize(max = nil, options = {})
24
24
  options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :io => STDERR, :severity => Log.severity, :frequency => 2
25
- depth, num_reports, desc, io, severity, file, bytes, frequency = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency
25
+ depth, num_reports, desc, io, severity, file, bytes, frequency, process, callback = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency, :process, :callback
26
26
 
27
27
  @max = max
28
28
  @ticks = 0
@@ -34,6 +34,8 @@ module Log
34
34
  @desc = desc.nil? ? "" : desc.gsub(/\n/,' ')
35
35
  @file = file
36
36
  @bytes = bytes
37
+ @process = process
38
+ @callback = callback
37
39
  end
38
40
 
39
41
  def percent
@@ -80,5 +82,18 @@ module Log
80
82
  step = pos - (@ticks || 0)
81
83
  tick(step)
82
84
  end
85
+
86
+ def process(elem)
87
+ case res = @process.call(elem)
88
+ when FalseClass
89
+ nil
90
+ when TrueClass
91
+ tick
92
+ when Integer
93
+ pos(res)
94
+ when Float
95
+ pos(res * max)
96
+ end
97
+ end
83
98
  end
84
99
  end
@@ -124,6 +124,29 @@ module Log
124
124
  str
125
125
  end
126
126
 
127
+ def load(info)
128
+ info.each do |key, value|
129
+ case key.to_sym
130
+ when :start
131
+ @start = value
132
+ when :last_time
133
+ @last_time = value
134
+ when :last_count
135
+ @last_count = value
136
+ when :last_percent
137
+ @last_percent = value
138
+ when :desc
139
+ @desc = value
140
+ when :ticks
141
+ @ticks = value
142
+ when :max
143
+ @max = value
144
+ when :mean
145
+ @mean = value
146
+ end
147
+ end
148
+ end
149
+
127
150
  def save
128
151
  info = {:start => @start, :last_time => @last_time, :last_count => @last_count, :last_percent => @last_percent, :desc => @desc, :ticks => @ticks, :max => @max, :mean => @mean}
129
152
  info.delete_if{|k,v| v.nil?}
@@ -154,7 +177,7 @@ module Log
154
177
  bars << self unless BARS.include? self
155
178
 
156
179
  print(io, Log.up_lines(bars.length) << Log.color(:magenta, "···Progress\n") << Log.down_lines(bars.length+1)) if Log::ProgressBar.offset == 0
157
- print(io, Log.up_lines(@depth) << report_msg << Log.down_lines(@depth))
180
+ print(io, Log.up_lines(@depth) << report_msg << "\n" << Log.down_lines(@depth - 1))
158
181
  @last_time = Time.now
159
182
  @last_count = ticks
160
183
  @last_percent = percent if max and max > 0
@@ -175,7 +198,10 @@ module Log
175
198
  done_msg << " - " << thr_msg
176
199
  done_msg << Log.color(:magenta, " · " << desc)
177
200
  print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth))
178
- Open.rm file if file and Open.exists? file
201
+
202
+ Open.rm file if file and Open.exists?(file)
203
+
204
+ @callback.call self if @callback
179
205
  end
180
206
 
181
207
  def error(io = STDERR)
@@ -192,7 +218,14 @@ module Log
192
218
  done_msg << " - " << thr_msg
193
219
  done_msg << Log.color(:magenta, " · " << desc)
194
220
  print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth))
195
- Open.rm file if file and Open.exists? file
221
+
222
+ Open.rm file if file and Open.exists?(file)
223
+
224
+ begin
225
+ @callback.call self
226
+ rescue
227
+ Log.debug "Callback failed for filed progress bar: #{$!.message}"
228
+ end if @callback
196
229
  end
197
230
  end
198
231
  end
@@ -312,11 +312,23 @@ module Misc
312
312
  end
313
313
  end
314
314
 
315
+ def self.sort_genomic_locations_by_contig(stream, contigs, sep = ":")
316
+ ext_stream = TSV.traverse stream, :type => :array, :into => :stream do |line|
317
+ chr = line.partition(sep).first
318
+ num = contigs.index chr
319
+ num.to_s + sep + line
320
+ end
321
+
322
+ TSV.traverse sort_stream(ext_stream, '#', "-k1,1n -k3,3n -t#{sep}"), :type => :array, :into => :stream do |line|
323
+ line.partition(sep).last
324
+ end
325
+ end
326
+
315
327
  def self.sort_genomic_locations_strict(stream, sep = ":")
316
328
  sort_stream(stream, '#', "-k1,1V -k2,2n -t#{sep}")
317
329
  end
318
330
 
319
- def self.sort_genomic_locations(stream)
331
+ def self.sort_genomic_locations(stream, sep = ":")
320
332
  sort_stream(stream, '#', "-k1,1 -k2,2n -t#{sep}")
321
333
  end
322
334
 
@@ -386,6 +398,7 @@ module Misc
386
398
  cmp
387
399
  end
388
400
  end
401
+
389
402
  def self.intersect_streams_cmp_chr(chr1, chr2)
390
403
  chr1 <=> chr2
391
404
  end
@@ -492,4 +505,50 @@ module Misc
492
505
  end
493
506
  end
494
507
  end
508
+
509
+ def self.genomic_mutations_to_BED(mutations, chr_prefix = false, sort_order = :normal)
510
+ io = if Array === sort_order
511
+
512
+ case chr_prefix.to_s.downcase
513
+ when "remove"
514
+ sort_order = sort_order.collect{|chr| "chr" + chr } unless sort_order.first.include?('chr')
515
+ when "true", "add"
516
+ sort_order = sort_order.collect{|chr| chr.sub('chr', '') } if sort_order.first.include?('chr')
517
+ end
518
+
519
+ sort_genomic_locations_by_contig(mutations, sort_order)
520
+
521
+ else
522
+
523
+ case sort_order.to_s
524
+ when 'strict'
525
+ sort_genomic_locations_strict(mutations)
526
+ else
527
+ sort_genomic_locations(mutations)
528
+ end
529
+
530
+ end
531
+
532
+ TSV.traverse io, :type => :array, :into => :stream do |mutation|
533
+ chr, pos, mut, *rest = mutation.split(":")
534
+ size = case mut
535
+ when nil
536
+ 1
537
+ when /^\+(.*)/
538
+ 1 + $1.length
539
+ when /^\-(.*)/
540
+ $1.length
541
+ else
542
+ mut.length
543
+ end
544
+
545
+ case chr_prefix.to_s.downcase
546
+ when "true", "add"
547
+ chr = "chr" + chr if ! chr.include?('chr')
548
+ when "remove"
549
+ chr = chr.sub("chr", '') if chr.include?('chr')
550
+ end
551
+ [chr, pos.to_i - 1, pos.to_i - 1 + size, mutation] * "\t"
552
+ end
553
+ end
495
554
  end
@@ -250,6 +250,8 @@ module Workflow
250
250
  end
251
251
 
252
252
  def assign_dep_inputs(_inputs, options, all_d, task_info)
253
+ IndiferentHash.setup(_inputs)
254
+
253
255
  options.each{|i,v|
254
256
  next if i == :compute or i == "compute"
255
257
  case v
@@ -259,13 +261,16 @@ module Workflow
259
261
  rec_dependency = all_d.flatten.select{|d| d.task_name.to_sym == v }.first
260
262
 
261
263
  if rec_dependency.nil?
262
- if _inputs.include? v
263
- _inputs[i] = _inputs.delete(v)
264
+ if _inputs.include?(v)
265
+ #_inputs[i] = _inputs.delete(v)
266
+ _inputs[i] = _inputs[v] unless _inputs.include? i #_inputs.delete(v)
264
267
  else
265
268
  _inputs[i] = v unless _inputs.include? i
266
269
  end
267
270
  else
268
271
  input_options = task_info[:input_options][i] || {}
272
+
273
+ #ToDo why was this always true?
269
274
  if input_options[:stream] or true
270
275
  #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
271
276
  _inputs[i] = rec_dependency
@@ -79,9 +79,10 @@ module Workflow
79
79
  dep = dependencies.last.join
80
80
  raise dep.get_exception if dep.error?
81
81
  set_info :result_type, dep.info[:result_type]
82
- forget = config :forget_dep_tasks, :forget_dep_tasks, :default => FORGET_DEP_TASKS
82
+ forget = config :forget_dep_tasks, "forget_dep_tasks", "key:forget_dep_tasks", :default => FORGET_DEP_TASKS
83
83
  if forget
84
- remove = config :remove_dep_tasks, :remove_dep_tasks, :default => REMOVE_DEP_TASKS
84
+ remove = config :remove_dep_tasks, "remove_dep_tasks", "key:remove_dep_tasks", :default => REMOVE_DEP_TASKS
85
+
85
86
  self.archive_deps
86
87
  self.copy_files_dir
87
88
  self.dependencies = self.dependencies - [dep]
@@ -92,7 +93,10 @@ module Workflow
92
93
  when 'true'
93
94
  dep.clean
94
95
  when 'recursive'
95
- dep.recursive_clean
96
+ dep.rec_dependencies.each do |d|
97
+ d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
98
+ end
99
+ dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
96
100
  end
97
101
  else
98
102
  if Open.exists?(dep.files_dir)
@@ -623,6 +623,15 @@ class Step
623
623
  Log.warn "Exception removing result of aborted job: #{$!.message}"
624
624
  end
625
625
  end
626
+
627
+ if Open.exists?(tmp_path) && status != :done
628
+ Log.warn "Aborted job had finished. Removing tmp result -- #{ tmp_path }"
629
+ begin
630
+ Open.rm tmp_path
631
+ rescue Exception
632
+ Log.warn "Exception removing tmp result of aborted job: #{$!.message}"
633
+ end
634
+ end
626
635
  end
627
636
 
628
637
  def _abort
@@ -51,7 +51,7 @@ class Step
51
51
  end
52
52
  end
53
53
 
54
- def self.job_files_for_archive(files, recursive = false)
54
+ def self.job_files_for_archive(files, recursive = false, skip_overriden = false)
55
55
  job_files = Set.new
56
56
 
57
57
  jobs = files.collect do |file|
@@ -65,6 +65,8 @@ class Step
65
65
 
66
66
  jobs.each do |step|
67
67
  next unless File.exists?(step.path)
68
+ next if skip_overriden && step.overriden
69
+
68
70
  job_files << step.path
69
71
  job_files << step.info_file if File.exists?(step.info_file)
70
72
  job_files << Step.md5_file(step.path) if File.exists?(Step.md5_file step.path)
@@ -258,9 +260,9 @@ puts resource[path].find(search_path)
258
260
  end
259
261
  end
260
262
 
261
- def self.purge(path, recursive = false)
263
+ def self.purge(path, recursive = false, skip_overriden = true)
262
264
  path = [path] if String === path
263
- job_files = job_files_for_archive path, recursive
265
+ job_files = job_files_for_archive path, recursive, skip_overriden
264
266
 
265
267
  job_files.each do |file|
266
268
  begin
@@ -1,23 +1,28 @@
1
1
  class Step
2
+
3
+ def self.status_color(status)
4
+ case status.to_sym
5
+ when :error, :aborted, :missing, :dead, :unsync
6
+ :red
7
+ when :streaming, :started
8
+ :cyan
9
+ when :done, :noinfo
10
+ :green
11
+ when :dependencies, :waiting, :setup
12
+ :yellow
13
+ when :notfound, :cleaned
14
+ :blue
15
+ else
16
+ if status.to_s.index ">"
17
+ :cyan
18
+ else
19
+ :cyan
20
+ end
21
+ end
22
+ end
23
+
2
24
  def self.prov_status_msg(status)
3
- color = case status.to_sym
4
- when :error, :aborted, :missing, :dead, :unsync
5
- :red
6
- when :streaming, :started
7
- :cyan
8
- when :done, :noinfo
9
- :green
10
- when :dependencies, :waiting, :setup
11
- :yellow
12
- when :notfound, :cleaned
13
- :blue
14
- else
15
- if status.to_s.index ">"
16
- :cyan
17
- else
18
- :cyan
19
- end
20
- end
25
+ color = status_color(status)
21
26
  Log.color(color, status.to_s)
22
27
  end
23
28
 
@@ -25,7 +30,7 @@ class Step
25
30
  parts = path.sub(/\{.*/,'').split "/"
26
31
 
27
32
  parts.pop
28
-
33
+
29
34
  task = Log.color(:yellow, parts.pop)
30
35
  workflow = Log.color(:magenta, parts.pop)
31
36
  #if status.to_s == 'noinfo' && parts.last != 'jobs'
@@ -89,7 +94,7 @@ class Step
89
94
  str << prov_report(dep, offset + 1, task, seen, expand_repeats)
90
95
  else
91
96
  if expand_repeats
92
- str << Log.color(:green, Log.uncolor(prov_report(dep, offset+1, task)))
97
+ str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
93
98
  else
94
99
  info = dep.info || {}
95
100
  status = info[:status] || :missing
@@ -98,7 +103,7 @@ class Step
98
103
  status = :unsync if status == :done and not Open.exist?(path)
99
104
  status = :notfound if status == :noinfo and not Open.exist?(path)
100
105
 
101
- str << Log.color(status == :notfound ? :blue : :green, " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
106
+ str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info)))
102
107
  end
103
108
  end
104
109
  end if step.dependencies
@@ -20,6 +20,7 @@ $ rbbt mnl [options]
20
20
  -j--job* Job ids
21
21
  -s--search* Regular expression
22
22
  -t--tail* Show the last lines of the STDERR
23
+ -p--progress Report progress of job and the dependencies
23
24
  -SBP--sbatch_parameters show sbatch parameters
24
25
  -PERF--procpath_performance show Procpath performance summary
25
26
  -sacct--sacct_peformance show sacct performance summary
@@ -34,8 +35,8 @@ if options[:help]
34
35
  exit 0
35
36
  end
36
37
 
37
- Log.severity = 4
38
- done, error, running, queued, aborted, jobid, search, tail = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail
38
+ #Log.severity = 4
39
+ done, error, running, queued, aborted, jobid, search, tail, progress = options.values_at :done, :error, :running, :queued, :aborted, :job, :search, :tail, :progress
39
40
 
40
41
  workdir = File.expand_path('~/rbbt-slurm')
41
42
  Path.setup(workdir)
@@ -234,7 +235,24 @@ workdir.glob("**/command.slurm").sort_by{|f| File.mtime(f)}.each do |fcmd|
234
235
  puts CMD.cmd("grep -i -w 'Completed step' #{File.join(dir, 'std.err')} | grep -v 'Retrying dep.' | tail -n #{tail.to_i}", :no_fail => true).read
235
236
  else
236
237
  puts Log.color(:magenta, "Log tail: ")
237
- puts CMD.cmd("tail -n #{tail.to_i} #{File.join(dir, 'std.err')}").read
238
+ puts CMD.cmd(" cat #{File.join(dir, 'std.err')} | grep -v '^[^\\s:]*\\[3.m' | tail -n #{tail.to_i} ").read
239
+ end
240
+ end
241
+
242
+ if options[:progress]
243
+ step_line = Open.read(fcmd).split("\n").select{|line| line =~ /^#STEP_PATH:/}.first
244
+ if step_line
245
+ require 'rbbt/workflow'
246
+ step_path = step_line.split(": ").last.strip
247
+ step = Step.new step_path
248
+ step.load_dependencies_from_info
249
+ (step.rec_dependencies + [step]).reverse.each do |j|
250
+ next if j.done? || ! j.running?
251
+ next unless j.file(:progress).exists?
252
+ bar = Log::ProgressBar.new
253
+ bar.load(j.file(:progress).yaml)
254
+ puts Log.color(:magenta, "Progress: ") + bar.report_msg + " " + Log.color(:yellow, j.task_signature)
255
+ end
238
256
  end
239
257
  end
240
258
 
@@ -16,6 +16,8 @@ $ rbbt workflow server [options] <Workflow>
16
16
  -B--Bind* Bind IP
17
17
  -p--port* TCP port
18
18
  -s--server* Server type: thin, webrick, unicorn, etc
19
+ -w--workers* Number of workers for cluster mode (puma)
20
+ -so--server_options* Additional options for server (e.g. option1=value1;option2=value2)
19
21
  -f--finder Start server with finder functionality
20
22
  -RS--Rserve_session* Rserve session to use, otherwise start new one
21
23
  -wd--workdir* Change the working directory of the workflow
@@ -32,7 +34,6 @@ $ rbbt workflow server [options] <Workflow>
32
34
  --export_synchronous* Export workflow tasks as synchronous
33
35
  --export_exec* Export workflow tasks as exec
34
36
  --export_stream* Export workflow tasks as stream
35
- --options* Additional options for server (e.g. option1=value1;option2=value2)
36
37
  EOF
37
38
 
38
39
  if options[:help]
@@ -108,20 +109,24 @@ TmpFile.with_file do |app_dir|
108
109
  config_ru_file = File.exist?('./workflow_config.ru') ? './workflow_config.ru' : Rbbt.share['workflow_config.ru'].find
109
110
 
110
111
 
111
- if options[:options]
112
- options[:options].split(";").each do |pair|
112
+ if options[:server_options]
113
+ options[:server_options].split(";").each do |pair|
113
114
  name, _sep, value = pair.partition("=")
114
115
  name = name[1..-1].to_sym if name[0] == ':'
116
+ value = value.to_i if value =~ /^\d+$/
117
+ value = true if value == "true"
118
+ value = false if value == "false"
115
119
  options[name] = value
116
120
  end
121
+ options.delete :server_options
117
122
  end
118
123
 
119
124
  case server
120
125
  when 'unicorn'
121
126
  `env RBBT_LOG=#{Log.severity.to_s} unicorn -c #{ Rbbt.share['unicorn.rb'].find } '#{config_ru_file}' -p #{options[:Port] || "2887"}`
122
- when 'puma_alt'
127
+ when 'puma_production'
123
128
  #`puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 3 -t 8:32 --preload`
124
- `env RBBT_LOG=#{Log.severity.to_s} puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 20 -t 10:160 --preload`
129
+ CMD.cmd_log("env RBBT_LOG=#{Log.severity.to_s} puma '#{config_ru_file}' -p #{options[:Port] || "2887"} -w 20 -t 10:160 --preload")
125
130
  else
126
131
  options[:config] = config_ru_file
127
132
 
@@ -5,8 +5,9 @@ class TestConfig < Test::Unit::TestCase
5
5
  def setup
6
6
  Rbbt::Config.set({:cpus => 30}, :test_config, :test)
7
7
  Rbbt::Config.set(:cpus, 5, "slow::2", :test)
8
- Rbbt::Config.set({:token => "token"}, "token", "key:token")
8
+ Rbbt::Config.set({:token => "token"}, "token")
9
9
  Rbbt::Config.set(:notoken, "no_token")
10
+ Rbbt::Config.set({:emptytoken => "empty"})
10
11
  end
11
12
 
12
13
  def test_simple
@@ -19,8 +20,12 @@ class TestConfig < Test::Unit::TestCase
19
20
 
20
21
  def test_simple_no_token
21
22
  assert_equal "token", Rbbt::Config.get("token", "token")
22
- assert_equal "token", Rbbt::Config.get("token")
23
- assert_equal "no_token", Rbbt::Config.get("notoken")
23
+ assert_equal "no_token", Rbbt::Config.get("notoken", "key:notoken")
24
+ assert_equal 'token', Rbbt::Config.get("token", "key:token")
25
+ assert_equal 'token', Rbbt::Config.get("token")
26
+ assert_equal nil, Rbbt::Config.get("token", "someotherthing")
27
+ assert_equal "default_token", Rbbt::Config.get("token", 'unknown', :default => 'default_token')
28
+ assert_equal 'empty', Rbbt::Config.get("emptytoken", 'key:emptytoken')
24
29
  end
25
30
 
26
31
  def test_prio
@@ -54,6 +59,11 @@ class TestConfig < Test::Unit::TestCase
54
59
  assert_equal "V1", Rbbt::Config.get('key', 'token1', 'token2')
55
60
  end
56
61
 
62
+ def test_default
63
+ Rbbt::Config.add_entry 'key', 'V1', 'token1'
64
+ assert_equal "V3", Rbbt::Config.get('key', 'token2', :default => 'V3')
65
+ end
66
+
57
67
 
58
68
  end
59
69
 
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.30.7
4
+ version: 5.30.12
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-02-10 00:00:00.000000000 Z
11
+ date: 2021-02-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake