rbbt-util 5.32.28 → 5.33.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a680ce3f6d91a1b0ff8060ee18b8e68efbd43df56f238869b30f6f88617d23d9
4
- data.tar.gz: d6c8c724f15d4ce4c910ff9db6becbd0587c8dc04598abcc934884a8368dd93e
3
+ metadata.gz: 70ba4440be0bdca6f80b4aa78393ca4fbec5dbc8d8a1cc72b6393e73127b5340
4
+ data.tar.gz: 42021bd242233b5e4e3a268c27291953d84c8bad85b87d8feaf6389685f043bf
5
5
  SHA512:
6
- metadata.gz: 9b90c640f51de96d469af4b442d9ba66aa89a592c7776196058da3b20e13b07ef50f67e07018435b12bed13492c3bf9d9abf4de565902094a3d58be4ab628fd5
7
- data.tar.gz: 523d7234c59bc1781d47827daebb07870913b8dec9851110d5c2e0c92441903601965f9a7bb4b494b0271d64e09ad0f3b80b254176ee3495744c2bbf1f34f714
6
+ metadata.gz: d6a229aa39440cf3b0b42a505e32c699d791d8dd1e1328f42cd6500c947560b0d9343312080af78475a6f4835077e53c029d862e9d4c90addd84408925754b50
7
+ data.tar.gz: e31505e92ca6425095a90bdcd8fe95d435c8435bc929dcb1c0c77083a1286439c952d7a98af1091664fcee7a57ba3f86b99258a84958714d630ab4a3678f1ee3
@@ -18,12 +18,24 @@ module HPC
18
18
 
19
19
  def self.job_workload(job)
20
20
  workload = []
21
+ path_jobs = {}
22
+
23
+ path_jobs[job.path] = job
24
+
21
25
  heap = []
22
- heap << job
23
- while job = heap.pop
26
+ heap << job.path
27
+ while job_path = heap.pop
28
+ job = path_jobs[job_path]
24
29
  next if job.done?
25
30
  workload << job
26
- heap.concat job_dependencies(job)
31
+
32
+ deps = job_dependencies(job)
33
+
34
+ deps.each do |d|
35
+ path_jobs[d.path] ||= d
36
+ end
37
+
38
+ heap.concat deps.collect(&:path)
27
39
  heap.uniq!
28
40
  end
29
41
  workload.uniq
@@ -68,8 +80,15 @@ module HPC
68
80
  batches.each do |batch|
69
81
  jobs = batch[:jobs]
70
82
  all_deps = jobs.collect{|d| job_dependencies(d) }.flatten.uniq
83
+
84
+ minimum = all_deps
85
+ all_deps.each do |dep|
86
+ minimum -= job_dependencies(dep)
87
+ end
88
+
89
+ all_deps = minimum
71
90
  deps = all_deps.collect do |d|
72
- (batches - [batch]).select{|batch| batch[:jobs].include? d }
91
+ (batches - [batch]).select{|batch| batch[:jobs].collect(&:path).include? d.path }
73
92
  end.flatten.uniq
74
93
  batch[:deps] = deps
75
94
  end
@@ -130,7 +149,7 @@ module HPC
130
149
  def self.job_batches(rules, job)
131
150
  job_chains = self.job_chains(rules, job)
132
151
 
133
- workload = job_workload(job)
152
+ workload = job_workload(job).uniq
134
153
 
135
154
  batches = chain_batches(rules, job_chains, workload)
136
155
  batches = add_batch_deps(batches)
@@ -26,7 +26,29 @@ module HPC
26
26
  raise "No batch without unmet dependencies" if top.nil?
27
27
  batches.delete top
28
28
  job_options = options.merge(top[:rules])
29
- job_options.merge!(:batch_dependencies => top[:deps].nil? ? [] : top[:deps].collect{|d| batch_ids[d] })
29
+
30
+ if top[:deps].nil?
31
+ batch_dependencies = []
32
+ else
33
+ top_jobs = top[:jobs]
34
+
35
+ batch_dependencies = top[:deps].collect{|d|
36
+ target = d[:top_level]
37
+ canfail = false
38
+
39
+ top_jobs.each do |job|
40
+ canfail = true if job.canfail_paths.include?(target.path)
41
+ end
42
+
43
+ if canfail
44
+ 'canfail:' + batch_ids[d].to_s
45
+ else
46
+ batch_ids[d].to_s
47
+ end
48
+ }
49
+ end
50
+
51
+ job_options.merge!(:batch_dependencies => batch_dependencies )
30
52
  job_options.merge!(:manifest => top[:jobs].collect{|d| d.task_signature })
31
53
 
32
54
  if options[:dry_run]
@@ -111,6 +111,31 @@ module Persist
111
111
  end
112
112
  end
113
113
 
114
+ def with_read(&block)
115
+ if read? || write?
116
+ return yield
117
+ else
118
+ read_and_close &block
119
+ end
120
+ end
121
+
122
+ def with_write(&block)
123
+ if write?
124
+ return yield
125
+ else
126
+ if self.read?
127
+ self.write_and_read do
128
+ return yield
129
+ end
130
+ else
131
+ self.write_and_close do
132
+ return yield
133
+ end
134
+ end
135
+ end
136
+ end
137
+
138
+
114
139
  def read_and_close
115
140
  if read? || write?
116
141
  begin
data/lib/rbbt/persist.rb CHANGED
@@ -423,6 +423,8 @@ module Persist
423
423
  end
424
424
  end
425
425
 
426
+ repo.read
427
+
426
428
  case
427
429
  when (keys.length == 1 and keys.first == subkey + 'NIL')
428
430
  nil
@@ -430,19 +432,19 @@ module Persist
430
432
  []
431
433
  when (keys.length == 1 and keys.first =~ /:SINGLE$/)
432
434
  key = keys.first
433
- values = repo.read_and_close do
435
+ values = repo.with_read do
434
436
  repo[key]
435
437
  end
436
438
  Annotated.load_tsv_values(key, values, "literal", "annotation_types", "JSON")
437
439
  when (keys.any? and not keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
438
- repo.read_and_close do
440
+ repo.with_read do
439
441
  keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
440
442
  v = repo[key]
441
443
  Annotated.load_tsv_values(key, v, "literal", "annotation_types", "JSON")
442
444
  }
443
445
  end
444
446
  when (keys.any? and keys.first =~ /ANNOTATED_DOUBLE_ARRAY/)
445
- repo.read_and_close do
447
+ repo.with_read do
446
448
 
447
449
  res = keys.sort_by{|k| k.split(":").last.to_i}.collect{|key|
448
450
  v = repo[key]
@@ -457,7 +459,7 @@ module Persist
457
459
  else
458
460
  entities = yield
459
461
 
460
- repo.write_and_close do
462
+ repo.write_and_read do
461
463
  case
462
464
  when entities.nil?
463
465
  repo[subkey + "NIL"] = nil
@@ -462,4 +462,8 @@ module Path
462
462
  Path.setup relative_file , @pkgdir, @resource
463
463
  end
464
464
  end
465
+
466
+ def clean_annotations
467
+ "" << self.to_s
468
+ end
465
469
  end
data/lib/rbbt/tsv/util.rb CHANGED
@@ -297,6 +297,8 @@ module TSV
297
297
  when :double
298
298
  if field.nil?
299
299
  through do |k,v| new[k] = v.first end
300
+ elsif field == :all
301
+ through do |k,v| new[k] = v.flatten.compact end
300
302
  else
301
303
  pos = identify_field field
302
304
  through do |k,v| new[k] = v[pos] end
@@ -313,7 +315,16 @@ module TSV
313
315
  end
314
316
  end
315
317
  self.annotate(new)
316
- new.fields = new.fields[0..0] if new.fields
318
+ if new.fields
319
+ case field
320
+ when nil
321
+ new.fields = new.fields[0..0]
322
+ when :all
323
+ new.fields = [new.fields * "+"]
324
+ else
325
+ new.fields = [field]
326
+ end
327
+ end
317
328
  new.type = :flat
318
329
  new
319
330
  end
@@ -205,7 +205,8 @@ class RbbtProcessQueue
205
205
  end
206
206
 
207
207
  @queue.close_read
208
- Log.info "Cpu process (#{num_processes}) started with master: #{@master_pid}"
208
+ Log.info "Cpu process #{@master_pid} with #{num_processes} workers."
209
+ Log.low "Signal #{@master_pid} USR1/USR2 (#10/#12) to increase/decrease workers."
209
210
  end
210
211
 
211
212
  def init(&block)
@@ -49,7 +49,7 @@ puts resource[path].find(search_path)
49
49
  test_str = options[:test] ? '-nv' : ''
50
50
 
51
51
  real_paths.each do |source_path|
52
- Log.medium "Migrating #{source_path} #{options[:files].length} files to #{target} - #{Misc.fingerprint(options[:files])}}"
52
+ Log.medium "Migrating #{source_path} #{options[:files].length} files to #{target} - #{Misc.fingerprint(options[:files])}}" if options[:files]
53
53
  if File.directory?(source_path) || source_path =~ /\/$/
54
54
  source_path += "/" unless source_path[-1] == "/"
55
55
  target += "/" unless target[-1] == "/"
@@ -32,8 +32,6 @@ class DependencyError < Aborted
32
32
  def initialize(msg)
33
33
  if defined? Step and Step === msg
34
34
  step = msg
35
- workflow = step.path.split("/")[-3]
36
- new_msg = [workflow, step.short_path, step.messages.last] * " - "
37
35
  new_msg = [step.path, step.messages.last] * ": "
38
36
  super(new_msg)
39
37
  else
@@ -42,6 +40,21 @@ class DependencyError < Aborted
42
40
  end
43
41
  end
44
42
 
43
+ class DependencyRbbtException < RbbtException
44
+ def initialize(msg)
45
+ if defined? Step and Step === msg
46
+ step = msg
47
+
48
+ new_msg = nil
49
+ new_msg = [step.path, step.messages.last] * ": "
50
+
51
+ super(new_msg)
52
+ else
53
+ super(msg)
54
+ end
55
+ end
56
+ end
57
+
45
58
  class DontClose < Exception; end
46
59
 
47
60
  class KeepLocked < Exception
@@ -64,4 +77,3 @@ class StopInsist < Exception
64
77
  @exception = exception
65
78
  end
66
79
  end
67
-
@@ -44,7 +44,7 @@ class Step
44
44
  end
45
45
 
46
46
  def self.files_dir(path)
47
- path.nil? ? nil : path + '.files'
47
+ path.nil? ? nil : Path.setup(path + '.files')
48
48
  end
49
49
 
50
50
  def self.info_file(path)
@@ -154,8 +154,12 @@ class Step
154
154
  value = Annotated.purge value if defined? Annotated
155
155
  Open.lock(info_file, :lock => info_lock) do
156
156
  i = info(false).dup
157
+ value = Annotated.purge(value)
158
+
157
159
  i[key] = value
160
+
158
161
  dump = Step.serialize_info(i)
162
+
159
163
  @info_cache = IndiferentHash.setup(i)
160
164
  Misc.sensiblewrite(info_file, dump, :force => true, :lock => false) if Open.exists?(info_file)
161
165
  @info_cache_time = Time.now
@@ -107,7 +107,26 @@ class Step
107
107
  canfail = ComputeDependency === job && job.canfail?
108
108
  end
109
109
 
110
- raise DependencyError, job if job.error? and not canfail
110
+ raise_dependency_error(job) if job.error? and not canfail
111
+ end
112
+
113
+ def self.raise_dependency_error(job)
114
+ begin
115
+ if job.get_exception
116
+ klass = job.get_exception.class
117
+ else
118
+ klass = Kernel.const_get(info[:exception][:class])
119
+ end
120
+ rescue
121
+ Log.exception $!
122
+ raise DependencyError, job
123
+ end
124
+
125
+ if (klass <= RbbtException)
126
+ raise DependencyRbbtException, job
127
+ else
128
+ raise DependencyError, job
129
+ end
111
130
  end
112
131
 
113
132
  def log_dependency_exec(dependency, action)
@@ -169,7 +188,7 @@ class Step
169
188
 
170
189
  if dependency.error?
171
190
  log_dependency_exec(dependency, :error)
172
- raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
191
+ raise_dependency_error dependency
173
192
  end
174
193
 
175
194
  if dependency.streaming?
@@ -368,7 +387,7 @@ class Step
368
387
  seen_paths << step.path
369
388
  begin
370
389
  Step.prepare_for_execution(step) unless step == self
371
- rescue DependencyError
390
+ rescue DependencyError, DependencyRbbtException
372
391
  raise $! unless canfail_paths.include? step.path
373
392
  end
374
393
  next unless step.dependencies and step.dependencies.any?
@@ -101,7 +101,7 @@ class Step
101
101
  seen = []
102
102
  while path = deps.pop
103
103
  dep_info = archived_info[path]
104
- if dep_info
104
+ if Hash === dep_info
105
105
  dep_info[:inputs].each do |k,v|
106
106
  all_inputs[k] = v unless all_inputs.include?(k)
107
107
  end if dep_info[:inputs]
@@ -285,7 +285,7 @@ class Step
285
285
  begin
286
286
  return true unless info[:exception]
287
287
  klass = Kernel.const_get(info[:exception][:class])
288
- ! (klass <= RbbtException)
288
+ ! (klass <= RbbtException )
289
289
  rescue Exception
290
290
  true
291
291
  end
@@ -437,7 +437,7 @@ class Step
437
437
  end
438
438
  end # END SYNC
439
439
  res
440
- rescue DependencyError
440
+ rescue DependencyError, DependencyRbbtException
441
441
  exception $!
442
442
  rescue LockInterrupted
443
443
  raise $!
@@ -484,7 +484,7 @@ class Step
484
484
  if dofork
485
485
  fork(true) unless started?
486
486
 
487
- join unless done?
487
+ join unless done? or dofork == :nowait
488
488
  else
489
489
  run(true) unless started?
490
490
 
@@ -496,6 +496,7 @@ class Step
496
496
 
497
497
  def fork(no_load = false, semaphore = nil)
498
498
  raise "Can not fork: Step is waiting for proces #{@pid} to finish" if not @pid.nil? and not Process.pid == @pid and Misc.pid_exists?(@pid) and not done? and info[:forked]
499
+ Log.debug "Fork to run #{self.path}"
499
500
  sout, sin = Misc.pipe if no_load == :stream
500
501
  @pid = Process.fork do
501
502
  Signal.trap(:TERM) do
@@ -24,7 +24,15 @@ module Workflow
24
24
 
25
25
  type = :io if file.split(".").last == 'as_io'
26
26
 
27
+ type = :path if file.split(".").last == 'as_path'
28
+
29
+ type = :nofile if file.split(".").last == 'nofile'
30
+
27
31
  case type
32
+ when :nofile
33
+ inputs[input.to_sym] = Open.realpath(file)
34
+ when :path
35
+ inputs[input.to_sym] = Open.realpath(Open.read(file).strip)
28
36
  when :io
29
37
  inputs[input.to_sym] = Open.open(Open.realpath(file))
30
38
  when :file, :binary
@@ -96,12 +104,21 @@ class Step
96
104
  Open.write(path, value.to_s)
97
105
  when Step === value
98
106
  Open.ln_s(value.path, path)
107
+ when type.to_s == "binary"
108
+ if String === value && File.exists?(value)
109
+ value = File.expand_path(value)
110
+ Open.ln_s(value, path)
111
+ elsif String === value && Misc.is_filename?(value, false)
112
+ Open.write(path + '.as_path' , value)
113
+ else
114
+ Open.write(path, value, :mode => 'wb')
115
+ end
99
116
  when type.to_s == "file"
100
117
  if String === value && File.exists?(value)
101
118
  value = File.expand_path(value)
102
119
  Open.ln_s(value, path)
103
120
  else
104
- value = value.collect{|v| v = "#{v}" if Path === v; v }if Array === value
121
+ value = value.collect{|v| v = "#{v}" if Path === v; v } if Array === value
105
122
  value = "#{value}" if Path === value
106
123
  Open.write(path + '.yaml', value.to_yaml)
107
124
  end
@@ -65,9 +65,13 @@ module Workflow
65
65
 
66
66
  IndiferentHash.setup(resources)
67
67
 
68
- default_resources = rules["default_resources"] || rules["defaults"]["resources"]
68
+ default_resources = rules["default_resources"]
69
+ default_resources ||= rules["defaults"]["resources"] if rules["defaults"]
70
+ default_resources ||= {}
71
+
69
72
  default_resources.each{|k,v| resources[k] ||= v } if default_resources
70
73
 
74
+ resources = {:cpus => 1} if resources.empty?
71
75
  resources
72
76
  end
73
77
 
@@ -98,9 +102,14 @@ module Workflow
98
102
  candidates
99
103
  end
100
104
 
105
+ def self.process(*args)
106
+ self.new.process(*args)
107
+ end
108
+
101
109
  attr_accessor :available_resources, :resources_requested, :resources_used, :timer
102
110
 
103
- def initialize(timer = 5, available_resources = {})
111
+ def initialize(timer = 5, available_resources = nil)
112
+ available_resources = {:cpus => Etc.nprocessors } if available_resources.nil?
104
113
  @timer = timer
105
114
  @available_resources = IndiferentHash.setup(available_resources)
106
115
  @resources_requested = IndiferentHash.setup({})
@@ -148,7 +157,7 @@ module Workflow
148
157
  log = job_rules[:log] if job_rules
149
158
  log = Log.severity if log.nil?
150
159
  Log.with_severity log do
151
- job.produce(false, true)
160
+ job.produce(false, :nowait)
152
161
  end
153
162
  end
154
163
  end
@@ -176,7 +185,9 @@ module Workflow
176
185
  end
177
186
  end
178
187
 
179
- def process(rules, jobs)
188
+ def process(rules, jobs = nil)
189
+ jobs, rules = rules, {} if jobs.nil?
190
+ jobs = [jobs] if Step === jobs
180
191
  begin
181
192
 
182
193
  workload = Orchestrator.workload(jobs)
@@ -193,6 +204,7 @@ module Workflow
193
204
  when (job.error? || job.aborted?)
194
205
  begin
195
206
  if job.recoverable_error?
207
+ iif [:CLEAN, job, job.status, job.info[:exception]]
196
208
  job.clean
197
209
  raise TryAgain
198
210
  else
@@ -219,8 +231,9 @@ module Workflow
219
231
 
220
232
  new_workload = {}
221
233
  workload.each do |k,v|
222
- next if k.done?
223
- new_workload[k] = v.reject{|d| d.done? || (d.error? && ! d.recoverable_error?)}
234
+ next if k.done? || k.error? || k.aborted?
235
+ #new_workload[k] = v.reject{|d| d.done? || ((d.error? || d.aborted?) && ! d.recoverable_error?)}
236
+ new_workload[k] = v.reject{|d| d.done? || d.error? || d.aborted?}
224
237
  end
225
238
  workload = new_workload
226
239
  sleep timer
data/lib/rbbt/workflow.rb CHANGED
@@ -423,7 +423,7 @@ module Workflow
423
423
 
424
424
  # jobname => true sets the value of the input to the name of the job
425
425
  if task.input_options
426
- jobname_input = task.input_options.select{|i,o| o[:jobname]}.collect{|i,o| i }.first
426
+ jobname_input = task.input_options.select{|i,o| o[:jobname] }.collect{|i,o| i }.first
427
427
  else
428
428
  jobname_input = nil
429
429
  end
@@ -476,14 +476,14 @@ module Workflow
476
476
  end
477
477
  end
478
478
 
479
+ input_values = task.take_input_values(inputs)
479
480
  if real_inputs.empty? && Workflow::TAG != :inputs && ! overriden
480
481
  step_path = step_path taskname, jobname, [], [], extension
481
- input_values = task.take_input_values(inputs)
482
482
  else
483
- input_values = task.take_input_values(inputs)
484
483
  step_path = step_path taskname, jobname, input_values, dependencies, extension
485
484
  end
486
485
 
486
+
487
487
  job = get_job_step step_path, task, input_values, dependencies
488
488
  job.workflow = self
489
489
  job.clean_name = jobname
@@ -503,8 +503,6 @@ module Workflow
503
503
 
504
504
  def _job(taskname, jobname = nil, inputs = {})
505
505
 
506
- _inputs = IndiferentHash.setup(inputs.dup)
507
-
508
506
  task_info = task_info(taskname)
509
507
  task_inputs = task_info[:inputs]
510
508
  persist_inputs = inputs.values_at(*task_inputs)
@@ -191,6 +191,7 @@ The `recursive_clean` cleans all the job dependency steps recursively.
191
191
  -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
192
192
  -uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
193
193
  --fork Run job asyncronously and monitor progress. It monitors detached processes as well
194
+ --orchestrate* Run the job through the orchestrator
194
195
  --detach Run job asyncronously and detach process
195
196
  --exec Run job with no persistence
196
197
  -O--output* Save job result into file
@@ -438,6 +439,19 @@ begin
438
439
  end
439
440
 
440
441
  job.fork
442
+ elsif options[:orchestrate]
443
+ require 'rbbt/workflow/util/orchestrator'
444
+ rules = case options[:orchestrate]
445
+ when 'none', 'open', 'default'
446
+ nil
447
+ else
448
+ YAML.parse(Open.read(options[:orchestrate]))
449
+ end
450
+ if rules
451
+ Workflow::Orchestrator.process rules, job
452
+ else
453
+ Workflow::Orchestrator.process job
454
+ end unless job.done?
441
455
  else
442
456
  job.run(:stream)
443
457
  res = job
@@ -573,10 +587,10 @@ when Step
573
587
  elsif detach
574
588
  exit! 0
575
589
  else
590
+ res.join
576
591
  if %w(float integer string boolean).include?(res.result_type.to_s)
577
592
  out.puts res.load
578
593
  else
579
- res.join
580
594
  Open.open(res.path, :mode => 'rb') do |io|
581
595
  Misc.consume_stream(io, false, out)
582
596
  end if Open.exist?(res.path) || Open.remote?(res.path) || Open.ssh?(res.path)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.32.28
4
+ version: 5.33.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-02-11 00:00:00.000000000 Z
11
+ date: 2022-02-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake