rbbt-util 5.23.18 → 5.23.19

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 39526da82633a38054bbce4d640d5006f0e2f3d2
4
- data.tar.gz: 2bb4766c1d2aea82fa9cdbccfa4ebc8f6ef49deb
3
+ metadata.gz: 5ea281c4b995f87831e4ad034423d3ca5ff1b6f5
4
+ data.tar.gz: 4e8c74787a866cbfdf0bc0b45d18b89f043751a2
5
5
  SHA512:
6
- metadata.gz: bc8a05f50a9615be78b3cf65cac028208e02d8a00b0fdca73ddd389258e1e836cc573d25dcf90fb45805a322a204794f137ebb58f270f351c8646c6e2ec014f7
7
- data.tar.gz: 1a9a0b9baf29b552cf4d19f0343d1154dccc87ecbe522226f49c6090b6c631ae1680505eedaa483aa117020583562679c3211c5ad5c4b0376ed559e187e5cb03
6
+ metadata.gz: 6c62ba3da73a0cf27a1aa41cb82b6b9d32a97dac3dafe62efbd7552a7a6cc58f69b577708b377f1b34f6b50d748a06e95b8e34e2b95b69a1fb392a1292adedff
7
+ data.tar.gz: 74c2fbcee1ada86a02a540232bec94581048b585dcb55015ae59b1988b9d0de8e4b460bffabf5100f3b043486c35f04b2b430da1c39521332d3c1d3a7360946a
data/lib/rbbt/persist.rb CHANGED
@@ -371,7 +371,7 @@ module Persist
371
371
  when type.to_sym == :memory
372
372
  repo = persist_options[:repo] || Persist::MEMORY
373
373
  path = path.find if Path === path
374
- repo[path] = yield
374
+ repo[path] ||= yield
375
375
 
376
376
  when (type.to_sym == :annotations and persist_options.include? :annotation_repo)
377
377
 
@@ -468,6 +468,7 @@ module Persist
468
468
  when String
469
469
  persist name, :memory, :file => name + "_" << options, &block
470
470
  else
471
+ options = options.dup
471
472
  file = name
472
473
  repo = options.delete :repo if options and options.any?
473
474
  file << "_" << (options[:key] ? options[:key] : Misc.hash2md5(options)) if options and options.any?
@@ -195,7 +195,7 @@ class RbbtProcessQueue
195
195
 
196
196
  @pid = Process.fork do
197
197
  Misc.pre_fork
198
- Log::ProgressBar.add_offset if @offset
198
+ Log::ProgressBar.add_offset @offset if @offset
199
199
 
200
200
  @cleanup.call if @cleanup
201
201
  @queue.close_write
@@ -212,7 +212,7 @@ class RbbtProcessQueue
212
212
  else
213
213
  run
214
214
  end
215
- Log::ProgressBar.remove_offset if @offset
215
+ Log::ProgressBar.remove_offset @offset if @offset
216
216
  end
217
217
  end
218
218
 
@@ -118,16 +118,17 @@ class RbbtProcessQueue
118
118
  while @count > 0
119
119
  @count -= 1
120
120
  @total += 1
121
- processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, @offset, &@init_block)
121
+ processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, (@offset ? @total : false), &@init_block)
122
122
  Log.warn "Added process #{processes.last.pid} to #{Process.pid} (#{processes.length})"
123
123
  end
124
124
 
125
125
  while @count < 0
126
126
  @count += 1
127
+ @total -= 1
127
128
  next unless processes.length > 1
128
- first = processes.shift
129
- first.stop
130
- Log.warn "Removed process #{first.pid} from #{Process.pid} (#{processes.length})"
129
+ last = processes.last
130
+ last.stop
131
+ Log.warn "Removed process #{last.pid} from #{Process.pid} (#{processes.length})"
131
132
  end
132
133
  end
133
134
  end
@@ -148,7 +149,7 @@ class RbbtProcessQueue
148
149
  num_processes.times do |i|
149
150
  @total += 1
150
151
  process_mutex.synchronize do
151
- processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, @offset, &@init_block)
152
+ processes << RbbtProcessQueueWorker.new(@queue, @callback_queue, @cleanup, @respawn, (@offset ? @total : false), &@init_block)
152
153
  end
153
154
  end
154
155
 
@@ -189,6 +190,7 @@ class RbbtProcessQueue
189
190
 
190
191
  RbbtSemaphore.post_semaphore(@sem)
191
192
 
193
+ Log.low "Process monitor #{Process.pid} joining threads"
192
194
  begin
193
195
  @monitor_thread.join
194
196
  @manager_thread.raise TryAgain if @manager_thread.alive?
@@ -197,6 +199,7 @@ class RbbtProcessQueue
197
199
  rescue Exception
198
200
  Kernel.exit -1
199
201
  end
202
+ Log.low "Process monitor #{Process.pid} threads joined successfully, now exit"
200
203
 
201
204
  Kernel.exit 0
202
205
  end
@@ -111,15 +111,19 @@ module Log
111
111
  if short_mean
112
112
  thr = short_mean
113
113
  else
114
- thr = @ticks / (Time.now - @start)
114
+ thr = begin
115
+ (@ticks || 1) / (Time.now - @start)
116
+ rescue
117
+ 1
118
+ end
115
119
  end
116
120
 
117
121
  if mean.nil? or mean.to_i > 1
118
122
  str = "#{ Log.color :blue, thr.to_i.to_s } per sec."
119
- str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
123
+ #str << " #{ Log.color :yellow, mean.to_i.to_s } avg. #{Log.color :yellow, @mean_max.to_i.to_s} max." if @mean_max > 0
120
124
  else
121
125
  str = "#{ Log.color :blue, (1/thr).ceil.to_s } secs each"
122
- str << " #{ Log.color :yellow, (1/mean).ceil.to_s } avg. #{Log.color :yellow, (1/@mean_max).ceil.to_s} min." if @mean_max > 0
126
+ #str << " #{ Log.color :yellow, (1/mean).ceil.to_s } avg. #{Log.color :yellow, (1/@mean_max).ceil.to_s} min." if @mean_max > 0
123
127
  end
124
128
 
125
129
  str
@@ -158,12 +162,12 @@ module Log
158
162
  end
159
163
 
160
164
  def report_msg
161
- str = Log.color :magenta, desc
165
+ str = Log.color(:magenta, "·")
162
166
  if @ticks == 0
163
167
  if @max
164
- return str << " " << Log.color(:yellow, "waiting on #{@max} #{bytes ? 'bytes' : 'items'}")
168
+ return str << " " << Log.color(:magenta, "waiting on #{@max} #{bytes ? 'bytes' : 'items'}") << Log.color(:magenta, " · " << desc)
165
169
  else
166
- return str << " " << Log.color(:yellow, "waiting - PID: #{Process.pid}")
170
+ return str << " " << Log.color(:magenta, "waiting - PID: #{Process.pid}") << Log.color(:magenta, " · " << desc)
167
171
  end
168
172
  end
169
173
  str << " " << thr_msg
@@ -172,6 +176,7 @@ module Log
172
176
  else
173
177
  str << Log.color(:blue, " -- ") << ticks.to_s << " #{bytes ? 'bytes' : 'items'}"
174
178
  end
179
+ str << Log.color(:magenta, " · " << desc)
175
180
  str
176
181
  end
177
182
 
@@ -183,14 +188,20 @@ module Log
183
188
 
184
189
  def report(io = STDERR)
185
190
  if Log::LAST != "progress"
186
- length = Log::ProgressBar.cleanup_bars
187
191
  bars = BARS
188
- print(io, Log.color(:yellow, "...Progress\n"))
189
- bars.sort_by{|b| b.depth }.reverse.each do |bar|
190
- if SILENCED.include? bar
191
- print(io, Log.color(:yellow ,bar.report_msg) << "\n")
192
- else
193
- print(io, "\n")
192
+ if Log::LAST == "new_bar"
193
+ Log::LAST.replace "progress"
194
+ bar = bars.sort_by{|b| b.depth }.first
195
+ print(io, Log.color(:magenta ,bar.report_msg) << "\n")
196
+ else
197
+ length = Log::ProgressBar.cleanup_bars
198
+ print(io, Log.color(:magenta, "···Progress\n"))
199
+ bars.sort_by{|b| b.depth }.reverse.each do |bar|
200
+ if SILENCED.include? bar
201
+ print(io, Log.color(:magenta, "·\n"))
202
+ else
203
+ print(io, Log.color(:magenta ,bar.report_msg) << "\n")
204
+ end
194
205
  end
195
206
  end
196
207
  else
@@ -198,7 +209,7 @@ module Log
198
209
  end
199
210
  bars << self unless BARS.include? self
200
211
 
201
- print(io, Log.up_lines(bars.length+1) << Log.color(:yellow, "...Progress\n") << Log.down_lines(bars.length+1))
212
+ print(io, Log.up_lines(bars.length) << Log.color(:magenta, "···Progress\n") << Log.down_lines(bars.length+1)) if Log::ProgressBar.offset == 0
202
213
  print(io, Log.up_lines(@depth) << report_msg << Log.down_lines(@depth))
203
214
  @last_time = Time.now
204
215
  @last_count = ticks
@@ -207,7 +218,7 @@ module Log
207
218
  end
208
219
 
209
220
  def done(io = STDERR)
210
- done_msg = Log.color(:magenta, desc) << " " << Log.color(:green, "done")
221
+ done_msg = Log.color(:magenta, "· ") << Log.color(:green, "done")
211
222
  if @start
212
223
  ellapsed = (Time.now - @start).to_i
213
224
  else
@@ -217,13 +228,14 @@ module Log
217
228
  done_msg << " " << Log.color(:blue, (@ticks).to_s) << " #{bytes ? 'bytes' : 'items'} in " << Log.color(:green, ellapsed)
218
229
  @last_count = 0
219
230
  @last_time = @start
220
- done_msg << " (" << thr_msg << ")"
231
+ done_msg << " - " << thr_msg
232
+ done_msg << Log.color(:magenta, " · " << desc)
221
233
  print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth))
222
234
  Open.rm @file if @file and Open.exists? @file
223
235
  end
224
236
 
225
237
  def error(io = STDERR)
226
- done_msg = Log.color(:magenta, desc) << " " << Log.color(:red, "error")
238
+ done_msg = Log.color(:magenta, "· ") << Log.color(:red, "error")
227
239
  if @start
228
240
  ellapsed = (Time.now - @start).to_i
229
241
  else
@@ -233,7 +245,8 @@ module Log
233
245
  done_msg << " " << Log.color(:blue, (@ticks).to_s) << " in " << Log.color(:green, ellapsed)
234
246
  @last_count = 0
235
247
  @last_time = @start
236
- done_msg << " (" << thr_msg << ")"
248
+ done_msg << " - " << thr_msg
249
+ done_msg << Log.color(:magenta, " · " << desc)
237
250
  print(io, Log.up_lines(@depth) << done_msg << Log.down_lines(@depth))
238
251
  Open.rm @file if @file and Open.exists? @file
239
252
  end
@@ -5,14 +5,16 @@ module Log
5
5
  REMOVE = []
6
6
  SILENCED = []
7
7
 
8
- def self.add_offset
9
- @@offset = offset + 1
8
+ def self.add_offset(value = 1)
9
+ value = 1 if TrueClass === value
10
+ @@offset = offset + value.to_i
10
11
  @@offset = 0 if @@offset < 0
11
12
  @@offset
12
13
  end
13
14
 
14
- def self.remove_offset
15
- @@offset = offset - 1
15
+ def self.remove_offset(value = 1)
16
+ value = 1 if TrueClass === value
17
+ @@offset = offset - value.to_i
16
18
  @@offset = 0 if @@offset < 0
17
19
  @@offset
18
20
  end
@@ -27,7 +29,7 @@ module Log
27
29
  def self.new_bar(max, options = {})
28
30
  cleanup_bars
29
31
  BAR_MUTEX.synchronize do
30
- #Log::LAST.replace "new_bar" if Log::LAST == "progress"
32
+ Log::LAST.replace "new_bar" if Log::LAST == "progress"
31
33
  options = Misc.add_defaults options, :depth => BARS.length + Log::ProgressBar.offset
32
34
  BARS << (bar = ProgressBar.new(max, options))
33
35
  bar
@@ -69,6 +71,7 @@ module Log
69
71
  BAR_MUTEX.synchronize do
70
72
  REMOVE << bar
71
73
  end
74
+ Log::LAST.replace "remove_bar" if Log::LAST == "progress"
72
75
  end
73
76
 
74
77
  def self.with_bar(max, options = {})
@@ -6,7 +6,7 @@ module Log
6
6
 
7
7
  attr_accessor :max, :ticks, :frequency, :depth, :desc, :file, :bytes
8
8
  def initialize(max = nil, options = {})
9
- options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :desc => "Progress", :io => STDERR, :severity => Log.severity, :frequency => 2
9
+ options = Misc.add_defaults options, :depth => 0, :num_reports => 100, :io => STDERR, :severity => Log.severity, :frequency => 2
10
10
  depth, num_reports, desc, io, severity, file, bytes, frequency = Misc.process_options options, :depth, :num_reports, :desc, :io, :severity, :file, :bytes, :frequency
11
11
 
12
12
  @max = max
@@ -16,7 +16,7 @@ module Log
16
16
  @last_count = nil
17
17
  @last_percent = nil
18
18
  @depth = depth
19
- @desc = desc.nil? ? "Progress" : desc.gsub(/\n/,' ')
19
+ @desc = desc.nil? ? "" : desc.gsub(/\n/,' ')
20
20
  @file = file
21
21
  @bytes = bytes
22
22
  end
@@ -85,7 +85,7 @@ module ConcurrentStream
85
85
  Log.low "Not failing on exception joining thread in ConcurrenStream: #{filename}"
86
86
  else
87
87
  Log.low "Exception joining thread in ConcurrenStream: #{filename}"
88
- raise $!
88
+ stream_raise_exception $!
89
89
  end
90
90
  end
91
91
  end
@@ -98,7 +98,7 @@ module ConcurrentStream
98
98
  @pids.each do |pid|
99
99
  begin
100
100
  Process.waitpid(pid, Process::WUNTRACED)
101
- raise ProcessFailed.new "Error joining process #{pid} in #{self.filename || self.inspect}" unless $?.success? or no_fail
101
+ stream_raise_exception ProcessFailed.new "Error joining process #{pid} in #{self.filename || self.inspect}" unless $?.success? or no_fail
102
102
  rescue Errno::ECHILD
103
103
  end
104
104
  end
@@ -203,7 +203,7 @@ module ConcurrentStream
203
203
  Log.exception $!
204
204
  self.abort
205
205
  self.join
206
- raise $!
206
+ stream_raise_exception $!
207
207
  ensure
208
208
  self.join if self.closed? or self.eof?
209
209
  end
@@ -220,16 +220,12 @@ module ConcurrentStream
220
220
  end
221
221
  end
222
222
 
223
- def raise(exception)
224
- begin
225
- threads.each do |thread|
226
- thread.raise exception
227
- end
228
-
229
- self.abort
230
- ensure
231
- Kernel.raise exception
223
+ def stream_raise_exception(exception)
224
+ threads.each do |thread|
225
+ thread.raise exception
232
226
  end
227
+
228
+ self.abort
233
229
  end
234
230
 
235
231
  end
@@ -45,6 +45,18 @@ end
45
45
  end
46
46
  end
47
47
 
48
+ def self.with_env(var, value, &block)
49
+ var = var.to_s
50
+ value = value.to_s
51
+ current = ENV[var]
52
+ begin
53
+ ENV[var] = value
54
+ yield
55
+ ensure
56
+ ENV[var] = current
57
+ end
58
+ end
59
+
48
60
  def self.path_relative_to(basedir, path)
49
61
  path = File.expand_path(path) unless path[0] == "/"
50
62
  basedir = File.expand_path(basedir) unless basedir[0] == "/"
@@ -106,18 +106,19 @@ class Step
106
106
  def info_lock
107
107
  @info_lock = begin
108
108
  path = Persist.persistence_path(info_file + '.lock', {:dir => Step.lock_dir})
109
- Lockfile.new path, :refresh => false, :dont_use_lock_id => true
109
+ #Lockfile.new path, :refresh => false, :dont_use_lock_id => true
110
+ Lockfile.new path
110
111
  end if @info_lock.nil?
111
112
  @info_lock
112
113
  end
113
114
 
114
115
  def status_lock
115
116
  return @mutex
116
- @status_lock = begin
117
- path = Persist.persistence_path(info_file + '.status.lock', {:dir => Step.lock_dir})
118
- Lockfile.new path, :refresh => false, :dont_use_lock_id => true
119
- end if @status_lock.nil?
120
- @status_lock
117
+ #@status_lock = begin
118
+ # path = Persist.persistence_path(info_file + '.status.lock', {:dir => Step.lock_dir})
119
+ # Lockfile.new path, :refresh => false, :dont_use_lock_id => true
120
+ # end if @status_lock.nil?
121
+ #@status_lock
121
122
  end
122
123
 
123
124
  def info(check_lock = true)
@@ -949,7 +950,7 @@ module Workflow
949
950
  _inputs = assign_dep_inputs(_inputs, options, all_d, workflow.task_info(dep_task))
950
951
  jobname = _inputs[:jobname] if _inputs.include? :jobname
951
952
 
952
- job = workflow.job(dep_task, jobname, _inputs)
953
+ job = workflow._job(dep_task, jobname, _inputs)
953
954
  ComputeDependency.setup(job, compute) if compute
954
955
  job
955
956
  end
@@ -963,7 +964,7 @@ module Workflow
963
964
  d_.overriden = true
964
965
  d_
965
966
  else
966
- job(dependency, jobname, _inputs)
967
+ _job(dependency, jobname, _inputs)
967
968
  end
968
969
  when Proc
969
970
  if DependencyBlock === dependency
@@ -994,7 +995,7 @@ module Workflow
994
995
  task_info = d[:workflow].task_info(d[:task])
995
996
 
996
997
  inputs = assign_dep_inputs({}, options.merge(d[:inputs] || {}), real_dependencies, task_info)
997
- d = d[:workflow].job(d[:task], d[:jobname], inputs)
998
+ d = d[:workflow]._job(d[:task], d[:jobname], inputs)
998
999
  end
999
1000
  end
1000
1001
  ComputeDependency.setup(d, compute) if compute
@@ -1014,7 +1015,7 @@ module Workflow
1014
1015
  else
1015
1016
  task_info = (dep[:task] && dep[:workflow]) ? dep[:workflow].task_info(dep[:task]) : nil
1016
1017
  inputs = assign_dep_inputs({}, dep[:inputs], real_dependencies, task_info)
1017
- dep = dep[:workflow].job(dep[:task], dep[:jobname], inputs)
1018
+ dep = dep[:workflow]._job(dep[:task], dep[:jobname], inputs)
1018
1019
  end
1019
1020
  end
1020
1021
  end
@@ -236,7 +236,7 @@ class Step
236
236
  when :bootstrap
237
237
  cpus = rest.nil? ? nil : rest.first
238
238
 
239
- cpus = config('dep_cpus', 'bootstrap', :default => [5, list.length / 2].min) if cpus.nil?
239
+ cpus = config('dep_cpus', 'bootstrap', :default => [5, list.length / 2].min) if cpus.nil? || cpus.to_i == 0
240
240
 
241
241
  respawn = rest && rest.include?(:respawn)
242
242
  respawn = false if rest && rest.include?(:norespawn)
@@ -256,9 +256,13 @@ class Step
256
256
  Log.warn "Error in bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
257
257
 
258
258
  rescue Aborted
259
- dep.abort
260
- Log.warn "Aborted bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
261
- raise $!
259
+ ex = $!
260
+ begin
261
+ dep.abort
262
+ Log.warn "Aborted bootstrap dependency #{dep.path}: #{dep.messages.last}" if dep.error? or dep.aborted?
263
+ rescue
264
+ end
265
+ raise StopInsist.new(ex)
262
266
 
263
267
  rescue RbbtException
264
268
  if canfail || dep.canfail?
@@ -267,7 +271,10 @@ class Step
267
271
  Log.warn "NOT Allowing failing of #{dep.path}: #{dep.messages.last}"
268
272
  dep.exception $!
269
273
  if dep.recoverable_error?
270
- dep.abort
274
+ begin
275
+ dep.abort
276
+ rescue
277
+ end
271
278
  raise $!
272
279
  else
273
280
  raise StopInsist.new($!)
@@ -290,6 +297,7 @@ class Step
290
297
 
291
298
  def canfail_paths
292
299
  return Set.new if ! File.exists?(info_file)
300
+
293
301
  if info[:canfail_paths]
294
302
  Set.new(info[:canfail_paths])
295
303
  else
@@ -104,22 +104,36 @@ class Step
104
104
  (no_load or ENV["RBBT_NO_STREAM"]) ? @result : prepare_result(@result, @task.result_description)
105
105
  end
106
106
 
107
+ def updatable?
108
+ (ENV["RBBT_UPDATE_ALL_JOBS"] == 'true' || Open.exists?(info_file)) && ! relocated?
109
+ end
110
+
107
111
  def dependency_checks
108
112
  rec_dependencies.
109
113
  select{|dependency| ! (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency) }.
110
114
  select{|dependency| ! Open.remote?(dependency.path) }.
111
- select{|dependency| Open.exists?(dependency.info_file) && ! dependency.relocated? }.
115
+ select{|dependency| dependency.updatable? }.
112
116
  select{|dependency| ! dependency.error? }
113
117
  end
114
118
 
115
119
  def input_checks
116
- inputs.select{|i| Step === i }
120
+ inputs.select{|i| Step === i }.
121
+ select{|dependency| dependency.updatable? }
117
122
  end
118
123
 
119
124
  def checks
120
125
  (dependency_checks + input_checks).uniq
121
126
  end
122
127
 
128
+ def persist_checks
129
+ canfail_paths = self.canfail_paths
130
+ checks.collect do |dep|
131
+ path = dep.path
132
+ next if ! dep.done? && canfail_paths.include?(path)
133
+ path
134
+ end.compact
135
+ end
136
+
123
137
  def out_of_date
124
138
 
125
139
  checks = self.checks
@@ -128,8 +142,7 @@ class Step
128
142
  outdated_dep = []
129
143
  canfail_paths = self.canfail_paths
130
144
  checks.each do |dep|
131
- next unless Open.exists?(dep.info_file)
132
- next if dep.relocated?
145
+ next unless dep.updatable?
133
146
 
134
147
  begin
135
148
  if dep.done? && self.done? && Open.exists?(dep.path) && Open.exists?(self.path) && (File.mtime(dep.path) > File.mtime(self.path))
@@ -183,7 +196,7 @@ class Step
183
196
  no_load = :stream if no_load
184
197
 
185
198
  Open.write(pid_file, Process.pid.to_s) unless Open.exists?(path) or Open.exists?(pid_file)
186
- result = Persist.persist "Job", @task.result_type, :file => path, :check => checks.collect{|dep| dep.path}, :no_load => no_load do
199
+ result = Persist.persist "Job", @task.result_type, :file => path, :check => persist_checks, :no_load => no_load do
187
200
  if Step === Step.log_relay_step and not self == Step.log_relay_step
188
201
  relay_log(Step.log_relay_step) unless self.respond_to? :relay_step and self.relay_step
189
202
  end
@@ -310,7 +310,7 @@ class Step
310
310
  self
311
311
  end
312
312
 
313
- def rec_dependencies(need_run = false)
313
+ def rec_dependencies(need_run = false, seen = [])
314
314
 
315
315
  # A step result with no info_file means that it was manually
316
316
  # placed. In that case, do not consider its dependencies
@@ -320,9 +320,10 @@ class Step
320
320
 
321
321
  new_dependencies = []
322
322
  dependencies.each{|step|
323
- next if self.done? && need_run && Open.exists?(step.info_file)
323
+ next if seen.include? step
324
+ next if self.done? && need_run && ! updatable?
324
325
 
325
- r = step.rec_dependencies(need_run)
326
+ r = step.rec_dependencies(need_run, new_dependencies)
326
327
  new_dependencies.concat r
327
328
  new_dependencies << step
328
329
  }
data/lib/rbbt/workflow.rb CHANGED
@@ -295,11 +295,10 @@ module Workflow
295
295
  @_m
296
296
  end
297
297
 
298
- def job(taskname, jobname = nil, inputs = {})
298
+ def __job(taskname, jobname = nil, inputs = {})
299
299
  taskname = taskname.to_sym
300
300
  return remote_tasks[taskname].job(taskname, jobname, inputs) if remote_tasks and remote_tasks.include? taskname
301
301
 
302
-
303
302
  task = tasks[taskname]
304
303
  raise "Task not found: #{ taskname }" if task.nil?
305
304
 
@@ -308,8 +307,9 @@ module Workflow
308
307
  Workflow.resolve_locals(inputs)
309
308
 
310
309
  task_info = task_info(taskname)
311
- task_inputs = task.inputs
310
+ task_inputs = task_info[:inputs]
312
311
  #defaults = IndiferentHash.setup(task_info[:input_defaults]).merge(task.input_defaults)
312
+ all_defaults = IndiferentHash.setup(task_info[:input_defaults])
313
313
  defaults = IndiferentHash.setup(task.input_defaults)
314
314
 
315
315
  missing_inputs = []
@@ -339,14 +339,14 @@ module Workflow
339
339
 
340
340
  inputs.each do |k,v|
341
341
  next unless (task_inputs.include?(k.to_sym) or task_inputs.include?(k.to_s))
342
- default = defaults[k]
342
+ default = all_defaults[k]
343
343
  next if default == v
344
344
  next if (String === default and Symbol === v and v.to_s == default)
345
345
  next if (Symbol === default and String === v and v == default.to_s)
346
346
  real_inputs[k] = v
347
347
  end
348
348
 
349
- jobname_input_value = inputs[jobname_input] || defaults[jobname_input]
349
+ jobname_input_value = inputs[jobname_input] || all_defaults[jobname_input]
350
350
  if jobname_input && jobname.nil? && String === jobname_input_value && ! jobname_input_value.include?('/')
351
351
  jobname = jobname_input_value
352
352
  end
@@ -371,6 +371,26 @@ module Workflow
371
371
  job
372
372
  end
373
373
 
374
+ def _job(taskname, jobname = nil, inputs = {})
375
+
376
+ _inputs = IndiferentHash.setup(inputs.dup)
377
+
378
+ task_info = task_info(taskname)
379
+ task_inputs = task_info[:inputs]
380
+ Persist.memory("STEP", :taskname => taskname, :jobname => jobname, :inputs => inputs.values_at(*task_inputs), :repo => step_cache) do
381
+ __job(taskname, jobname, inputs)
382
+ end
383
+ end
384
+
385
+ def job(taskname, jobname = nil, inputs = {})
386
+ begin
387
+ _job(taskname, jobname, inputs)
388
+ ensure
389
+ step_cache.clear
390
+ end
391
+ end
392
+
393
+
374
394
  def set_step_dependencies(step)
375
395
  if step.info[:dependencies]
376
396
  Misc.insist do
@@ -10,6 +10,16 @@ OPT_BUILD_DIR="$SOFTWARE_DIR/.build"; [ -d $OPT_BUILD_DIR ] || mkdir -p $OPT_BUI
10
10
 
11
11
  mkdir -p "$OPT_BUILD_DIR"
12
12
 
13
+ function link(){
14
+ local source="$1"
15
+ local target="$2"
16
+ local rel_source=$(realpath --relative-to="$(dirname $target)" "$source")
17
+
18
+ [ -h "$target" ] && rm "$target"
19
+ echo ln -s "$rel_source" "$target"
20
+ ln -s "$rel_source" "$target"
21
+ }
22
+
13
23
  function expand_path(){
14
24
  name=$(basename $1)
15
25
  dir=$(dirname $1)
@@ -77,7 +87,7 @@ get_svn(){
77
87
  clean_build
78
88
 
79
89
  cd $OPT_BUILD_DIR
80
- ln -s "$OPT_SCM_DIR/$name" "$name"
90
+ link "$OPT_SCM_DIR/$name" "$name"
81
91
 
82
92
  cd "$old_pwd"
83
93
 
@@ -107,7 +117,7 @@ get_git(){
107
117
  clean_build
108
118
 
109
119
  cd $OPT_BUILD_DIR
110
- ln -s "$OPT_SCM_DIR/$name" "$name"
120
+ link "$OPT_SCM_DIR/$name" "$name"
111
121
 
112
122
  cd "$old_pwd"
113
123
  }
@@ -204,10 +214,17 @@ opt_dir(){
204
214
 
205
215
  move_opt(){
206
216
  local name="$1"
207
- local pkg_dir="`opt_dir \"$name\"`"
217
+ local pkg_dir=$(opt_dir "$name")
218
+ local source=$(build_dir)
208
219
 
209
220
  mkdir -p $(dirname "$pkg_dir")
210
- mv "`build_dir`" "$pkg_dir"
221
+ if [ -h "$source" ]; then
222
+ local real_source=$(realpath "$source")
223
+ echo link "$real_source" "$pkg_dir"
224
+ link "$real_source" "$pkg_dir"
225
+ else
226
+ mv "$source" "$pkg_dir"
227
+ fi
211
228
  }
212
229
 
213
230
  setup(){
@@ -216,7 +233,7 @@ setup(){
216
233
 
217
234
  if versioned? "$name"; then
218
235
  rm -f "$(dirname $pkg_dir)/current"
219
- ln -sf "$pkg_dir" "$(dirname $pkg_dir)/current"
236
+ link "$pkg_dir" "$(dirname $pkg_dir)/current"
220
237
  pkg_dir="$(dirname $pkg_dir)/current"
221
238
  fi
222
239
 
@@ -227,7 +244,7 @@ setup(){
227
244
  for exe in ` find "$pkg_dir/bin/" -maxdepth 1 -type f -executable`; do
228
245
  exe=$(basename $exe)
229
246
  rm -f ./$exe
230
- ln -s "$pkg_dir/bin/$exe" . 2>/dev/null
247
+ link "$pkg_dir/bin/$exe" . 2>/dev/null
231
248
  done
232
249
 
233
250
  cd "$old_pwd"
@@ -238,7 +255,7 @@ setup(){
238
255
  for exe in ` find "$pkg_dir/" -maxdepth 1 -type f -executable`; do
239
256
  exe=$(basename $exe)
240
257
  rm -f ./$exe
241
- ln -s "$pkg_dir/$exe" . 2>/dev/null
258
+ link "$pkg_dir/$exe" . 2>/dev/null
242
259
  done
243
260
 
244
261
  cd "$old_pwd"
@@ -298,7 +315,7 @@ install_jar(){
298
315
 
299
316
  [ -d "$OPT_DIR/$name/" ] || mkdir -p "$OPT_DIR/$name/"
300
317
  wget "$url" -O "$OPT_DIR/$name/$name.jar"
301
- ln -s "$OPT_DIR/$name/$name.jar" "$OPT_JAR_DIR/$name.jar"
318
+ link "$OPT_DIR/$name/$name.jar" "$OPT_JAR_DIR/$name.jar"
302
319
  }
303
320
 
304
321
  jar2bin(){
@@ -190,6 +190,7 @@ the job dependencies recursively.
190
190
  -cl--clean Clean the last step of the job so that it gets recomputed
191
191
  -ct--clean_task* Clean a particular dependency task
192
192
  -rcl--recursive_clean Clean the last step and its dependencies to recompute the job completely
193
+ -uaj--update_all_jobs Consider all dependencies when checking for updates, even when they have no info files
193
194
  --fork Run job asyncronously and monitor progress. It monitors detached processes as well
194
195
  --detach Run job asyncronously and detach process
195
196
  --exec Run job with no persistence
@@ -234,6 +235,10 @@ end
234
235
 
235
236
  workflow = Workflow.require_workflow workflow
236
237
 
238
+ if options[:update_all_jobs]
239
+ ENV["RBBT_UPDATE_ALL_JOBS"] = 'true'
240
+ end
241
+
237
242
  if options[:workflows]
238
243
  require 'rbbt/workflow'
239
244
  workflows = options[:workflows].split(',')
@@ -285,11 +290,11 @@ name = options.delete(:jobname)
285
290
 
286
291
  # get job args
287
292
  sopt_option_string = SOPT_options(workflow, task)
293
+ job_options = SOPT.get sopt_option_string
294
+
288
295
  if options[:load_inputs]
289
296
  task_info = workflow.task_info(task_name)
290
- job_options = Workflow.load_inputs(options[:load_inputs], task_info[:inputs], task_info[:input_types]).merge(SOPT.get(sopt_option_string))
291
- else
292
- job_options = SOPT.get sopt_option_string
297
+ job_options = Workflow.load_inputs(options[:load_inputs], task_info[:inputs], task_info[:input_types]).merge(SOPT.get(sopt_option_string)).merge(job_options)
293
298
  end
294
299
  job_options = fix_options(workflow, task, job_options)
295
300
  saved_job_options = job_options
@@ -304,4 +304,21 @@ class TestWorkflow < Test::Unit::TestCase
304
304
  assert_equal real_other, Workflow.relocate(real, other)
305
305
  end
306
306
 
307
+ def test_delete_dep
308
+ job = TestWF.job(:t3).recursive_clean
309
+ job.run
310
+ assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
311
+ job = TestWF.job(:t3)
312
+ job.step(:t1).clean
313
+ assert job.checks.select{|d| d.task_name.to_s == "t1" }.empty?
314
+ job = TestWF.job(:t3).recursive_clean
315
+ job.run
316
+ assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
317
+ job = TestWF.job(:t3)
318
+ job.step(:t1).clean
319
+ Misc.with_env "RBBT_UPDATE_ALL_JOBS", "true" do
320
+ assert job.checks.select{|d| d.task_name.to_s == "t1" }.any?
321
+ end
322
+ end
323
+
307
324
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.23.18
4
+ version: 5.23.19
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-08-16 00:00:00.000000000 Z
11
+ date: 2018-08-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake