rbbt-util 5.20.2 → 5.20.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,156 +1,12 @@
1
+ require 'rbbt/workflow/step/dependencies'
2
+
1
3
  class Step
2
4
 
3
5
  attr_reader :stream, :dupped, :saved_stream, :inputs
4
6
 
5
- STREAM_CACHE = {}
6
- STREAM_CACHE_MUTEX = Mutex.new
7
- def self.purge_stream_cache
8
- Log.medium "Purging dup. stream cache"
9
- STREAM_CACHE_MUTEX.synchronize do
10
- #STREAM_CACHE.collect{|k,s|
11
- # next
12
- # Thread.new do
13
- # Misc.consume_stream s
14
- # end
15
- #}
16
- STREAM_CACHE.clear
17
- end
18
- end
19
-
20
- def self.dup_stream(stream)
21
- case stream
22
- when IO, File, Step
23
- return stream if stream.respond_to?(:closed?) and stream.closed?
24
- return stream if stream.respond_to?(:done?) and stream.done?
25
-
26
- STREAM_CACHE_MUTEX.synchronize do
27
- stream_key = Misc.fingerprint(stream)
28
- current = STREAM_CACHE[stream_key]
29
- case current
30
- when nil
31
- Log.medium "Not duplicating stream #{stream_key}"
32
- STREAM_CACHE[stream_key] = stream
33
- when File
34
- if Open.exists? current.path
35
- Log.medium "Reopening file #{stream_key}"
36
- Open.open(current.path)
37
- else
38
- new = Misc.dup_stream(current)
39
- Log.medium "Duplicating file #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
40
- new
41
- end
42
- when Step
43
- job = current
44
- current = job.result
45
- new = Misc.dup_stream(current)
46
- job.result = current
47
- Log.medium "Duplicating step #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
48
- new
49
- else
50
- new = Misc.dup_stream(current)
51
- Log.medium "Duplicating stream #{stream_key} #{ Misc.fingerprint(stream) } => #{Misc.fingerprint(new)}"
52
- new
53
- end
54
- end
55
- when TSV::Dumper#, TSV::Parser
56
- stream = stream.stream
57
- return stream if stream.closed?
58
-
59
- STREAM_CACHE_MUTEX.synchronize do
60
- if STREAM_CACHE[stream].nil?
61
- Log.high "Not duplicating dumper #{ stream.inspect }"
62
- STREAM_CACHE[stream] = stream
63
- else
64
- new = Misc.dup_stream(STREAM_CACHE[stream])
65
- Log.high "Duplicating dumper #{ stream.inspect } into #{new.inspect}"
66
- new
67
- end
68
- end
69
- else
70
- stream
71
- end
72
- end
73
-
74
- def self.prepare_for_execution(job)
75
- return if (job.done? and not job.dirty?) or
76
- (job.streaming? and job.running?) or
77
- (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === job and not (job.error? or job.aborted?))
78
-
79
- job.clean if job.aborted? or (job.started? and not job.running? and not job.error?)
80
-
81
- raise DependencyError, job if job.error?
82
- end
83
-
84
- def execute_dependency(dependency)
85
- task_name = self.task_name
86
- begin
87
-
88
- if dependency.done?
89
- Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} done -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
90
- return
91
- end
92
-
93
- if not dependency.started?
94
- Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} starting -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
95
- dependency.run(:stream)
96
- raise TryAgain
97
- end
98
-
99
- dependency.grace
100
-
101
- if dependency.aborted?
102
- Log.warn "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} aborted (clean and retry) -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
103
- dependency.clean
104
- raise TryAgain
105
- end
106
-
107
- if dependency.error?
108
- Log.error "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} error -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
109
- raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
110
- end
111
-
112
- if dependency.streaming?
113
- Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} streaming -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
114
- return
115
- end
116
-
117
- begin
118
- Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joining -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
119
- dependency.join
120
- raise TryAgain unless dependency.done?
121
- Log.info "#{Log.color :cyan, "dependency"} #{Log.color :yellow, task_name.to_s || ""} => #{Log.color :yellow, dependency.task_name.to_s || ""} joined -- #{Log.color :blue, dependency.path} -- #{Log.color :yellow, self.short_path}"
122
- rescue Aborted
123
- raise TryAgain
124
- end
125
-
126
- rescue TryAgain
127
- retry
128
- rescue Aborted
129
- Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
130
- raise $!
131
- rescue Interrupt
132
- Log.error "Interrupted while in dep. #{Log.color :red, dependency.task_name.to_s}"
133
- raise $!
134
- rescue Exception
135
- Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s }"
136
- Log.exception $!
137
- raise $!
138
- end
139
- end
140
-
141
- def dup_inputs
142
- return if @dupped or ENV["RBBT_NO_STREAM"] == 'true'
143
- Log.low "Dupping inputs for #{path}"
144
- dupped_inputs = @inputs.collect do |input|
145
- Step.dup_stream input
146
- end
147
- @inputs.replace dupped_inputs
148
- @dupped = true
149
- end
150
-
151
7
  def get_stream
152
8
  @mutex.synchronize do
153
- Log.low "Getting stream from #{path} #{!@saved_stream}"
9
+ Log.low "Getting stream from #{path} #{!@saved_stream} [#{object_id}]"
154
10
  begin
155
11
  return nil if @saved_stream
156
12
  if IO === @result
@@ -162,14 +18,33 @@ class Step
162
18
  end
163
19
  end
164
20
 
21
+ def resolve_input_steps
22
+ step = false
23
+ new_inputs = @inputs.collect do |i|
24
+ if Step === i
25
+ step = true
26
+ if i.done?
27
+ i.load
28
+ elsif i.streaming?
29
+ TSV.get_stream i
30
+ else
31
+ i.join
32
+ i.load
33
+ end
34
+ else
35
+ i
36
+ end
37
+ end
38
+ @inputs.replace new_inputs if step
39
+ end
165
40
 
166
41
  def _exec
42
+ resolve_input_steps
167
43
  @exec = true if @exec.nil?
168
44
  @task.exec_in((bindings ? bindings : self), *@inputs)
169
45
  end
170
46
 
171
47
  def exec(no_load=false)
172
- dup_inputs
173
48
  dependencies.each{|dependency| dependency.exec(no_load) }
174
49
  @mutex.synchronize do
175
50
  @result = self._exec
@@ -182,7 +57,6 @@ class Step
182
57
  rec_dependencies.collect{|dependency| (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === dependency) ? nil : dependency.path }.compact.uniq
183
58
  end
184
59
 
185
-
186
60
  def kill_children
187
61
  begin
188
62
  children_pids = info[:children_pids]
@@ -202,60 +76,6 @@ class Step
202
76
  end
203
77
  end
204
78
 
205
- def run_dependencies
206
- @seen ||= []
207
- seen_paths ||= Set.new
208
-
209
- dependencies.uniq.each do |dependency|
210
- dependency_path = dependency.path
211
- next if seen_paths.include? dependency_path
212
- @seen.concat dependency.rec_dependencies
213
- seen_paths.union(dependency.rec_dependencies.collect{|d| d.path})
214
- @seen << dependency
215
- seen_paths << dependency_path
216
- end
217
-
218
- @seen.uniq!
219
- @seen.delete self
220
-
221
- return if @seen.empty?
222
-
223
- log :dependencies, "#{Log.color :magenta, "Dependencies"} for step #{Log.color :yellow, task.name.to_s || ""}"
224
-
225
- @seen.each do |dependency|
226
- Step.prepare_for_execution(dependency)
227
- end
228
-
229
- pre_deps = []
230
- last_deps = []
231
- @seen.each do |dependency|
232
- if dependencies.include? dependency
233
- if dependency.inputs.flatten.select{|i| Step === i}.any?
234
- last_deps << dependency
235
- else
236
- pre_deps << dependency
237
- end
238
-
239
- else
240
- pre_deps << dependency
241
- end
242
- end
243
-
244
- pre_deps.each do |dependency|
245
- dependency.dup_inputs
246
- execute_dependency(dependency)
247
- end
248
-
249
- last_deps.each do |dependency|
250
- dependency.dup_inputs
251
- end
252
-
253
- last_deps.each do |dependency|
254
- execute_dependency(dependency)
255
- end
256
-
257
- end
258
-
259
79
  def run(no_load = false)
260
80
  result = nil
261
81
 
@@ -280,7 +100,7 @@ class Step
280
100
  :clean_name => clean_name,
281
101
  })
282
102
 
283
- dup_inputs
103
+ #dup_inputs
284
104
  begin
285
105
  run_dependencies
286
106
  rescue Exception
@@ -294,11 +114,11 @@ class Step
294
114
  set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
295
115
 
296
116
  set_info :started, (start_time = Time.now)
297
- log :started, "#{Log.color :magenta, "Starting"} step #{Log.color :yellow, task.name.to_s || ""}"
117
+ log :started, "Starting step #{Log.color :yellow, task.name.to_s || ""}"
298
118
 
299
119
  begin
300
120
  result = _exec
301
- rescue Aborted
121
+ rescue Aborted, Interrupt
302
122
  log(:aborted, "Aborted")
303
123
  raise $!
304
124
  rescue Exception
@@ -326,7 +146,7 @@ class Step
326
146
  end
327
147
 
328
148
  if stream
329
- log :streaming, "#{Log.color :magenta, "Streaming"} step #{Log.color :yellow, task.name.to_s || ""}"
149
+ log :streaming, "Streaming step #{Log.color :yellow, task.name.to_s || ""}"
330
150
  ConcurrentStream.setup stream do
331
151
  begin
332
152
  if status != :done
@@ -334,14 +154,14 @@ class Step
334
154
  set_info :done, (done_time = Time.now)
335
155
  set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)
336
156
  set_info :time_elapsed, (time_elapsed = done_time - start_time)
337
- log :done, "#{Log.color :magenta, "Completed"} step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
157
+ log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
338
158
  end
339
159
  end
340
160
  rescue
341
161
  Log.exception $!
342
162
  ensure
343
163
  join
344
- Step.purge_stream_cache
164
+ #Step.purge_stream_cache
345
165
  FileUtils.rm pid_file if File.exists?(pid_file)
346
166
  end
347
167
  end
@@ -358,8 +178,8 @@ class Step
358
178
  set_info :done, (done_time = Time.now)
359
179
  set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)
360
180
  set_info :time_elapsed, (time_elapsed = done_time - start_time)
361
- log :done, "#{Log.color :magenta, "Completed"} step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
362
- Step.purge_stream_cache
181
+ log :done, "Completed step #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i}+#{(total_time_elapsed - time_elapsed).to_i} sec."
182
+ #Step.purge_stream_cache
363
183
  FileUtils.rm pid_file if File.exists?(pid_file)
364
184
  end
365
185
 
@@ -370,10 +190,14 @@ class Step
370
190
  @result ||= result
371
191
  self
372
192
  else
373
- Step.purge_stream_cache
193
+ #Step.purge_stream_cache
374
194
  @result = prepare_result result, @task.result_description
375
195
  end
376
196
  end
197
+ rescue Aborted, Interrupt
198
+ abort
199
+ stop_dependencies
200
+ raise $!
377
201
  rescue Exception
378
202
  exception $!
379
203
  stop_dependencies
@@ -392,9 +216,9 @@ class Step
392
216
  end
393
217
  end
394
218
 
395
- clean if dirty? or not running?
219
+ clean if dirty? or (not running? and not done?)
396
220
 
397
- run(true) unless started?
221
+ run(:stream) unless started?
398
222
 
399
223
  join unless done?
400
224
 
@@ -412,7 +236,7 @@ class Step
412
236
  @forked = true
413
237
  res = run true
414
238
  set_info :forked, true
415
- rescue Aborted
239
+ rescue Aborted, Interrupt
416
240
  Log.debug{"Forked process aborted: #{path}"}
417
241
  log :aborted, "Job aborted (#{Process.pid})"
418
242
  raise $!
@@ -452,13 +276,6 @@ class Step
452
276
  self
453
277
  end
454
278
 
455
- def stop_dependencies
456
- dependencies.each do |dep|
457
- dep.abort
458
- end
459
- kill_children
460
- end
461
-
462
279
  def abort_pid
463
280
  @pid ||= info[:pid]
464
281
 
@@ -491,7 +308,7 @@ class Step
491
308
  Log.medium "Aborting job stream #{stream.inspect} -- #{Log.color :blue, path}"
492
309
  stream.abort
493
310
  #stream.close unless stream.closed?
494
- rescue Aborted
311
+ rescue Aborted, Interrupt
495
312
  Log.medium "Aborting job stream #{stream.inspect} ABORTED RETRY -- #{Log.color :blue, path}"
496
313
  Log.exception $!
497
314
  retry
@@ -507,11 +324,12 @@ class Step
507
324
  begin
508
325
  stop_dependencies
509
326
  abort_stream
510
- abort_pid if defined? @forked and @forked
511
- rescue Aborted
327
+ abort_pid if defined? @forked and @forked and running?
328
+ rescue Aborted, Interrupt
512
329
  Log.medium{"#{Log.color :red, "Aborting ABORTED RETRY"} #{Log.color :blue, path}"}
513
330
  retry
514
331
  rescue Exception
332
+ Log.exception $!
515
333
  retry
516
334
  ensure
517
335
  if Open.exists? path
@@ -523,7 +341,6 @@ class Step
523
341
  end
524
342
  end
525
343
  end
526
- Log.medium{"#{Log.color :red, "Aborted"} #{Log.color :blue, path}"}
527
344
  end
528
345
 
529
346
  def abort
@@ -75,7 +75,13 @@ module Task
75
75
  seen = []
76
76
  task_inputs = {}
77
77
  deps.each do |dep|
78
- wf, task = (Array === dep ? [dep.first, dep.first.tasks[dep[1].to_sym]] : [workflow, workflow.tasks[dep.to_sym]])
78
+ if Array === dep and dep.first
79
+ wf, task = (Array === dep ? [dep.first, dep.first.tasks[dep[1].to_sym]] : [workflow, workflow.tasks[dep.to_sym]])
80
+ elsif Symbol === dep
81
+ wf, task = [workflow, workflow.tasks[dep.to_sym]]
82
+ else
83
+ next
84
+ end
79
85
  maps = (Array === dep and Hash === dep.last) ? dep.last.keys : []
80
86
  raise "Dependency task not found: #{dep}" if task.nil?
81
87
  next if seen.include? [wf, task.name]
@@ -36,9 +36,14 @@ matches = file.subset(source, target)
36
36
  if options[:tsv]
37
37
  puts matches.tsv.to_s
38
38
  else
39
+ fields = file.fields
39
40
  matches.each do |item|
40
41
  puts Log.color(:magenta, item)
41
- info = file.fields.zip(file[item])
42
+ if fields.any?
43
+ info = file.fields.zip(file[item])
44
+ else
45
+ info = {}
46
+ end
42
47
  source,_sep, target = item.partition "~"
43
48
  puts " " << Misc.format_definition_list_item("source", source)
44
49
  puts " " << Misc.format_definition_list_item("target", target)
@@ -433,9 +433,13 @@ case res
433
433
  when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
434
434
  res = job.result
435
435
  if res.respond_to? :gets
436
- while block = res.read(2048) do
437
- out.write block
438
- end #unless io.closed?
436
+ begin
437
+ Misc.consume_stream(res, false, out)
438
+ #while block = res.readpartial(Misc::BLOCK_SIZE) do
439
+ # out.write block
440
+ #end #unless io.closed?
441
+ rescue EOFError, IOError
442
+ end
439
443
  res.join if res.respond_to? :join
440
444
  else
441
445
  puts res.to_s
@@ -443,16 +447,24 @@ when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
443
447
  when Step
444
448
  if res.streaming?
445
449
  io = TSV.get_stream res
446
- while block = io.read(2048) do
447
- out.write block
448
- end #unless io.closed?
450
+ Misc.consume_stream(io, false, out)
451
+ #begin
452
+ # while block = io.readpartial(Misc::BLOCK_SIZE) do
453
+ # out.write block
454
+ # end #unless io.closed?
455
+ #rescue EOFError, IOError
456
+ #end
449
457
  io.join if io.respond_to? :join
450
458
  elsif IO === res.result
451
459
  begin
452
460
  io = res.get_stream
453
- while block = io.read(2048) do
454
- out.write block
455
- end #unless io.closed?
461
+ Misc.consume_stream(io, false, out)
462
+ #begin
463
+ # while block = io.readpartial(Misc::BLOCK_SIZE) do
464
+ # out.write block
465
+ # end #unless io.closed?
466
+ #rescue EOFError, IOError
467
+ #end
456
468
  io.join if io.respond_to? :join
457
469
  rescue Aborted, Interrupt
458
470
  Log.error "Process interrupted. Aborting step"