rbbt-util 5.20.2 → 5.20.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -467,7 +467,7 @@ module Open
467
467
  begin
468
468
  File.open(file, mode) do |f|
469
469
  f.flock(File::LOCK_EX)
470
- while block = content.read(2014)
470
+ while block = content.read(Misc::BLOCK_SIZE)
471
471
  f.write block
472
472
  end
473
473
  f.flock(File::LOCK_UN)
data/lib/rbbt/workflow.rb CHANGED
@@ -70,6 +70,7 @@ module Workflow
70
70
  case
71
71
  when (defined?(Rbbt) and Rbbt.etc.workflow_dir.exists?)
72
72
  dir = Rbbt.etc.workflow_dir.read.strip
73
+ dir = File.expand_path(dir)
73
74
  Path.setup(dir)
74
75
  when defined?(Rbbt)
75
76
  Rbbt.workflows
@@ -1,6 +1,14 @@
1
1
  require 'rbbt/util/open'
2
2
  require 'yaml'
3
3
 
4
+ module ComputeDependency
5
+ attr_accessor :compute
6
+ def self.setup(dep, value)
7
+ dep.extend ComputeDependency
8
+ dep.compute = value
9
+ end
10
+ end
11
+
4
12
  class Step
5
13
 
6
14
 
@@ -493,7 +501,7 @@ module Workflow
493
501
  wf, t, o = dep
494
502
  wf.rec_dependencies(t).each do |d|
495
503
  if Array === d
496
- new = d
504
+ new = d.dup
497
505
  else
498
506
  new = [dep.first, d]
499
507
  end
@@ -513,10 +521,12 @@ module Workflow
513
521
  all_deps << dep.dependency if dep.dependency
514
522
  case dep.dependency
515
523
  when Array
516
- dep_wf, dep_task = dep.dependency
517
- dep_rec_dependencies = dep_wf.rec_dependencies(dep_task.to_sym)
518
- dep_rec_dependencies.collect!{|d| Array === d ? d : [dep_wf, d]}
519
- all_deps.concat dep_rec_dependencies
524
+ dep_wf, dep_task, dep_options = dep.dependency
525
+ if dep_task === Symbol
526
+ dep_rec_dependencies = dep_wf.rec_dependencies(dep_task.to_sym)
527
+ dep_rec_dependencies.collect!{|d| Array === d ? d : [dep_wf, d]}
528
+ all_deps.concat dep_rec_dependencies
529
+ end
520
530
  when Symbol, String
521
531
  all_deps.concat rec_dependencies(dep.dependency.to_sym)
522
532
  end
@@ -553,43 +563,84 @@ module Workflow
553
563
  end
554
564
 
555
565
  def rec_input_defaults(taskname)
566
+ rec_inputs = rec_inputs(taskname)
556
567
  [taskname].concat(rec_dependencies(taskname)).inject(IndiferentHash.setup({})){|acc, tn|
557
- new = (Array === tn ? tn.first.tasks[tn[1].to_sym] : tasks[tn.to_sym]).input_defaults
568
+ if Array === tn and tn.first
569
+ new = tn.first.tasks[tn[1].to_sym].input_types
570
+ elsif Symbol === tn
571
+ new = tasks[tn.to_sym].input_types
572
+ else
573
+ next acc
574
+ end
558
575
  acc = new.merge(acc)
576
+ acc.delete_if{|input,defaults| not rec_inputs.include? input}
577
+ acc
559
578
  }.tap{|h| IndiferentHash.setup(h)}
560
579
  end
561
580
 
562
581
  def rec_input_types(taskname)
582
+ rec_inputs = rec_inputs(taskname)
563
583
  [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
564
- new = (Array === tn ? tn.first.tasks[tn[1].to_sym] : tasks[tn.to_sym]).input_types
584
+ if Array === tn and tn.first
585
+ new = tn.first.tasks[tn[1].to_sym].input_types
586
+ elsif Symbol === tn
587
+ new = tasks[tn.to_sym].input_types
588
+ else
589
+ next acc
590
+ end
565
591
  acc = new.merge(acc)
592
+ acc.delete_if{|input,defaults| not rec_inputs.include? input}
593
+ acc
566
594
  }.tap{|h| IndiferentHash.setup(h)}
567
595
  end
568
596
 
569
597
  def rec_input_descriptions(taskname)
598
+ rec_inputs = rec_inputs(taskname)
570
599
  [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
571
- new = (Array === tn ? tn.first.tasks[tn[1].to_sym] : tasks[tn.to_sym]).input_descriptions
600
+ if Array === tn and tn.first
601
+ new = tn.first.tasks[tn[1].to_sym].input_types
602
+ elsif Symbol === tn
603
+ new = tasks[tn.to_sym].input_types
604
+ else
605
+ next acc
606
+ end
572
607
  acc = new.merge(acc)
608
+ acc.delete_if{|input,defaults| not rec_inputs.include? input}
609
+ acc
573
610
  }.tap{|h| IndiferentHash.setup(h)}
574
611
  end
575
612
 
576
613
  def rec_input_options(taskname)
614
+ rec_inputs = rec_inputs(taskname)
577
615
  [taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
578
- new = (Array === tn ? tn.first.tasks[tn[1].to_sym] : tasks[tn.to_sym]).input_options
616
+ if Array === tn and tn.first
617
+ new = tn.first.tasks[tn[1].to_sym].input_types
618
+ elsif Symbol === tn
619
+ new = tasks[tn.to_sym].input_types
620
+ else
621
+ next acc
622
+ end
579
623
  acc = new.merge(acc)
624
+ acc = acc.delete_if{|input,defaults| not rec_inputs.include? input}
625
+ acc
580
626
  }.tap{|h| IndiferentHash.setup(h)}
581
627
  end
582
628
 
583
629
  def real_dependencies(task, jobname, inputs, dependencies)
584
630
  real_dependencies = []
631
+ path_deps = {}
585
632
  dependencies.each do |dependency|
586
- real_dependencies << case dependency
633
+ real_dep = case dependency
587
634
  when Array
588
635
  workflow, dep_task, options = dependency
589
636
 
590
637
  _inputs = IndiferentHash.setup(inputs.dup)
638
+ compute = options[:compute] if options
591
639
  options.each{|i,v|
640
+ next if i == :compute or i == "compute"
592
641
  case v
642
+ when :compute
643
+ compute = v
593
644
  when Symbol
594
645
  all_d = (real_dependencies + real_dependencies.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
595
646
  rec_dependency = all_d.select{|d| d.task_name.to_sym == v }.first
@@ -598,7 +649,7 @@ module Workflow
598
649
  _inputs[i] = v unless _inputs.include? i
599
650
  else
600
651
  input_options = workflow.task_info(dep_task)[:input_options][i] || {}
601
- if input_options[:stream]
652
+ if true or input_options[:stream]
602
653
  #rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
603
654
  _inputs[i] = rec_dependency
604
655
  else
@@ -617,8 +668,9 @@ module Workflow
617
668
  end
618
669
  } if options
619
670
 
620
- res = workflow.job(dep_task, jobname, _inputs)
621
- res
671
+ job = workflow.job(dep_task, jobname, _inputs)
672
+ ComputeDependency.setup(job, compute) if compute
673
+ job
622
674
  when Step
623
675
  dependency
624
676
  when Symbol
@@ -626,10 +678,28 @@ module Workflow
626
678
  job(dependency, jobname, _inputs)
627
679
  when Proc
628
680
  _inputs = IndiferentHash.setup(inputs.dup)
629
- dependency.call jobname, _inputs, real_dependencies
681
+ dep = dependency.call jobname, _inputs, real_dependencies
682
+
683
+ if DependencyBlock === dependency
684
+ orig_dep = dependency.dependency
685
+ if Hash === orig_dep.last
686
+ options = orig_dep.last
687
+ compute = options[:compute]
688
+
689
+ if Array === dep
690
+ dep.each{|d| ComputeDependency.setup(d, compute)}
691
+ elsif dep
692
+ ComputeDependency.setup(dep, compute)
693
+ end if compute
694
+ end
695
+ end
696
+
697
+ dep
630
698
  else
631
699
  raise "Dependency for #{task.name} not understood: #{Misc.fingerprint dependency}"
632
700
  end
701
+
702
+ real_dependencies << real_dep
633
703
  end
634
704
  real_dependencies.flatten.compact
635
705
  end
@@ -641,7 +711,7 @@ module Workflow
641
711
  if inputs.any? or dependencies.any?
642
712
  tagged_jobname = case TAG
643
713
  when :hash
644
- hash_str = Misc.obj2md5({:inputs => inputs, :dependencies => dependencies})
714
+ hash_str = Misc.obj2digest({:inputs => inputs, :dependencies => dependencies})
645
715
  jobname + '_' << hash_str
646
716
  else
647
717
  jobname
@@ -679,4 +749,5 @@ module Workflow
679
749
  dir = File.dirname(path)
680
750
  Misc.path_relative_to(workdir_find, dir).sub(/([^\/]+)\/.*/,'\1')
681
751
  end
752
+
682
753
  end
@@ -34,25 +34,16 @@ module Workflow
34
34
  @result_description = description
35
35
  end
36
36
 
37
- #def dep(*dependency_list, &block)
38
- # @dependencies ||= []
39
- # if Module === dependency_list.first or Hash === dependency_list.last
40
- # @dependencies << dependency_list
41
- # else
42
- # @dependency_list ||= []
43
- # if block_given?
44
- # dependency_list.unshift self if dependency_list.length == 1
45
- # dependency_list << block
46
- # end
47
- # dependencies.concat dependency_list
48
- # end
49
- #end
50
-
51
37
  def dep(*dependency, &block)
52
38
  @dependencies ||= []
53
39
  if block_given?
54
- dependency.unshift self if dependency.length == 1
55
- DependencyBlock.setup block, dependency if dependency.any?
40
+ if dependency.any?
41
+ wf, task, opt = dependency
42
+
43
+ opt, wf = wf, nil if Hash === wf
44
+ wf, task = self, wf if task.nil? and wf
45
+ DependencyBlock.setup block, [wf, task, opt]
46
+ end
56
47
  @dependencies << block
57
48
  else
58
49
  if Module === dependency.first or
@@ -76,7 +67,7 @@ module Workflow
76
67
  end
77
68
 
78
69
  name = name.to_sym
79
-
70
+
80
71
  block = self.method(name) unless block_given?
81
72
 
82
73
  task_info = {
@@ -117,4 +108,6 @@ module Workflow
117
108
  synchronous_exports.uniq!
118
109
  synchronous_exports
119
110
  end
111
+
112
+ alias export export_asynchronous
120
113
  end
@@ -155,7 +155,7 @@ class Step
155
155
 
156
156
  dep_hash ||= begin
157
157
  h = {}
158
- rec_dependencies.each{|dep| h[dep.task.name.to_s] ||= dep }
158
+ rec_dependencies.each{|dep| h[dep.task_name.to_s] ||= dep }
159
159
  h
160
160
  end
161
161
  dep = dep_hash[a]
@@ -204,7 +204,7 @@ class Step
204
204
  pid_file = Step.pid_file path
205
205
  files_dir = Step.files_dir path
206
206
 
207
- if Open.exists?(path) or Open.exists?(pid_file)
207
+ if Open.exists?(path) or Open.exists?(pid_file) or Open.exists?(info_file)
208
208
 
209
209
  @result = nil
210
210
  @pid = nil
@@ -245,22 +245,10 @@ class Step
245
245
  end
246
246
 
247
247
  def recursive_clean
248
- clean
249
- rec_dependencies.each do |step|
250
- if Open.exists?(step.info_file)
251
- step.clean
252
- end
253
- end
254
- self
255
- end
256
-
257
- def recursive_clean
258
- clean
259
248
  dependencies.each do |step|
260
- if Open.exists?(step.info_file)
261
- step.recursive_clean
262
- end
249
+ step.recursive_clean
263
250
  end
251
+ clean if Open.exists?(self.info_file)
264
252
  self
265
253
  end
266
254
 
@@ -0,0 +1,451 @@
1
+
2
+ class Step
3
+
4
+ ##STREAM_CACHE = {}
5
+ ##STREAM_CACHE_MUTEX = Mutex.new
6
+ ##def self.purge_stream_cache
7
+ ## Log.medium "Purging dup. stream cache"
8
+ ## STREAM_CACHE_MUTEX.synchronize do
9
+ ## #STREAM_CACHE.collect{|k,s|
10
+ ## # Thread.new do
11
+ ## # Misc.consume_stream s
12
+ ## # end
13
+ ## #}
14
+ ## STREAM_CACHE.clear
15
+ ## end
16
+ ##end
17
+
18
+ ##def self.dup_stream(stream)
19
+ ## case stream
20
+ ## when IO, File, Step
21
+ ## return stream if stream.respond_to?(:closed?) and stream.closed?
22
+ ## return stream if stream.respond_to?(:done?) and stream.done?
23
+
24
+ ## STREAM_CACHE_MUTEX.synchronize do
25
+ ## stream_key = Misc.fingerprint(stream)
26
+ ## current = STREAM_CACHE[stream_key]
27
+ ## case current
28
+ ## when nil
29
+ ## Log.medium "Not duplicating stream #{stream_key}"
30
+ ## STREAM_CACHE[stream_key] = stream
31
+ ## when File
32
+ ## if Open.exists? current.path
33
+ ## Log.medium "Reopening file #{stream_key}"
34
+ ## Open.open(current.path)
35
+ ## else
36
+ ## new = Misc.dup_stream(current)
37
+ ## Log.medium "Duplicating file #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
38
+ ## new
39
+ ## end
40
+ ## when Step
41
+ ## job = current
42
+ ## current = job.result
43
+ ## new = Misc.dup_stream(current)
44
+ ## job.result = current
45
+ ## Log.medium "Duplicating step #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
46
+ ## new
47
+ ## else
48
+ ## new = Misc.dup_stream(current)
49
+ ## Log.medium "Duplicating stream #{stream_key} #{ Misc.fingerprint(stream) } => #{Misc.fingerprint(new)}"
50
+ ## new
51
+ ## end
52
+ ## end
53
+ ## when TSV::Dumper#, TSV::Parser
54
+ ## stream = stream.stream
55
+ ## return stream if stream.closed?
56
+
57
+ ## STREAM_CACHE_MUTEX.synchronize do
58
+ ## if STREAM_CACHE[stream].nil?
59
+ ## Log.high "Not duplicating dumper #{ stream.inspect }"
60
+ ## STREAM_CACHE[stream] = stream
61
+ ## else
62
+ ## new = Misc.dup_stream(STREAM_CACHE[stream])
63
+ ## Log.high "Duplicating dumper #{ stream.inspect } into #{new.inspect}"
64
+ ## new
65
+ ## end
66
+ ## end
67
+ ## else
68
+ ## stream
69
+ ## end
70
+ ##end
71
+
72
+ def self.prepare_for_execution(job)
73
+ return if (job.done? and not job.dirty?) or
74
+ (job.streaming? and job.running?) or
75
+ (defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === job and not (job.error? or job.aborted?))
76
+
77
+ job.clean if job.error? or job.aborted? or (job.started? and not job.running? and not job.error?)
78
+
79
+ raise DependencyError, job if job.error?
80
+ end
81
+
82
+ def log_dependency_exec(dependency, action)
83
+ task_name = self.task_name
84
+
85
+ str = Log.color(:reset, "")
86
+ str << Log.color(:yellow, task_name.to_s || "")
87
+ str << " "
88
+ str << Log.color(:magenta, action.to_s)
89
+ str << " "
90
+ str << Log.color(:yellow, dependency.task_name.to_s || "")
91
+ str << " -- "
92
+ str << "#{Log.color :blue, dependency.path}"
93
+
94
+ Log.info str
95
+ end
96
+
97
+ def execute_dependency(dependency, log = true)
98
+ task_name = self.task_name
99
+ begin
100
+
101
+ dependency.resolve_input_steps
102
+
103
+ if dependency.done?
104
+ log_dependency_exec(dependency, :done) if log
105
+ return
106
+ end
107
+
108
+ if not dependency.started?
109
+ log_dependency_exec(dependency, :starting)
110
+ dependency.run(:stream)
111
+ raise TryAgain
112
+ end
113
+
114
+ dependency.grace
115
+
116
+ if dependency.aborted?
117
+ log_dependency_exec(dependency, "aborted (clean)")
118
+ dependency.clean
119
+ raise TryAgain
120
+ end
121
+
122
+ if dependency.error?
123
+ log_dependency_exec(dependency, :error)
124
+ raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
125
+ end
126
+
127
+ if dependency.streaming?
128
+ log_dependency_exec(dependency, :streaming) if log
129
+ return
130
+ end
131
+
132
+ begin
133
+ log_dependency_exec(dependency, :joining)
134
+ dependency.join
135
+ raise TryAgain unless dependency.done?
136
+ rescue Aborted
137
+ raise TryAgain
138
+ end
139
+
140
+ rescue TryAgain
141
+ retry
142
+ rescue Aborted, Interrupt
143
+ Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
144
+ raise $!
145
+ rescue Interrupt
146
+ Log.error "Interrupted while in dep. #{Log.color :red, dependency.task_name.to_s}"
147
+ raise $!
148
+ rescue Exception
149
+ Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s }"
150
+ Log.exception $!
151
+ raise $!
152
+ end
153
+ end
154
+
155
+ #def dup_inputs
156
+ # return if true or @dupped or ENV["RBBT_NO_STREAM"] == 'true'
157
+ # Log.low "Dupping inputs for #{path}"
158
+ # dupped_inputs = @inputs.collect do |input|
159
+ # Step.dup_stream input
160
+ # end
161
+ # @inputs.replace dupped_inputs
162
+ # @dupped = true
163
+ #end
164
+
165
+ def consolidate_dependencies(path_deps = {})
166
+ return false if @consolidated or dependencies.nil? or dependencies.empty?
167
+ consolidated_deps = dependencies.collect do |dep|
168
+ dep.consolidate_dependencies(path_deps)
169
+ path = dep.path
170
+ path_deps[path] ||= dep
171
+ end
172
+ dependencies.replace consolidated_deps
173
+ @consolidated = true
174
+ end
175
+
176
+ #def prepare_dependencies
177
+ # dep_step = {}
178
+
179
+ # all_deps = rec_dependencies + [self]
180
+
181
+ # seen_paths = Set.new
182
+ # all_deps.uniq.each do |step|
183
+ # next if seen_paths.include? step.path
184
+ # Step.prepare_for_execution(step)
185
+ # seen_paths << step.path
186
+ # step.dependencies.each do |step_dep|
187
+ # dep_step[step_dep.path] ||= []
188
+ # dep_step[step_dep.path] << step_dep
189
+ # end
190
+ # end
191
+
192
+ # seen_paths = Set.new
193
+ # rec_dependencies.uniq.each do |step|
194
+ # next if seen_paths.include? step.path
195
+ # = {} seen_paths << step.path
196
+ # execute_dependency(step)
197
+ # if step.streaming? and step.result
198
+ # if dep_step[step.path] and dep_step[step.path].length > 1
199
+ # stream = step.result
200
+ # other_steps = dep_step[step.path] - [step]
201
+ # copies = Misc.dup_stream_multiple(stream, other_steps.length)
202
+ # other_steps.zip(copies).each do |other,dupped_stream|
203
+ # other.instance_variable_set(:@result, dupped_stream)
204
+ # end
205
+ # end
206
+ # end
207
+ # end
208
+
209
+ # Step.purge_stream_cache
210
+ #end
211
+
212
+ def execute_and_dup(step, dep_step, log = true)
213
+ dup = ! step.result
214
+ execute_dependency(step, log)
215
+ if dup and step.streaming? and step.result
216
+ if dep_step[step.path] and dep_step[step.path].length > 1
217
+ stream = step.result
218
+ other_steps = dep_step[step.path] - [step]
219
+ copies = Misc.dup_stream_multiple(stream, other_steps.length)
220
+ log_dependency_exec(step, "duplicating #{copies.length}")
221
+ other_steps.zip(copies).each do |other,dupped_stream|
222
+ other.instance_variable_set(:@result, dupped_stream)
223
+ end
224
+ end
225
+ end
226
+ end
227
+
228
+ def run_compute_dependencies(type, list, dep_step = {})
229
+ if Array === type
230
+ type, *rest = type
231
+ end
232
+
233
+ case type
234
+ when :produce, :no_dup
235
+ list.each do |step|
236
+ step.produce
237
+ nil
238
+ end
239
+ when :bootstrap
240
+ cpus = rest.nil? ? nil : rest.first
241
+ cpus = 30 if cpus.nil?
242
+ cpus = list.length / 2 if cpus > list.length / 2
243
+
244
+ Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :_respawn => :always) do |dep|
245
+ dep.produce
246
+ nil
247
+ end
248
+ else
249
+ list.each do |step|
250
+ execute_and_dup(step, dep_step, false)
251
+ end
252
+ end
253
+ end
254
+
255
+ def run_dependencies
256
+ dep_step = {}
257
+
258
+ all_deps = rec_dependencies + [self]
259
+
260
+ dependencies.each do |dep|
261
+ next unless ComputeDependency === dep
262
+ if dep.compute == :produce
263
+ dep.produce
264
+ end
265
+ end
266
+
267
+ compute_deps = rec_dependencies.collect do |dep|
268
+ next unless ComputeDependency === dep
269
+ dep.rec_dependencies
270
+ end.compact.flatten.uniq
271
+
272
+ seen_paths = Set.new
273
+ all_deps.uniq.each do |step|
274
+ next if seen_paths.include? step.path
275
+ seen_paths << step.path
276
+ Step.prepare_for_execution(step) unless step == self
277
+ next unless step.dependencies and step.dependencies.any?
278
+ step.dependencies.each do |step_dep|
279
+ next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and step_dep.compute == :nodup)
280
+ dep_step[step_dep.path] ||= []
281
+ dep_step[step_dep.path] << step_dep
282
+ end
283
+ end
284
+
285
+ required_dep_paths = []
286
+ dep_step.each do |path,list|
287
+ required_dep_paths << path if list.length > 1
288
+ end
289
+
290
+ required_dep_paths.concat dependencies.collect{|dep| dep.path }
291
+
292
+ log :dependencies, "Dependencies for step #{Log.color :yellow, task.name.to_s || ""}"
293
+
294
+ pre_deps = []
295
+ compute_pre_deps = {}
296
+ last_deps = []
297
+ compute_last_deps = {}
298
+ seen_paths = Set.new
299
+ rec_dependencies.uniq.each do |step|
300
+ next if seen_paths.include? step.path
301
+ seen_paths << step.path
302
+ next unless required_dep_paths.include? step.path
303
+ if dependencies.include?(step) and step.inputs.flatten.select{|i| Step === i}.any?
304
+ if ComputeDependency === step
305
+ compute_last_deps[step.compute] ||= []
306
+ compute_last_deps[step.compute] << step
307
+ else
308
+ last_deps << step
309
+ end
310
+ else
311
+ if ComputeDependency === step
312
+ compute_pre_deps[step.compute] ||= []
313
+ compute_pre_deps[step.compute] << step
314
+ else
315
+ pre_deps << step #if dependencies.include?(step)
316
+ end
317
+ end
318
+ end
319
+
320
+ pre_deps.each do |step|
321
+ next if compute_deps.include? step
322
+ execute_and_dup(step, dep_step, false)
323
+ end
324
+
325
+ compute_pre_deps.each do |type,list|
326
+ run_compute_dependencies(type, list, dep_step)
327
+ end
328
+
329
+ last_deps.each do |step|
330
+ next if compute_deps.include? step
331
+ execute_and_dup(step, dep_step)
332
+ end
333
+
334
+ compute_last_deps.each do |type,list|
335
+ run_compute_dependencies(type, list, dep_step)
336
+ end
337
+
338
+ end
339
+
340
+ def stop_dependencies
341
+ dependencies.each do |dep|
342
+ dep.abort
343
+ end
344
+ kill_children
345
+ end
346
+
347
+ #def run_dependencies
348
+ # @seen ||= []
349
+ # seen_paths ||= Set.new
350
+ #
351
+ # consolidate_dependencies
352
+ # dependencies.uniq.each do |dependency|
353
+ # dependency_path = dependency.path
354
+ # next if seen_paths.include? dependency_path
355
+ # @seen.concat dependency.rec_dependencies
356
+ # seen_paths.union(dependency.rec_dependencies.collect{|d| d.path})
357
+ # @seen << dependency
358
+ # seen_paths << dependency_path
359
+ # end
360
+
361
+ # @seen.uniq!
362
+ # @seen.delete self
363
+
364
+ # return if @seen.empty?
365
+
366
+ # log :dependencies, "#{Log.color :magenta, "Dependencies"} for step #{Log.color :yellow, task.name.to_s || ""}"
367
+
368
+ # @seen.each do |dependency|
369
+ # Step.prepare_for_execution(dependency)
370
+ # end
371
+
372
+ # pre_deps = []
373
+ # compute_pre_deps = {}
374
+ # last_deps = []
375
+ # compute_last_deps = {}
376
+ # @seen.each do |dependency|
377
+ # if dependencies.include?(dependency) and dependency.inputs.flatten.select{|i| Step === i}.any?
378
+ # if ComputeDependency === dependency
379
+ # compute_last_deps[dependency.compute] ||= []
380
+ # compute_last_deps[dependency.compute] << dependency
381
+ # else
382
+ # last_deps << dependency
383
+ # end
384
+ # else
385
+ # if ComputeDependency === dependency
386
+ # compute_pre_deps[dependency.compute] ||= []
387
+ # compute_pre_deps[dependency.compute] << dependency
388
+ # else
389
+ # pre_deps << dependency if dependencies.include?(dependency)
390
+ # end
391
+ # end
392
+ # end
393
+
394
+ # pre_deps.each do |dependency|
395
+ # dependency.dup_inputs
396
+ # execute_dependency(dependency)
397
+ # end
398
+
399
+ # compute_pre_deps.each do |type,list|
400
+ # if Array === type
401
+ # type, *rest = type
402
+ # end
403
+
404
+ # case type
405
+ # when :bootstrap
406
+ # cpus = rest.nil? ? nil : rest.first
407
+ # cpus = 10 if cpus.nil?
408
+
409
+ # list.each do |dependency|
410
+ # dependency.dup_inputs
411
+ # end
412
+
413
+ # Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :_respawn => :always) do |dep|
414
+ # dep.produce
415
+ # nil
416
+ # end
417
+ # else
418
+ # list.each do |dependency|
419
+ # dependency.dup_inputs
420
+ # execute_dependency(dependency)
421
+ # end
422
+ # end
423
+ # end
424
+
425
+ # last_deps.each do |dependency|
426
+ # dependency.dup_inputs
427
+ # end
428
+
429
+ # last_deps.each do |dependency|
430
+ # execute_dependency(dependency)
431
+ # end
432
+
433
+ # compute_last_deps.each do |type,list|
434
+ # case type
435
+ # when :_bootstrap
436
+ # list.each do |dependency|
437
+ # dependency.dup_inputs
438
+ # end
439
+ # Misc.bootstrap(list, 3, :bar => "Boostrapping dependencies for #{path}", :respawn => :always) do |dependency|
440
+ # dependency.produce
441
+ # nil
442
+ # end
443
+ # else
444
+ # list.each do |dependency|
445
+ # dependency.dup_inputs
446
+ # execute_dependency(dependency)
447
+ # end
448
+ # end
449
+ # end
450
+ #end
451
+ end