rbbt-util 5.20.2 → 5.20.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/association/index.rb +3 -4
- data/lib/rbbt/tsv/accessor.rb +1 -1
- data/lib/rbbt/tsv/manipulate.rb +0 -1
- data/lib/rbbt/tsv/parallel/traverse.rb +8 -2
- data/lib/rbbt/util/cmd.rb +1 -1
- data/lib/rbbt/util/concurrency/processes.rb +4 -3
- data/lib/rbbt/util/concurrency/processes/worker.rb +4 -2
- data/lib/rbbt/util/log/progress/util.rb +20 -1
- data/lib/rbbt/util/misc/bgzf.rb +1 -1
- data/lib/rbbt/util/misc/inspect.rb +7 -3
- data/lib/rbbt/util/misc/multipart_payload.rb +8 -8
- data/lib/rbbt/util/misc/pipes.rb +115 -51
- data/lib/rbbt/util/open.rb +1 -1
- data/lib/rbbt/workflow.rb +1 -0
- data/lib/rbbt/workflow/accessor.rb +86 -15
- data/lib/rbbt/workflow/definition.rb +10 -17
- data/lib/rbbt/workflow/step.rb +4 -16
- data/lib/rbbt/workflow/step/dependencies.rb +451 -0
- data/lib/rbbt/workflow/step/run.rb +43 -226
- data/lib/rbbt/workflow/task.rb +7 -1
- data/share/rbbt_commands/association/subset +6 -1
- data/share/rbbt_commands/workflow/task +21 -9
- data/test/rbbt/util/misc/test_pipes.rb +54 -0
- data/test/rbbt/workflow/step/test_dependencies.rb +206 -0
- metadata +5 -2
data/lib/rbbt/util/open.rb
CHANGED
data/lib/rbbt/workflow.rb
CHANGED
@@ -1,6 +1,14 @@
|
|
1
1
|
require 'rbbt/util/open'
|
2
2
|
require 'yaml'
|
3
3
|
|
4
|
+
module ComputeDependency
|
5
|
+
attr_accessor :compute
|
6
|
+
def self.setup(dep, value)
|
7
|
+
dep.extend ComputeDependency
|
8
|
+
dep.compute = value
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
4
12
|
class Step
|
5
13
|
|
6
14
|
|
@@ -493,7 +501,7 @@ module Workflow
|
|
493
501
|
wf, t, o = dep
|
494
502
|
wf.rec_dependencies(t).each do |d|
|
495
503
|
if Array === d
|
496
|
-
new = d
|
504
|
+
new = d.dup
|
497
505
|
else
|
498
506
|
new = [dep.first, d]
|
499
507
|
end
|
@@ -513,10 +521,12 @@ module Workflow
|
|
513
521
|
all_deps << dep.dependency if dep.dependency
|
514
522
|
case dep.dependency
|
515
523
|
when Array
|
516
|
-
dep_wf, dep_task = dep.dependency
|
517
|
-
|
518
|
-
|
519
|
-
|
524
|
+
dep_wf, dep_task, dep_options = dep.dependency
|
525
|
+
if dep_task === Symbol
|
526
|
+
dep_rec_dependencies = dep_wf.rec_dependencies(dep_task.to_sym)
|
527
|
+
dep_rec_dependencies.collect!{|d| Array === d ? d : [dep_wf, d]}
|
528
|
+
all_deps.concat dep_rec_dependencies
|
529
|
+
end
|
520
530
|
when Symbol, String
|
521
531
|
all_deps.concat rec_dependencies(dep.dependency.to_sym)
|
522
532
|
end
|
@@ -553,43 +563,84 @@ module Workflow
|
|
553
563
|
end
|
554
564
|
|
555
565
|
def rec_input_defaults(taskname)
|
566
|
+
rec_inputs = rec_inputs(taskname)
|
556
567
|
[taskname].concat(rec_dependencies(taskname)).inject(IndiferentHash.setup({})){|acc, tn|
|
557
|
-
|
568
|
+
if Array === tn and tn.first
|
569
|
+
new = tn.first.tasks[tn[1].to_sym].input_types
|
570
|
+
elsif Symbol === tn
|
571
|
+
new = tasks[tn.to_sym].input_types
|
572
|
+
else
|
573
|
+
next acc
|
574
|
+
end
|
558
575
|
acc = new.merge(acc)
|
576
|
+
acc.delete_if{|input,defaults| not rec_inputs.include? input}
|
577
|
+
acc
|
559
578
|
}.tap{|h| IndiferentHash.setup(h)}
|
560
579
|
end
|
561
580
|
|
562
581
|
def rec_input_types(taskname)
|
582
|
+
rec_inputs = rec_inputs(taskname)
|
563
583
|
[taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
|
564
|
-
|
584
|
+
if Array === tn and tn.first
|
585
|
+
new = tn.first.tasks[tn[1].to_sym].input_types
|
586
|
+
elsif Symbol === tn
|
587
|
+
new = tasks[tn.to_sym].input_types
|
588
|
+
else
|
589
|
+
next acc
|
590
|
+
end
|
565
591
|
acc = new.merge(acc)
|
592
|
+
acc.delete_if{|input,defaults| not rec_inputs.include? input}
|
593
|
+
acc
|
566
594
|
}.tap{|h| IndiferentHash.setup(h)}
|
567
595
|
end
|
568
596
|
|
569
597
|
def rec_input_descriptions(taskname)
|
598
|
+
rec_inputs = rec_inputs(taskname)
|
570
599
|
[taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
|
571
|
-
|
600
|
+
if Array === tn and tn.first
|
601
|
+
new = tn.first.tasks[tn[1].to_sym].input_types
|
602
|
+
elsif Symbol === tn
|
603
|
+
new = tasks[tn.to_sym].input_types
|
604
|
+
else
|
605
|
+
next acc
|
606
|
+
end
|
572
607
|
acc = new.merge(acc)
|
608
|
+
acc.delete_if{|input,defaults| not rec_inputs.include? input}
|
609
|
+
acc
|
573
610
|
}.tap{|h| IndiferentHash.setup(h)}
|
574
611
|
end
|
575
612
|
|
576
613
|
def rec_input_options(taskname)
|
614
|
+
rec_inputs = rec_inputs(taskname)
|
577
615
|
[taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
|
578
|
-
|
616
|
+
if Array === tn and tn.first
|
617
|
+
new = tn.first.tasks[tn[1].to_sym].input_types
|
618
|
+
elsif Symbol === tn
|
619
|
+
new = tasks[tn.to_sym].input_types
|
620
|
+
else
|
621
|
+
next acc
|
622
|
+
end
|
579
623
|
acc = new.merge(acc)
|
624
|
+
acc = acc.delete_if{|input,defaults| not rec_inputs.include? input}
|
625
|
+
acc
|
580
626
|
}.tap{|h| IndiferentHash.setup(h)}
|
581
627
|
end
|
582
628
|
|
583
629
|
def real_dependencies(task, jobname, inputs, dependencies)
|
584
630
|
real_dependencies = []
|
631
|
+
path_deps = {}
|
585
632
|
dependencies.each do |dependency|
|
586
|
-
|
633
|
+
real_dep = case dependency
|
587
634
|
when Array
|
588
635
|
workflow, dep_task, options = dependency
|
589
636
|
|
590
637
|
_inputs = IndiferentHash.setup(inputs.dup)
|
638
|
+
compute = options[:compute] if options
|
591
639
|
options.each{|i,v|
|
640
|
+
next if i == :compute or i == "compute"
|
592
641
|
case v
|
642
|
+
when :compute
|
643
|
+
compute = v
|
593
644
|
when Symbol
|
594
645
|
all_d = (real_dependencies + real_dependencies.collect{|d| d.rec_dependencies} ).flatten.compact.uniq
|
595
646
|
rec_dependency = all_d.select{|d| d.task_name.to_sym == v }.first
|
@@ -598,7 +649,7 @@ module Workflow
|
|
598
649
|
_inputs[i] = v unless _inputs.include? i
|
599
650
|
else
|
600
651
|
input_options = workflow.task_info(dep_task)[:input_options][i] || {}
|
601
|
-
if input_options[:stream]
|
652
|
+
if true or input_options[:stream]
|
602
653
|
#rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
|
603
654
|
_inputs[i] = rec_dependency
|
604
655
|
else
|
@@ -617,8 +668,9 @@ module Workflow
|
|
617
668
|
end
|
618
669
|
} if options
|
619
670
|
|
620
|
-
|
621
|
-
|
671
|
+
job = workflow.job(dep_task, jobname, _inputs)
|
672
|
+
ComputeDependency.setup(job, compute) if compute
|
673
|
+
job
|
622
674
|
when Step
|
623
675
|
dependency
|
624
676
|
when Symbol
|
@@ -626,10 +678,28 @@ module Workflow
|
|
626
678
|
job(dependency, jobname, _inputs)
|
627
679
|
when Proc
|
628
680
|
_inputs = IndiferentHash.setup(inputs.dup)
|
629
|
-
dependency.call jobname, _inputs, real_dependencies
|
681
|
+
dep = dependency.call jobname, _inputs, real_dependencies
|
682
|
+
|
683
|
+
if DependencyBlock === dependency
|
684
|
+
orig_dep = dependency.dependency
|
685
|
+
if Hash === orig_dep.last
|
686
|
+
options = orig_dep.last
|
687
|
+
compute = options[:compute]
|
688
|
+
|
689
|
+
if Array === dep
|
690
|
+
dep.each{|d| ComputeDependency.setup(d, compute)}
|
691
|
+
elsif dep
|
692
|
+
ComputeDependency.setup(dep, compute)
|
693
|
+
end if compute
|
694
|
+
end
|
695
|
+
end
|
696
|
+
|
697
|
+
dep
|
630
698
|
else
|
631
699
|
raise "Dependency for #{task.name} not understood: #{Misc.fingerprint dependency}"
|
632
700
|
end
|
701
|
+
|
702
|
+
real_dependencies << real_dep
|
633
703
|
end
|
634
704
|
real_dependencies.flatten.compact
|
635
705
|
end
|
@@ -641,7 +711,7 @@ module Workflow
|
|
641
711
|
if inputs.any? or dependencies.any?
|
642
712
|
tagged_jobname = case TAG
|
643
713
|
when :hash
|
644
|
-
hash_str = Misc.
|
714
|
+
hash_str = Misc.obj2digest({:inputs => inputs, :dependencies => dependencies})
|
645
715
|
jobname + '_' << hash_str
|
646
716
|
else
|
647
717
|
jobname
|
@@ -679,4 +749,5 @@ module Workflow
|
|
679
749
|
dir = File.dirname(path)
|
680
750
|
Misc.path_relative_to(workdir_find, dir).sub(/([^\/]+)\/.*/,'\1')
|
681
751
|
end
|
752
|
+
|
682
753
|
end
|
@@ -34,25 +34,16 @@ module Workflow
|
|
34
34
|
@result_description = description
|
35
35
|
end
|
36
36
|
|
37
|
-
#def dep(*dependency_list, &block)
|
38
|
-
# @dependencies ||= []
|
39
|
-
# if Module === dependency_list.first or Hash === dependency_list.last
|
40
|
-
# @dependencies << dependency_list
|
41
|
-
# else
|
42
|
-
# @dependency_list ||= []
|
43
|
-
# if block_given?
|
44
|
-
# dependency_list.unshift self if dependency_list.length == 1
|
45
|
-
# dependency_list << block
|
46
|
-
# end
|
47
|
-
# dependencies.concat dependency_list
|
48
|
-
# end
|
49
|
-
#end
|
50
|
-
|
51
37
|
def dep(*dependency, &block)
|
52
38
|
@dependencies ||= []
|
53
39
|
if block_given?
|
54
|
-
|
55
|
-
|
40
|
+
if dependency.any?
|
41
|
+
wf, task, opt = dependency
|
42
|
+
|
43
|
+
opt, wf = wf, nil if Hash === wf
|
44
|
+
wf, task = self, wf if task.nil? and wf
|
45
|
+
DependencyBlock.setup block, [wf, task, opt]
|
46
|
+
end
|
56
47
|
@dependencies << block
|
57
48
|
else
|
58
49
|
if Module === dependency.first or
|
@@ -76,7 +67,7 @@ module Workflow
|
|
76
67
|
end
|
77
68
|
|
78
69
|
name = name.to_sym
|
79
|
-
|
70
|
+
|
80
71
|
block = self.method(name) unless block_given?
|
81
72
|
|
82
73
|
task_info = {
|
@@ -117,4 +108,6 @@ module Workflow
|
|
117
108
|
synchronous_exports.uniq!
|
118
109
|
synchronous_exports
|
119
110
|
end
|
111
|
+
|
112
|
+
alias export export_asynchronous
|
120
113
|
end
|
data/lib/rbbt/workflow/step.rb
CHANGED
@@ -155,7 +155,7 @@ class Step
|
|
155
155
|
|
156
156
|
dep_hash ||= begin
|
157
157
|
h = {}
|
158
|
-
rec_dependencies.each{|dep| h[dep.
|
158
|
+
rec_dependencies.each{|dep| h[dep.task_name.to_s] ||= dep }
|
159
159
|
h
|
160
160
|
end
|
161
161
|
dep = dep_hash[a]
|
@@ -204,7 +204,7 @@ class Step
|
|
204
204
|
pid_file = Step.pid_file path
|
205
205
|
files_dir = Step.files_dir path
|
206
206
|
|
207
|
-
if Open.exists?(path) or Open.exists?(pid_file)
|
207
|
+
if Open.exists?(path) or Open.exists?(pid_file) or Open.exists?(info_file)
|
208
208
|
|
209
209
|
@result = nil
|
210
210
|
@pid = nil
|
@@ -245,22 +245,10 @@ class Step
|
|
245
245
|
end
|
246
246
|
|
247
247
|
def recursive_clean
|
248
|
-
clean
|
249
|
-
rec_dependencies.each do |step|
|
250
|
-
if Open.exists?(step.info_file)
|
251
|
-
step.clean
|
252
|
-
end
|
253
|
-
end
|
254
|
-
self
|
255
|
-
end
|
256
|
-
|
257
|
-
def recursive_clean
|
258
|
-
clean
|
259
248
|
dependencies.each do |step|
|
260
|
-
|
261
|
-
step.recursive_clean
|
262
|
-
end
|
249
|
+
step.recursive_clean
|
263
250
|
end
|
251
|
+
clean if Open.exists?(self.info_file)
|
264
252
|
self
|
265
253
|
end
|
266
254
|
|
@@ -0,0 +1,451 @@
|
|
1
|
+
|
2
|
+
class Step
|
3
|
+
|
4
|
+
##STREAM_CACHE = {}
|
5
|
+
##STREAM_CACHE_MUTEX = Mutex.new
|
6
|
+
##def self.purge_stream_cache
|
7
|
+
## Log.medium "Purging dup. stream cache"
|
8
|
+
## STREAM_CACHE_MUTEX.synchronize do
|
9
|
+
## #STREAM_CACHE.collect{|k,s|
|
10
|
+
## # Thread.new do
|
11
|
+
## # Misc.consume_stream s
|
12
|
+
## # end
|
13
|
+
## #}
|
14
|
+
## STREAM_CACHE.clear
|
15
|
+
## end
|
16
|
+
##end
|
17
|
+
|
18
|
+
##def self.dup_stream(stream)
|
19
|
+
## case stream
|
20
|
+
## when IO, File, Step
|
21
|
+
## return stream if stream.respond_to?(:closed?) and stream.closed?
|
22
|
+
## return stream if stream.respond_to?(:done?) and stream.done?
|
23
|
+
|
24
|
+
## STREAM_CACHE_MUTEX.synchronize do
|
25
|
+
## stream_key = Misc.fingerprint(stream)
|
26
|
+
## current = STREAM_CACHE[stream_key]
|
27
|
+
## case current
|
28
|
+
## when nil
|
29
|
+
## Log.medium "Not duplicating stream #{stream_key}"
|
30
|
+
## STREAM_CACHE[stream_key] = stream
|
31
|
+
## when File
|
32
|
+
## if Open.exists? current.path
|
33
|
+
## Log.medium "Reopening file #{stream_key}"
|
34
|
+
## Open.open(current.path)
|
35
|
+
## else
|
36
|
+
## new = Misc.dup_stream(current)
|
37
|
+
## Log.medium "Duplicating file #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
|
38
|
+
## new
|
39
|
+
## end
|
40
|
+
## when Step
|
41
|
+
## job = current
|
42
|
+
## current = job.result
|
43
|
+
## new = Misc.dup_stream(current)
|
44
|
+
## job.result = current
|
45
|
+
## Log.medium "Duplicating step #{stream_key} #{current.inspect} => #{Misc.fingerprint(new)}"
|
46
|
+
## new
|
47
|
+
## else
|
48
|
+
## new = Misc.dup_stream(current)
|
49
|
+
## Log.medium "Duplicating stream #{stream_key} #{ Misc.fingerprint(stream) } => #{Misc.fingerprint(new)}"
|
50
|
+
## new
|
51
|
+
## end
|
52
|
+
## end
|
53
|
+
## when TSV::Dumper#, TSV::Parser
|
54
|
+
## stream = stream.stream
|
55
|
+
## return stream if stream.closed?
|
56
|
+
|
57
|
+
## STREAM_CACHE_MUTEX.synchronize do
|
58
|
+
## if STREAM_CACHE[stream].nil?
|
59
|
+
## Log.high "Not duplicating dumper #{ stream.inspect }"
|
60
|
+
## STREAM_CACHE[stream] = stream
|
61
|
+
## else
|
62
|
+
## new = Misc.dup_stream(STREAM_CACHE[stream])
|
63
|
+
## Log.high "Duplicating dumper #{ stream.inspect } into #{new.inspect}"
|
64
|
+
## new
|
65
|
+
## end
|
66
|
+
## end
|
67
|
+
## else
|
68
|
+
## stream
|
69
|
+
## end
|
70
|
+
##end
|
71
|
+
|
72
|
+
def self.prepare_for_execution(job)
|
73
|
+
return if (job.done? and not job.dirty?) or
|
74
|
+
(job.streaming? and job.running?) or
|
75
|
+
(defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === job and not (job.error? or job.aborted?))
|
76
|
+
|
77
|
+
job.clean if job.error? or job.aborted? or (job.started? and not job.running? and not job.error?)
|
78
|
+
|
79
|
+
raise DependencyError, job if job.error?
|
80
|
+
end
|
81
|
+
|
82
|
+
def log_dependency_exec(dependency, action)
|
83
|
+
task_name = self.task_name
|
84
|
+
|
85
|
+
str = Log.color(:reset, "")
|
86
|
+
str << Log.color(:yellow, task_name.to_s || "")
|
87
|
+
str << " "
|
88
|
+
str << Log.color(:magenta, action.to_s)
|
89
|
+
str << " "
|
90
|
+
str << Log.color(:yellow, dependency.task_name.to_s || "")
|
91
|
+
str << " -- "
|
92
|
+
str << "#{Log.color :blue, dependency.path}"
|
93
|
+
|
94
|
+
Log.info str
|
95
|
+
end
|
96
|
+
|
97
|
+
def execute_dependency(dependency, log = true)
|
98
|
+
task_name = self.task_name
|
99
|
+
begin
|
100
|
+
|
101
|
+
dependency.resolve_input_steps
|
102
|
+
|
103
|
+
if dependency.done?
|
104
|
+
log_dependency_exec(dependency, :done) if log
|
105
|
+
return
|
106
|
+
end
|
107
|
+
|
108
|
+
if not dependency.started?
|
109
|
+
log_dependency_exec(dependency, :starting)
|
110
|
+
dependency.run(:stream)
|
111
|
+
raise TryAgain
|
112
|
+
end
|
113
|
+
|
114
|
+
dependency.grace
|
115
|
+
|
116
|
+
if dependency.aborted?
|
117
|
+
log_dependency_exec(dependency, "aborted (clean)")
|
118
|
+
dependency.clean
|
119
|
+
raise TryAgain
|
120
|
+
end
|
121
|
+
|
122
|
+
if dependency.error?
|
123
|
+
log_dependency_exec(dependency, :error)
|
124
|
+
raise DependencyError, [dependency.path, dependency.messages.last] * ": " if dependency.error?
|
125
|
+
end
|
126
|
+
|
127
|
+
if dependency.streaming?
|
128
|
+
log_dependency_exec(dependency, :streaming) if log
|
129
|
+
return
|
130
|
+
end
|
131
|
+
|
132
|
+
begin
|
133
|
+
log_dependency_exec(dependency, :joining)
|
134
|
+
dependency.join
|
135
|
+
raise TryAgain unless dependency.done?
|
136
|
+
rescue Aborted
|
137
|
+
raise TryAgain
|
138
|
+
end
|
139
|
+
|
140
|
+
rescue TryAgain
|
141
|
+
retry
|
142
|
+
rescue Aborted, Interrupt
|
143
|
+
Log.error "Aborted dep. #{Log.color :red, dependency.task_name.to_s}"
|
144
|
+
raise $!
|
145
|
+
rescue Interrupt
|
146
|
+
Log.error "Interrupted while in dep. #{Log.color :red, dependency.task_name.to_s}"
|
147
|
+
raise $!
|
148
|
+
rescue Exception
|
149
|
+
Log.error "Exception in dep. #{ Log.color :red, dependency.task_name.to_s }"
|
150
|
+
Log.exception $!
|
151
|
+
raise $!
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
#def dup_inputs
|
156
|
+
# return if true or @dupped or ENV["RBBT_NO_STREAM"] == 'true'
|
157
|
+
# Log.low "Dupping inputs for #{path}"
|
158
|
+
# dupped_inputs = @inputs.collect do |input|
|
159
|
+
# Step.dup_stream input
|
160
|
+
# end
|
161
|
+
# @inputs.replace dupped_inputs
|
162
|
+
# @dupped = true
|
163
|
+
#end
|
164
|
+
|
165
|
+
def consolidate_dependencies(path_deps = {})
|
166
|
+
return false if @consolidated or dependencies.nil? or dependencies.empty?
|
167
|
+
consolidated_deps = dependencies.collect do |dep|
|
168
|
+
dep.consolidate_dependencies(path_deps)
|
169
|
+
path = dep.path
|
170
|
+
path_deps[path] ||= dep
|
171
|
+
end
|
172
|
+
dependencies.replace consolidated_deps
|
173
|
+
@consolidated = true
|
174
|
+
end
|
175
|
+
|
176
|
+
#def prepare_dependencies
|
177
|
+
# dep_step = {}
|
178
|
+
|
179
|
+
# all_deps = rec_dependencies + [self]
|
180
|
+
|
181
|
+
# seen_paths = Set.new
|
182
|
+
# all_deps.uniq.each do |step|
|
183
|
+
# next if seen_paths.include? step.path
|
184
|
+
# Step.prepare_for_execution(step)
|
185
|
+
# seen_paths << step.path
|
186
|
+
# step.dependencies.each do |step_dep|
|
187
|
+
# dep_step[step_dep.path] ||= []
|
188
|
+
# dep_step[step_dep.path] << step_dep
|
189
|
+
# end
|
190
|
+
# end
|
191
|
+
|
192
|
+
# seen_paths = Set.new
|
193
|
+
# rec_dependencies.uniq.each do |step|
|
194
|
+
# next if seen_paths.include? step.path
|
195
|
+
# = {} seen_paths << step.path
|
196
|
+
# execute_dependency(step)
|
197
|
+
# if step.streaming? and step.result
|
198
|
+
# if dep_step[step.path] and dep_step[step.path].length > 1
|
199
|
+
# stream = step.result
|
200
|
+
# other_steps = dep_step[step.path] - [step]
|
201
|
+
# copies = Misc.dup_stream_multiple(stream, other_steps.length)
|
202
|
+
# other_steps.zip(copies).each do |other,dupped_stream|
|
203
|
+
# other.instance_variable_set(:@result, dupped_stream)
|
204
|
+
# end
|
205
|
+
# end
|
206
|
+
# end
|
207
|
+
# end
|
208
|
+
|
209
|
+
# Step.purge_stream_cache
|
210
|
+
#end
|
211
|
+
|
212
|
+
def execute_and_dup(step, dep_step, log = true)
|
213
|
+
dup = ! step.result
|
214
|
+
execute_dependency(step, log)
|
215
|
+
if dup and step.streaming? and step.result
|
216
|
+
if dep_step[step.path] and dep_step[step.path].length > 1
|
217
|
+
stream = step.result
|
218
|
+
other_steps = dep_step[step.path] - [step]
|
219
|
+
copies = Misc.dup_stream_multiple(stream, other_steps.length)
|
220
|
+
log_dependency_exec(step, "duplicating #{copies.length}")
|
221
|
+
other_steps.zip(copies).each do |other,dupped_stream|
|
222
|
+
other.instance_variable_set(:@result, dupped_stream)
|
223
|
+
end
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
|
228
|
+
def run_compute_dependencies(type, list, dep_step = {})
|
229
|
+
if Array === type
|
230
|
+
type, *rest = type
|
231
|
+
end
|
232
|
+
|
233
|
+
case type
|
234
|
+
when :produce, :no_dup
|
235
|
+
list.each do |step|
|
236
|
+
step.produce
|
237
|
+
nil
|
238
|
+
end
|
239
|
+
when :bootstrap
|
240
|
+
cpus = rest.nil? ? nil : rest.first
|
241
|
+
cpus = 30 if cpus.nil?
|
242
|
+
cpus = list.length / 2 if cpus > list.length / 2
|
243
|
+
|
244
|
+
Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :_respawn => :always) do |dep|
|
245
|
+
dep.produce
|
246
|
+
nil
|
247
|
+
end
|
248
|
+
else
|
249
|
+
list.each do |step|
|
250
|
+
execute_and_dup(step, dep_step, false)
|
251
|
+
end
|
252
|
+
end
|
253
|
+
end
|
254
|
+
|
255
|
+
def run_dependencies
|
256
|
+
dep_step = {}
|
257
|
+
|
258
|
+
all_deps = rec_dependencies + [self]
|
259
|
+
|
260
|
+
dependencies.each do |dep|
|
261
|
+
next unless ComputeDependency === dep
|
262
|
+
if dep.compute == :produce
|
263
|
+
dep.produce
|
264
|
+
end
|
265
|
+
end
|
266
|
+
|
267
|
+
compute_deps = rec_dependencies.collect do |dep|
|
268
|
+
next unless ComputeDependency === dep
|
269
|
+
dep.rec_dependencies
|
270
|
+
end.compact.flatten.uniq
|
271
|
+
|
272
|
+
seen_paths = Set.new
|
273
|
+
all_deps.uniq.each do |step|
|
274
|
+
next if seen_paths.include? step.path
|
275
|
+
seen_paths << step.path
|
276
|
+
Step.prepare_for_execution(step) unless step == self
|
277
|
+
next unless step.dependencies and step.dependencies.any?
|
278
|
+
step.dependencies.each do |step_dep|
|
279
|
+
next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and step_dep.compute == :nodup)
|
280
|
+
dep_step[step_dep.path] ||= []
|
281
|
+
dep_step[step_dep.path] << step_dep
|
282
|
+
end
|
283
|
+
end
|
284
|
+
|
285
|
+
required_dep_paths = []
|
286
|
+
dep_step.each do |path,list|
|
287
|
+
required_dep_paths << path if list.length > 1
|
288
|
+
end
|
289
|
+
|
290
|
+
required_dep_paths.concat dependencies.collect{|dep| dep.path }
|
291
|
+
|
292
|
+
log :dependencies, "Dependencies for step #{Log.color :yellow, task.name.to_s || ""}"
|
293
|
+
|
294
|
+
pre_deps = []
|
295
|
+
compute_pre_deps = {}
|
296
|
+
last_deps = []
|
297
|
+
compute_last_deps = {}
|
298
|
+
seen_paths = Set.new
|
299
|
+
rec_dependencies.uniq.each do |step|
|
300
|
+
next if seen_paths.include? step.path
|
301
|
+
seen_paths << step.path
|
302
|
+
next unless required_dep_paths.include? step.path
|
303
|
+
if dependencies.include?(step) and step.inputs.flatten.select{|i| Step === i}.any?
|
304
|
+
if ComputeDependency === step
|
305
|
+
compute_last_deps[step.compute] ||= []
|
306
|
+
compute_last_deps[step.compute] << step
|
307
|
+
else
|
308
|
+
last_deps << step
|
309
|
+
end
|
310
|
+
else
|
311
|
+
if ComputeDependency === step
|
312
|
+
compute_pre_deps[step.compute] ||= []
|
313
|
+
compute_pre_deps[step.compute] << step
|
314
|
+
else
|
315
|
+
pre_deps << step #if dependencies.include?(step)
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
320
|
+
pre_deps.each do |step|
|
321
|
+
next if compute_deps.include? step
|
322
|
+
execute_and_dup(step, dep_step, false)
|
323
|
+
end
|
324
|
+
|
325
|
+
compute_pre_deps.each do |type,list|
|
326
|
+
run_compute_dependencies(type, list, dep_step)
|
327
|
+
end
|
328
|
+
|
329
|
+
last_deps.each do |step|
|
330
|
+
next if compute_deps.include? step
|
331
|
+
execute_and_dup(step, dep_step)
|
332
|
+
end
|
333
|
+
|
334
|
+
compute_last_deps.each do |type,list|
|
335
|
+
run_compute_dependencies(type, list, dep_step)
|
336
|
+
end
|
337
|
+
|
338
|
+
end
|
339
|
+
|
340
|
+
def stop_dependencies
|
341
|
+
dependencies.each do |dep|
|
342
|
+
dep.abort
|
343
|
+
end
|
344
|
+
kill_children
|
345
|
+
end
|
346
|
+
|
347
|
+
#def run_dependencies
|
348
|
+
# @seen ||= []
|
349
|
+
# seen_paths ||= Set.new
|
350
|
+
#
|
351
|
+
# consolidate_dependencies
|
352
|
+
# dependencies.uniq.each do |dependency|
|
353
|
+
# dependency_path = dependency.path
|
354
|
+
# next if seen_paths.include? dependency_path
|
355
|
+
# @seen.concat dependency.rec_dependencies
|
356
|
+
# seen_paths.union(dependency.rec_dependencies.collect{|d| d.path})
|
357
|
+
# @seen << dependency
|
358
|
+
# seen_paths << dependency_path
|
359
|
+
# end
|
360
|
+
|
361
|
+
# @seen.uniq!
|
362
|
+
# @seen.delete self
|
363
|
+
|
364
|
+
# return if @seen.empty?
|
365
|
+
|
366
|
+
# log :dependencies, "#{Log.color :magenta, "Dependencies"} for step #{Log.color :yellow, task.name.to_s || ""}"
|
367
|
+
|
368
|
+
# @seen.each do |dependency|
|
369
|
+
# Step.prepare_for_execution(dependency)
|
370
|
+
# end
|
371
|
+
|
372
|
+
# pre_deps = []
|
373
|
+
# compute_pre_deps = {}
|
374
|
+
# last_deps = []
|
375
|
+
# compute_last_deps = {}
|
376
|
+
# @seen.each do |dependency|
|
377
|
+
# if dependencies.include?(dependency) and dependency.inputs.flatten.select{|i| Step === i}.any?
|
378
|
+
# if ComputeDependency === dependency
|
379
|
+
# compute_last_deps[dependency.compute] ||= []
|
380
|
+
# compute_last_deps[dependency.compute] << dependency
|
381
|
+
# else
|
382
|
+
# last_deps << dependency
|
383
|
+
# end
|
384
|
+
# else
|
385
|
+
# if ComputeDependency === dependency
|
386
|
+
# compute_pre_deps[dependency.compute] ||= []
|
387
|
+
# compute_pre_deps[dependency.compute] << dependency
|
388
|
+
# else
|
389
|
+
# pre_deps << dependency if dependencies.include?(dependency)
|
390
|
+
# end
|
391
|
+
# end
|
392
|
+
# end
|
393
|
+
|
394
|
+
# pre_deps.each do |dependency|
|
395
|
+
# dependency.dup_inputs
|
396
|
+
# execute_dependency(dependency)
|
397
|
+
# end
|
398
|
+
|
399
|
+
# compute_pre_deps.each do |type,list|
|
400
|
+
# if Array === type
|
401
|
+
# type, *rest = type
|
402
|
+
# end
|
403
|
+
|
404
|
+
# case type
|
405
|
+
# when :bootstrap
|
406
|
+
# cpus = rest.nil? ? nil : rest.first
|
407
|
+
# cpus = 10 if cpus.nil?
|
408
|
+
|
409
|
+
# list.each do |dependency|
|
410
|
+
# dependency.dup_inputs
|
411
|
+
# end
|
412
|
+
|
413
|
+
# Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :_respawn => :always) do |dep|
|
414
|
+
# dep.produce
|
415
|
+
# nil
|
416
|
+
# end
|
417
|
+
# else
|
418
|
+
# list.each do |dependency|
|
419
|
+
# dependency.dup_inputs
|
420
|
+
# execute_dependency(dependency)
|
421
|
+
# end
|
422
|
+
# end
|
423
|
+
# end
|
424
|
+
|
425
|
+
# last_deps.each do |dependency|
|
426
|
+
# dependency.dup_inputs
|
427
|
+
# end
|
428
|
+
|
429
|
+
# last_deps.each do |dependency|
|
430
|
+
# execute_dependency(dependency)
|
431
|
+
# end
|
432
|
+
|
433
|
+
# compute_last_deps.each do |type,list|
|
434
|
+
# case type
|
435
|
+
# when :_bootstrap
|
436
|
+
# list.each do |dependency|
|
437
|
+
# dependency.dup_inputs
|
438
|
+
# end
|
439
|
+
# Misc.bootstrap(list, 3, :bar => "Boostrapping dependencies for #{path}", :respawn => :always) do |dependency|
|
440
|
+
# dependency.produce
|
441
|
+
# nil
|
442
|
+
# end
|
443
|
+
# else
|
444
|
+
# list.each do |dependency|
|
445
|
+
# dependency.dup_inputs
|
446
|
+
# execute_dependency(dependency)
|
447
|
+
# end
|
448
|
+
# end
|
449
|
+
# end
|
450
|
+
#end
|
451
|
+
end
|