rbbt-util 5.20.7 → 5.20.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/persist.rb +1 -1
- data/lib/rbbt/tsv/stream.rb +1 -1
- data/lib/rbbt/util/misc/inspect.rb +1 -1
- data/lib/rbbt/workflow.rb +1 -0
- data/lib/rbbt/workflow/accessor.rb +31 -11
- data/lib/rbbt/workflow/definition.rb +6 -4
- data/lib/rbbt/workflow/step/dependencies.rb +5 -2
- data/lib/rbbt/workflow/task.rb +14 -4
- data/test/rbbt/test_workflow.rb +27 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 72d3825a7d4644c54b8498b2649e8820d610bef2
|
4
|
+
data.tar.gz: 02157c505e0f582a7b3cf4517ec590b124734602
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: e04bcbe557031df6d184336aa9ac8c2dfab9c33f509f4b81d8cf8fd6264a5fb01fc9c360fa9c35a5855a0a485642acdd29e201956f1ffe38f1a2af7f23b19c42
|
7
|
+
data.tar.gz: 040f5b2771c9b0037e64fc2b31f26e77855cab32c40406f1a978ec52cd8379df348b3feef701dc9e051db65b931bc8764a7e3067c9491547298365f34ca448ce
|
data/lib/rbbt/persist.rb
CHANGED
@@ -45,7 +45,7 @@ module Persist
|
|
45
45
|
if Array === check
|
46
46
|
newer = check.select{|file| newer? path, file}
|
47
47
|
if newer.any?
|
48
|
-
Log.medium "Persistence check for #{path} failed in: #{ newer
|
48
|
+
Log.medium "Persistence check for #{path} failed in: #{ Misc.fingerprint(newer)}"
|
49
49
|
return false
|
50
50
|
end
|
51
51
|
else
|
data/lib/rbbt/tsv/stream.rb
CHANGED
data/lib/rbbt/workflow.rb
CHANGED
@@ -493,30 +493,45 @@ module Workflow
|
|
493
493
|
@rec_dependencies ||= {}
|
494
494
|
@rec_dependencies[taskname] ||= begin
|
495
495
|
if task_dependencies.include? taskname
|
496
|
+
|
496
497
|
deps = task_dependencies[taskname]
|
497
|
-
|
498
|
+
|
499
|
+
#all_deps = deps.select{|dep| String === dep or Symbol === dep or Array === dep}
|
500
|
+
|
501
|
+
all_deps = []
|
498
502
|
deps.each do |dep|
|
503
|
+
if DependencyBlock === dep
|
504
|
+
all_deps << dep.dependency if dep.dependency
|
505
|
+
else
|
506
|
+
all_deps << dep unless Proc === dep
|
507
|
+
end
|
499
508
|
case dep
|
500
509
|
when Array
|
501
510
|
wf, t, o = dep
|
511
|
+
|
502
512
|
wf.rec_dependencies(t).each do |d|
|
503
513
|
if Array === d
|
504
514
|
new = d.dup
|
505
515
|
else
|
506
516
|
new = [dep.first, d]
|
507
517
|
end
|
518
|
+
|
508
519
|
if Hash === o and not o.empty?
|
509
520
|
if Hash === new.last
|
510
|
-
hash = new.last
|
521
|
+
hash = new.last.dup
|
511
522
|
o.each{|k,v| hash[k] ||= v}
|
523
|
+
new[new.length-1] = hash
|
512
524
|
else
|
513
|
-
new.push o
|
525
|
+
new.push o.dup
|
514
526
|
end
|
515
527
|
end
|
528
|
+
|
516
529
|
all_deps << new
|
517
530
|
end
|
531
|
+
|
518
532
|
when String, Symbol
|
519
|
-
|
533
|
+
rec_deps = rec_dependencies(dep.to_sym)
|
534
|
+
all_deps.concat rec_deps
|
520
535
|
when DependencyBlock
|
521
536
|
all_deps << dep.dependency if dep.dependency
|
522
537
|
case dep.dependency
|
@@ -558,14 +573,15 @@ module Workflow
|
|
558
573
|
|
559
574
|
def rec_inputs(taskname)
|
560
575
|
task = task_from_dep(taskname)
|
561
|
-
|
576
|
+
deps = rec_dependencies(taskname)
|
577
|
+
dep_inputs = task.dep_inputs deps, self
|
562
578
|
task.inputs + dep_inputs.values.flatten
|
563
579
|
end
|
564
580
|
|
565
581
|
def rec_input_defaults(taskname)
|
566
582
|
rec_inputs = rec_inputs(taskname)
|
567
583
|
[taskname].concat(rec_dependencies(taskname)).inject(IndiferentHash.setup({})){|acc, tn|
|
568
|
-
if Array === tn and tn
|
584
|
+
if Array === tn and tn[0] and tn[1]
|
569
585
|
new = tn.first.tasks[tn[1].to_sym].input_defaults
|
570
586
|
elsif Symbol === tn
|
571
587
|
new = tasks[tn.to_sym].input_defaults
|
@@ -581,7 +597,7 @@ module Workflow
|
|
581
597
|
def rec_input_types(taskname)
|
582
598
|
rec_inputs = rec_inputs(taskname)
|
583
599
|
[taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
|
584
|
-
if Array === tn and tn
|
600
|
+
if Array === tn and tn[0] and tn[1]
|
585
601
|
new = tn.first.tasks[tn[1].to_sym].input_types
|
586
602
|
elsif Symbol === tn
|
587
603
|
new = tasks[tn.to_sym].input_types
|
@@ -597,7 +613,7 @@ module Workflow
|
|
597
613
|
def rec_input_descriptions(taskname)
|
598
614
|
rec_inputs = rec_inputs(taskname)
|
599
615
|
[taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
|
600
|
-
if Array === tn and tn
|
616
|
+
if Array === tn and tn[0] and tn[1]
|
601
617
|
new = tn.first.tasks[tn[1].to_sym].input_descriptions
|
602
618
|
elsif Symbol === tn
|
603
619
|
new = tasks[tn.to_sym].input_descriptions
|
@@ -613,7 +629,7 @@ module Workflow
|
|
613
629
|
def rec_input_options(taskname)
|
614
630
|
rec_inputs = rec_inputs(taskname)
|
615
631
|
[taskname].concat(rec_dependencies(taskname)).inject({}){|acc, tn|
|
616
|
-
if Array === tn and tn
|
632
|
+
if Array === tn and tn[0] and tn[1]
|
617
633
|
new = tn.first.tasks[tn[1].to_sym].input_options
|
618
634
|
elsif Symbol === tn
|
619
635
|
new = tasks[tn.to_sym].input_options
|
@@ -646,10 +662,14 @@ module Workflow
|
|
646
662
|
rec_dependency = all_d.select{|d| d.task_name.to_sym == v }.first
|
647
663
|
|
648
664
|
if rec_dependency.nil?
|
649
|
-
|
665
|
+
if inputs.include? v
|
666
|
+
_inputs[i] = _inputs.delete(v)
|
667
|
+
else
|
668
|
+
_inputs[i] = v unless _inputs.include? i
|
669
|
+
end
|
650
670
|
else
|
651
671
|
input_options = workflow.task_info(dep_task)[:input_options][i] || {}
|
652
|
-
if
|
672
|
+
if input_options[:stream]
|
653
673
|
#rec_dependency.run(true).grace unless rec_dependency.done? or rec_dependency.running?
|
654
674
|
_inputs[i] = rec_dependency
|
655
675
|
else
|
@@ -38,11 +38,13 @@ module Workflow
|
|
38
38
|
@dependencies ||= []
|
39
39
|
if block_given?
|
40
40
|
if dependency.any?
|
41
|
-
wf, task, opt = dependency
|
42
41
|
|
43
|
-
|
44
|
-
|
45
|
-
|
42
|
+
wf, task_name, options = dependency
|
43
|
+
options, task_name = task_name, nil if Hash === task_name
|
44
|
+
options, wf = wf, nil if Hash === wf
|
45
|
+
task_name, wf = wf, self if task_name.nil?
|
46
|
+
|
47
|
+
DependencyBlock.setup block, [wf, task_name, options]
|
46
48
|
end
|
47
49
|
@dependencies << block
|
48
50
|
else
|
@@ -77,7 +77,9 @@ class Step
|
|
77
77
|
(job.streaming? and job.running?) or
|
78
78
|
(defined? WorkflowRESTClient and WorkflowRESTClient::RemoteStep === job and not (job.error? or job.aborted?))
|
79
79
|
|
80
|
-
|
80
|
+
if job.error? or job.aborted? or (job.started? and not job.running? and not job.error?)
|
81
|
+
job.clean
|
82
|
+
end
|
81
83
|
|
82
84
|
job.dup_inputs unless job.done? or job.started?
|
83
85
|
|
@@ -165,9 +167,10 @@ class Step
|
|
165
167
|
stream = step.result
|
166
168
|
other_steps = dep_step[step.path]
|
167
169
|
return unless other_steps.length > 1
|
170
|
+
log_dependency_exec(step, "duplicating #{other_steps.length}")
|
168
171
|
copies = Misc.tee_stream_thread_multiple(stream, other_steps.length)
|
169
|
-
log_dependency_exec(step, "duplicating #{copies.length}")
|
170
172
|
other_steps.zip(copies).each do |other,dupped_stream|
|
173
|
+
stream.annotate(dupped_stream) if stream.respond_to?(:annotate)
|
171
174
|
other.instance_variable_set("@result", dupped_stream)
|
172
175
|
end
|
173
176
|
end
|
data/lib/rbbt/workflow/task.rb
CHANGED
@@ -75,10 +75,15 @@ module Task
|
|
75
75
|
seen = []
|
76
76
|
task_inputs = {}
|
77
77
|
deps.each do |dep|
|
78
|
-
if
|
79
|
-
wf, task = (Array === dep ? [dep.first, dep.first.tasks[dep[1].to_sym]] : [workflow, workflow.tasks[dep.to_sym]])
|
80
|
-
elsif Symbol === dep
|
78
|
+
if Symbol === dep
|
81
79
|
wf, task = [workflow, workflow.tasks[dep.to_sym]]
|
80
|
+
elsif Array === dep and dep.first
|
81
|
+
wf, task_name, options = dep
|
82
|
+
options, task_name = task_name, nil if Hash === task_name
|
83
|
+
options, wf = wf, nil if Hash === wf
|
84
|
+
task_name, wf = wf, workflow if task_name.nil? and Symbol === wf or String === wf
|
85
|
+
next if task_name.nil?
|
86
|
+
task = wf.tasks[task_name.to_sym]
|
82
87
|
else
|
83
88
|
next
|
84
89
|
end
|
@@ -88,12 +93,17 @@ module Task
|
|
88
93
|
seen << [wf, task.name]
|
89
94
|
new_inputs = task.inputs - maps
|
90
95
|
next unless new_inputs.any?
|
91
|
-
task_inputs[task]
|
96
|
+
if task_inputs[task].nil?
|
97
|
+
task_inputs[task] = new_inputs
|
98
|
+
else
|
99
|
+
task_inputs[task] = (task_inputs[task] + new_inputs).uniq
|
100
|
+
end
|
92
101
|
end
|
93
102
|
task_inputs
|
94
103
|
end
|
95
104
|
|
96
105
|
def dep_inputs(deps, workflow = nil)
|
106
|
+
return {} if deps.empty?
|
97
107
|
task_inputs = Task.dep_inputs deps, workflow
|
98
108
|
task_inputs.each do |task, inputs|
|
99
109
|
inputs.replace (inputs - self.inputs)
|
data/test/rbbt/test_workflow.rb
CHANGED
@@ -57,6 +57,25 @@ Returns numer * 2 lines containing TEST
|
|
57
57
|
|
58
58
|
export_synchronous :double
|
59
59
|
export_asynchronous :repeat2
|
60
|
+
|
61
|
+
input :letter, :string, "Letter", "D"
|
62
|
+
task :letter => :string do |l|
|
63
|
+
l
|
64
|
+
end
|
65
|
+
|
66
|
+
dep :letter
|
67
|
+
task :letter_repeat => :string do |l|
|
68
|
+
([step(:letter).load] * 2) * ""
|
69
|
+
end
|
70
|
+
|
71
|
+
dep :letter
|
72
|
+
dep :letter_repeat, :letter => "A"
|
73
|
+
task :two_letters => :string do
|
74
|
+
dependencies.collect{|d| d.load } * ":"
|
75
|
+
end
|
76
|
+
|
77
|
+
|
78
|
+
|
60
79
|
end
|
61
80
|
|
62
81
|
TestWF.workdir = Rbbt.tmp.test.workflow
|
@@ -125,4 +144,12 @@ class TestWorkflow < Test::Unit::TestCase
|
|
125
144
|
assert_equal 'bar', job.exec
|
126
145
|
end
|
127
146
|
|
147
|
+
def test_letter
|
148
|
+
assert_equal "D", TestWF.job(:letter).run
|
149
|
+
assert_equal "B", TestWF.job(:letter, nil, :letter => "B").run
|
150
|
+
assert_equal "BB", TestWF.job(:letter_repeat, nil, :letter => "B").run
|
151
|
+
job = TestWF.job(:two_letters, nil, :letter => "V")
|
152
|
+
assert_equal "V:AA", job.run
|
153
|
+
end
|
154
|
+
|
128
155
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.20.
|
4
|
+
version: 5.20.8
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-06-
|
11
|
+
date: 2016-06-08 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|