scout-gear 7.3.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +20 -9
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +26 -23
- data/lib/scout/config.rb +1 -1
- data/lib/scout/log/color.rb +4 -1
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +58 -54
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +11 -0
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +90 -15
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +28 -12
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +72 -46
- data/lib/scout/tsv/index.rb +69 -13
- data/lib/scout/tsv/open.rb +138 -84
- data/lib/scout/tsv/parser.rb +135 -80
- data/lib/scout/tsv/path.rb +1 -2
- data/lib/scout/tsv/persist/adapter.rb +15 -45
- data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
- data/lib/scout/tsv/persist.rb +4 -0
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +96 -92
- data/lib/scout/tsv/util/filter.rb +9 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +78 -33
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +60 -11
- data/lib/scout/tsv.rb +26 -3
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +21 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +36 -5
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +100 -34
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +81 -46
- data/lib/scout/workflow/usage.rb +8 -6
- data/scout-gear.gemspec +24 -20
- data/scout_commands/workflow/task +34 -7
- data/test/scout/open/test_stream.rb +60 -58
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +6 -0
- data/test/scout/test_tsv.rb +212 -2
- data/test/scout/test_work_queue.rb +21 -19
- data/test/scout/tsv/persist/test_adapter.rb +1 -1
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +35 -3
- data/test/scout/tsv/test_open.rb +160 -2
- data/test/scout/tsv/test_parser.rb +19 -2
- data/test/scout/tsv/test_persist.rb +2 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +88 -3
- data/test/scout/tsv/test_util.rb +1 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +25 -11
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +3 -3
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- metadata +20 -6
@@ -5,7 +5,7 @@ class Step
|
|
5
5
|
|
6
6
|
def self.status_color(status)
|
7
7
|
case status.to_sym
|
8
|
-
when :error, :aborted, :
|
8
|
+
when :error, :aborted, :dead, :unsync
|
9
9
|
:red
|
10
10
|
when :streaming, :started
|
11
11
|
:cyan
|
@@ -13,7 +13,7 @@ class Step
|
|
13
13
|
:green
|
14
14
|
when :dependencies, :waiting, :setup
|
15
15
|
:yellow
|
16
|
-
when :notfound, :cleaned
|
16
|
+
when :notfound, :cleaned, :missing
|
17
17
|
:blue
|
18
18
|
else
|
19
19
|
if status.to_s.index ">"
|
@@ -91,6 +91,7 @@ class Step
|
|
91
91
|
info[:task_name] = task
|
92
92
|
path = step.path
|
93
93
|
status = info[:status] || :missing
|
94
|
+
status = :noinfo if status == :missing && Open.exist?(path)
|
94
95
|
status = "remote" if Open.remote?(path) || Open.ssh?(path)
|
95
96
|
name = info[:name] || File.basename(path)
|
96
97
|
status = :unsync if status == :done and not Open.exist?(path)
|
@@ -103,9 +104,9 @@ class Step
|
|
103
104
|
step.dependencies.each do |dep|
|
104
105
|
if dep.input_dependencies.any?
|
105
106
|
dep.input_dependencies.each do |id|
|
106
|
-
input_name, _dep = dep.recursive_inputs.
|
107
|
+
input_name, _dep = dep.recursive_inputs.select{|f,d|
|
107
108
|
d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
|
108
|
-
}.last
|
109
|
+
}.keys.last
|
109
110
|
if input_name
|
110
111
|
input_dependencies[id] ||= []
|
111
112
|
input_dependencies[id] << [dep, input_name]
|
@@ -115,10 +116,10 @@ class Step
|
|
115
116
|
end if step.dependencies
|
116
117
|
|
117
118
|
str = ""
|
118
|
-
str = " " * offset + this_step_msg if ENV["
|
119
|
+
str = " " * offset + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
|
119
120
|
|
120
121
|
step.dependencies.dup.tap{|l|
|
121
|
-
l.reverse! if ENV["
|
122
|
+
l.reverse! if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
|
122
123
|
}.each do |dep|
|
123
124
|
path = dep.path
|
124
125
|
new = ! seen.include?(path)
|
@@ -141,7 +142,7 @@ class Step
|
|
141
142
|
end
|
142
143
|
end if step.dependencies
|
143
144
|
|
144
|
-
str += (" " * offset) + this_step_msg unless ENV["
|
145
|
+
str += (" " * offset) + this_step_msg unless ENV["SCOUT_ORIGINAL_STACK"] == 'true'
|
145
146
|
|
146
147
|
str
|
147
148
|
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
class Step
|
2
|
+
def abort(exception = nil)
|
3
|
+
while @result && streaming? && stream = self.stream
|
4
|
+
stream.abort(exception)
|
5
|
+
end
|
6
|
+
end
|
7
|
+
|
8
|
+
def recoverable_error?
|
9
|
+
self.error? && ! (ScoutException === self.exception)
|
10
|
+
end
|
11
|
+
|
12
|
+
def updated?
|
13
|
+
return false if self.error? && self.recoverable_error?
|
14
|
+
return true unless ENV["SCOUT_UPDATE"]
|
15
|
+
newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
16
|
+
newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
17
|
+
|
18
|
+
newer.empty?
|
19
|
+
end
|
20
|
+
|
21
|
+
def clean
|
22
|
+
@take_stream = nil
|
23
|
+
@result = nil
|
24
|
+
@info = nil
|
25
|
+
@info_load_time = nil
|
26
|
+
Open.rm path if Open.exist?(path)
|
27
|
+
Open.rm info_file if Open.exist?(info_file)
|
28
|
+
Open.rm_rf files_dir if Open.exist?(files_dir)
|
29
|
+
end
|
30
|
+
|
31
|
+
def present?
|
32
|
+
Open.exist?(path) &&
|
33
|
+
Open.exist?(info_file) &&
|
34
|
+
Open.exist?(files_dir)
|
35
|
+
end
|
36
|
+
|
37
|
+
|
38
|
+
def recursive_clean
|
39
|
+
dependencies.each do |dep|
|
40
|
+
dep.recursive_clean
|
41
|
+
end
|
42
|
+
clean
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
data/lib/scout/workflow/step.rb
CHANGED
@@ -1,6 +1,7 @@
|
|
1
1
|
require_relative '../path'
|
2
2
|
require_relative '../persist'
|
3
3
|
require_relative 'step/info'
|
4
|
+
require_relative 'step/status'
|
4
5
|
require_relative 'step/load'
|
5
6
|
require_relative 'step/file'
|
6
7
|
require_relative 'step/dependencies'
|
@@ -10,11 +11,13 @@ require_relative 'step/progress'
|
|
10
11
|
|
11
12
|
class Step
|
12
13
|
|
13
|
-
attr_accessor :path, :inputs, :dependencies, :task, :tee_copies
|
14
|
-
def initialize(path, inputs = nil, dependencies = nil, &task)
|
14
|
+
attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs
|
15
|
+
def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, &task)
|
15
16
|
@path = path
|
16
17
|
@inputs = inputs
|
17
18
|
@dependencies = dependencies
|
19
|
+
@id = id
|
20
|
+
@non_default_inputs = non_default_inputs
|
18
21
|
@task = task
|
19
22
|
@mutex = Mutex.new
|
20
23
|
@tee_copies = 1
|
@@ -26,7 +29,7 @@ class Step
|
|
26
29
|
|
27
30
|
def inputs
|
28
31
|
@inputs ||= begin
|
29
|
-
if Open.exists?(info_file)
|
32
|
+
if info_file && Open.exists?(info_file)
|
30
33
|
info[:inputs]
|
31
34
|
else
|
32
35
|
[]
|
@@ -55,6 +58,13 @@ class Step
|
|
55
58
|
@name ||= File.basename(@path)
|
56
59
|
end
|
57
60
|
|
61
|
+
def clean_name
|
62
|
+
return @id if @id
|
63
|
+
return info[:clean_name] if info.include? :clean_name
|
64
|
+
return m[1] if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
|
65
|
+
return name.split(".").first
|
66
|
+
end
|
67
|
+
|
58
68
|
def task_name
|
59
69
|
@task_name ||= @task.name if @task.respond_to?(:name)
|
60
70
|
end
|
@@ -64,26 +74,65 @@ class Step
|
|
64
74
|
end
|
65
75
|
|
66
76
|
def exec
|
77
|
+
|
78
|
+
if inputs
|
79
|
+
if Task === task
|
80
|
+
types = task.inputs.collect{|name,type| type }
|
81
|
+
new_inputs = inputs.zip(types).collect{|input,info|
|
82
|
+
type, desc, default, options = info
|
83
|
+
next input unless Step === input
|
84
|
+
input.join if input.streaming?
|
85
|
+
Task.format_input(input.join.path, type, options)
|
86
|
+
}
|
87
|
+
else
|
88
|
+
new_inputs = inputs.collect{|input|
|
89
|
+
Step === input ? input.load : input
|
90
|
+
}
|
91
|
+
end
|
92
|
+
inputs = new_inputs
|
93
|
+
end
|
94
|
+
|
67
95
|
@result = self.instance_exec(*inputs, &task)
|
68
96
|
end
|
69
97
|
|
98
|
+
def tmp_path
|
99
|
+
@tmp_path ||= begin
|
100
|
+
basename = File.basename(@path)
|
101
|
+
dirname = File.dirname(@path)
|
102
|
+
tmp_path = File.join(dirname, '.' + basename)
|
103
|
+
@path.setup(tmp_path) if Path === @path
|
104
|
+
tmp_path
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
70
108
|
attr_reader :result
|
71
|
-
def run
|
109
|
+
def run(stream = false)
|
72
110
|
return @result || self.load if done?
|
73
111
|
prepare_dependencies
|
74
112
|
run_dependencies
|
75
|
-
@result =
|
113
|
+
@result =
|
76
114
|
begin
|
77
|
-
|
78
|
-
|
79
|
-
:
|
80
|
-
|
115
|
+
Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
|
116
|
+
clear_info
|
117
|
+
merge_info :status => :start, :start => Time.now,
|
118
|
+
:pid => Process.pid, :pid_hostname => ENV["HOSTNAME"],
|
119
|
+
:inputs => inputs, :type => type,
|
120
|
+
:dependencies => dependencies.collect{|d| d.path }
|
81
121
|
|
82
|
-
|
83
|
-
|
84
|
-
|
122
|
+
|
123
|
+
@result = exec
|
124
|
+
|
125
|
+
if @result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
|
126
|
+
Open.mv self.tmp_path, self.path
|
127
|
+
else
|
128
|
+
@result = @result.stream if @result.respond_to?(:stream)
|
129
|
+
end
|
130
|
+
|
131
|
+
@result
|
132
|
+
end
|
85
133
|
rescue Exception => e
|
86
|
-
merge_info :status => :error, :exception => e
|
134
|
+
merge_info :status => :error, :exception => e, :end => Time.now
|
135
|
+
abort_dependencies
|
87
136
|
raise e
|
88
137
|
ensure
|
89
138
|
if ! (error? || aborted?)
|
@@ -106,8 +155,18 @@ class Step
|
|
106
155
|
end
|
107
156
|
end
|
108
157
|
end
|
158
|
+
|
159
|
+
if stream && ENV["SCOUT_NO_STREAM"].nil?
|
160
|
+
@result
|
161
|
+
else
|
162
|
+
if IO === @result || @result.respond_to?(:stream)
|
163
|
+
join
|
164
|
+
@result = nil
|
165
|
+
self.load
|
166
|
+
else
|
167
|
+
@result
|
168
|
+
end
|
109
169
|
end
|
110
|
-
@result
|
111
170
|
end
|
112
171
|
|
113
172
|
def done?
|
@@ -140,9 +199,23 @@ class Step
|
|
140
199
|
end
|
141
200
|
end
|
142
201
|
|
202
|
+
def consume_all_streams
|
203
|
+
threads = []
|
204
|
+
while @result && streaming? && stream = self.stream
|
205
|
+
threads << Open.consume_stream(stream, true)
|
206
|
+
end
|
207
|
+
threads.each do |t|
|
208
|
+
begin
|
209
|
+
t.join
|
210
|
+
rescue
|
211
|
+
threads.each{|t| t.raise(Aborted); t.join }
|
212
|
+
raise $!
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
143
217
|
def join
|
144
|
-
|
145
|
-
Open.consume_stream(io, false) if io
|
218
|
+
consume_all_streams
|
146
219
|
self
|
147
220
|
end
|
148
221
|
|
@@ -157,31 +230,24 @@ class Step
|
|
157
230
|
done? ? Persist.load(path, type) : exec
|
158
231
|
end
|
159
232
|
|
160
|
-
def clean
|
161
|
-
@take_stream = nil
|
162
|
-
@result = nil
|
163
|
-
@info = nil
|
164
|
-
@info_load_time = nil
|
165
|
-
Open.rm path if Open.exist?(path)
|
166
|
-
Open.rm info_file if Open.exist?(info_file)
|
167
|
-
Open.rm_rf files_dir if Open.exist?(files_dir)
|
168
|
-
end
|
169
|
-
|
170
|
-
def recursive_clean
|
171
|
-
dependencies.each do |dep|
|
172
|
-
dep.recursive_clean
|
173
|
-
end
|
174
|
-
clean
|
175
|
-
end
|
176
|
-
|
177
233
|
def step(task_name)
|
178
234
|
dependencies.each do |dep|
|
179
235
|
return dep if dep.task_name == task_name
|
236
|
+
rec_dep = dep.step(task_name)
|
237
|
+
return rec_dep if rec_dep
|
180
238
|
end
|
181
239
|
nil
|
182
240
|
end
|
183
241
|
|
242
|
+
def short_path
|
243
|
+
Scout.identify @path
|
244
|
+
end
|
245
|
+
|
184
246
|
def digest_str
|
185
|
-
|
247
|
+
"Step: " + short_path
|
248
|
+
end
|
249
|
+
|
250
|
+
def fingerprint
|
251
|
+
digest_str
|
186
252
|
end
|
187
253
|
end
|
@@ -1,11 +1,10 @@
|
|
1
|
+
require_relative '../../named_array'
|
1
2
|
module Task
|
3
|
+
def self.format_input(value, type, options = {})
|
4
|
+
return value if IO === value || StringIO === value || Step === value
|
2
5
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
value = value.load if Step === value
|
7
|
-
if String === value && ! [:path, :file].include?(type)
|
8
|
-
if Open.exists?(value)
|
6
|
+
if String === value && ! [:path, :file, :folder].include?(type) && (options.nil? || (! options[:noload] || options[:stream]))
|
7
|
+
if Open.exists?(value) && ! Open.directory?(value)
|
9
8
|
Persist.load(value, type)
|
10
9
|
else
|
11
10
|
Persist.deserialize(value, type)
|
@@ -13,7 +12,7 @@ module Task
|
|
13
12
|
else
|
14
13
|
if m = type.to_s.match(/(.*)_array/)
|
15
14
|
if Array === value
|
16
|
-
value.collect{|v| format_input(v, m[1].to_sym, options) }
|
15
|
+
value.collect{|v| self.format_input(v, m[1].to_sym, options) }
|
17
16
|
end
|
18
17
|
else
|
19
18
|
value
|
@@ -34,35 +33,30 @@ module Task
|
|
34
33
|
IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
|
35
34
|
|
36
35
|
input_array = []
|
36
|
+
input_names = []
|
37
37
|
non_default_inputs = []
|
38
38
|
self.inputs.each_with_index do |p,i|
|
39
39
|
name, type, desc, value, options = p
|
40
|
+
input_names << name
|
40
41
|
provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
|
41
|
-
provided = format_input(provided, type, options || {})
|
42
|
+
provided = Task.format_input(provided, type, options || {})
|
42
43
|
if ! provided.nil? && provided != value
|
43
|
-
non_default_inputs << name
|
44
|
+
non_default_inputs << name.to_sym
|
44
45
|
input_array << provided
|
45
46
|
else
|
46
47
|
input_array << value
|
47
48
|
end
|
48
49
|
end
|
49
50
|
|
51
|
+
NamedArray.setup(input_array, input_names)
|
52
|
+
|
50
53
|
[input_array, non_default_inputs]
|
51
54
|
end
|
52
55
|
|
53
|
-
def digest_inputs(provided_inputs = {})
|
54
|
-
input_array, non_default_inputs = assign_inputs(provided_inputs)
|
55
|
-
if Array === provided_inputs
|
56
|
-
Misc.digest(input_array)
|
57
|
-
else
|
58
|
-
Misc.digest(input_array)
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
56
|
def process_inputs(provided_inputs = {})
|
63
57
|
input_array, non_default_inputs = assign_inputs provided_inputs
|
64
|
-
|
65
|
-
[input_array, non_default_inputs,
|
58
|
+
digest_str = Misc.digest_str(input_array)
|
59
|
+
[input_array, non_default_inputs, digest_str]
|
66
60
|
end
|
67
61
|
|
68
62
|
def save_file_input(orig_file, directory)
|
data/lib/scout/workflow/task.rb
CHANGED
@@ -44,27 +44,45 @@ module Task
|
|
44
44
|
|
45
45
|
provided_inputs ||= {}
|
46
46
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
47
|
+
# Helper function
|
48
|
+
load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
|
49
|
+
task = step_options.delete(:task) if step_options.include?(:task)
|
50
|
+
workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
|
51
|
+
id = step_options.delete(:id) if step_options.include?(:id)
|
52
|
+
id = step_options.delete(:jobname) if step_options.include?(:jobname)
|
51
53
|
|
52
|
-
|
53
|
-
|
54
|
+
step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
|
55
|
+
step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
|
54
56
|
|
55
57
|
resolved_inputs = {}
|
56
|
-
|
58
|
+
step_inputs.each do |k,v|
|
57
59
|
if Symbol === v
|
58
60
|
input_dep = dependencies.select{|d| d.task_name == v }.first
|
59
|
-
resolved_inputs[k] = input_dep ||
|
61
|
+
resolved_inputs[k] = input_dep || step_inputs[v] || k
|
60
62
|
else
|
61
63
|
resolved_inputs[k] = v
|
62
64
|
end
|
63
65
|
end
|
64
|
-
workflow.job(task, id, resolved_inputs)
|
66
|
+
[workflow.job(task, id, resolved_inputs), step_inputs]
|
65
67
|
end
|
66
68
|
|
67
|
-
|
69
|
+
# Helper function
|
70
|
+
find_dep_non_default_inputs = proc do |dep,definition_options,step_inputs={}|
|
71
|
+
dep_non_default_inputs = dep.non_default_inputs
|
72
|
+
dep_non_default_inputs.select do |name|
|
73
|
+
step_inputs.include?(name)
|
74
|
+
end
|
75
|
+
dep_non_default_inputs.reject! do |name|
|
76
|
+
definition_options.include?(name) &&
|
77
|
+
(definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
|
78
|
+
end
|
79
|
+
|
80
|
+
dep_non_default_inputs
|
81
|
+
end
|
82
|
+
|
83
|
+
deps.each do |workflow,task,definition_options,block=nil|
|
84
|
+
definition_options[:id] = definition_options.delete(:jobname) if definition_options.include?(:jobname)
|
85
|
+
|
68
86
|
if provided_inputs.include?(overriden = [workflow.name, task] * "#")
|
69
87
|
dep = provided_inputs[overriden]
|
70
88
|
dep = Step.new dep unless Step === dep
|
@@ -74,58 +92,52 @@ module Task
|
|
74
92
|
next
|
75
93
|
end
|
76
94
|
|
77
|
-
|
95
|
+
definition_options ||= {}
|
96
|
+
|
78
97
|
if block
|
79
|
-
|
98
|
+
fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
|
99
|
+
self.inputs.each do |name,type,desc,value|
|
100
|
+
fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
|
101
|
+
end
|
102
|
+
fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
|
103
|
+
block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
|
80
104
|
|
81
|
-
res = block.call id,
|
105
|
+
res = block.call id, block_options, dependencies
|
82
106
|
|
83
107
|
case res
|
84
108
|
when Step
|
85
109
|
dep = res
|
86
110
|
dependencies << dep
|
87
|
-
dep_non_default_inputs =
|
88
|
-
non_default_inputs.concat(dep_non_default_inputs
|
111
|
+
dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
|
112
|
+
non_default_inputs.concat(dep_non_default_inputs)
|
89
113
|
when Hash
|
90
|
-
|
91
|
-
dep = load_dep.call(id, workflow, task,
|
114
|
+
step_options = block_options.merge(res)
|
115
|
+
dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
|
92
116
|
dependencies << dep
|
93
|
-
dep_non_default_inputs =
|
94
|
-
dep_non_default_inputs -= options.keys
|
95
|
-
if new_options.include?(:inputs)
|
96
|
-
dep_non_default_inputs -= new_options[:inputs].keys
|
97
|
-
else
|
98
|
-
dep_non_default_inputs -= new_options.keys
|
99
|
-
end
|
117
|
+
dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
|
100
118
|
non_default_inputs.concat(dep_non_default_inputs)
|
101
119
|
when Array
|
102
120
|
res.each do |_res|
|
103
121
|
if Hash === _res
|
104
|
-
|
105
|
-
dep = load_dep.call(id, workflow, task,
|
122
|
+
step_options = block_options.merge(_res)
|
123
|
+
dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
|
106
124
|
dependencies << dep
|
107
|
-
dep_non_default_inputs =
|
108
|
-
dep_non_default_inputs -= options.keys
|
109
|
-
if new_options.include?(:inputs)
|
110
|
-
dep_non_default_inputs -= new_options[:inputs].keys
|
111
|
-
else
|
112
|
-
dep_non_default_inputs -= new_options.keys
|
113
|
-
end
|
125
|
+
dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
|
114
126
|
non_default_inputs.concat(dep_non_default_inputs)
|
115
127
|
else
|
116
128
|
dep = _res
|
117
129
|
dependencies << dep
|
118
|
-
dep_non_default_inputs =
|
119
|
-
non_default_inputs.concat(dep_non_default_inputs
|
130
|
+
dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
|
131
|
+
non_default_inputs.concat(dep_non_default_inputs)
|
120
132
|
end
|
121
133
|
end
|
122
134
|
end
|
123
135
|
else
|
124
|
-
|
125
|
-
dep = load_dep.call(id, workflow, task,
|
136
|
+
step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
|
137
|
+
dep, step_inputs = load_dep.call(id, workflow, task, step_options, definition_options, dependencies)
|
126
138
|
dependencies << dep
|
127
|
-
dep_non_default_inputs =
|
128
|
-
non_default_inputs.concat(dep_non_default_inputs
|
139
|
+
dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
|
140
|
+
non_default_inputs.concat(dep_non_default_inputs)
|
129
141
|
end
|
130
142
|
end
|
131
143
|
|
@@ -137,24 +149,47 @@ module Task
|
|
137
149
|
provided_inputs = {} if provided_inputs.nil?
|
138
150
|
id = DEFAULT_NAME if id.nil?
|
139
151
|
|
140
|
-
inputs, non_default_inputs,
|
152
|
+
inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
|
141
153
|
|
142
154
|
dependencies = dependencies(id, provided_inputs, non_default_inputs)
|
143
155
|
|
144
156
|
non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
|
145
157
|
|
146
158
|
if non_default_inputs.any?
|
147
|
-
hash = Misc.digest(:inputs =>
|
148
|
-
|
149
|
-
|
159
|
+
hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
|
160
|
+
name = [id, hash] * "_"
|
161
|
+
else
|
162
|
+
name = id
|
150
163
|
end
|
151
164
|
|
152
|
-
|
165
|
+
extension = self.extension
|
166
|
+
if extension == :dep_task
|
167
|
+
extension = nil
|
168
|
+
if dependencies.any?
|
169
|
+
dep_basename = File.basename(dependencies.last.path)
|
170
|
+
if dep_basename.include? "."
|
171
|
+
parts = dep_basename.split(".")
|
172
|
+
extension = [parts.pop]
|
173
|
+
while parts.last.length <= 4
|
174
|
+
extension << parts.pop
|
175
|
+
end
|
176
|
+
extension = extension.reverse * "."
|
177
|
+
end
|
178
|
+
end
|
179
|
+
end
|
180
|
+
|
181
|
+
path = directory[name]
|
182
|
+
|
183
|
+
path = path.set_extension(extension) if extension
|
153
184
|
|
154
185
|
Persist.memory(path) do
|
155
|
-
|
186
|
+
if hash
|
187
|
+
Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
|
188
|
+
else
|
189
|
+
Log.debug "ID #{self.name} #{id} - Clean"
|
190
|
+
end
|
156
191
|
NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
|
157
|
-
Step.new path.find, inputs, dependencies, &self
|
192
|
+
Step.new path.find, inputs, dependencies, id, non_default_inputs, &self
|
158
193
|
end
|
159
194
|
end
|
160
195
|
end
|
data/lib/scout/workflow/usage.rb
CHANGED
@@ -16,14 +16,16 @@ module Task
|
|
16
16
|
str.puts
|
17
17
|
end
|
18
18
|
|
19
|
+
deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
|
19
20
|
if deps and deps.any?
|
20
21
|
seen = inputs.collect{|name,_| name }
|
21
22
|
dep_inputs = {}
|
22
|
-
deps.each do |dep_workflow,task_name|
|
23
|
+
deps.each do |dep_workflow,task_name,options|
|
23
24
|
next if task_name.nil?
|
24
25
|
task = dep_workflow.tasks[task_name]
|
25
26
|
next if task.inputs.nil?
|
26
27
|
inputs = task.inputs.reject{|name, _| seen.include? name }
|
28
|
+
inputs = task.inputs.reject{|name, _| options.include? name }
|
27
29
|
next unless inputs.any?
|
28
30
|
dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
|
29
31
|
dep_inputs[dep] = inputs
|
@@ -109,25 +111,25 @@ end
|
|
109
111
|
|
110
112
|
module Workflow
|
111
113
|
|
112
|
-
def dep_tree(task_name, seen = nil)
|
114
|
+
def dep_tree(task_name, seen = nil, seen_options = nil)
|
113
115
|
@dep_tree ||= {}
|
114
116
|
key = [self, task_name]
|
115
117
|
|
116
118
|
return @dep_tree[key] if @dep_tree.include?(key)
|
117
119
|
save = seen.nil?
|
118
120
|
seen = Set.new if seen.nil?
|
121
|
+
seen_options = {} if seen_options.nil?
|
119
122
|
|
120
123
|
dep_tree = {}
|
121
124
|
task = self.tasks[task_name]
|
122
|
-
task.deps.each do |
|
125
|
+
task.deps.each do |workflow, task, options|
|
123
126
|
next if seen.include? dep
|
124
|
-
seen <<
|
125
|
-
workflow, task, *rest = dep
|
127
|
+
seen << [workflow, task, options.merge(seen_options)]
|
126
128
|
next if task.nil?
|
127
129
|
|
128
130
|
key = [workflow, task]
|
129
131
|
|
130
|
-
dep_tree[key] = workflow.dep_tree(task, seen)
|
132
|
+
dep_tree[key] = workflow.dep_tree(task, seen, options.merge(seen_options))
|
131
133
|
end if task.deps
|
132
134
|
|
133
135
|
@dep_tree[key] = dep_tree if save
|