scout-gear 7.3.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +20 -9
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +26 -23
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/log/color.rb +4 -1
  10. data/lib/scout/log/progress/report.rb +1 -1
  11. data/lib/scout/log/progress/util.rb +58 -54
  12. data/lib/scout/log/progress.rb +1 -1
  13. data/lib/scout/log/trap.rb +107 -0
  14. data/lib/scout/log.rb +56 -21
  15. data/lib/scout/meta_extension.rb +13 -6
  16. data/lib/scout/misc/digest.rb +1 -1
  17. data/lib/scout/misc/format.rb +12 -0
  18. data/lib/scout/misc/insist.rb +1 -1
  19. data/lib/scout/misc/monitor.rb +11 -0
  20. data/lib/scout/misc/system.rb +10 -0
  21. data/lib/scout/named_array.rb +65 -3
  22. data/lib/scout/open/lock/lockfile.rb +587 -0
  23. data/lib/scout/open/lock.rb +28 -2
  24. data/lib/scout/open/remote.rb +4 -0
  25. data/lib/scout/open/stream.rb +90 -15
  26. data/lib/scout/open/util.rb +13 -3
  27. data/lib/scout/path/find.rb +9 -1
  28. data/lib/scout/path/util.rb +35 -0
  29. data/lib/scout/persist/serialize.rb +18 -5
  30. data/lib/scout/persist.rb +28 -12
  31. data/lib/scout/resource/path.rb +53 -0
  32. data/lib/scout/resource/produce.rb +0 -8
  33. data/lib/scout/resource/util.rb +2 -1
  34. data/lib/scout/tmpfile.rb +7 -8
  35. data/lib/scout/tsv/attach.rb +177 -0
  36. data/lib/scout/tsv/change_id.rb +40 -0
  37. data/lib/scout/tsv/dumper.rb +72 -46
  38. data/lib/scout/tsv/index.rb +69 -13
  39. data/lib/scout/tsv/open.rb +138 -84
  40. data/lib/scout/tsv/parser.rb +135 -80
  41. data/lib/scout/tsv/path.rb +1 -2
  42. data/lib/scout/tsv/persist/adapter.rb +15 -45
  43. data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
  44. data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
  45. data/lib/scout/tsv/persist.rb +4 -0
  46. data/lib/scout/tsv/transformer.rb +141 -0
  47. data/lib/scout/tsv/traverse.rb +96 -92
  48. data/lib/scout/tsv/util/filter.rb +9 -0
  49. data/lib/scout/tsv/util/reorder.rb +81 -0
  50. data/lib/scout/tsv/util/select.rb +78 -33
  51. data/lib/scout/tsv/util/unzip.rb +86 -0
  52. data/lib/scout/tsv/util.rb +60 -11
  53. data/lib/scout/tsv.rb +26 -3
  54. data/lib/scout/work_queue/socket.rb +6 -1
  55. data/lib/scout/work_queue/worker.rb +5 -2
  56. data/lib/scout/work_queue.rb +15 -8
  57. data/lib/scout/workflow/definition.rb +21 -2
  58. data/lib/scout/workflow/step/dependencies.rb +24 -4
  59. data/lib/scout/workflow/step/info.rb +36 -5
  60. data/lib/scout/workflow/step/provenance.rb +8 -7
  61. data/lib/scout/workflow/step/status.rb +45 -0
  62. data/lib/scout/workflow/step.rb +100 -34
  63. data/lib/scout/workflow/task/inputs.rb +14 -20
  64. data/lib/scout/workflow/task.rb +81 -46
  65. data/lib/scout/workflow/usage.rb +8 -6
  66. data/scout-gear.gemspec +24 -20
  67. data/scout_commands/workflow/task +34 -7
  68. data/test/scout/open/test_stream.rb +60 -58
  69. data/test/scout/path/test_find.rb +10 -1
  70. data/test/scout/resource/test_produce.rb +15 -0
  71. data/test/scout/test_meta_extension.rb +25 -0
  72. data/test/scout/test_named_array.rb +18 -0
  73. data/test/scout/test_persist.rb +6 -0
  74. data/test/scout/test_tsv.rb +212 -2
  75. data/test/scout/test_work_queue.rb +21 -19
  76. data/test/scout/tsv/persist/test_adapter.rb +1 -1
  77. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  78. data/test/scout/tsv/test_attach.rb +227 -0
  79. data/test/scout/tsv/test_change_id.rb +98 -0
  80. data/test/scout/tsv/test_dumper.rb +1 -1
  81. data/test/scout/tsv/test_index.rb +35 -3
  82. data/test/scout/tsv/test_open.rb +160 -2
  83. data/test/scout/tsv/test_parser.rb +19 -2
  84. data/test/scout/tsv/test_persist.rb +2 -0
  85. data/test/scout/tsv/test_transformer.rb +108 -0
  86. data/test/scout/tsv/test_traverse.rb +88 -3
  87. data/test/scout/tsv/test_util.rb +1 -0
  88. data/test/scout/tsv/util/test_reorder.rb +94 -0
  89. data/test/scout/tsv/util/test_select.rb +25 -11
  90. data/test/scout/tsv/util/test_unzip.rb +112 -0
  91. data/test/scout/work_queue/test_socket.rb +0 -1
  92. data/test/scout/workflow/step/test_status.rb +31 -0
  93. data/test/scout/workflow/task/test_inputs.rb +14 -14
  94. data/test/scout/workflow/test_step.rb +3 -3
  95. data/test/scout/workflow/test_task.rb +168 -32
  96. data/test/scout/workflow/test_usage.rb +33 -6
  97. metadata +20 -6
@@ -5,7 +5,7 @@ class Step
5
5
 
6
6
  def self.status_color(status)
7
7
  case status.to_sym
8
- when :error, :aborted, :missing, :dead, :unsync
8
+ when :error, :aborted, :dead, :unsync
9
9
  :red
10
10
  when :streaming, :started
11
11
  :cyan
@@ -13,7 +13,7 @@ class Step
13
13
  :green
14
14
  when :dependencies, :waiting, :setup
15
15
  :yellow
16
- when :notfound, :cleaned
16
+ when :notfound, :cleaned, :missing
17
17
  :blue
18
18
  else
19
19
  if status.to_s.index ">"
@@ -91,6 +91,7 @@ class Step
91
91
  info[:task_name] = task
92
92
  path = step.path
93
93
  status = info[:status] || :missing
94
+ status = :noinfo if status == :missing && Open.exist?(path)
94
95
  status = "remote" if Open.remote?(path) || Open.ssh?(path)
95
96
  name = info[:name] || File.basename(path)
96
97
  status = :unsync if status == :done and not Open.exist?(path)
@@ -103,9 +104,9 @@ class Step
103
104
  step.dependencies.each do |dep|
104
105
  if dep.input_dependencies.any?
105
106
  dep.input_dependencies.each do |id|
106
- input_name, _dep = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d|
107
+ input_name, _dep = dep.recursive_inputs.select{|f,d|
107
108
  d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
108
- }.last
109
+ }.keys.last
109
110
  if input_name
110
111
  input_dependencies[id] ||= []
111
112
  input_dependencies[id] << [dep, input_name]
@@ -115,10 +116,10 @@ class Step
115
116
  end if step.dependencies
116
117
 
117
118
  str = ""
118
- str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'
119
+ str = " " * offset + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
119
120
 
120
121
  step.dependencies.dup.tap{|l|
121
- l.reverse! if ENV["RBBT_ORIGINAL_STACK"] == 'true'
122
+ l.reverse! if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
122
123
  }.each do |dep|
123
124
  path = dep.path
124
125
  new = ! seen.include?(path)
@@ -141,7 +142,7 @@ class Step
141
142
  end
142
143
  end if step.dependencies
143
144
 
144
- str += (" " * offset) + this_step_msg unless ENV["RBBT_ORIGINAL_STACK"] == 'true'
145
+ str += (" " * offset) + this_step_msg unless ENV["SCOUT_ORIGINAL_STACK"] == 'true'
145
146
 
146
147
  str
147
148
  end
@@ -0,0 +1,45 @@
1
+ class Step
2
+ def abort(exception = nil)
3
+ while @result && streaming? && stream = self.stream
4
+ stream.abort(exception)
5
+ end
6
+ end
7
+
8
+ def recoverable_error?
9
+ self.error? && ! (ScoutException === self.exception)
10
+ end
11
+
12
+ def updated?
13
+ return false if self.error? && self.recoverable_error?
14
+ return true unless ENV["SCOUT_UPDATE"]
15
+ newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
16
+ newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
17
+
18
+ newer.empty?
19
+ end
20
+
21
+ def clean
22
+ @take_stream = nil
23
+ @result = nil
24
+ @info = nil
25
+ @info_load_time = nil
26
+ Open.rm path if Open.exist?(path)
27
+ Open.rm info_file if Open.exist?(info_file)
28
+ Open.rm_rf files_dir if Open.exist?(files_dir)
29
+ end
30
+
31
+ def present?
32
+ Open.exist?(path) &&
33
+ Open.exist?(info_file) &&
34
+ Open.exist?(files_dir)
35
+ end
36
+
37
+
38
+ def recursive_clean
39
+ dependencies.each do |dep|
40
+ dep.recursive_clean
41
+ end
42
+ clean
43
+ end
44
+
45
+ end
@@ -1,6 +1,7 @@
1
1
  require_relative '../path'
2
2
  require_relative '../persist'
3
3
  require_relative 'step/info'
4
+ require_relative 'step/status'
4
5
  require_relative 'step/load'
5
6
  require_relative 'step/file'
6
7
  require_relative 'step/dependencies'
@@ -10,11 +11,13 @@ require_relative 'step/progress'
10
11
 
11
12
  class Step
12
13
 
13
- attr_accessor :path, :inputs, :dependencies, :task, :tee_copies
14
- def initialize(path, inputs = nil, dependencies = nil, &task)
14
+ attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs
15
+ def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, &task)
15
16
  @path = path
16
17
  @inputs = inputs
17
18
  @dependencies = dependencies
19
+ @id = id
20
+ @non_default_inputs = non_default_inputs
18
21
  @task = task
19
22
  @mutex = Mutex.new
20
23
  @tee_copies = 1
@@ -26,7 +29,7 @@ class Step
26
29
 
27
30
  def inputs
28
31
  @inputs ||= begin
29
- if Open.exists?(info_file)
32
+ if info_file && Open.exists?(info_file)
30
33
  info[:inputs]
31
34
  else
32
35
  []
@@ -55,6 +58,13 @@ class Step
55
58
  @name ||= File.basename(@path)
56
59
  end
57
60
 
61
+ def clean_name
62
+ return @id if @id
63
+ return info[:clean_name] if info.include? :clean_name
64
+ return m[1] if m = name.match(/(.*?)(?:_[a-z0-9]{32})?(?:\..*)?/)
65
+ return name.split(".").first
66
+ end
67
+
58
68
  def task_name
59
69
  @task_name ||= @task.name if @task.respond_to?(:name)
60
70
  end
@@ -64,26 +74,65 @@ class Step
64
74
  end
65
75
 
66
76
  def exec
77
+
78
+ if inputs
79
+ if Task === task
80
+ types = task.inputs.collect{|name,type| type }
81
+ new_inputs = inputs.zip(types).collect{|input,info|
82
+ type, desc, default, options = info
83
+ next input unless Step === input
84
+ input.join if input.streaming?
85
+ Task.format_input(input.join.path, type, options)
86
+ }
87
+ else
88
+ new_inputs = inputs.collect{|input|
89
+ Step === input ? input.load : input
90
+ }
91
+ end
92
+ inputs = new_inputs
93
+ end
94
+
67
95
  @result = self.instance_exec(*inputs, &task)
68
96
  end
69
97
 
98
+ def tmp_path
99
+ @tmp_path ||= begin
100
+ basename = File.basename(@path)
101
+ dirname = File.dirname(@path)
102
+ tmp_path = File.join(dirname, '.' + basename)
103
+ @path.setup(tmp_path) if Path === @path
104
+ tmp_path
105
+ end
106
+ end
107
+
70
108
  attr_reader :result
71
- def run
109
+ def run(stream = false)
72
110
  return @result || self.load if done?
73
111
  prepare_dependencies
74
112
  run_dependencies
75
- @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
113
+ @result =
76
114
  begin
77
- merge_info :status => :start, :start => Time.now,
78
- :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"],
79
- :inputs => inputs, :type => type,
80
- :dependencies => dependencies.collect{|d| d.path }
115
+ Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
116
+ clear_info
117
+ merge_info :status => :start, :start => Time.now,
118
+ :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"],
119
+ :inputs => inputs, :type => type,
120
+ :dependencies => dependencies.collect{|d| d.path }
81
121
 
82
- @result = exec
83
- @result = @result.respond_to?(:stream) ? @result.stream : @result
84
- @result
122
+
123
+ @result = exec
124
+
125
+ if @result.nil? && File.exist?(self.tmp_path) && ! File.exist?(self.path)
126
+ Open.mv self.tmp_path, self.path
127
+ else
128
+ @result = @result.stream if @result.respond_to?(:stream)
129
+ end
130
+
131
+ @result
132
+ end
85
133
  rescue Exception => e
86
- merge_info :status => :error, :exception => e
134
+ merge_info :status => :error, :exception => e, :end => Time.now
135
+ abort_dependencies
87
136
  raise e
88
137
  ensure
89
138
  if ! (error? || aborted?)
@@ -106,8 +155,18 @@ class Step
106
155
  end
107
156
  end
108
157
  end
158
+
159
+ if stream && ENV["SCOUT_NO_STREAM"].nil?
160
+ @result
161
+ else
162
+ if IO === @result || @result.respond_to?(:stream)
163
+ join
164
+ @result = nil
165
+ self.load
166
+ else
167
+ @result
168
+ end
109
169
  end
110
- @result
111
170
  end
112
171
 
113
172
  def done?
@@ -140,9 +199,23 @@ class Step
140
199
  end
141
200
  end
142
201
 
202
+ def consume_all_streams
203
+ threads = []
204
+ while @result && streaming? && stream = self.stream
205
+ threads << Open.consume_stream(stream, true)
206
+ end
207
+ threads.each do |t|
208
+ begin
209
+ t.join
210
+ rescue
211
+ threads.each{|t| t.raise(Aborted); t.join }
212
+ raise $!
213
+ end
214
+ end
215
+ end
216
+
143
217
  def join
144
- io = self.stream if streaming?
145
- Open.consume_stream(io, false) if io
218
+ consume_all_streams
146
219
  self
147
220
  end
148
221
 
@@ -157,31 +230,24 @@ class Step
157
230
  done? ? Persist.load(path, type) : exec
158
231
  end
159
232
 
160
- def clean
161
- @take_stream = nil
162
- @result = nil
163
- @info = nil
164
- @info_load_time = nil
165
- Open.rm path if Open.exist?(path)
166
- Open.rm info_file if Open.exist?(info_file)
167
- Open.rm_rf files_dir if Open.exist?(files_dir)
168
- end
169
-
170
- def recursive_clean
171
- dependencies.each do |dep|
172
- dep.recursive_clean
173
- end
174
- clean
175
- end
176
-
177
233
  def step(task_name)
178
234
  dependencies.each do |dep|
179
235
  return dep if dep.task_name == task_name
236
+ rec_dep = dep.step(task_name)
237
+ return rec_dep if rec_dep
180
238
  end
181
239
  nil
182
240
  end
183
241
 
242
+ def short_path
243
+ Scout.identify @path
244
+ end
245
+
184
246
  def digest_str
185
- path.dup
247
+ "Step: " + short_path
248
+ end
249
+
250
+ def fingerprint
251
+ digest_str
186
252
  end
187
253
  end
@@ -1,11 +1,10 @@
1
+ require_relative '../../named_array'
1
2
  module Task
3
+ def self.format_input(value, type, options = {})
4
+ return value if IO === value || StringIO === value || Step === value
2
5
 
3
- def format_input(value, type, options = {})
4
- return value if IO === value || StringIO === value
5
-
6
- value = value.load if Step === value
7
- if String === value && ! [:path, :file].include?(type)
8
- if Open.exists?(value)
6
+ if String === value && ! [:path, :file, :folder].include?(type) && (options.nil? || (! options[:noload] || options[:stream]))
7
+ if Open.exists?(value) && ! Open.directory?(value)
9
8
  Persist.load(value, type)
10
9
  else
11
10
  Persist.deserialize(value, type)
@@ -13,7 +12,7 @@ module Task
13
12
  else
14
13
  if m = type.to_s.match(/(.*)_array/)
15
14
  if Array === value
16
- value.collect{|v| format_input(v, m[1].to_sym, options) }
15
+ value.collect{|v| self.format_input(v, m[1].to_sym, options) }
17
16
  end
18
17
  else
19
18
  value
@@ -34,35 +33,30 @@ module Task
34
33
  IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
35
34
 
36
35
  input_array = []
36
+ input_names = []
37
37
  non_default_inputs = []
38
38
  self.inputs.each_with_index do |p,i|
39
39
  name, type, desc, value, options = p
40
+ input_names << name
40
41
  provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
41
- provided = format_input(provided, type, options || {})
42
+ provided = Task.format_input(provided, type, options || {})
42
43
  if ! provided.nil? && provided != value
43
- non_default_inputs << name
44
+ non_default_inputs << name.to_sym
44
45
  input_array << provided
45
46
  else
46
47
  input_array << value
47
48
  end
48
49
  end
49
50
 
51
+ NamedArray.setup(input_array, input_names)
52
+
50
53
  [input_array, non_default_inputs]
51
54
  end
52
55
 
53
- def digest_inputs(provided_inputs = {})
54
- input_array, non_default_inputs = assign_inputs(provided_inputs)
55
- if Array === provided_inputs
56
- Misc.digest(input_array)
57
- else
58
- Misc.digest(input_array)
59
- end
60
- end
61
-
62
56
  def process_inputs(provided_inputs = {})
63
57
  input_array, non_default_inputs = assign_inputs provided_inputs
64
- digest = Misc.digest(input_array)
65
- [input_array, non_default_inputs, digest]
58
+ digest_str = Misc.digest_str(input_array)
59
+ [input_array, non_default_inputs, digest_str]
66
60
  end
67
61
 
68
62
  def save_file_input(orig_file, directory)
@@ -44,27 +44,45 @@ module Task
44
44
 
45
45
  provided_inputs ||= {}
46
46
 
47
- load_dep = proc do |id, workflow, task, inputs, hash_options, dependencies|
48
- task = hash_options[:task] if hash_options.include?(:task)
49
- workflow = hash_options[:workflow] if hash_options.include?(:workflow)
50
- id = hash_options[:id] if hash_options.include? :id
47
+ # Helper function
48
+ load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
49
+ task = step_options.delete(:task) if step_options.include?(:task)
50
+ workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
51
+ id = step_options.delete(:id) if step_options.include?(:id)
52
+ id = step_options.delete(:jobname) if step_options.include?(:jobname)
51
53
 
52
- hash_inputs = hash_options.include?(:inputs)? hash_options[:inputs] : hash_options
53
- inputs = IndiferentHash.add_defaults hash_inputs, inputs
54
+ step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
55
+ step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
54
56
 
55
57
  resolved_inputs = {}
56
- inputs.each do |k,v|
58
+ step_inputs.each do |k,v|
57
59
  if Symbol === v
58
60
  input_dep = dependencies.select{|d| d.task_name == v }.first
59
- resolved_inputs[k] = input_dep || inputs[v] || k
61
+ resolved_inputs[k] = input_dep || step_inputs[v] || k
60
62
  else
61
63
  resolved_inputs[k] = v
62
64
  end
63
65
  end
64
- workflow.job(task, id, resolved_inputs)
66
+ [workflow.job(task, id, resolved_inputs), step_inputs]
65
67
  end
66
68
 
67
- deps.each do |workflow,task,options,block=nil|
69
+ # Helper function
70
+ find_dep_non_default_inputs = proc do |dep,definition_options,step_inputs={}|
71
+ dep_non_default_inputs = dep.non_default_inputs
72
+ dep_non_default_inputs.select do |name|
73
+ step_inputs.include?(name)
74
+ end
75
+ dep_non_default_inputs.reject! do |name|
76
+ definition_options.include?(name) &&
77
+ (definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
78
+ end
79
+
80
+ dep_non_default_inputs
81
+ end
82
+
83
+ deps.each do |workflow,task,definition_options,block=nil|
84
+ definition_options[:id] = definition_options.delete(:jobname) if definition_options.include?(:jobname)
85
+
68
86
  if provided_inputs.include?(overriden = [workflow.name, task] * "#")
69
87
  dep = provided_inputs[overriden]
70
88
  dep = Step.new dep unless Step === dep
@@ -74,58 +92,52 @@ module Task
74
92
  next
75
93
  end
76
94
 
77
- options ||= {}
95
+ definition_options ||= {}
96
+
78
97
  if block
79
- inputs = IndiferentHash.add_defaults options.dup, provided_inputs
98
+ fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
99
+ self.inputs.each do |name,type,desc,value|
100
+ fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
101
+ end
102
+ fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
103
+ block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
80
104
 
81
- res = block.call id, inputs, dependencies
105
+ res = block.call id, block_options, dependencies
82
106
 
83
107
  case res
84
108
  when Step
85
109
  dep = res
86
110
  dependencies << dep
87
- dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
88
- non_default_inputs.concat(dep_non_default_inputs - options.keys)
111
+ dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
112
+ non_default_inputs.concat(dep_non_default_inputs)
89
113
  when Hash
90
- new_options = res
91
- dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
114
+ step_options = block_options.merge(res)
115
+ dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
92
116
  dependencies << dep
93
- dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
94
- dep_non_default_inputs -= options.keys
95
- if new_options.include?(:inputs)
96
- dep_non_default_inputs -= new_options[:inputs].keys
97
- else
98
- dep_non_default_inputs -= new_options.keys
99
- end
117
+ dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
100
118
  non_default_inputs.concat(dep_non_default_inputs)
101
119
  when Array
102
120
  res.each do |_res|
103
121
  if Hash === _res
104
- new_options = _res
105
- dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
122
+ step_options = block_options.merge(_res)
123
+ dep, step_inputs = load_dep.call(id, workflow, task, step_options, block_options, dependencies)
106
124
  dependencies << dep
107
- dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
108
- dep_non_default_inputs -= options.keys
109
- if new_options.include?(:inputs)
110
- dep_non_default_inputs -= new_options[:inputs].keys
111
- else
112
- dep_non_default_inputs -= new_options.keys
113
- end
125
+ dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
114
126
  non_default_inputs.concat(dep_non_default_inputs)
115
127
  else
116
128
  dep = _res
117
129
  dependencies << dep
118
- dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
119
- non_default_inputs.concat(dep_non_default_inputs - options.keys)
130
+ dep_non_default_inputs = find_dep_non_default_inputs.call(dep, block_options)
131
+ non_default_inputs.concat(dep_non_default_inputs)
120
132
  end
121
133
  end
122
134
  end
123
135
  else
124
- inputs = IndiferentHash.add_defaults options.dup, provided_inputs
125
- dep = load_dep.call(id, workflow, task, inputs, {}, dependencies)
136
+ step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
137
+ dep, step_inputs = load_dep.call(id, workflow, task, step_options, definition_options, dependencies)
126
138
  dependencies << dep
127
- dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
128
- non_default_inputs.concat(dep_non_default_inputs - options.keys)
139
+ dep_non_default_inputs = find_dep_non_default_inputs.call(dep, definition_options, step_inputs)
140
+ non_default_inputs.concat(dep_non_default_inputs)
129
141
  end
130
142
  end
131
143
 
@@ -137,24 +149,47 @@ module Task
137
149
  provided_inputs = {} if provided_inputs.nil?
138
150
  id = DEFAULT_NAME if id.nil?
139
151
 
140
- inputs, non_default_inputs, input_hash = process_inputs provided_inputs
152
+ inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs
141
153
 
142
154
  dependencies = dependencies(id, provided_inputs, non_default_inputs)
143
155
 
144
156
  non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
145
157
 
146
158
  if non_default_inputs.any?
147
- hash = Misc.digest(:inputs => input_hash, :dependencies => dependencies)
148
- Log.debug "Hash #{name} - #{hash}: #{Log.fingerprint(:inputs => inputs, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
149
- id = [id, hash] * "_"
159
+ hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
160
+ name = [id, hash] * "_"
161
+ else
162
+ name = id
150
163
  end
151
164
 
152
- path = directory[id]
165
+ extension = self.extension
166
+ if extension == :dep_task
167
+ extension = nil
168
+ if dependencies.any?
169
+ dep_basename = File.basename(dependencies.last.path)
170
+ if dep_basename.include? "."
171
+ parts = dep_basename.split(".")
172
+ extension = [parts.pop]
173
+ while parts.last.length <= 4
174
+ extension << parts.pop
175
+ end
176
+ extension = extension.reverse * "."
177
+ end
178
+ end
179
+ end
180
+
181
+ path = directory[name]
182
+
183
+ path = path.set_extension(extension) if extension
153
184
 
154
185
  Persist.memory(path) do
155
- Log.debug "Creating job #{path} #{Log.fingerprint inputs} #{Log.fingerprint dependencies}"
186
+ if hash
187
+ Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
188
+ else
189
+ Log.debug "ID #{self.name} #{id} - Clean"
190
+ end
156
191
  NamedArray.setup(inputs, @inputs.collect{|i| i[0] }) if @inputs
157
- Step.new path.find, inputs, dependencies, &self
192
+ Step.new path.find, inputs, dependencies, id, non_default_inputs, &self
158
193
  end
159
194
  end
160
195
  end
@@ -16,14 +16,16 @@ module Task
16
16
  str.puts
17
17
  end
18
18
 
19
+ deps = workflow ? workflow.recursive_deps(self.name) : self.deps if deps.nil?
19
20
  if deps and deps.any?
20
21
  seen = inputs.collect{|name,_| name }
21
22
  dep_inputs = {}
22
- deps.each do |dep_workflow,task_name|
23
+ deps.each do |dep_workflow,task_name,options|
23
24
  next if task_name.nil?
24
25
  task = dep_workflow.tasks[task_name]
25
26
  next if task.inputs.nil?
26
27
  inputs = task.inputs.reject{|name, _| seen.include? name }
28
+ inputs = task.inputs.reject{|name, _| options.include? name }
27
29
  next unless inputs.any?
28
30
  dep = workflow.nil? || dep_workflow.name != workflow.name ? ["#{dep_workflow.name}", task_name.to_s] *"#" : task_name.to_s
29
31
  dep_inputs[dep] = inputs
@@ -109,25 +111,25 @@ end
109
111
 
110
112
  module Workflow
111
113
 
112
- def dep_tree(task_name, seen = nil)
114
+ def dep_tree(task_name, seen = nil, seen_options = nil)
113
115
  @dep_tree ||= {}
114
116
  key = [self, task_name]
115
117
 
116
118
  return @dep_tree[key] if @dep_tree.include?(key)
117
119
  save = seen.nil?
118
120
  seen = Set.new if seen.nil?
121
+ seen_options = {} if seen_options.nil?
119
122
 
120
123
  dep_tree = {}
121
124
  task = self.tasks[task_name]
122
- task.deps.each do |dep|
125
+ task.deps.each do |workflow, task, options|
123
126
  next if seen.include? dep
124
- seen << dep
125
- workflow, task, *rest = dep
127
+ seen << [workflow, task, options.merge(seen_options)]
126
128
  next if task.nil?
127
129
 
128
130
  key = [workflow, task]
129
131
 
130
- dep_tree[key] = workflow.dep_tree(task, seen)
132
+ dep_tree[key] = workflow.dep_tree(task, seen, options.merge(seen_options))
131
133
  end if task.deps
132
134
 
133
135
  @dep_tree[key] = dep_tree if save