scout-gear 10.7.1 → 10.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +6 -0
  3. data/VERSION +1 -1
  4. data/lib/scout/association/index.rb +1 -1
  5. data/lib/scout/association.rb +21 -5
  6. data/lib/scout/entity/format.rb +9 -4
  7. data/lib/scout/entity/identifiers.rb +2 -2
  8. data/lib/scout/entity/named_array.rb +13 -0
  9. data/lib/scout/entity/property.rb +2 -1
  10. data/lib/scout/entity.rb +7 -4
  11. data/lib/scout/persist/tsv/adapter/base.rb +13 -1
  12. data/lib/scout/persist/tsv.rb +2 -1
  13. data/lib/scout/tsv/attach.rb +10 -2
  14. data/lib/scout/tsv/change_id.rb +3 -0
  15. data/lib/scout/tsv/dumper.rb +34 -30
  16. data/lib/scout/tsv/open.rb +1 -0
  17. data/lib/scout/tsv/parser.rb +21 -10
  18. data/lib/scout/tsv/path.rb +8 -0
  19. data/lib/scout/tsv/stream.rb +15 -8
  20. data/lib/scout/tsv/traverse.rb +12 -2
  21. data/lib/scout/tsv/util/process.rb +4 -1
  22. data/lib/scout/tsv/util/select.rb +8 -2
  23. data/lib/scout/tsv/util/sort.rb +23 -15
  24. data/lib/scout/tsv/util.rb +11 -2
  25. data/lib/scout/tsv.rb +23 -11
  26. data/lib/scout/workflow/definition.rb +3 -3
  27. data/lib/scout/workflow/deployment/orchestrator.rb +8 -5
  28. data/lib/scout/workflow/step/dependencies.rb +35 -11
  29. data/lib/scout/workflow/step/file.rb +2 -1
  30. data/lib/scout/workflow/step/info.rb +14 -2
  31. data/lib/scout/workflow/step/load.rb +5 -3
  32. data/lib/scout/workflow/step/progress.rb +6 -0
  33. data/lib/scout/workflow/step/provenance.rb +1 -1
  34. data/lib/scout/workflow/step/status.rb +10 -4
  35. data/lib/scout/workflow/step.rb +32 -12
  36. data/lib/scout/workflow/task/dependencies.rb +33 -24
  37. data/lib/scout/workflow/task/inputs.rb +40 -12
  38. data/lib/scout/workflow/task.rb +22 -10
  39. data/lib/scout/workflow/usage.rb +2 -2
  40. data/lib/scout/workflow.rb +1 -1
  41. data/scout-gear.gemspec +10 -4
  42. data/scout_commands/db/query +83 -0
  43. data/scout_commands/db/register +43 -0
  44. data/scout_commands/db/show +47 -0
  45. data/test/scout/entity/test_named_array.rb +21 -0
  46. data/test/scout/persist/test_tsv.rb +20 -0
  47. data/test/scout/persist/tsv/adapter/test_base.rb +20 -0
  48. data/test/scout/test_tsv.rb +40 -0
  49. data/test/scout/tsv/test_dumper.rb +24 -0
  50. data/test/scout/tsv/test_path.rb +24 -0
  51. data/test/scout/tsv/test_stream.rb +93 -0
  52. data/test/scout/tsv/test_traverse.rb +99 -0
  53. data/test/scout/tsv/test_util.rb +2 -0
  54. data/test/scout/tsv/util/test_select.rb +22 -0
  55. data/test/scout/tsv/util/test_sort.rb +24 -0
  56. data/test/scout/workflow/step/test_dependencies.rb +26 -0
  57. data/test/scout/workflow/step/test_info.rb +35 -0
  58. data/test/scout/workflow/task/test_dependencies.rb +67 -1
  59. data/test/scout/workflow/task/test_inputs.rb +24 -7
  60. data/test/scout/workflow/test_task.rb +36 -0
  61. data/test/scout/workflow/test_usage.rb +0 -1
  62. metadata +9 -3
@@ -1,4 +1,10 @@
1
+ require 'scout/entity'
2
+
1
3
  module TSV
4
+ def prepare_entity(...)
5
+ Entity.prepare_entity(...)
6
+ end
7
+
2
8
  def sort_by(field = nil, just_keys = false, &block)
3
9
  field = :all if field.nil?
4
10
 
@@ -8,16 +14,18 @@ module TSV
8
14
  elems = []
9
15
  case type
10
16
  when :single
11
- through :key, field do |key, field|
12
- elems << [key, field]
17
+ through :key, field do |key, value|
18
+ elems << [key, value]
13
19
  end
14
20
  when :list, :flat
15
- through :key, field do |key, fields|
16
- elems << [key, fields.first]
21
+ through :key, field do |key, value|
22
+ v = field == :key ? key : value[0]
23
+ elems << [key, v]
17
24
  end
18
25
  when :double
19
- through :key, field do |key, fields|
20
- elems << [key, fields.first]
26
+ through :key, field do |key, value|
27
+ v = field == :key ? key : value[0]
28
+ elems << [key, v]
21
29
  end
22
30
  end
23
31
  end
@@ -81,16 +89,16 @@ module TSV
81
89
  elems = []
82
90
  case type
83
91
  when :single
84
- through :key, field do |key, field|
85
- elems << [key, field]
92
+ through :key, field do |key, value|
93
+ elems << [key, value]
86
94
  end
87
95
  when :list, :flat
88
- through :key, field do |key, fields|
89
- elems << [key, fields.first]
96
+ through :key, field do |key, values|
97
+ elems << [key, values[0]]
90
98
  end
91
99
  when :double
92
- through :key, field do |key, fields|
93
- elems << [key, fields.first]
100
+ through :key, field do |key, values|
101
+ elems << [key, values[0]]
94
102
  end
95
103
  end
96
104
  end
@@ -99,7 +107,7 @@ module TSV
99
107
  if fields == :all
100
108
  if just_keys
101
109
  keys = elems.sort_by{|key, value| key }.collect{|key, values| key}
102
- keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
110
+ keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true))
103
111
  else
104
112
  elems.sort_by{|key, value| key }
105
113
  end
@@ -128,7 +136,7 @@ module TSV
128
136
  end
129
137
  if just_keys
130
138
  keys = sorted.collect{|key, value| key}
131
- keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
139
+ keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
132
140
  keys
133
141
  else
134
142
  sorted.collect{|key, value| [key, self[key]]}
@@ -137,7 +145,7 @@ module TSV
137
145
  else
138
146
  if just_keys
139
147
  keys = elems.sort(&block).collect{|key, value| key}
140
- keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
148
+ keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
141
149
  keys
142
150
  else
143
151
  elems.sort(&block).collect{|key, value| [key, self[key]]}
@@ -32,6 +32,7 @@ module TSV
32
32
  text
33
33
  end
34
34
 
35
+ path = path.find if Path === path
35
36
  TmpFile.with_file(values.uniq * "\n", false) do |value_file|
36
37
  cmd = "cat '#{ path }' | sed 's/\\t/\\tHEADERNOMATCH/' | grep -w -F -f '#{ value_file }' | sed 's/HEADERNOMATCH//' |sort -u|cut -f 2 |sort|uniq -c|sed 's/^ *//;s/ /\t/'"
37
38
  begin
@@ -84,8 +85,9 @@ module TSV
84
85
 
85
86
  def each(*args, &block)
86
87
  if block_given?
88
+ actual_unnamed = @unnamed.nil? ? true : @unnamed
87
89
  super(*args) do |k,v|
88
- NamedArray.setup(v, @fields) unless @unnamed || @type == :flat || ! (Array === v)
90
+ NamedArray.setup(v, @fields) unless actual_unnamed || @type == :flat || ! (Array === v)
89
91
  block.call(k, v)
90
92
  end
91
93
  else
@@ -105,7 +107,8 @@ module TSV
105
107
  end
106
108
  end
107
109
 
108
- def with_unnamed(unnamed = true)
110
+ def with_unnamed(unnamed = nil)
111
+ unnamed = true if unnamed.nil?
109
112
  begin
110
113
  old_unnamed = @unnamed
111
114
  @unnamed = unnamed
@@ -177,4 +180,10 @@ Example:
177
180
  self.annotate(super(other))
178
181
  end
179
182
 
183
+ def merge_zip(other)
184
+ other.each do |k,v|
185
+ self.zip_new k, v
186
+ end
187
+ end
188
+
180
189
  end
data/lib/scout/tsv.rb CHANGED
@@ -16,7 +16,8 @@ require_relative 'tsv/csv'
16
16
 
17
17
  module TSV
18
18
  extend Annotation
19
- annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer
19
+ annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer, :entity_options
20
+
20
21
 
21
22
  def self.str2options(str)
22
23
  field_options,_sep, rest = str.partition("#")
@@ -34,7 +35,6 @@ module TSV
34
35
  alias original_setup setup
35
36
 
36
37
  def setup(obj, *rest, &block)
37
-
38
38
  if rest.length == 1 && String === rest.first
39
39
  options = TSV.str2options(rest.first)
40
40
  if Array === obj
@@ -44,7 +44,7 @@ module TSV
44
44
  when :single
45
45
  nil
46
46
  end
47
- obj = Misc.array2hash(obj, default_value)
47
+ obj = IndiferentHash.array2hash(obj, default_value)
48
48
  end
49
49
  original_setup(obj, options, &block)
50
50
  else
@@ -57,7 +57,7 @@ module TSV
57
57
  when :single
58
58
  nil
59
59
  end
60
- obj = Misc.array2hash(obj, default_value)
60
+ obj = IndiferentHash.array2hash(obj, default_value)
61
61
  end
62
62
  original_setup(obj, *rest, &block)
63
63
  end
@@ -74,11 +74,17 @@ module TSV
74
74
  end
75
75
 
76
76
  def self.open(file, options = {})
77
- grep, invert_grep, nocache, monitor, entity_options = IndiferentHash.process_options options, :grep, :invert_grep, :nocache, :monitor, :entity_options
77
+ grep, invert_grep, fixed_grep, nocache, monitor, entity_options, unnamed, field = IndiferentHash.process_options options, :grep, :invert_grep, :fixed_grep, :nocache, :monitor, :entity_options, :unnamed, :field
78
+
79
+ if field and options[:field].nil?
80
+ options[:fields] = [field]
81
+ options[:type] ||= :single
82
+ end
78
83
 
79
84
  persist_options = IndiferentHash.pull_keys options, :persist
80
85
  persist_options = IndiferentHash.add_defaults persist_options, prefix: "TSV", type: :HDB, persist: false
81
86
  persist_options[:data] ||= options[:data]
87
+ persist_options[:update] = options.delete(:update) if options.include?(:update)
82
88
 
83
89
  file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
84
90
 
@@ -113,14 +119,20 @@ module TSV
113
119
  end
114
120
 
115
121
  tsv = if TSV::Parser === file
116
- TSV.parse(file, **options)
117
- else
118
- options[:tsv_invert_grep] ||= invert_grep if invert_grep
119
- Open.open(file, grep: grep, invert_grep: invert_grep, nocache: nocache) do |f|
120
- TSV.parse(f, **options)
121
- end
122
+ TSV.parse(file, **options)
123
+ else
124
+ options[:tsv_invert_grep] ||= invert_grep if invert_grep
125
+ Open.open(file, grep: grep, invert_grep: invert_grep, fixed_grep: fixed_grep, nocache: nocache) do |f|
126
+ TSV.parse(f, **options)
127
+ end
128
+ end
129
+
130
+ if tsv.identifiers.nil? and Path === tsv.filename and tsv.filename.identifier_file_path
131
+ tsv.identifiers = tsv.filename.identifier_file_path.find if tsv.filename.identifier_file_path.exists?
122
132
  end
123
133
 
134
+ tsv.unnamed = unnamed unless unnamed.nil?
135
+
124
136
  tsv
125
137
  end
126
138
  end
@@ -164,9 +164,9 @@ module Workflow
164
164
  Step.wait_for_jobs dependencies.select{|d| d.streaming? }
165
165
  dep = dependencies.last
166
166
  dep.join
167
- raise dep.get_exception if dep.error?
167
+ raise dep.exception if dep.error?
168
168
  raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
169
- set_info :type, dep.info[:type]
169
+ merge_info type: dep.info[:type], task_alias: true
170
170
 
171
171
  forget = config :forget_task_alias, "forget_task_alias"
172
172
  forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_TASK_ALIAS if forget.nil?
@@ -181,7 +181,7 @@ module Workflow
181
181
  self.copy_linked_files_dir
182
182
  self.dependencies = self.dependencies - [dep]
183
183
  Open.rm_rf self.files_dir if Open.exist? self.files_dir
184
- FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
184
+ Open.link_dir dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
185
185
 
186
186
  if dep.overriden?
187
187
  Open.link dep.path, self.tmp_path
@@ -34,14 +34,16 @@ module Workflow
34
34
  end
35
35
 
36
36
  def self.job_rules(rules, job)
37
+ IndiferentHash.setup(rules)
37
38
  workflow = job.workflow.name
38
39
  task_name = job.task_name.to_s
39
40
  defaults = rules["defaults"] || {}
40
41
 
41
42
  return IndiferentHash.setup(defaults) unless rules[workflow]
42
- return IndiferentHash.setup(defaults) unless rules[workflow][task_name]
43
+ workflow_rules = IndiferentHash.setup(rules[workflow])
44
+ return IndiferentHash.setup(defaults) unless workflow_rules[task_name]
45
+ job_rules = IndiferentHash.setup(workflow_rules[task_name])
43
46
 
44
- job_rules = IndiferentHash.setup(rules[workflow][task_name])
45
47
  defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults
46
48
  job_rules
47
49
  end
@@ -85,7 +87,7 @@ module Workflow
85
87
  candidates = workload.
86
88
  select{|k,v| v.empty? }.
87
89
  collect{|k,v| k }.
88
- reject{|k| k.done? || k.running? }
90
+ reject{|k| k.done? || k.running? || (k.error? && ! k.recoverable_error?) }
89
91
  else
90
92
  candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.
91
93
  select{|k,v| v.empty? }.
@@ -107,7 +109,7 @@ module Workflow
107
109
 
108
110
  attr_accessor :available_resources, :resources_requested, :resources_used, :timer
109
111
 
110
- def initialize(timer = 5, available_resources = {})
112
+ def initialize(timer = 5, available_resources = nil)
111
113
  available_resources = {:cpus => Etc.nprocessors } if available_resources.nil?
112
114
  @timer = timer
113
115
  @available_resources = IndiferentHash.setup(available_resources)
@@ -192,7 +194,7 @@ module Workflow
192
194
  workload = Orchestrator.workload(jobs)
193
195
  all_jobs = workload.keys
194
196
 
195
- all_jobs.each{|job| job.clean unless job.done? && job.updated? }
197
+ all_jobs.each{|job| job.clean unless (job.done? && job.updated?) || (job.error? && ! job.recoverable_error?) }
196
198
 
197
199
  top_level_jobs = jobs.collect{|job| job.path }
198
200
  failed_jobs = []
@@ -216,6 +218,7 @@ module Workflow
216
218
  raise TryAgain
217
219
  end
218
220
  else
221
+ Log.warn "Non-recoverable error in #{job.path}"
219
222
  next
220
223
  end
221
224
  ensure
@@ -1,20 +1,26 @@
1
1
  class Step
2
2
  def rec_dependencies(connected = false, seen = [])
3
- direct_deps = []
4
- dependencies.each do |dep|
5
- next if seen.include? dep.path
6
- next if connected && dep.done? && dep.updated?
7
- direct_deps << dep
8
- end if dependencies
9
- seen.concat direct_deps.collect{|d| d.path }
10
- direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
3
+ @rec_dependencies = {}
4
+ @rec_dependencies[connected] ||= begin
5
+ direct_deps = []
6
+ dependencies.each do |dep|
7
+ next if seen.include? dep.path
8
+ next if connected && dep.done? && dep.updated?
9
+ direct_deps << dep
10
+ end if dependencies
11
+ seen.concat direct_deps.collect{|d| d.path }
12
+ seen.uniq!
13
+ direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
14
+ direct_deps.uniq!
15
+ direct_deps
16
+ end
11
17
  end
12
18
 
13
19
  def recursive_inputs
14
20
  recursive_inputs = NamedArray === inputs ? inputs.to_hash : {}
15
21
  return recursive_inputs if dependencies.nil?
16
22
  dependencies.inject(recursive_inputs) do |acc,dep|
17
- acc.merge(dep.recursive_inputs)
23
+ acc = dep.recursive_inputs.merge(acc)
18
24
  end
19
25
  end
20
26
 
@@ -41,6 +47,7 @@ class Step
41
47
  end
42
48
 
43
49
  next if dep.done?
50
+ next if dep.error? && ! dep.recoverable_error?
44
51
 
45
52
  if dep.dependencies
46
53
  dep.dependencies.each do |d|
@@ -72,9 +79,13 @@ class Step
72
79
  def run_dependencies
73
80
  all_dependencies.each do |dep|
74
81
  next if dep.running? || dep.done?
82
+ next if dep.error? && ! dep.recoverable_error?
83
+
75
84
  compute_options = compute[dep.path] if compute
76
85
  compute_options = [] if compute_options.nil?
77
86
 
87
+ next if compute_options.include?(false)
88
+
78
89
  stream = compute_options.include?(:stream)
79
90
  stream = true unless ENV["SCOUT_EXPLICIT_STREAMING"] == 'true'
80
91
  stream = :no_load if compute_options.include?(:produce)
@@ -95,12 +106,25 @@ class Step
95
106
  all_dependencies.each{|dep| dep.abort if dep.running? }
96
107
  end
97
108
 
98
- def self.wait_for_jobs(jobs)
109
+ def self.wait_for_jobs(jobs, canfail=false)
99
110
  threads = []
100
111
  jobs.each do |job|
101
112
  threads << Thread.new do
102
113
  Thread.current.report_on_exception = false
103
- job.join
114
+ begin
115
+ job.join
116
+ rescue Exception
117
+ case canfail
118
+ when TrueClass
119
+ next
120
+ else
121
+ if canfail === $!
122
+ next
123
+ else
124
+ raise $!
125
+ end
126
+ end
127
+ end
104
128
  end
105
129
  end
106
130
  threads.each do |t|
@@ -7,6 +7,7 @@ class Step
7
7
  else
8
8
  Path.setup(dir)
9
9
  end
10
+ dir = dir.find
10
11
  dir.pkgdir = self
11
12
  dir
12
13
  end
@@ -34,7 +35,7 @@ class Step
34
35
  begin
35
36
  realpath = Open.realpath(self.files_dir)
36
37
  Open.rm self.files_dir
37
- Open.cp realpath, self.files_dir
38
+ Open.link_dir realpath, self.files_dir
38
39
  rescue
39
40
  Log.warn "Copy files_dir for #{self.workflow_short_path} failed: " + $!.message
40
41
  end
@@ -24,7 +24,7 @@ class Step
24
24
  end
25
25
 
26
26
  def save_info(info = nil)
27
- Persist.save(info, info_file, SERIALIZER)
27
+ Persist.save(@info = info, info_file, SERIALIZER)
28
28
  @info_load_time = Time.now
29
29
  end
30
30
 
@@ -121,6 +121,7 @@ class Step
121
121
  if message.nil?
122
122
  Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
123
123
  else
124
+ message = Log.fingerprint(message).sub(/^'/,'').sub(/'$/,'')
124
125
  Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
125
126
  end
126
127
  end
@@ -143,6 +144,10 @@ class Step
143
144
  info[:messages]
144
145
  end
145
146
 
147
+ def message(message)
148
+ merge_info :message => message
149
+ end
150
+
146
151
  def status
147
152
  info[:status].tap{|s| s.nil? ? s : s.to_sym }
148
153
  end
@@ -165,7 +170,14 @@ class Step
165
170
  end
166
171
 
167
172
  def overriden_deps
168
- rec_dependencies.select{|d| d.overriden? }
173
+ dependencies.select{|d| d.overriden? }
174
+ end
175
+
176
+ def recursive_overriden_deps
177
+ overriden_deps = dependencies.select{|d|
178
+ d.overriden?
179
+ }
180
+ (overriden_deps + overriden_deps.collect{|d| d.recursive_overriden_deps }).flatten.uniq
169
181
  end
170
182
 
171
183
  def exception
@@ -4,9 +4,11 @@ class Step
4
4
  Path.setup(path) unless Path === path
5
5
  relocated = path.relocate
6
6
  return relocated if Open.exists?(relocated)
7
- subpath = path.split("/")[-3..-1] * "/"
8
- relocated = Path.setup("var/jobs")[subpath]
9
- return relocated if Open.exists?(relocated)
7
+ if path.scan("/").length >= 2
8
+ subpath = path.split("/")[-3..-1] * "/"
9
+ relocated = Path.setup("var/jobs")[subpath]
10
+ return relocated if Open.exists?(relocated)
11
+ end
10
12
  path
11
13
  end
12
14
 
@@ -19,5 +19,11 @@ class Step
19
19
  bar
20
20
  end
21
21
  end
22
+
23
+ def traverse(obj, desc: nil , **kwargs, &block)
24
+ desc = "Processing #{self.short_path}" if desc.nil?
25
+ kwargs[:bar] = self.progress_bar(desc) unless kwargs.include?(:bar)
26
+ TSV.traverse obj, **kwargs, &block
27
+ end
22
28
  end
23
29
 
@@ -62,7 +62,7 @@ class Step
62
62
  end
63
63
 
64
64
  if $inputs and $inputs.any?
65
- job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
65
+ job_inputs = Step.new(path).recursive_inputs.to_hash
66
66
  IndiferentHash.setup(job_inputs)
67
67
 
68
68
  $inputs.each do |input|
@@ -20,11 +20,17 @@ class Step
20
20
  self.error? && ! (ScoutException === self.exception)
21
21
  end
22
22
 
23
- def updated?
24
- return false if self.error? && self.recoverable_error?
25
- return true if self.done? && ! ENV["SCOUT_UPDATE"]
23
+ def newer_dependencies
26
24
  newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
27
25
  newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
26
+ newer += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| Path.newer?(self.path, dep.path) }
27
+ newer
28
+ end
29
+
30
+ def updated?
31
+ return false if self.error? && self.recoverable_error?
32
+ return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"]
33
+ newer = newer_dependencies
28
34
 
29
35
  Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
30
36
  newer.empty?
@@ -56,7 +62,7 @@ class Step
56
62
  end
57
63
 
58
64
  def canfail?
59
- @compute && @compute.include?(:canfail)
65
+ @compute && @compute[self.path] && @compute[self.path].include?(:canfail)
60
66
  end
61
67
 
62
68
  def started?
@@ -138,18 +138,32 @@ class Step
138
138
 
139
139
  attr_reader :result
140
140
  def run(stream = false)
141
+ case stream
142
+ when TrueClass, :stream
143
+ no_load = :stream
144
+ when :no_load
145
+ no_load = true
146
+ else
147
+ no_load = false
148
+ end
149
+
150
+ if done?
151
+ if no_load
152
+ if no_load == :stream
153
+ return self.stream
154
+ else
155
+ return self.path
156
+ end
157
+ else
158
+ return @result || self.load
159
+ end
160
+ end
161
+
162
+
141
163
  return @result || self.load if done?
142
164
  prepare_dependencies
143
165
  begin
144
166
 
145
- case stream
146
- when TrueClass, :stream
147
- no_load = :stream
148
- when :no_load
149
- no_load = true
150
- else
151
- no_load = false
152
- end
153
167
 
154
168
  @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies, no_load: no_load) do
155
169
  input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
@@ -206,6 +220,7 @@ class Step
206
220
  end
207
221
 
208
222
  @result.abort_callback = proc do |exception|
223
+ Open.rm self.path
209
224
  if exception.nil? || Aborted === exception || Interrupt === exception
210
225
  merge_info :status => :aborted, :end => Time.now
211
226
  else
@@ -240,6 +255,7 @@ class Step
240
255
  run(noload)
241
256
  end
242
257
  join
258
+ exit! 0
243
259
  end
244
260
  Process.detach pid
245
261
  grace
@@ -350,6 +366,10 @@ class Step
350
366
 
351
367
  def step(task_name)
352
368
  task_name = task_name.to_sym
369
+ dependencies.each do |dep|
370
+ return dep if dep.task_name && dep.task_name.to_sym == task_name
371
+ return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
372
+ end
353
373
  dependencies.each do |dep|
354
374
  return dep if dep.task_name && dep.task_name.to_sym == task_name
355
375
  return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
@@ -359,12 +379,12 @@ class Step
359
379
  nil
360
380
  end
361
381
 
362
- def short_path
363
- Resource.identify @path
382
+ def identify_path
383
+ Resource.identify path
364
384
  end
365
385
 
366
386
  def digest_str
367
- "Step: " + short_path
387
+ "Step: " + identify_path
368
388
  end
369
389
 
370
390
  def fingerprint
@@ -373,7 +393,7 @@ class Step
373
393
 
374
394
  def task_signature
375
395
  workflow_name = String === workflow ? workflow : workflow.name
376
- [workflow, task_name] * "#"
396
+ [workflow_name, task_name] * "#"
377
397
  end
378
398
 
379
399
  def alias?