scout-gear 10.7.1 → 10.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +6 -0
- data/VERSION +1 -1
- data/lib/scout/association/index.rb +1 -1
- data/lib/scout/association.rb +21 -5
- data/lib/scout/entity/format.rb +9 -4
- data/lib/scout/entity/identifiers.rb +2 -2
- data/lib/scout/entity/named_array.rb +13 -0
- data/lib/scout/entity/property.rb +2 -1
- data/lib/scout/entity.rb +7 -4
- data/lib/scout/persist/tsv/adapter/base.rb +13 -1
- data/lib/scout/persist/tsv.rb +2 -1
- data/lib/scout/tsv/attach.rb +10 -2
- data/lib/scout/tsv/change_id.rb +3 -0
- data/lib/scout/tsv/dumper.rb +34 -30
- data/lib/scout/tsv/open.rb +1 -0
- data/lib/scout/tsv/parser.rb +21 -10
- data/lib/scout/tsv/path.rb +8 -0
- data/lib/scout/tsv/stream.rb +15 -8
- data/lib/scout/tsv/traverse.rb +12 -2
- data/lib/scout/tsv/util/process.rb +4 -1
- data/lib/scout/tsv/util/select.rb +8 -2
- data/lib/scout/tsv/util/sort.rb +23 -15
- data/lib/scout/tsv/util.rb +11 -2
- data/lib/scout/tsv.rb +23 -11
- data/lib/scout/workflow/definition.rb +3 -3
- data/lib/scout/workflow/deployment/orchestrator.rb +8 -5
- data/lib/scout/workflow/step/dependencies.rb +35 -11
- data/lib/scout/workflow/step/file.rb +2 -1
- data/lib/scout/workflow/step/info.rb +14 -2
- data/lib/scout/workflow/step/load.rb +5 -3
- data/lib/scout/workflow/step/progress.rb +6 -0
- data/lib/scout/workflow/step/provenance.rb +1 -1
- data/lib/scout/workflow/step/status.rb +10 -4
- data/lib/scout/workflow/step.rb +32 -12
- data/lib/scout/workflow/task/dependencies.rb +33 -24
- data/lib/scout/workflow/task/inputs.rb +40 -12
- data/lib/scout/workflow/task.rb +22 -10
- data/lib/scout/workflow/usage.rb +2 -2
- data/lib/scout/workflow.rb +1 -1
- data/scout-gear.gemspec +10 -4
- data/scout_commands/db/query +83 -0
- data/scout_commands/db/register +43 -0
- data/scout_commands/db/show +47 -0
- data/test/scout/entity/test_named_array.rb +21 -0
- data/test/scout/persist/test_tsv.rb +20 -0
- data/test/scout/persist/tsv/adapter/test_base.rb +20 -0
- data/test/scout/test_tsv.rb +40 -0
- data/test/scout/tsv/test_dumper.rb +24 -0
- data/test/scout/tsv/test_path.rb +24 -0
- data/test/scout/tsv/test_stream.rb +93 -0
- data/test/scout/tsv/test_traverse.rb +99 -0
- data/test/scout/tsv/test_util.rb +2 -0
- data/test/scout/tsv/util/test_select.rb +22 -0
- data/test/scout/tsv/util/test_sort.rb +24 -0
- data/test/scout/workflow/step/test_dependencies.rb +26 -0
- data/test/scout/workflow/step/test_info.rb +35 -0
- data/test/scout/workflow/task/test_dependencies.rb +67 -1
- data/test/scout/workflow/task/test_inputs.rb +24 -7
- data/test/scout/workflow/test_task.rb +36 -0
- data/test/scout/workflow/test_usage.rb +0 -1
- metadata +9 -3
data/lib/scout/tsv/util/sort.rb
CHANGED
@@ -1,4 +1,10 @@
|
|
1
|
+
require 'scout/entity'
|
2
|
+
|
1
3
|
module TSV
|
4
|
+
def prepare_entity(...)
|
5
|
+
Entity.prepare_entity(...)
|
6
|
+
end
|
7
|
+
|
2
8
|
def sort_by(field = nil, just_keys = false, &block)
|
3
9
|
field = :all if field.nil?
|
4
10
|
|
@@ -8,16 +14,18 @@ module TSV
|
|
8
14
|
elems = []
|
9
15
|
case type
|
10
16
|
when :single
|
11
|
-
through :key, field do |key,
|
12
|
-
elems << [key,
|
17
|
+
through :key, field do |key, value|
|
18
|
+
elems << [key, value]
|
13
19
|
end
|
14
20
|
when :list, :flat
|
15
|
-
through :key, field do |key,
|
16
|
-
|
21
|
+
through :key, field do |key, value|
|
22
|
+
v = field == :key ? key : value[0]
|
23
|
+
elems << [key, v]
|
17
24
|
end
|
18
25
|
when :double
|
19
|
-
through :key, field do |key,
|
20
|
-
|
26
|
+
through :key, field do |key, value|
|
27
|
+
v = field == :key ? key : value[0]
|
28
|
+
elems << [key, v]
|
21
29
|
end
|
22
30
|
end
|
23
31
|
end
|
@@ -81,16 +89,16 @@ module TSV
|
|
81
89
|
elems = []
|
82
90
|
case type
|
83
91
|
when :single
|
84
|
-
through :key, field do |key,
|
85
|
-
elems << [key,
|
92
|
+
through :key, field do |key, value|
|
93
|
+
elems << [key, value]
|
86
94
|
end
|
87
95
|
when :list, :flat
|
88
|
-
through :key, field do |key,
|
89
|
-
elems << [key,
|
96
|
+
through :key, field do |key, values|
|
97
|
+
elems << [key, values[0]]
|
90
98
|
end
|
91
99
|
when :double
|
92
|
-
through :key, field do |key,
|
93
|
-
elems << [key,
|
100
|
+
through :key, field do |key, values|
|
101
|
+
elems << [key, values[0]]
|
94
102
|
end
|
95
103
|
end
|
96
104
|
end
|
@@ -99,7 +107,7 @@ module TSV
|
|
99
107
|
if fields == :all
|
100
108
|
if just_keys
|
101
109
|
keys = elems.sort_by{|key, value| key }.collect{|key, values| key}
|
102
|
-
keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true))
|
110
|
+
keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true))
|
103
111
|
else
|
104
112
|
elems.sort_by{|key, value| key }
|
105
113
|
end
|
@@ -128,7 +136,7 @@ module TSV
|
|
128
136
|
end
|
129
137
|
if just_keys
|
130
138
|
keys = sorted.collect{|key, value| key}
|
131
|
-
keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
|
139
|
+
keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
|
132
140
|
keys
|
133
141
|
else
|
134
142
|
sorted.collect{|key, value| [key, self[key]]}
|
@@ -137,7 +145,7 @@ module TSV
|
|
137
145
|
else
|
138
146
|
if just_keys
|
139
147
|
keys = elems.sort(&block).collect{|key, value| key}
|
140
|
-
keys = prepare_entity(keys, key_field, entity_options.merge(:dup_array => true)) unless @unnamed
|
148
|
+
keys = prepare_entity(keys, key_field, (entity_options || {}).merge(:dup_array => true)) unless @unnamed
|
141
149
|
keys
|
142
150
|
else
|
143
151
|
elems.sort(&block).collect{|key, value| [key, self[key]]}
|
data/lib/scout/tsv/util.rb
CHANGED
@@ -32,6 +32,7 @@ module TSV
|
|
32
32
|
text
|
33
33
|
end
|
34
34
|
|
35
|
+
path = path.find if Path === path
|
35
36
|
TmpFile.with_file(values.uniq * "\n", false) do |value_file|
|
36
37
|
cmd = "cat '#{ path }' | sed 's/\\t/\\tHEADERNOMATCH/' | grep -w -F -f '#{ value_file }' | sed 's/HEADERNOMATCH//' |sort -u|cut -f 2 |sort|uniq -c|sed 's/^ *//;s/ /\t/'"
|
37
38
|
begin
|
@@ -84,8 +85,9 @@ module TSV
|
|
84
85
|
|
85
86
|
def each(*args, &block)
|
86
87
|
if block_given?
|
88
|
+
actual_unnamed = @unnamed.nil? ? true : @unnamed
|
87
89
|
super(*args) do |k,v|
|
88
|
-
NamedArray.setup(v, @fields) unless
|
90
|
+
NamedArray.setup(v, @fields) unless actual_unnamed || @type == :flat || ! (Array === v)
|
89
91
|
block.call(k, v)
|
90
92
|
end
|
91
93
|
else
|
@@ -105,7 +107,8 @@ module TSV
|
|
105
107
|
end
|
106
108
|
end
|
107
109
|
|
108
|
-
def with_unnamed(unnamed =
|
110
|
+
def with_unnamed(unnamed = nil)
|
111
|
+
unnamed = true if unnamed.nil?
|
109
112
|
begin
|
110
113
|
old_unnamed = @unnamed
|
111
114
|
@unnamed = unnamed
|
@@ -177,4 +180,10 @@ Example:
|
|
177
180
|
self.annotate(super(other))
|
178
181
|
end
|
179
182
|
|
183
|
+
def merge_zip(other)
|
184
|
+
other.each do |k,v|
|
185
|
+
self.zip_new k, v
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
180
189
|
end
|
data/lib/scout/tsv.rb
CHANGED
@@ -16,7 +16,8 @@ require_relative 'tsv/csv'
|
|
16
16
|
|
17
17
|
module TSV
|
18
18
|
extend Annotation
|
19
|
-
annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer
|
19
|
+
annotation :key_field, :fields, :type, :cast, :filename, :namespace, :unnamed, :identifiers, :serializer, :entity_options
|
20
|
+
|
20
21
|
|
21
22
|
def self.str2options(str)
|
22
23
|
field_options,_sep, rest = str.partition("#")
|
@@ -34,7 +35,6 @@ module TSV
|
|
34
35
|
alias original_setup setup
|
35
36
|
|
36
37
|
def setup(obj, *rest, &block)
|
37
|
-
|
38
38
|
if rest.length == 1 && String === rest.first
|
39
39
|
options = TSV.str2options(rest.first)
|
40
40
|
if Array === obj
|
@@ -44,7 +44,7 @@ module TSV
|
|
44
44
|
when :single
|
45
45
|
nil
|
46
46
|
end
|
47
|
-
obj =
|
47
|
+
obj = IndiferentHash.array2hash(obj, default_value)
|
48
48
|
end
|
49
49
|
original_setup(obj, options, &block)
|
50
50
|
else
|
@@ -57,7 +57,7 @@ module TSV
|
|
57
57
|
when :single
|
58
58
|
nil
|
59
59
|
end
|
60
|
-
obj =
|
60
|
+
obj = IndiferentHash.array2hash(obj, default_value)
|
61
61
|
end
|
62
62
|
original_setup(obj, *rest, &block)
|
63
63
|
end
|
@@ -74,11 +74,17 @@ module TSV
|
|
74
74
|
end
|
75
75
|
|
76
76
|
def self.open(file, options = {})
|
77
|
-
grep, invert_grep, nocache, monitor, entity_options = IndiferentHash.process_options options, :grep, :invert_grep, :nocache, :monitor, :entity_options
|
77
|
+
grep, invert_grep, fixed_grep, nocache, monitor, entity_options, unnamed, field = IndiferentHash.process_options options, :grep, :invert_grep, :fixed_grep, :nocache, :monitor, :entity_options, :unnamed, :field
|
78
|
+
|
79
|
+
if field and options[:field].nil?
|
80
|
+
options[:fields] = [field]
|
81
|
+
options[:type] ||= :single
|
82
|
+
end
|
78
83
|
|
79
84
|
persist_options = IndiferentHash.pull_keys options, :persist
|
80
85
|
persist_options = IndiferentHash.add_defaults persist_options, prefix: "TSV", type: :HDB, persist: false
|
81
86
|
persist_options[:data] ||= options[:data]
|
87
|
+
persist_options[:update] = options.delete(:update) if options.include?(:update)
|
82
88
|
|
83
89
|
file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
|
84
90
|
|
@@ -113,14 +119,20 @@ module TSV
|
|
113
119
|
end
|
114
120
|
|
115
121
|
tsv = if TSV::Parser === file
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
+
TSV.parse(file, **options)
|
123
|
+
else
|
124
|
+
options[:tsv_invert_grep] ||= invert_grep if invert_grep
|
125
|
+
Open.open(file, grep: grep, invert_grep: invert_grep, fixed_grep: fixed_grep, nocache: nocache) do |f|
|
126
|
+
TSV.parse(f, **options)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
if tsv.identifiers.nil? and Path === tsv.filename and tsv.filename.identifier_file_path
|
131
|
+
tsv.identifiers = tsv.filename.identifier_file_path.find if tsv.filename.identifier_file_path.exists?
|
122
132
|
end
|
123
133
|
|
134
|
+
tsv.unnamed = unnamed unless unnamed.nil?
|
135
|
+
|
124
136
|
tsv
|
125
137
|
end
|
126
138
|
end
|
@@ -164,9 +164,9 @@ module Workflow
|
|
164
164
|
Step.wait_for_jobs dependencies.select{|d| d.streaming? }
|
165
165
|
dep = dependencies.last
|
166
166
|
dep.join
|
167
|
-
raise dep.
|
167
|
+
raise dep.exception if dep.error?
|
168
168
|
raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
|
169
|
-
|
169
|
+
merge_info type: dep.info[:type], task_alias: true
|
170
170
|
|
171
171
|
forget = config :forget_task_alias, "forget_task_alias"
|
172
172
|
forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_TASK_ALIAS if forget.nil?
|
@@ -181,7 +181,7 @@ module Workflow
|
|
181
181
|
self.copy_linked_files_dir
|
182
182
|
self.dependencies = self.dependencies - [dep]
|
183
183
|
Open.rm_rf self.files_dir if Open.exist? self.files_dir
|
184
|
-
|
184
|
+
Open.link_dir dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
|
185
185
|
|
186
186
|
if dep.overriden?
|
187
187
|
Open.link dep.path, self.tmp_path
|
@@ -34,14 +34,16 @@ module Workflow
|
|
34
34
|
end
|
35
35
|
|
36
36
|
def self.job_rules(rules, job)
|
37
|
+
IndiferentHash.setup(rules)
|
37
38
|
workflow = job.workflow.name
|
38
39
|
task_name = job.task_name.to_s
|
39
40
|
defaults = rules["defaults"] || {}
|
40
41
|
|
41
42
|
return IndiferentHash.setup(defaults) unless rules[workflow]
|
42
|
-
|
43
|
+
workflow_rules = IndiferentHash.setup(rules[workflow])
|
44
|
+
return IndiferentHash.setup(defaults) unless workflow_rules[task_name]
|
45
|
+
job_rules = IndiferentHash.setup(workflow_rules[task_name])
|
43
46
|
|
44
|
-
job_rules = IndiferentHash.setup(rules[workflow][task_name])
|
45
47
|
defaults.each{|k,v| job_rules[k] = v if job_rules[k].nil? } if defaults
|
46
48
|
job_rules
|
47
49
|
end
|
@@ -85,7 +87,7 @@ module Workflow
|
|
85
87
|
candidates = workload.
|
86
88
|
select{|k,v| v.empty? }.
|
87
89
|
collect{|k,v| k }.
|
88
|
-
reject{|k| k.done? || k.running? }
|
90
|
+
reject{|k| k.done? || k.running? || (k.error? && ! k.recoverable_error?) }
|
89
91
|
else
|
90
92
|
candidates = workload. #select{|k,v| Orchestrator.job_rules(rules, k) }.
|
91
93
|
select{|k,v| v.empty? }.
|
@@ -107,7 +109,7 @@ module Workflow
|
|
107
109
|
|
108
110
|
attr_accessor :available_resources, :resources_requested, :resources_used, :timer
|
109
111
|
|
110
|
-
def initialize(timer = 5, available_resources =
|
112
|
+
def initialize(timer = 5, available_resources = nil)
|
111
113
|
available_resources = {:cpus => Etc.nprocessors } if available_resources.nil?
|
112
114
|
@timer = timer
|
113
115
|
@available_resources = IndiferentHash.setup(available_resources)
|
@@ -192,7 +194,7 @@ module Workflow
|
|
192
194
|
workload = Orchestrator.workload(jobs)
|
193
195
|
all_jobs = workload.keys
|
194
196
|
|
195
|
-
all_jobs.each{|job| job.clean unless job.done? && job.updated? }
|
197
|
+
all_jobs.each{|job| job.clean unless (job.done? && job.updated?) || (job.error? && ! job.recoverable_error?) }
|
196
198
|
|
197
199
|
top_level_jobs = jobs.collect{|job| job.path }
|
198
200
|
failed_jobs = []
|
@@ -216,6 +218,7 @@ module Workflow
|
|
216
218
|
raise TryAgain
|
217
219
|
end
|
218
220
|
else
|
221
|
+
Log.warn "Non-recoverable error in #{job.path}"
|
219
222
|
next
|
220
223
|
end
|
221
224
|
ensure
|
@@ -1,20 +1,26 @@
|
|
1
1
|
class Step
|
2
2
|
def rec_dependencies(connected = false, seen = [])
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
3
|
+
@rec_dependencies = {}
|
4
|
+
@rec_dependencies[connected] ||= begin
|
5
|
+
direct_deps = []
|
6
|
+
dependencies.each do |dep|
|
7
|
+
next if seen.include? dep.path
|
8
|
+
next if connected && dep.done? && dep.updated?
|
9
|
+
direct_deps << dep
|
10
|
+
end if dependencies
|
11
|
+
seen.concat direct_deps.collect{|d| d.path }
|
12
|
+
seen.uniq!
|
13
|
+
direct_deps.inject(direct_deps){|acc,d| acc.concat(d.rec_dependencies(connected, seen)); acc }
|
14
|
+
direct_deps.uniq!
|
15
|
+
direct_deps
|
16
|
+
end
|
11
17
|
end
|
12
18
|
|
13
19
|
def recursive_inputs
|
14
20
|
recursive_inputs = NamedArray === inputs ? inputs.to_hash : {}
|
15
21
|
return recursive_inputs if dependencies.nil?
|
16
22
|
dependencies.inject(recursive_inputs) do |acc,dep|
|
17
|
-
acc.merge(
|
23
|
+
acc = dep.recursive_inputs.merge(acc)
|
18
24
|
end
|
19
25
|
end
|
20
26
|
|
@@ -41,6 +47,7 @@ class Step
|
|
41
47
|
end
|
42
48
|
|
43
49
|
next if dep.done?
|
50
|
+
next if dep.error? && ! dep.recoverable_error?
|
44
51
|
|
45
52
|
if dep.dependencies
|
46
53
|
dep.dependencies.each do |d|
|
@@ -72,9 +79,13 @@ class Step
|
|
72
79
|
def run_dependencies
|
73
80
|
all_dependencies.each do |dep|
|
74
81
|
next if dep.running? || dep.done?
|
82
|
+
next if dep.error? && ! dep.recoverable_error?
|
83
|
+
|
75
84
|
compute_options = compute[dep.path] if compute
|
76
85
|
compute_options = [] if compute_options.nil?
|
77
86
|
|
87
|
+
next if compute_options.include?(false)
|
88
|
+
|
78
89
|
stream = compute_options.include?(:stream)
|
79
90
|
stream = true unless ENV["SCOUT_EXPLICIT_STREAMING"] == 'true'
|
80
91
|
stream = :no_load if compute_options.include?(:produce)
|
@@ -95,12 +106,25 @@ class Step
|
|
95
106
|
all_dependencies.each{|dep| dep.abort if dep.running? }
|
96
107
|
end
|
97
108
|
|
98
|
-
def self.wait_for_jobs(jobs)
|
109
|
+
def self.wait_for_jobs(jobs, canfail=false)
|
99
110
|
threads = []
|
100
111
|
jobs.each do |job|
|
101
112
|
threads << Thread.new do
|
102
113
|
Thread.current.report_on_exception = false
|
103
|
-
|
114
|
+
begin
|
115
|
+
job.join
|
116
|
+
rescue Exception
|
117
|
+
case canfail
|
118
|
+
when TrueClass
|
119
|
+
next
|
120
|
+
else
|
121
|
+
if canfail === $!
|
122
|
+
next
|
123
|
+
else
|
124
|
+
raise $!
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
104
128
|
end
|
105
129
|
end
|
106
130
|
threads.each do |t|
|
@@ -7,6 +7,7 @@ class Step
|
|
7
7
|
else
|
8
8
|
Path.setup(dir)
|
9
9
|
end
|
10
|
+
dir = dir.find
|
10
11
|
dir.pkgdir = self
|
11
12
|
dir
|
12
13
|
end
|
@@ -34,7 +35,7 @@ class Step
|
|
34
35
|
begin
|
35
36
|
realpath = Open.realpath(self.files_dir)
|
36
37
|
Open.rm self.files_dir
|
37
|
-
Open.
|
38
|
+
Open.link_dir realpath, self.files_dir
|
38
39
|
rescue
|
39
40
|
Log.warn "Copy files_dir for #{self.workflow_short_path} failed: " + $!.message
|
40
41
|
end
|
@@ -24,7 +24,7 @@ class Step
|
|
24
24
|
end
|
25
25
|
|
26
26
|
def save_info(info = nil)
|
27
|
-
Persist.save(info, info_file, SERIALIZER)
|
27
|
+
Persist.save(@info = info, info_file, SERIALIZER)
|
28
28
|
@info_load_time = Time.now
|
29
29
|
end
|
30
30
|
|
@@ -121,6 +121,7 @@ class Step
|
|
121
121
|
if message.nil?
|
122
122
|
Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
|
123
123
|
else
|
124
|
+
message = Log.fingerprint(message).sub(/^'/,'').sub(/'$/,'')
|
124
125
|
Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
|
125
126
|
end
|
126
127
|
end
|
@@ -143,6 +144,10 @@ class Step
|
|
143
144
|
info[:messages]
|
144
145
|
end
|
145
146
|
|
147
|
+
def message(message)
|
148
|
+
merge_info :message => message
|
149
|
+
end
|
150
|
+
|
146
151
|
def status
|
147
152
|
info[:status].tap{|s| s.nil? ? s : s.to_sym }
|
148
153
|
end
|
@@ -165,7 +170,14 @@ class Step
|
|
165
170
|
end
|
166
171
|
|
167
172
|
def overriden_deps
|
168
|
-
|
173
|
+
dependencies.select{|d| d.overriden? }
|
174
|
+
end
|
175
|
+
|
176
|
+
def recursive_overriden_deps
|
177
|
+
overriden_deps = dependencies.select{|d|
|
178
|
+
d.overriden?
|
179
|
+
}
|
180
|
+
(overriden_deps + overriden_deps.collect{|d| d.recursive_overriden_deps }).flatten.uniq
|
169
181
|
end
|
170
182
|
|
171
183
|
def exception
|
@@ -4,9 +4,11 @@ class Step
|
|
4
4
|
Path.setup(path) unless Path === path
|
5
5
|
relocated = path.relocate
|
6
6
|
return relocated if Open.exists?(relocated)
|
7
|
-
|
8
|
-
|
9
|
-
|
7
|
+
if path.scan("/").length >= 2
|
8
|
+
subpath = path.split("/")[-3..-1] * "/"
|
9
|
+
relocated = Path.setup("var/jobs")[subpath]
|
10
|
+
return relocated if Open.exists?(relocated)
|
11
|
+
end
|
10
12
|
path
|
11
13
|
end
|
12
14
|
|
@@ -19,5 +19,11 @@ class Step
|
|
19
19
|
bar
|
20
20
|
end
|
21
21
|
end
|
22
|
+
|
23
|
+
def traverse(obj, desc: nil , **kwargs, &block)
|
24
|
+
desc = "Processing #{self.short_path}" if desc.nil?
|
25
|
+
kwargs[:bar] = self.progress_bar(desc) unless kwargs.include?(:bar)
|
26
|
+
TSV.traverse obj, **kwargs, &block
|
27
|
+
end
|
22
28
|
end
|
23
29
|
|
@@ -20,11 +20,17 @@ class Step
|
|
20
20
|
self.error? && ! (ScoutException === self.exception)
|
21
21
|
end
|
22
22
|
|
23
|
-
def
|
24
|
-
return false if self.error? && self.recoverable_error?
|
25
|
-
return true if self.done? && ! ENV["SCOUT_UPDATE"]
|
23
|
+
def newer_dependencies
|
26
24
|
newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
27
25
|
newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
26
|
+
newer += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| Path.newer?(self.path, dep.path) }
|
27
|
+
newer
|
28
|
+
end
|
29
|
+
|
30
|
+
def updated?
|
31
|
+
return false if self.error? && self.recoverable_error?
|
32
|
+
return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"]
|
33
|
+
newer = newer_dependencies
|
28
34
|
|
29
35
|
Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
|
30
36
|
newer.empty?
|
@@ -56,7 +62,7 @@ class Step
|
|
56
62
|
end
|
57
63
|
|
58
64
|
def canfail?
|
59
|
-
@compute && @compute.include?(:canfail)
|
65
|
+
@compute && @compute[self.path] && @compute[self.path].include?(:canfail)
|
60
66
|
end
|
61
67
|
|
62
68
|
def started?
|
data/lib/scout/workflow/step.rb
CHANGED
@@ -138,18 +138,32 @@ class Step
|
|
138
138
|
|
139
139
|
attr_reader :result
|
140
140
|
def run(stream = false)
|
141
|
+
case stream
|
142
|
+
when TrueClass, :stream
|
143
|
+
no_load = :stream
|
144
|
+
when :no_load
|
145
|
+
no_load = true
|
146
|
+
else
|
147
|
+
no_load = false
|
148
|
+
end
|
149
|
+
|
150
|
+
if done?
|
151
|
+
if no_load
|
152
|
+
if no_load == :stream
|
153
|
+
return self.stream
|
154
|
+
else
|
155
|
+
return self.path
|
156
|
+
end
|
157
|
+
else
|
158
|
+
return @result || self.load
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
|
141
163
|
return @result || self.load if done?
|
142
164
|
prepare_dependencies
|
143
165
|
begin
|
144
166
|
|
145
|
-
case stream
|
146
|
-
when TrueClass, :stream
|
147
|
-
no_load = :stream
|
148
|
-
when :no_load
|
149
|
-
no_load = true
|
150
|
-
else
|
151
|
-
no_load = false
|
152
|
-
end
|
153
167
|
|
154
168
|
@result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies, no_load: no_load) do
|
155
169
|
input_names = (task.respond_to?(:inputs) && task.inputs) ? task.inputs.collect{|name,_| name} : []
|
@@ -206,6 +220,7 @@ class Step
|
|
206
220
|
end
|
207
221
|
|
208
222
|
@result.abort_callback = proc do |exception|
|
223
|
+
Open.rm self.path
|
209
224
|
if exception.nil? || Aborted === exception || Interrupt === exception
|
210
225
|
merge_info :status => :aborted, :end => Time.now
|
211
226
|
else
|
@@ -240,6 +255,7 @@ class Step
|
|
240
255
|
run(noload)
|
241
256
|
end
|
242
257
|
join
|
258
|
+
exit! 0
|
243
259
|
end
|
244
260
|
Process.detach pid
|
245
261
|
grace
|
@@ -350,6 +366,10 @@ class Step
|
|
350
366
|
|
351
367
|
def step(task_name)
|
352
368
|
task_name = task_name.to_sym
|
369
|
+
dependencies.each do |dep|
|
370
|
+
return dep if dep.task_name && dep.task_name.to_sym == task_name
|
371
|
+
return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
|
372
|
+
end
|
353
373
|
dependencies.each do |dep|
|
354
374
|
return dep if dep.task_name && dep.task_name.to_sym == task_name
|
355
375
|
return dep if dep.overriden_task && dep.overriden_task.to_sym == task_name
|
@@ -359,12 +379,12 @@ class Step
|
|
359
379
|
nil
|
360
380
|
end
|
361
381
|
|
362
|
-
def
|
363
|
-
Resource.identify
|
382
|
+
def identify_path
|
383
|
+
Resource.identify path
|
364
384
|
end
|
365
385
|
|
366
386
|
def digest_str
|
367
|
-
"Step: " +
|
387
|
+
"Step: " + identify_path
|
368
388
|
end
|
369
389
|
|
370
390
|
def fingerprint
|
@@ -373,7 +393,7 @@ class Step
|
|
373
393
|
|
374
394
|
def task_signature
|
375
395
|
workflow_name = String === workflow ? workflow : workflow.name
|
376
|
-
[
|
396
|
+
[workflow_name, task_name] * "#"
|
377
397
|
end
|
378
398
|
|
379
399
|
def alias?
|