scout-gear 10.11.6 → 10.11.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +16 -2
- data/VERSION +1 -1
- data/bin/scout +10 -10
- data/lib/scout/association/fields.rb +15 -15
- data/lib/scout/association/index.rb +6 -6
- data/lib/scout/association/item.rb +18 -8
- data/lib/scout/association.rb +4 -4
- data/lib/scout/entity/identifiers.rb +5 -5
- data/lib/scout/entity/property.rb +2 -2
- data/lib/scout/entity.rb +1 -1
- data/lib/scout/knowledge_base/description.rb +10 -10
- data/lib/scout/knowledge_base/entity.rb +6 -6
- data/lib/scout/knowledge_base/list.rb +1 -1
- data/lib/scout/knowledge_base/query.rb +4 -4
- data/lib/scout/knowledge_base/registry.rb +6 -6
- data/lib/scout/knowledge_base/traverse.rb +7 -40
- data/lib/scout/persist/engine/fix_width_table.rb +6 -6
- data/lib/scout/persist/engine/packed_index.rb +2 -2
- data/lib/scout/persist/engine/sharder.rb +4 -4
- data/lib/scout/persist/engine/tkrzw.rb +1 -1
- data/lib/scout/persist/engine/tokyocabinet.rb +2 -2
- data/lib/scout/persist/tsv/adapter/fix_width_table.rb +1 -1
- data/lib/scout/persist/tsv/adapter/packed_index.rb +1 -1
- data/lib/scout/persist/tsv/adapter/tkrzw.rb +1 -1
- data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +3 -3
- data/lib/scout/persist/tsv/serialize.rb +3 -3
- data/lib/scout/persist/tsv.rb +1 -1
- data/lib/scout/semaphore.rb +78 -3
- data/lib/scout/tsv/annotation/repo.rb +4 -4
- data/lib/scout/tsv/annotation.rb +2 -2
- data/lib/scout/tsv/attach.rb +7 -7
- data/lib/scout/tsv/change_id/translate.rb +1 -1
- data/lib/scout/tsv/csv.rb +3 -3
- data/lib/scout/tsv/dumper.rb +8 -8
- data/lib/scout/tsv/index.rb +1 -1
- data/lib/scout/tsv/open.rb +3 -3
- data/lib/scout/tsv/stream.rb +2 -2
- data/lib/scout/tsv/traverse.rb +4 -4
- data/lib/scout/tsv/util/filter.rb +9 -9
- data/lib/scout/tsv/util/process.rb +2 -2
- data/lib/scout/tsv/util/reorder.rb +2 -2
- data/lib/scout/tsv/util/select.rb +3 -3
- data/lib/scout/tsv/util/unzip.rb +2 -2
- data/lib/scout/tsv/util.rb +1 -1
- data/lib/scout/tsv.rb +2 -2
- data/lib/scout/work_queue/socket.rb +2 -2
- data/lib/scout/work_queue/worker.rb +4 -4
- data/lib/scout/work_queue.rb +5 -5
- data/lib/scout/workflow/definition.rb +18 -16
- data/lib/scout/workflow/deployment/local.rb +81 -62
- data/lib/scout/workflow/deployment/orchestrator/batches.rb +66 -5
- data/lib/scout/workflow/deployment/orchestrator/chains.rb +47 -30
- data/lib/scout/workflow/deployment/orchestrator/rules.rb +3 -3
- data/lib/scout/workflow/deployment/orchestrator/workload.rb +11 -22
- data/lib/scout/workflow/deployment/scheduler/job.rb +34 -36
- data/lib/scout/workflow/deployment/scheduler/lfs.rb +1 -1
- data/lib/scout/workflow/deployment/scheduler/pbs.rb +4 -4
- data/lib/scout/workflow/deployment/scheduler/slurm.rb +2 -2
- data/lib/scout/workflow/deployment/scheduler.rb +23 -12
- data/lib/scout/workflow/deployment/trace.rb +2 -2
- data/lib/scout/workflow/documentation.rb +4 -4
- data/lib/scout/workflow/export.rb +1 -1
- data/lib/scout/workflow/path.rb +2 -2
- data/lib/scout/workflow/step/children.rb +1 -1
- data/lib/scout/workflow/step/dependencies.rb +36 -3
- data/lib/scout/workflow/step/info.rb +5 -19
- data/lib/scout/workflow/step/inputs.rb +1 -1
- data/lib/scout/workflow/step/progress.rb +2 -2
- data/lib/scout/workflow/step/provenance.rb +4 -4
- data/lib/scout/workflow/step/status.rb +23 -9
- data/lib/scout/workflow/step.rb +19 -17
- data/lib/scout/workflow/task/dependencies.rb +10 -3
- data/lib/scout/workflow/task/info.rb +3 -3
- data/lib/scout/workflow/task/inputs.rb +8 -8
- data/lib/scout/workflow/task.rb +37 -22
- data/lib/scout/workflow/usage.rb +13 -13
- data/lib/scout/workflow/util.rb +1 -1
- data/lib/scout/workflow.rb +6 -6
- data/scout-gear.gemspec +3 -3
- data/scout_commands/alias +1 -1
- data/scout_commands/batch/clean +12 -12
- data/scout_commands/batch/list +26 -25
- data/scout_commands/batch/tail +9 -5
- data/scout_commands/cat +1 -1
- data/scout_commands/doc +2 -2
- data/scout_commands/entity +4 -4
- data/scout_commands/find +1 -1
- data/scout_commands/kb/config +1 -1
- data/scout_commands/kb/entities +1 -1
- data/scout_commands/kb/list +1 -1
- data/scout_commands/kb/query +2 -2
- data/scout_commands/kb/register +1 -1
- data/scout_commands/kb/show +1 -1
- data/scout_commands/kb/traverse +1 -1
- data/scout_commands/log +6 -6
- data/scout_commands/resource/produce +2 -2
- data/scout_commands/resource/sync +1 -1
- data/scout_commands/system/clean +7 -7
- data/scout_commands/system/status +4 -4
- data/scout_commands/template +1 -1
- data/scout_commands/update +1 -1
- data/scout_commands/workflow/info +1 -1
- data/scout_commands/workflow/install +1 -1
- data/scout_commands/workflow/list +2 -2
- data/scout_commands/workflow/process +2 -2
- data/scout_commands/workflow/prov +3 -3
- data/scout_commands/workflow/task +36 -11
- data/scout_commands/workflow/trace +1 -1
- data/scout_commands/workflow/write_info +2 -2
- data/share/templates/command +1 -1
- data/test/scout/association/test_item.rb +5 -0
- data/test/scout/entity/test_property.rb +3 -3
- data/test/scout/knowledge_base/test_description.rb +1 -1
- data/test/scout/knowledge_base/test_traverse.rb +2 -2
- data/test/scout/persist/engine/test_packed_index.rb +6 -6
- data/test/scout/persist/test_tsv.rb +4 -4
- data/test/scout/persist/tsv/adapter/test_packed_index.rb +4 -4
- data/test/scout/persist/tsv/adapter/test_sharder.rb +23 -23
- data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +1 -1
- data/test/scout/persist/tsv/test_serialize.rb +1 -1
- data/test/scout/test_association.rb +1 -1
- data/test/scout/test_tsv.rb +2 -2
- data/test/scout/test_workflow.rb +2 -2
- data/test/scout/tsv/test_annotation.rb +4 -4
- data/test/scout/tsv/test_index.rb +1 -1
- data/test/scout/tsv/test_open.rb +2 -2
- data/test/scout/tsv/test_parser.rb +2 -2
- data/test/scout/tsv/test_stream.rb +1 -1
- data/test/scout/tsv/test_transformer.rb +1 -1
- data/test/scout/tsv/util/test_filter.rb +1 -1
- data/test/scout/tsv/util/test_melt.rb +1 -1
- data/test/scout/tsv/util/test_reorder.rb +1 -1
- data/test/scout/work_queue/test_socket.rb +3 -3
- data/test/scout/work_queue/test_worker.rb +2 -2
- data/test/scout/workflow/deployment/orchestrator/test_batches.rb +13 -3
- data/test/scout/workflow/deployment/orchestrator/test_chains.rb +15 -13
- data/test/scout/workflow/deployment/orchestrator/test_workload.rb +1 -1
- data/test/scout/workflow/deployment/test_local.rb +2 -2
- data/test/scout/workflow/deployment/test_scheduler.rb +1 -2
- data/test/scout/workflow/step/test_children.rb +1 -1
- data/test/scout/workflow/step/test_dependencies.rb +36 -1
- data/test/scout/workflow/step/test_info.rb +3 -35
- data/test/scout/workflow/step/test_load.rb +1 -1
- data/test/scout/workflow/step/test_provenance.rb +1 -1
- data/test/scout/workflow/step/test_status.rb +33 -1
- data/test/scout/workflow/task/test_dependencies.rb +9 -7
- data/test/scout/workflow/task/test_inputs.rb +1 -1
- data/test/scout/workflow/test_definition.rb +1 -1
- data/test/scout/workflow/test_documentation.rb +1 -1
- data/test/scout/workflow/test_entity.rb +2 -2
- data/test/scout/workflow/test_step.rb +13 -13
- data/test/scout/workflow/test_usage.rb +1 -1
- data/test/test_helper.rb +1 -1
- metadata +2 -2
|
@@ -30,7 +30,8 @@ class Step
|
|
|
30
30
|
end
|
|
31
31
|
|
|
32
32
|
def save_info(info = nil)
|
|
33
|
-
|
|
33
|
+
purged = Annotation.purge(@info = info)
|
|
34
|
+
Persist.save(purged, info_file, SERIALIZER)
|
|
34
35
|
@info_load_time = Time.now
|
|
35
36
|
end
|
|
36
37
|
|
|
@@ -82,6 +83,7 @@ class Step
|
|
|
82
83
|
|
|
83
84
|
if start && eend
|
|
84
85
|
time = eend - start
|
|
86
|
+
Log.warn "No issue time #{self.path}" if issued.nil?
|
|
85
87
|
total_time = eend - issued
|
|
86
88
|
if total_time - time > 1
|
|
87
89
|
time_str = "#{Misc.format_seconds_short(time)} (#{Misc.format_seconds_short(total_time)})"
|
|
@@ -93,7 +95,7 @@ class Step
|
|
|
93
95
|
message = Log.color(:time, time_str)
|
|
94
96
|
end
|
|
95
97
|
end
|
|
96
|
-
report_status value, message
|
|
98
|
+
report_status value, message
|
|
97
99
|
end
|
|
98
100
|
|
|
99
101
|
if key == :message
|
|
@@ -136,7 +138,7 @@ class Step
|
|
|
136
138
|
def set_info(key, value)
|
|
137
139
|
merge_info(key => value)
|
|
138
140
|
end
|
|
139
|
-
|
|
141
|
+
|
|
140
142
|
def report_status(status, message = nil)
|
|
141
143
|
if message.nil?
|
|
142
144
|
Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
|
|
@@ -184,22 +186,6 @@ class Step
|
|
|
184
186
|
! (done? && status == :done) && (info[:pid] && Misc.pid_alive?(info[:pid]))
|
|
185
187
|
end
|
|
186
188
|
|
|
187
|
-
def overriden?
|
|
188
|
-
@overriden = overriden_task || overriden_workflow || overriden_deps.any? if @overriden.nil?
|
|
189
|
-
@overriden
|
|
190
|
-
end
|
|
191
|
-
|
|
192
|
-
def overriden_deps
|
|
193
|
-
dependencies.select{|d| d.overriden? }
|
|
194
|
-
end
|
|
195
|
-
|
|
196
|
-
def recursive_overriden_deps
|
|
197
|
-
overriden_deps = dependencies.select{|d|
|
|
198
|
-
d.overriden?
|
|
199
|
-
}
|
|
200
|
-
(overriden_deps + overriden_deps.collect{|d| d.recursive_overriden_deps }).flatten.uniq
|
|
201
|
-
end
|
|
202
|
-
|
|
203
189
|
def exception
|
|
204
190
|
return nil unless info[:exception]
|
|
205
191
|
begin
|
|
@@ -3,7 +3,7 @@ class Step
|
|
|
3
3
|
if clean_name != name
|
|
4
4
|
#hash = name[clean_name.length..-1]
|
|
5
5
|
#inputs_dir += hash
|
|
6
|
-
Log.medium "Saving job inputs to: #{Log.fingerprint inputs_dir}"
|
|
6
|
+
Log.medium "Saving job inputs to: #{Log.fingerprint inputs_dir} #{Log.fingerprint provided_inputs}"
|
|
7
7
|
self.task.save_inputs(inputs_dir, provided_inputs)
|
|
8
8
|
else
|
|
9
9
|
Log.medium "Saving no input job: #{Log.fingerprint inputs_dir}"
|
|
@@ -58,10 +58,10 @@ class Step
|
|
|
58
58
|
str = if ! (Open.remote?(path) || Open.ssh?(path)) && (Open.exists?(path) && $main_mtime && path_mtime && ($main_mtime - path_mtime) < -2)
|
|
59
59
|
prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
|
|
60
60
|
else
|
|
61
|
-
prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " "
|
|
61
|
+
prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " "
|
|
62
62
|
end
|
|
63
63
|
|
|
64
|
-
if $inputs and $inputs.any?
|
|
64
|
+
if $inputs and $inputs.any?
|
|
65
65
|
job_inputs = Step.new(path).recursive_inputs.to_hash
|
|
66
66
|
IndiferentHash.setup(job_inputs)
|
|
67
67
|
|
|
@@ -119,7 +119,7 @@ class Step
|
|
|
119
119
|
step.dependencies.each do |dep|
|
|
120
120
|
if dep.input_dependencies.any?
|
|
121
121
|
dep.input_dependencies.each do |id|
|
|
122
|
-
input_name, _dep = dep.recursive_inputs.select{|f,d|
|
|
122
|
+
input_name, _dep = dep.recursive_inputs.select{|f,d|
|
|
123
123
|
d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
|
|
124
124
|
}.keys.last
|
|
125
125
|
if input_name
|
|
@@ -135,7 +135,7 @@ class Step
|
|
|
135
135
|
indent = prov_indent(step, offset, input_dependencies)
|
|
136
136
|
str << indent + this_step_msg if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
|
|
137
137
|
|
|
138
|
-
step.dependencies.dup.tap{|l|
|
|
138
|
+
step.dependencies.dup.tap{|l|
|
|
139
139
|
l.reverse! if ENV["SCOUT_ORIGINAL_STACK"] == 'true'
|
|
140
140
|
}.each do |dep|
|
|
141
141
|
path = dep.path
|
|
@@ -5,7 +5,7 @@ class Step
|
|
|
5
5
|
begin
|
|
6
6
|
s = Misc.abort_child pid, true
|
|
7
7
|
Log.medium "Aborted pid #{path} #{s}"
|
|
8
|
-
rescue
|
|
8
|
+
rescue
|
|
9
9
|
Log.debug("Aborted job #{pid} was not killed: #{$!.message}")
|
|
10
10
|
end
|
|
11
11
|
else
|
|
@@ -17,35 +17,49 @@ class Step
|
|
|
17
17
|
end
|
|
18
18
|
|
|
19
19
|
def recoverable_error?
|
|
20
|
-
self.error? && ! (ScoutException === self.exception)
|
|
20
|
+
self.error? && ! (ENV['SCOUT_NO_RECOVERABLE_ERROR'].to_s.downcase == 'true') && ! (ScoutException === self.exception)
|
|
21
21
|
end
|
|
22
22
|
|
|
23
23
|
def newer_dependencies
|
|
24
|
+
rec_dependencies = self.rec_dependencies
|
|
25
|
+
rec_dependencies = rec_dependencies.select{|dep| Open.exists?(dep.info_file) }
|
|
26
|
+
rec_dependencies = rec_dependencies.reject{|dep| dep.error? && ! dep.recoverable_error? }
|
|
24
27
|
newer = rec_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
|
25
28
|
newer += input_dependencies.select{|dep| Path.newer?(self.path, dep.path) }
|
|
26
29
|
newer += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| Path.newer?(self.path, dep.path) }
|
|
27
30
|
newer
|
|
28
31
|
end
|
|
29
32
|
|
|
33
|
+
def cleaned_dependencies
|
|
34
|
+
return []
|
|
35
|
+
rec_dependencies = self.rec_dependencies
|
|
36
|
+
cleaned = rec_dependencies.select{|dep| dep.info[:status] == :cleaned }
|
|
37
|
+
cleaned += input_dependencies.select{|dep| dep.info[:status] == :cleaned }
|
|
38
|
+
cleaned += rec_dependencies.collect{|dep| dep.input_dependencies }.flatten.select{|dep| dep.info[:status] == :cleaned }
|
|
39
|
+
cleaned
|
|
40
|
+
end
|
|
41
|
+
|
|
30
42
|
def updated?
|
|
31
43
|
return false if self.error? && self.recoverable_error?
|
|
32
|
-
return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"]
|
|
44
|
+
return true if (self.done? || (self.error? && ! self.recoverable_error?)) && ! ENV["SCOUT_UPDATE"].to_s.downcase == 'true'
|
|
33
45
|
newer = newer_dependencies
|
|
46
|
+
cleaned = cleaned_dependencies
|
|
34
47
|
|
|
35
48
|
Log.low "Newer deps found for #{Log.fingerprint self}: #{Log.fingerprint newer}" if newer.any?
|
|
36
|
-
|
|
49
|
+
Log.low "Cleaned deps found for #{Log.fingerprint self}: #{Log.fingerprint cleaned}" if cleaned.any?
|
|
50
|
+
newer.empty? && cleaned.empty?
|
|
37
51
|
end
|
|
38
52
|
|
|
39
53
|
def clean
|
|
40
54
|
Log.debug "Cleaning job files: #{path}"
|
|
41
|
-
@take_stream = nil
|
|
55
|
+
@take_stream = nil
|
|
42
56
|
@result = nil
|
|
43
57
|
@info = nil
|
|
44
58
|
@info_load_time = nil
|
|
45
59
|
@done = nil
|
|
46
|
-
Open.rm path if Open.exist_or_link?(path)
|
|
47
|
-
Open.rm tmp_path if Open.exist_or_link?(tmp_path)
|
|
48
|
-
Open.rm info_file if Open.exist_or_link?(info_file)
|
|
60
|
+
Open.rm path if Open.exist_or_link?(path) && Path.can_write?(path)
|
|
61
|
+
Open.rm tmp_path if Open.exist_or_link?(tmp_path) && Path.can_write?(tmp_path)
|
|
62
|
+
Open.rm info_file if Open.exist_or_link?(info_file) && Path.can_write?(info_file)
|
|
49
63
|
Open.rm_rf files_dir if Open.exist_or_link?(files_dir)
|
|
50
64
|
self
|
|
51
65
|
end
|
|
@@ -59,7 +73,7 @@ class Step
|
|
|
59
73
|
dependencies.each do |dep|
|
|
60
74
|
dep.recursive_clean
|
|
61
75
|
end
|
|
62
|
-
clean
|
|
76
|
+
clean if Open.exists?(self.info_file)
|
|
63
77
|
end
|
|
64
78
|
|
|
65
79
|
def canfail?
|
data/lib/scout/workflow/step.rb
CHANGED
|
@@ -13,17 +13,18 @@ require_relative 'step/inputs'
|
|
|
13
13
|
require_relative 'step/children'
|
|
14
14
|
require_relative 'step/archive'
|
|
15
15
|
|
|
16
|
-
class Step
|
|
16
|
+
class Step
|
|
17
17
|
|
|
18
18
|
attr_accessor :path, :inputs, :dependencies, :id, :task, :tee_copies, :non_default_inputs, :provided_inputs, :compute, :overriden_task, :overriden_workflow, :workflow, :exec_context, :overriden
|
|
19
|
-
def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, exec_context: nil, &task)
|
|
19
|
+
def initialize(path = nil, inputs = nil, dependencies = nil, id = nil, non_default_inputs = nil, provided_inputs = nil, compute = nil, overriden = nil, exec_context: nil, &task)
|
|
20
20
|
@path = path
|
|
21
21
|
@inputs = inputs
|
|
22
22
|
@dependencies = dependencies
|
|
23
23
|
@id = id
|
|
24
24
|
@non_default_inputs = non_default_inputs
|
|
25
25
|
@provided_inputs = provided_inputs
|
|
26
|
-
@compute = compute
|
|
26
|
+
@compute = compute
|
|
27
|
+
@overriden = overriden unless overriden.nil?
|
|
27
28
|
@task = task
|
|
28
29
|
@mutex = Mutex.new
|
|
29
30
|
@tee_copies = 1
|
|
@@ -36,12 +37,12 @@ class Step
|
|
|
36
37
|
|
|
37
38
|
def provided_inputs
|
|
38
39
|
@provided_inputs ||= begin
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
40
|
+
if info_file && Open.exists?(info_file)
|
|
41
|
+
info[:provided_inputs]
|
|
42
|
+
else
|
|
43
|
+
{}
|
|
44
|
+
end
|
|
45
|
+
end
|
|
45
46
|
end
|
|
46
47
|
|
|
47
48
|
def non_default_inputs
|
|
@@ -97,7 +98,7 @@ class Step
|
|
|
97
98
|
return @id if @id
|
|
98
99
|
return info[:clean_name] if info.include? :clean_name
|
|
99
100
|
if m = name.match(/(.+?)(?:_[a-z0-9]{32})?(?:\..*)?$/)
|
|
100
|
-
return m[1]
|
|
101
|
+
return m[1]
|
|
101
102
|
end
|
|
102
103
|
return name.split(".").first
|
|
103
104
|
end
|
|
@@ -189,9 +190,10 @@ class Step
|
|
|
189
190
|
end
|
|
190
191
|
end
|
|
191
192
|
|
|
192
|
-
|
|
193
193
|
return @result || self.load if done?
|
|
194
|
+
|
|
194
195
|
prepare_dependencies
|
|
196
|
+
|
|
195
197
|
begin
|
|
196
198
|
|
|
197
199
|
|
|
@@ -200,7 +202,7 @@ class Step
|
|
|
200
202
|
|
|
201
203
|
|
|
202
204
|
reset_info :status => :setup, :issued => Time.now,
|
|
203
|
-
:pid => Process.pid, :pid_hostname => Misc.hostname,
|
|
205
|
+
:pid => Process.pid, :pid_hostname => Misc.hostname,
|
|
204
206
|
:task_name => task_name, :workflow => workflow.to_s,
|
|
205
207
|
:provided_inputs => IndiferentHash.serializable(provided_inputs),
|
|
206
208
|
:non_default_inputs => non_default_inputs,
|
|
@@ -313,7 +315,7 @@ class Step
|
|
|
313
315
|
end
|
|
314
316
|
|
|
315
317
|
def streaming?
|
|
316
|
-
@take_stream || IO === @result || StringIO === @result
|
|
318
|
+
@take_stream || IO === @result || StringIO === @result
|
|
317
319
|
end
|
|
318
320
|
|
|
319
321
|
def stream
|
|
@@ -344,16 +346,16 @@ class Step
|
|
|
344
346
|
end
|
|
345
347
|
|
|
346
348
|
def consume_all_streams
|
|
347
|
-
threads = []
|
|
349
|
+
threads = []
|
|
348
350
|
while @result && streaming? && stream = self.stream
|
|
349
351
|
threads << Open.consume_stream(stream, true)
|
|
350
352
|
end
|
|
351
353
|
|
|
352
354
|
threads.compact!
|
|
353
355
|
|
|
354
|
-
threads.each do |t|
|
|
356
|
+
threads.each do |t|
|
|
355
357
|
begin
|
|
356
|
-
t.join
|
|
358
|
+
t.join
|
|
357
359
|
rescue Exception
|
|
358
360
|
threads.compact.each{|t| t.raise(Aborted); t.join }
|
|
359
361
|
raise $!
|
|
@@ -388,7 +390,7 @@ class Step
|
|
|
388
390
|
|
|
389
391
|
raise self.exception if self.exception
|
|
390
392
|
|
|
391
|
-
raise "Error in job #{self.path}" if self.error? or self.aborted?
|
|
393
|
+
raise "Error in job #{self.path}" if self.error? or self.aborted?
|
|
392
394
|
|
|
393
395
|
self
|
|
394
396
|
end
|
|
@@ -2,7 +2,7 @@ module Task
|
|
|
2
2
|
def dependencies(id, provided_inputs, non_default_inputs = [], compute = {})
|
|
3
3
|
return [] if deps.nil?
|
|
4
4
|
dependencies = []
|
|
5
|
-
|
|
5
|
+
|
|
6
6
|
provided_inputs ||= {}
|
|
7
7
|
|
|
8
8
|
# Helper function
|
|
@@ -66,8 +66,15 @@ module Task
|
|
|
66
66
|
|
|
67
67
|
if provided_inputs.include?(overriden = [workflow.name, task] * "#")
|
|
68
68
|
dep = provided_inputs[overriden]
|
|
69
|
-
dep =
|
|
70
|
-
|
|
69
|
+
dep = case dep
|
|
70
|
+
when Step
|
|
71
|
+
dep
|
|
72
|
+
when String
|
|
73
|
+
Step.new dep
|
|
74
|
+
when Symbol
|
|
75
|
+
dependencies.select{|dep| dep.task_name == dep }.last
|
|
76
|
+
end
|
|
77
|
+
|
|
71
78
|
dep.type = workflow.tasks[task].type
|
|
72
79
|
dep.overriden_task = task
|
|
73
80
|
dep.overriden_workflow = workflow
|
|
@@ -42,8 +42,8 @@ module Workflow
|
|
|
42
42
|
def rec_input_use(task_name)
|
|
43
43
|
input_use = {}
|
|
44
44
|
task = self.tasks[task_name]
|
|
45
|
-
task.inputs.each do |name,_|
|
|
46
|
-
input_use[name] ||= {}
|
|
45
|
+
task.inputs.each do |name,_|
|
|
46
|
+
input_use[name] ||= {}
|
|
47
47
|
input_use[name][self] ||= []
|
|
48
48
|
input_use[name][self] << task_name
|
|
49
49
|
end
|
|
@@ -69,7 +69,7 @@ module Workflow
|
|
|
69
69
|
raise "No '#{name}' task in '#{self.name}' Workflow" if task.nil?
|
|
70
70
|
id = File.join(self.name, name.to_s)
|
|
71
71
|
@task_info ||= {}
|
|
72
|
-
@task_info[id] ||= begin
|
|
72
|
+
@task_info[id] ||= begin
|
|
73
73
|
description = task.description
|
|
74
74
|
returns = task.returns
|
|
75
75
|
|
|
@@ -3,9 +3,9 @@ module Task
|
|
|
3
3
|
def self.format_input(value, type, options = {})
|
|
4
4
|
return value if IO === value || StringIO === value || Step === value
|
|
5
5
|
|
|
6
|
-
if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options && (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
|
|
6
|
+
if String === value && Path.is_filename?(value) && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options && (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
|
|
7
7
|
if Open.exists?(value) && ! Open.directory?(value)
|
|
8
|
-
Persist.load(value, type)
|
|
8
|
+
Persist.load(value, type)
|
|
9
9
|
else
|
|
10
10
|
Persist.deserialize(value, type)
|
|
11
11
|
end
|
|
@@ -96,7 +96,7 @@ module Task
|
|
|
96
96
|
basename += "-#{digest}"
|
|
97
97
|
end
|
|
98
98
|
new_file = File.join(directory, 'saved_input_files', basename)
|
|
99
|
-
relative_file = File.join('.', 'saved_input_files', basename)
|
|
99
|
+
relative_file = File.join('.', 'saved_input_files', basename)
|
|
100
100
|
Open.link orig_file, new_file
|
|
101
101
|
relative_file
|
|
102
102
|
end
|
|
@@ -104,7 +104,7 @@ module Task
|
|
|
104
104
|
def self.save_input(directory, name, type, value)
|
|
105
105
|
input_file = File.join(directory, name.to_s)
|
|
106
106
|
|
|
107
|
-
if Path.is_filename?(value)
|
|
107
|
+
if Path.is_filename?(value)
|
|
108
108
|
if type == :path
|
|
109
109
|
Open.write(input_file + ".as_path", value)
|
|
110
110
|
elsif Path.step_file?(value)
|
|
@@ -114,7 +114,7 @@ module Task
|
|
|
114
114
|
Open.write(input_file + ".as_file", relative_file)
|
|
115
115
|
end
|
|
116
116
|
elsif Step === value
|
|
117
|
-
Open.write(input_file + ".as_step", value.
|
|
117
|
+
Open.write(input_file + ".as_step", value.identify_path)
|
|
118
118
|
elsif type == :file
|
|
119
119
|
relative_file = save_file_input(value, directory)
|
|
120
120
|
Persist.save(relative_file, input_file, :file)
|
|
@@ -153,8 +153,8 @@ module Task
|
|
|
153
153
|
value.sub!(/^\./, File.dirname(filename)) if value.start_with?("./")
|
|
154
154
|
value
|
|
155
155
|
elsif filename.end_with?('.as_step')
|
|
156
|
-
value = Open.read(filename).strip
|
|
157
|
-
Step.load value
|
|
156
|
+
value = Path.setup Open.read(filename).strip
|
|
157
|
+
Step.load value.find
|
|
158
158
|
elsif filename.end_with?('.as_path')
|
|
159
159
|
value = Open.read(filename).strip
|
|
160
160
|
Path.setup value
|
|
@@ -183,7 +183,7 @@ module Task
|
|
|
183
183
|
self.recursive_inputs.each do |p|
|
|
184
184
|
name, type, desc, value, options = p
|
|
185
185
|
next if seen.include?(name)
|
|
186
|
-
filename = File.join(directory, name.to_s)
|
|
186
|
+
filename = File.join(directory, name.to_s)
|
|
187
187
|
value = Task.load_input_from_file(filename, type, options)
|
|
188
188
|
inputs[name] = value unless value.nil?
|
|
189
189
|
seen << name
|
data/lib/scout/workflow/task.rb
CHANGED
|
@@ -44,10 +44,23 @@ module Task
|
|
|
44
44
|
|
|
45
45
|
memory_inputs = nil if Array === memory_inputs && memory_inputs.compact.empty?
|
|
46
46
|
memory_inputs = nil if Hash === memory_inputs && memory_inputs.empty?
|
|
47
|
-
Persist.memory("Task job #{self.name}", repo: Workflow.job_cache, other: {task: self.name, id: id, provided_inputs: memory_inputs}) do
|
|
47
|
+
Persist.memory("Task job #{self.name}", repo: Workflow.job_cache, other: {workflow: self.workflow, task: self.name, id: id, provided_inputs: memory_inputs}) do
|
|
48
|
+
|
|
49
|
+
#{{{ Provided inputs and ID
|
|
48
50
|
provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
|
|
49
51
|
provided_inputs = {} if provided_inputs.nil?
|
|
50
|
-
IndiferentHash.setup(provided_inputs)
|
|
52
|
+
IndiferentHash.setup(provided_inputs) if Hash === provided_inputs
|
|
53
|
+
|
|
54
|
+
provided_inputs = load_inputs(provided_inputs.delete(:load_inputs)).merge(provided_inputs) if Hash === provided_inputs && provided_inputs[:load_inputs]
|
|
55
|
+
|
|
56
|
+
provided_input_names = case provided_inputs
|
|
57
|
+
when nil
|
|
58
|
+
[]
|
|
59
|
+
when Array
|
|
60
|
+
inputs.collect{|name,*| name }[0..provided_inputs.length]
|
|
61
|
+
when Hash
|
|
62
|
+
provided_inputs.keys
|
|
63
|
+
end
|
|
51
64
|
|
|
52
65
|
jobname_input = nil
|
|
53
66
|
inputs.each do |name,type,desc,default,input_options|
|
|
@@ -56,12 +69,14 @@ module Task
|
|
|
56
69
|
end
|
|
57
70
|
|
|
58
71
|
id = provided_inputs[jobname_input] if jobname_input && id.nil?
|
|
59
|
-
|
|
72
|
+
id = DEFAULT_NAME if id.nil?
|
|
73
|
+
id = Path.sanitize_filename(id, 150)
|
|
60
74
|
|
|
75
|
+
#{{{ Missing inputs
|
|
61
76
|
missing_inputs = []
|
|
62
77
|
self.inputs.each do |input,type,desc,val,options|
|
|
63
78
|
next unless options && options[:required]
|
|
64
|
-
missing_inputs << input unless
|
|
79
|
+
missing_inputs << input unless provided_input_names.include?(input.to_s) || provided_input_names.include?(input.to_sym)
|
|
65
80
|
end if self.inputs
|
|
66
81
|
|
|
67
82
|
if missing_inputs.length == 1
|
|
@@ -72,25 +87,32 @@ module Task
|
|
|
72
87
|
raise ParameterException, "Inputs #{Misc.humanize_list(missing_inputs)} are required but were not provided or are nil"
|
|
73
88
|
end
|
|
74
89
|
|
|
75
|
-
|
|
76
|
-
|
|
90
|
+
#{{{ Process inputs
|
|
77
91
|
job_inputs, non_default_inputs, input_digest_str = process_inputs provided_inputs, id
|
|
92
|
+
non_default_inputs.uniq!
|
|
93
|
+
NamedArray.setup(job_inputs, @inputs.collect{|i| i[0] }) if @inputs
|
|
94
|
+
step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
|
|
78
95
|
|
|
96
|
+
#{{{ Dependencies
|
|
79
97
|
compute = {}
|
|
80
98
|
dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
|
|
81
99
|
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
100
|
+
#{{{ Overrides
|
|
101
|
+
override_inputs = provided_input_names.select{|k| (String === k) && k.include?("#") }
|
|
102
|
+
overriden = override_inputs.any? && dependencies.select{|dep| dep.overrider? || dep.overriden? }.any?
|
|
85
103
|
|
|
86
|
-
|
|
104
|
+
non_default_inputs.delete_if{|input| override_inputs.include? input }
|
|
105
|
+
non_default_provided_inputs = non_default_inputs & provided_input_names
|
|
106
|
+
non_default_provided_inputs.delete jobname_input if provided_inputs[jobname_input] == id
|
|
87
107
|
|
|
88
|
-
|
|
89
|
-
if
|
|
108
|
+
#{{{ Hash and Path
|
|
109
|
+
if overriden || non_default_provided_inputs.any?
|
|
90
110
|
hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
|
|
91
|
-
name
|
|
111
|
+
Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies, overriden: override_inputs)}"
|
|
112
|
+
name = [id, hash] * "_"
|
|
92
113
|
else
|
|
93
|
-
name
|
|
114
|
+
Log.debug "ID #{self.name} #{id} - Clean"
|
|
115
|
+
name = id
|
|
94
116
|
end
|
|
95
117
|
|
|
96
118
|
extension = self.extension
|
|
@@ -110,17 +132,10 @@ module Task
|
|
|
110
132
|
end
|
|
111
133
|
|
|
112
134
|
path = directory[name]
|
|
113
|
-
|
|
114
135
|
path = path.set_extension(extension) if extension
|
|
115
136
|
|
|
116
|
-
if hash
|
|
117
|
-
Log.debug "ID #{self.name} #{id} - #{hash}: #{Log.fingerprint(:input_digest => input_digest_str, :non_default_inputs => non_default_inputs, :dependencies => dependencies)}"
|
|
118
|
-
else
|
|
119
|
-
Log.debug "ID #{self.name} #{id} - Clean"
|
|
120
|
-
end
|
|
121
|
-
NamedArray.setup(job_inputs, @inputs.collect{|i| i[0] }) if @inputs
|
|
122
137
|
step_provided_inputs = Hash === provided_inputs ? provided_inputs.slice(*non_default_inputs) : provided_inputs
|
|
123
|
-
Step.new path.find, job_inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, &self
|
|
138
|
+
Step.new path.find, job_inputs, dependencies, id, non_default_inputs, step_provided_inputs, compute, overriden, &self
|
|
124
139
|
end
|
|
125
140
|
end
|
|
126
141
|
|
data/lib/scout/workflow/usage.rb
CHANGED
|
@@ -5,19 +5,19 @@ module Task
|
|
|
5
5
|
str = StringIO.new
|
|
6
6
|
|
|
7
7
|
if description
|
|
8
|
-
title, paragraph = description.
|
|
8
|
+
title, _, paragraph = description.partition("\n\n")
|
|
9
9
|
if title.length < Misc::MAX_TTY_LINE_WIDTH
|
|
10
10
|
title = self.name.to_s + " - " + title
|
|
11
11
|
str.puts Log.color :yellow, title
|
|
12
12
|
str.puts Log.color :yellow, "-" * title.length
|
|
13
13
|
if paragraph
|
|
14
|
-
str.puts "\n" << Misc.format_paragraph(paragraph)
|
|
14
|
+
str.puts "\n" << Misc.format_paragraph(paragraph)
|
|
15
15
|
end
|
|
16
16
|
else
|
|
17
17
|
title = self.name.to_s
|
|
18
18
|
str.puts Log.color :yellow, title
|
|
19
19
|
str.puts Log.color :yellow, "-" * title.length
|
|
20
|
-
str.puts "\n" << Misc.format_paragraph(description)
|
|
20
|
+
str.puts "\n" << Misc.format_paragraph(description)
|
|
21
21
|
end
|
|
22
22
|
else
|
|
23
23
|
title = self.name.to_s
|
|
@@ -92,7 +92,7 @@ module Task
|
|
|
92
92
|
str.puts
|
|
93
93
|
str.puts Log.color(:magenta, "Input select options")
|
|
94
94
|
selects.collect{|p| p}.uniq.each do |input,options|
|
|
95
|
-
str.puts
|
|
95
|
+
str.puts
|
|
96
96
|
str.puts Log.color(:blue, input.to_s + ": ") << Misc.format_paragraph(options.collect{|o| Array === o ? o.first.to_s : o.to_s} * ", ") << "\n"
|
|
97
97
|
end
|
|
98
98
|
end
|
|
@@ -164,7 +164,7 @@ module Workflow
|
|
|
164
164
|
end
|
|
165
165
|
|
|
166
166
|
def _prov_tasks(tree)
|
|
167
|
-
tasks = []
|
|
167
|
+
tasks = []
|
|
168
168
|
heap = tree.values
|
|
169
169
|
heap = [tree]
|
|
170
170
|
while heap.any?
|
|
@@ -199,7 +199,7 @@ module Workflow
|
|
|
199
199
|
else
|
|
200
200
|
description += ";" + task_name.to_s
|
|
201
201
|
end
|
|
202
|
-
|
|
202
|
+
|
|
203
203
|
seen << [workflow, task_name]
|
|
204
204
|
end
|
|
205
205
|
description
|
|
@@ -212,9 +212,9 @@ module Workflow
|
|
|
212
212
|
|
|
213
213
|
offset_str = " " * offset
|
|
214
214
|
|
|
215
|
-
lines << offset_str
|
|
215
|
+
lines << offset_str
|
|
216
216
|
|
|
217
|
-
tree.each do |p,dtree|
|
|
217
|
+
tree.each do |p,dtree|
|
|
218
218
|
next if seen.include?(p)
|
|
219
219
|
seen.push(p)
|
|
220
220
|
workflow, task = p
|
|
@@ -233,7 +233,7 @@ module Workflow
|
|
|
233
233
|
str.puts Log.color :magenta, title
|
|
234
234
|
str.puts Log.color :magenta, "=" * title.length
|
|
235
235
|
else
|
|
236
|
-
str.puts Log.color :magenta, self.name
|
|
236
|
+
str.puts Log.color :magenta, self.name
|
|
237
237
|
str.puts Log.color :magenta, "=" * self.name.length
|
|
238
238
|
end
|
|
239
239
|
|
|
@@ -244,14 +244,14 @@ module Workflow
|
|
|
244
244
|
elsif task.nil?
|
|
245
245
|
|
|
246
246
|
if self.documentation[:description] and not self.documentation[:description].empty?
|
|
247
|
-
str.puts Misc.format_paragraph self.documentation[:description]
|
|
247
|
+
str.puts Misc.format_paragraph self.documentation[:description]
|
|
248
248
|
str.puts
|
|
249
249
|
end
|
|
250
250
|
|
|
251
251
|
str.puts Log.color :magenta, "## TASKS"
|
|
252
252
|
if self.documentation[:task_description] and not self.documentation[:task_description].empty?
|
|
253
253
|
str.puts
|
|
254
|
-
str.puts Misc.format_paragraph self.documentation[:task_description]
|
|
254
|
+
str.puts Misc.format_paragraph self.documentation[:task_description]
|
|
255
255
|
end
|
|
256
256
|
str.puts
|
|
257
257
|
|
|
@@ -276,7 +276,7 @@ module Workflow
|
|
|
276
276
|
|
|
277
277
|
prov_string = prov_string(dep_tree(name))
|
|
278
278
|
str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
|
|
279
|
-
end
|
|
279
|
+
end
|
|
280
280
|
|
|
281
281
|
else
|
|
282
282
|
|
|
@@ -302,7 +302,7 @@ module Workflow
|
|
|
302
302
|
offset, workflow, task_name = m.values_at 1, 2, 3
|
|
303
303
|
str.puts [offset, Log.color(:magenta, workflow), "#", Log.color(:yellow, task_name)] * ""
|
|
304
304
|
else
|
|
305
|
-
str.puts Log.color :blue, line
|
|
305
|
+
str.puts Log.color :blue, line
|
|
306
306
|
end
|
|
307
307
|
end
|
|
308
308
|
str.puts
|
data/lib/scout/workflow/util.rb
CHANGED
data/lib/scout/workflow.rb
CHANGED
|
@@ -27,9 +27,9 @@ module Workflow
|
|
|
27
27
|
end
|
|
28
28
|
|
|
29
29
|
def workflow_dir
|
|
30
|
-
@workflow_dir ||
|
|
31
|
-
ENV["SCOUT_WORKFLOW_DIR"] ||
|
|
32
|
-
begin
|
|
30
|
+
@workflow_dir ||
|
|
31
|
+
ENV["SCOUT_WORKFLOW_DIR"] ||
|
|
32
|
+
begin
|
|
33
33
|
workflow_dir_config = Path.setup("etc/workflow_dir")
|
|
34
34
|
if workflow_dir_config.exists?
|
|
35
35
|
Path.setup(workflow_dir_config.read.strip)
|
|
@@ -40,9 +40,9 @@ module Workflow
|
|
|
40
40
|
end
|
|
41
41
|
|
|
42
42
|
def workflow_repo
|
|
43
|
-
@workflow_repo ||
|
|
44
|
-
ENV["SCOUT_WORKFLOW_REPO"] ||
|
|
45
|
-
begin
|
|
43
|
+
@workflow_repo ||
|
|
44
|
+
ENV["SCOUT_WORKFLOW_REPO"] ||
|
|
45
|
+
begin
|
|
46
46
|
workflow_repo_config = Path.setup("etc/workflow_repo")
|
|
47
47
|
if workflow_repo_config.exists?
|
|
48
48
|
workflow_repo_config.read.strip
|
data/scout-gear.gemspec
CHANGED
|
@@ -2,11 +2,11 @@
|
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
|
4
4
|
# -*- encoding: utf-8 -*-
|
|
5
|
-
# stub: scout-gear 10.11.
|
|
5
|
+
# stub: scout-gear 10.11.7 ruby lib
|
|
6
6
|
|
|
7
7
|
Gem::Specification.new do |s|
|
|
8
8
|
s.name = "scout-gear".freeze
|
|
9
|
-
s.version = "10.11.
|
|
9
|
+
s.version = "10.11.7".freeze
|
|
10
10
|
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
|
12
12
|
s.require_paths = ["lib".freeze]
|
|
@@ -284,7 +284,7 @@ Gem::Specification.new do |s|
|
|
|
284
284
|
]
|
|
285
285
|
s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
|
|
286
286
|
s.licenses = ["MIT".freeze]
|
|
287
|
-
s.rubygems_version = "3.7.
|
|
287
|
+
s.rubygems_version = "3.7.2".freeze
|
|
288
288
|
s.summary = "basic gear for scouts".freeze
|
|
289
289
|
|
|
290
290
|
s.specification_version = 4
|