scout-gear 6.0.0 → 7.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +465 -432
- data/VERSION +1 -1
- data/bin/scout +5 -1
- data/lib/rbbt-scout.rb +5 -0
- data/lib/scout/concurrent_stream.rb +6 -2
- data/lib/scout/config.rb +168 -0
- data/lib/scout/exceptions.rb +9 -0
- data/lib/scout/indiferent_hash/options.rb +1 -0
- data/lib/scout/indiferent_hash.rb +4 -2
- data/lib/scout/log/color.rb +31 -2
- data/lib/scout/log/progress/report.rb +1 -0
- data/lib/scout/log/progress/util.rb +3 -1
- data/lib/scout/log/progress.rb +7 -3
- data/lib/scout/log.rb +8 -3
- data/lib/scout/misc/digest.rb +1 -3
- data/lib/scout/misc/monitor.rb +3 -0
- data/lib/scout/misc/system.rb +15 -0
- data/lib/scout/misc.rb +1 -0
- data/lib/scout/named_array.rb +68 -0
- data/lib/scout/open/stream.rb +58 -26
- data/lib/scout/path/find.rb +27 -3
- data/lib/scout/path/util.rb +7 -4
- data/lib/scout/persist/serialize.rb +7 -14
- data/lib/scout/persist.rb +21 -1
- data/lib/scout/resource/produce.rb +7 -94
- data/lib/scout/resource/software.rb +176 -0
- data/lib/scout/tsv/dumper.rb +107 -0
- data/lib/scout/tsv/index.rb +49 -0
- data/lib/scout/tsv/parser.rb +317 -0
- data/lib/scout/tsv/path.rb +13 -0
- data/lib/scout/tsv/persist/adapter.rb +348 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +113 -0
- data/lib/scout/tsv/persist.rb +15 -0
- data/lib/scout/tsv/traverse.rb +48 -0
- data/lib/scout/tsv/util.rb +24 -0
- data/lib/scout/tsv.rb +27 -0
- data/lib/scout/work_queue/worker.rb +16 -11
- data/lib/scout/work_queue.rb +63 -21
- data/lib/scout/workflow/definition.rb +93 -4
- data/lib/scout/workflow/step/config.rb +18 -0
- data/lib/scout/workflow/step/dependencies.rb +40 -0
- data/lib/scout/workflow/step/file.rb +15 -0
- data/lib/scout/workflow/step/info.rb +33 -6
- data/lib/scout/workflow/step/provenance.rb +148 -0
- data/lib/scout/workflow/step.rb +70 -20
- data/lib/scout/workflow/task.rb +5 -4
- data/lib/scout/workflow/usage.rb +1 -1
- data/lib/scout/workflow.rb +11 -3
- data/lib/scout-gear.rb +1 -0
- data/lib/scout.rb +1 -0
- data/scout-gear.gemspec +38 -3
- data/scout_commands/find +1 -1
- data/scout_commands/workflow/task +16 -10
- data/share/software/install_helpers +523 -0
- data/test/scout/log/test_progress.rb +0 -2
- data/test/scout/misc/test_system.rb +21 -0
- data/test/scout/open/test_stream.rb +160 -1
- data/test/scout/path/test_find.rb +14 -7
- data/test/scout/resource/test_software.rb +24 -0
- data/test/scout/test_config.rb +66 -0
- data/test/scout/test_meta_extension.rb +10 -0
- data/test/scout/test_named_array.rb +19 -0
- data/test/scout/test_persist.rb +35 -0
- data/test/scout/test_semaphore.rb +1 -1
- data/test/scout/test_tmpfile.rb +2 -2
- data/test/scout/test_tsv.rb +74 -0
- data/test/scout/test_work_queue.rb +63 -8
- data/test/scout/tsv/persist/test_adapter.rb +34 -0
- data/test/scout/tsv/persist/test_tokyocabinet.rb +92 -0
- data/test/scout/tsv/test_dumper.rb +44 -0
- data/test/scout/tsv/test_index.rb +64 -0
- data/test/scout/tsv/test_parser.rb +173 -0
- data/test/scout/tsv/test_persist.rb +36 -0
- data/test/scout/tsv/test_traverse.rb +9 -0
- data/test/scout/tsv/test_util.rb +0 -0
- data/test/scout/work_queue/test_worker.rb +49 -1
- data/test/scout/workflow/step/test_dependencies.rb +25 -0
- data/test/scout/workflow/step/test_info.rb +15 -17
- data/test/scout/workflow/step/test_load.rb +16 -18
- data/test/scout/workflow/step/test_provenance.rb +25 -0
- data/test/scout/workflow/test_step.rb +206 -10
- data/test/scout/workflow/test_task.rb +0 -3
- data/test/test_helper.rb +6 -0
- metadata +37 -2
data/lib/scout/work_queue.rb
CHANGED
@@ -35,52 +35,94 @@ class WorkQueue
|
|
35
35
|
end
|
36
36
|
|
37
37
|
def remove_worker(pid)
|
38
|
-
|
39
|
-
|
40
|
-
|
38
|
+
@worker_mutex.synchronize do
|
39
|
+
worker = @workers.index{|w| w.pid == pid}
|
40
|
+
if worker
|
41
|
+
Log.debug "Removed worker #{pid}"
|
42
|
+
@workers.delete_at(worker)
|
43
|
+
@removed_workers << pid
|
44
|
+
end
|
41
45
|
end
|
42
46
|
end
|
43
47
|
|
44
48
|
def process(&callback)
|
45
|
-
@
|
46
|
-
w.process @input, @output, &@worker_proc
|
47
|
-
end
|
48
|
-
@reader = Thread.new do
|
49
|
+
@reader = Thread.new do |parent|
|
49
50
|
begin
|
51
|
+
Thread.current.report_on_exception = false
|
52
|
+
Thread.current["name"] = "Output reader #{Process.pid}"
|
53
|
+
@done_workers ||= []
|
50
54
|
while true
|
51
55
|
obj = @output.read
|
52
56
|
if DoneProcessing === obj
|
53
|
-
|
57
|
+
|
58
|
+
done = @worker_mutex.synchronize do
|
59
|
+
Log.low "Worker #{obj.pid} done"
|
60
|
+
@done_workers << obj.pid
|
61
|
+
@closed && @done_workers.length == @removed_workers.length + @workers.length
|
62
|
+
end
|
63
|
+
|
64
|
+
break if done
|
65
|
+
elsif Exception === obj
|
66
|
+
raise obj
|
54
67
|
else
|
55
68
|
callback.call obj if callback
|
56
69
|
end
|
57
70
|
end
|
71
|
+
rescue DoneProcessing
|
58
72
|
rescue Aborted
|
73
|
+
rescue WorkerException
|
74
|
+
Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.message}"
|
75
|
+
self.abort
|
76
|
+
raise obj.worker_exception
|
77
|
+
rescue
|
78
|
+
Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
|
79
|
+
self.abort
|
80
|
+
raise $!
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
@workers.each do |w|
|
85
|
+
w.process @input, @output, &@worker_proc
|
86
|
+
end
|
87
|
+
|
88
|
+
Thread.pass until @reader["name"]
|
89
|
+
|
90
|
+
@waiter = Thread.new do
|
91
|
+
begin
|
92
|
+
Thread.current.report_on_exception = false
|
93
|
+
Thread.current["name"] = "Worker waiter #{Process.pid}"
|
94
|
+
while true
|
95
|
+
pid = Process.wait
|
96
|
+
remove_worker(pid)
|
97
|
+
break if @worker_mutex.synchronize{ @workers.empty? }
|
98
|
+
end
|
59
99
|
end
|
60
|
-
end
|
100
|
+
end
|
101
|
+
|
102
|
+
Thread.pass until @worker_mutex.synchronize{ @workers.select{|w| w.pid.nil? }.empty? }
|
103
|
+
Thread.pass until @waiter["name"]
|
61
104
|
end
|
62
105
|
|
63
106
|
def write(obj)
|
64
107
|
@input.write obj
|
65
108
|
end
|
66
109
|
|
110
|
+
def abort
|
111
|
+
Log.low "Aborting #{@workers.length} workers in queue #{Process.pid}"
|
112
|
+
@worker_mutex.synchronize do
|
113
|
+
@workers.each{|w| w.abort }
|
114
|
+
end
|
115
|
+
end
|
116
|
+
|
67
117
|
def close
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
pid = Process.wait
|
72
|
-
status = $?
|
73
|
-
worker = @worker_mutex.synchronize{ @removed_workers.delete_if{|w| w.pid == pid }.first }
|
74
|
-
worker.exit $?.exitstatus if worker
|
75
|
-
rescue Errno::ECHILD
|
76
|
-
Thread.pass until @workers.length == 0
|
77
|
-
break
|
78
|
-
end
|
118
|
+
@closed = true
|
119
|
+
@worker_mutex.synchronize{ @workers.length }.times do
|
120
|
+
@input.write DoneProcessing.new()
|
79
121
|
end
|
80
|
-
@reader.raise Aborted if @reader
|
81
122
|
end
|
82
123
|
|
83
124
|
def join
|
125
|
+
@waiter.join if @waiter
|
84
126
|
@reader.join if @reader
|
85
127
|
end
|
86
128
|
end
|
@@ -2,7 +2,7 @@ require_relative '../meta_extension'
|
|
2
2
|
|
3
3
|
module Workflow
|
4
4
|
extend MetaExtension
|
5
|
-
extension_attr :name, :tasks
|
5
|
+
extension_attr :name, :tasks, :helpers
|
6
6
|
|
7
7
|
class << self
|
8
8
|
attr_accessor :directory
|
@@ -17,6 +17,32 @@ module Workflow
|
|
17
17
|
@name ||= self.to_s
|
18
18
|
end
|
19
19
|
|
20
|
+
def helpers
|
21
|
+
@helpers ||= {}
|
22
|
+
end
|
23
|
+
|
24
|
+
def helper(name, *args, &block)
|
25
|
+
if block_given?
|
26
|
+
helpers[name] = block
|
27
|
+
else
|
28
|
+
raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
|
29
|
+
helpers[name].call(*args)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
33
|
+
def step_module
|
34
|
+
@_m ||= begin
|
35
|
+
m = Module.new
|
36
|
+
|
37
|
+
helpers.each do |name,block|
|
38
|
+
m.send(:define_method, name, &block)
|
39
|
+
end
|
40
|
+
|
41
|
+
m
|
42
|
+
end
|
43
|
+
@_m
|
44
|
+
end
|
45
|
+
|
20
46
|
attr_accessor :directory
|
21
47
|
def directory
|
22
48
|
@directory ||= Workflow.directory[name]
|
@@ -60,19 +86,82 @@ module Workflow
|
|
60
86
|
annotate_next_task(:inputs, args)
|
61
87
|
end
|
62
88
|
|
89
|
+
def desc(description)
|
90
|
+
annotate_next_task_single(:description, description)
|
91
|
+
end
|
92
|
+
|
93
|
+
def returns(type)
|
94
|
+
annotate_next_task_single(:returns, type)
|
95
|
+
end
|
96
|
+
|
97
|
+
def extension(extension)
|
98
|
+
annotate_next_task_single(:extension, extension)
|
99
|
+
end
|
100
|
+
|
63
101
|
def task(name_and_type, &block)
|
64
102
|
name, type = name_and_type.collect.first
|
65
103
|
@tasks ||= IndiferentHash.setup({})
|
66
104
|
begin
|
67
105
|
@annotate_next_task ||= {}
|
68
|
-
task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
|
106
|
+
task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name], workflow: self))
|
69
107
|
@tasks[name] = task
|
70
108
|
ensure
|
71
109
|
@annotate_next_task = {}
|
72
110
|
end
|
73
111
|
end
|
74
112
|
|
75
|
-
def
|
76
|
-
|
113
|
+
def task_alias(name, workflow, oname, *rest, &block)
|
114
|
+
dep(workflow, oname, *rest, &block)
|
115
|
+
extension :dep_task unless @extension
|
116
|
+
returns workflow.tasks[oname].returns if workflow.tasks.include?(oname) unless @returns
|
117
|
+
task name => nil do
|
118
|
+
raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
|
119
|
+
Step.wait_for_jobs dependencies.select{|d| d.streaming? }
|
120
|
+
dep = dependencies.last
|
121
|
+
dep.join
|
122
|
+
raise dep.get_exception if dep.error?
|
123
|
+
raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
|
124
|
+
set_info :result_type, dep.info[:result_type]
|
125
|
+
forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
|
126
|
+
if forget
|
127
|
+
remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
|
128
|
+
|
129
|
+
self.archive_deps
|
130
|
+
self.copy_files_dir
|
131
|
+
self.dependencies = self.dependencies - [dep]
|
132
|
+
Open.rm_rf self.files_dir if Open.exist? self.files_dir
|
133
|
+
FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
|
134
|
+
|
135
|
+
if dep.overriden || ! Workflow.job_path?(dep.path)
|
136
|
+
Open.link dep.path, self.tmp_path
|
137
|
+
else
|
138
|
+
Open.ln_h dep.path, self.tmp_path
|
139
|
+
|
140
|
+
case remove.to_s
|
141
|
+
when 'true'
|
142
|
+
dep.clean
|
143
|
+
when 'recursive'
|
144
|
+
(dep.dependencies + dep.rec_dependencies).uniq.each do |d|
|
145
|
+
next if d.overriden
|
146
|
+
d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
|
147
|
+
end
|
148
|
+
dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
|
149
|
+
end
|
150
|
+
end
|
151
|
+
else
|
152
|
+
if Open.exists?(dep.files_dir)
|
153
|
+
Open.rm_rf self.files_dir
|
154
|
+
Open.link dep.files_dir, self.files_dir
|
155
|
+
end
|
156
|
+
if defined?(RemoteStep) && RemoteStep === dep
|
157
|
+
Open.write(self.tmp_path, Open.read(dep.path))
|
158
|
+
else
|
159
|
+
Open.link dep.path, self.path
|
160
|
+
end
|
161
|
+
end
|
162
|
+
nil
|
163
|
+
end
|
77
164
|
end
|
165
|
+
|
166
|
+
alias dep_task task_alias
|
78
167
|
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
require_relative '../../config'
|
2
|
+
|
3
|
+
class Step
|
4
|
+
def config(key, *tokens)
|
5
|
+
options = tokens.pop if Hash === tokens.last
|
6
|
+
options ||= {}
|
7
|
+
|
8
|
+
new_tokens = []
|
9
|
+
if workflow
|
10
|
+
workflow_name = workflow.to_s
|
11
|
+
new_tokens << ("workflow:" << workflow_name)
|
12
|
+
new_tokens << ("task:" << workflow_name << "#" << task_name.to_s)
|
13
|
+
end
|
14
|
+
new_tokens << ("task:" << task_name.to_s)
|
15
|
+
|
16
|
+
Scout::Config.get(key, tokens + new_tokens, options)
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,40 @@
|
|
1
|
+
class Step
|
2
|
+
def recursive_inputs
|
3
|
+
dependencies.inject(@inputs.annotate(@inputs.dup)) do |acc,dep|
|
4
|
+
acc.concat(dep.inputs) if dep.inputs
|
5
|
+
acc
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
def input_dependencies
|
10
|
+
return [] unless inputs
|
11
|
+
inputs.select do |d|
|
12
|
+
Step === d
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def prepare_dependencies
|
17
|
+
inverse_dep = {}
|
18
|
+
dependencies.each{|dep|
|
19
|
+
next if dep.done?
|
20
|
+
if dep.dependencies
|
21
|
+
dep.dependencies.each do |d|
|
22
|
+
inverse_dep[d] ||= []
|
23
|
+
inverse_dep[d] << dep
|
24
|
+
end
|
25
|
+
end
|
26
|
+
input_dependencies.each do |d|
|
27
|
+
inverse_dep[d] ||= []
|
28
|
+
inverse_dep[d] << dep
|
29
|
+
end
|
30
|
+
}
|
31
|
+
inverse_dep.each do |dep,list|
|
32
|
+
dep.tee_copies = list.length
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def run_dependencies
|
37
|
+
dependencies.each{|dep| dep.run unless dep.running? || dep.done? }
|
38
|
+
end
|
39
|
+
|
40
|
+
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
class Step
|
2
|
-
SERIALIZER = :
|
2
|
+
SERIALIZER = :marshal
|
3
3
|
def info_file
|
4
4
|
@info_file ||= begin
|
5
5
|
info_file = @path + ".info"
|
@@ -20,7 +20,11 @@ class Step
|
|
20
20
|
end
|
21
21
|
|
22
22
|
def info
|
23
|
-
outdated =
|
23
|
+
outdated = begin
|
24
|
+
@info_load_time && (mtime = Open.mtime(info_file)) && mtime > @info_load_time
|
25
|
+
rescue
|
26
|
+
true
|
27
|
+
end
|
24
28
|
|
25
29
|
if @info.nil? || outdated
|
26
30
|
load_info
|
@@ -33,6 +37,19 @@ class Step
|
|
33
37
|
info = self.info
|
34
38
|
new_info.each do |key,value|
|
35
39
|
report_status new_info[:status], new_info[:message] if key == :status
|
40
|
+
if Exception === value
|
41
|
+
begin
|
42
|
+
Marshal.dump(value)
|
43
|
+
rescue TypeError
|
44
|
+
if ScoutException === value
|
45
|
+
new = ScoutException.new value.message
|
46
|
+
else
|
47
|
+
new = Exception.new value.message
|
48
|
+
end
|
49
|
+
new.set_backtrace(value.backtrace)
|
50
|
+
value = new
|
51
|
+
end
|
52
|
+
end
|
36
53
|
if info.include?(key)
|
37
54
|
case info[key]
|
38
55
|
when Array
|
@@ -55,14 +72,13 @@ class Step
|
|
55
72
|
|
56
73
|
def report_status(status, message = nil)
|
57
74
|
if message.nil?
|
58
|
-
Log.info Log.color(status, status
|
75
|
+
Log.info Log.color(:status, status, true) + " " + Log.color(:path, path)
|
59
76
|
else
|
60
|
-
Log.info Log.color(status, status
|
77
|
+
Log.info Log.color(:status, status, true) + " " + Log.color(:path, path) + " " + message
|
61
78
|
end
|
62
79
|
end
|
63
80
|
|
64
81
|
def log(status, message = nil)
|
65
|
-
report_status status, message
|
66
82
|
if message
|
67
83
|
merge_info :status => status, :messages => [message]
|
68
84
|
else
|
@@ -71,7 +87,18 @@ class Step
|
|
71
87
|
end
|
72
88
|
|
73
89
|
def status
|
74
|
-
info[:status]
|
90
|
+
info[:status].tap{|s| s.nil? ? s : s.to_sym }
|
91
|
+
end
|
92
|
+
|
93
|
+
def error?
|
94
|
+
status == :error
|
75
95
|
end
|
76
96
|
|
97
|
+
def aborted?
|
98
|
+
status == :aborted
|
99
|
+
end
|
100
|
+
|
101
|
+
def running?
|
102
|
+
! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
|
103
|
+
end
|
77
104
|
end
|
@@ -0,0 +1,148 @@
|
|
1
|
+
class Step
|
2
|
+
def self.job_path?(path)
|
3
|
+
path.split("/")[-4] == "jobs"
|
4
|
+
end
|
5
|
+
|
6
|
+
def self.status_color(status)
|
7
|
+
case status.to_sym
|
8
|
+
when :error, :aborted, :missing, :dead, :unsync
|
9
|
+
:red
|
10
|
+
when :streaming, :started
|
11
|
+
:cyan
|
12
|
+
when :done, :noinfo
|
13
|
+
:green
|
14
|
+
when :dependencies, :waiting, :setup
|
15
|
+
:yellow
|
16
|
+
when :notfound, :cleaned
|
17
|
+
:blue
|
18
|
+
else
|
19
|
+
if status.to_s.index ">"
|
20
|
+
:cyan
|
21
|
+
else
|
22
|
+
:cyan
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.prov_status_msg(status)
|
28
|
+
color = status_color(status)
|
29
|
+
Log.color(color, status.to_s)
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.prov_report_msg(status, name, path, info, input = nil)
|
33
|
+
parts = path.sub(/\{.*/,'').split "/"
|
34
|
+
|
35
|
+
parts.pop
|
36
|
+
|
37
|
+
task = Log.color(:yellow, parts.pop)
|
38
|
+
workflow = Log.color(:magenta, parts.pop)
|
39
|
+
|
40
|
+
if ! Step.job_path?(path)
|
41
|
+
task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:green, "file"), Log.color(:magenta, "-")
|
42
|
+
end
|
43
|
+
|
44
|
+
path_mtime = begin
|
45
|
+
Open.mtime(path)
|
46
|
+
rescue Exception
|
47
|
+
nil
|
48
|
+
end
|
49
|
+
|
50
|
+
if input.nil? || input.empty?
|
51
|
+
input_str = nil
|
52
|
+
else
|
53
|
+
input = input.reject{|dep,name| (input & dep.dependencies.collect{|d| [d,name]}).any? }
|
54
|
+
input = input.reject{|dep,name| (input & dep.input_dependencies.collect{|d| [d,name]}).any? }
|
55
|
+
input_str = Log.color(:magenta, "-> ") + input.collect{|dep,name| Log.color(:yellow, dep.task_name.to_s) + ":" + Log.color(:yellow, name) }.uniq * " "
|
56
|
+
end
|
57
|
+
|
58
|
+
str = if ! (Open.remote?(path) || Open.ssh?(path)) && (Open.exists?(path) && $main_mtime && path_mtime && ($main_mtime - path_mtime) < -2)
|
59
|
+
prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
|
60
|
+
else
|
61
|
+
prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " "
|
62
|
+
end
|
63
|
+
|
64
|
+
if $inputs and $inputs.any?
|
65
|
+
job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
|
66
|
+
IndiferentHash.setup(job_inputs)
|
67
|
+
|
68
|
+
$inputs.each do |input|
|
69
|
+
value = job_inputs[input]
|
70
|
+
next if value.nil?
|
71
|
+
value_str = Misc.fingerprint(value)
|
72
|
+
str << "\t#{Log.color :magenta, input}=#{value_str}"
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
if $info_fields and $info_fields.any?
|
77
|
+
$info_fields.each do |field|
|
78
|
+
IndiferentHash.setup(info)
|
79
|
+
value = info[field]
|
80
|
+
next if value.nil?
|
81
|
+
value_str = Misc.fingerprint(value)
|
82
|
+
str << "\t#{Log.color :magenta, field}=#{value_str}"
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
str << "\n"
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil)
|
90
|
+
info = step.info || {}
|
91
|
+
info[:task_name] = task
|
92
|
+
path = step.path
|
93
|
+
status = info[:status] || :missing
|
94
|
+
status = "remote" if Open.remote?(path) || Open.ssh?(path)
|
95
|
+
name = info[:name] || File.basename(path)
|
96
|
+
status = :unsync if status == :done and not Open.exist?(path)
|
97
|
+
status = :notfound if status == :noinfo and not Open.exist?(path)
|
98
|
+
|
99
|
+
|
100
|
+
this_step_msg = prov_report_msg(status, name, path, info, input)
|
101
|
+
|
102
|
+
input_dependencies = {}
|
103
|
+
step.dependencies.each do |dep|
|
104
|
+
if dep.input_dependencies.any?
|
105
|
+
dep.input_dependencies.each do |id|
|
106
|
+
input_name, _dep = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d|
|
107
|
+
d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
|
108
|
+
}.last
|
109
|
+
if input_name
|
110
|
+
input_dependencies[id] ||= []
|
111
|
+
input_dependencies[id] << [dep, input_name]
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end if step.dependencies
|
116
|
+
|
117
|
+
str = ""
|
118
|
+
str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'
|
119
|
+
|
120
|
+
step.dependencies.dup.tap{|l|
|
121
|
+
l.reverse! if ENV["RBBT_ORIGINAL_STACK"] == 'true'
|
122
|
+
}.each do |dep|
|
123
|
+
path = dep.path
|
124
|
+
new = ! seen.include?(path)
|
125
|
+
if new
|
126
|
+
seen << path
|
127
|
+
str << prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep])
|
128
|
+
else
|
129
|
+
if expand_repeats
|
130
|
+
str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
|
131
|
+
else
|
132
|
+
info = dep.info || {}
|
133
|
+
status = info[:status] || :missing
|
134
|
+
status = "remote" if Open.remote?(path) || Open.ssh?(path)
|
135
|
+
name = info[:name] || File.basename(path)
|
136
|
+
status = :unsync if status == :done and not Open.exist?(path)
|
137
|
+
status = :notfound if status == :noinfo and not Open.exist?(path)
|
138
|
+
|
139
|
+
str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end if step.dependencies
|
143
|
+
|
144
|
+
str += (" " * offset) + this_step_msg unless ENV["RBBT_ORIGINAL_STACK"] == 'true'
|
145
|
+
|
146
|
+
str
|
147
|
+
end
|
148
|
+
end
|
data/lib/scout/workflow/step.rb
CHANGED
@@ -2,15 +2,25 @@ require_relative '../path'
|
|
2
2
|
require_relative '../persist'
|
3
3
|
require_relative 'step/info'
|
4
4
|
require_relative 'step/load'
|
5
|
+
require_relative 'step/file'
|
6
|
+
require_relative 'step/dependencies'
|
7
|
+
require_relative 'step/provenance'
|
8
|
+
require_relative 'step/config'
|
5
9
|
|
6
10
|
class Step
|
7
11
|
|
8
|
-
attr_accessor :path, :inputs, :dependencies, :task
|
12
|
+
attr_accessor :path, :inputs, :dependencies, :task, :tee_copies
|
9
13
|
def initialize(path, inputs = nil, dependencies = nil, &task)
|
10
14
|
@path = path
|
11
15
|
@inputs = inputs
|
12
16
|
@dependencies = dependencies
|
13
17
|
@task = task
|
18
|
+
@mutex = Mutex.new
|
19
|
+
@tee_copies = 1
|
20
|
+
end
|
21
|
+
|
22
|
+
def synchronize(&block)
|
23
|
+
@mutex.synchronize(&block)
|
14
24
|
end
|
15
25
|
|
16
26
|
def inputs
|
@@ -48,30 +58,49 @@ class Step
|
|
48
58
|
@task_name ||= @task.name if @task.respond_to?(:name)
|
49
59
|
end
|
50
60
|
|
61
|
+
def workflow
|
62
|
+
@task.workflow if @task
|
63
|
+
end
|
64
|
+
|
51
65
|
def exec
|
52
|
-
self.instance_exec(*inputs, &task)
|
66
|
+
@result = self.instance_exec(*inputs, &task)
|
53
67
|
end
|
54
68
|
|
55
69
|
attr_reader :result
|
56
70
|
def run
|
57
71
|
return @result || self.load if done?
|
58
|
-
|
59
|
-
|
72
|
+
prepare_dependencies
|
73
|
+
run_dependencies
|
74
|
+
@result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
|
60
75
|
begin
|
61
76
|
merge_info :status => :start, :start => Time.now,
|
62
77
|
:pid => Process.pid, :pid_hostname => ENV["HOSTNAME"],
|
63
78
|
:inputs => inputs, :type => type,
|
64
79
|
:dependencies => dependencies.collect{|d| d.path }
|
65
80
|
|
66
|
-
|
81
|
+
exec
|
82
|
+
rescue Exception => e
|
83
|
+
merge_info :status => :error, :exception => e
|
84
|
+
raise e
|
67
85
|
ensure
|
68
|
-
if
|
69
|
-
|
86
|
+
if ! (error? || aborted?)
|
87
|
+
if streaming?
|
88
|
+
ConcurrentStream.setup(@result) do
|
89
|
+
merge_info :status => :done, :end => Time.now
|
90
|
+
end
|
91
|
+
|
92
|
+
@result.abort_callback = proc do |exception|
|
93
|
+
if Aborted === exception || Interrupt === exception
|
94
|
+
merge_info :status => :aborted, :end => Time.now
|
95
|
+
else
|
96
|
+
merge_info :status => :error, :exception => exception, :end => Time.now
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
log :streaming
|
101
|
+
else
|
70
102
|
merge_info :status => :done, :end => Time.now
|
71
103
|
end
|
72
|
-
log :streaming
|
73
|
-
else
|
74
|
-
merge_info :status => :done, :end => Time.now
|
75
104
|
end
|
76
105
|
end
|
77
106
|
end
|
@@ -82,19 +111,34 @@ class Step
|
|
82
111
|
end
|
83
112
|
|
84
113
|
def streaming?
|
85
|
-
IO === @result || StringIO === @result
|
114
|
+
@take_stream || IO === @result || StringIO === @result
|
86
115
|
end
|
87
116
|
|
88
|
-
def
|
89
|
-
|
90
|
-
|
117
|
+
def get_stream
|
118
|
+
synchronize do
|
119
|
+
if streaming? && ! @result.nil?
|
120
|
+
if @result.next
|
121
|
+
Log.debug "Taking result #{Log.fingerprint @result} next #{Log.fingerprint @result.next}"
|
122
|
+
else
|
123
|
+
Log.debug "Taking result #{Log.fingerprint @result}"
|
124
|
+
end
|
125
|
+
@take_stream, @result = @result, @result.next
|
126
|
+
@take_stream
|
127
|
+
elsif done?
|
128
|
+
Open.open(self.path)
|
129
|
+
else
|
130
|
+
if running?
|
131
|
+
nil
|
132
|
+
else
|
133
|
+
exec
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
91
137
|
end
|
92
138
|
|
93
139
|
def join
|
94
|
-
if streaming?
|
95
|
-
|
96
|
-
@result = nil
|
97
|
-
end
|
140
|
+
stream = get_stream if streaming?
|
141
|
+
Open.consume_stream(stream, false) if stream
|
98
142
|
end
|
99
143
|
|
100
144
|
def produce
|
@@ -109,7 +153,13 @@ class Step
|
|
109
153
|
end
|
110
154
|
|
111
155
|
def clean
|
112
|
-
|
156
|
+
@take_stream = nil
|
157
|
+
@result = nil
|
158
|
+
@info = nil
|
159
|
+
@info_load_time = nil
|
160
|
+
Open.rm path if Open.exist?(path)
|
161
|
+
Open.rm info_file if Open.exist?(info_file)
|
162
|
+
Open.rm_rf files_dir if Open.exist?(files_dir)
|
113
163
|
end
|
114
164
|
|
115
165
|
def recursive_clean
|
@@ -127,6 +177,6 @@ class Step
|
|
127
177
|
end
|
128
178
|
|
129
179
|
def digest_str
|
130
|
-
path
|
180
|
+
path.dup
|
131
181
|
end
|
132
182
|
end
|