scout-gear 6.0.0 → 7.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +465 -432
  3. data/VERSION +1 -1
  4. data/bin/scout +5 -1
  5. data/lib/rbbt-scout.rb +5 -0
  6. data/lib/scout/concurrent_stream.rb +6 -2
  7. data/lib/scout/config.rb +168 -0
  8. data/lib/scout/exceptions.rb +9 -0
  9. data/lib/scout/indiferent_hash/options.rb +1 -0
  10. data/lib/scout/indiferent_hash.rb +4 -2
  11. data/lib/scout/log/color.rb +31 -2
  12. data/lib/scout/log/progress/report.rb +1 -0
  13. data/lib/scout/log/progress/util.rb +3 -1
  14. data/lib/scout/log/progress.rb +7 -3
  15. data/lib/scout/log.rb +8 -3
  16. data/lib/scout/misc/digest.rb +1 -3
  17. data/lib/scout/misc/monitor.rb +3 -0
  18. data/lib/scout/misc/system.rb +15 -0
  19. data/lib/scout/misc.rb +1 -0
  20. data/lib/scout/named_array.rb +68 -0
  21. data/lib/scout/open/stream.rb +58 -26
  22. data/lib/scout/path/find.rb +27 -3
  23. data/lib/scout/path/util.rb +7 -4
  24. data/lib/scout/persist/serialize.rb +7 -14
  25. data/lib/scout/persist.rb +21 -1
  26. data/lib/scout/resource/produce.rb +7 -94
  27. data/lib/scout/resource/software.rb +176 -0
  28. data/lib/scout/tsv/dumper.rb +107 -0
  29. data/lib/scout/tsv/index.rb +49 -0
  30. data/lib/scout/tsv/parser.rb +317 -0
  31. data/lib/scout/tsv/path.rb +13 -0
  32. data/lib/scout/tsv/persist/adapter.rb +348 -0
  33. data/lib/scout/tsv/persist/tokyocabinet.rb +113 -0
  34. data/lib/scout/tsv/persist.rb +15 -0
  35. data/lib/scout/tsv/traverse.rb +48 -0
  36. data/lib/scout/tsv/util.rb +24 -0
  37. data/lib/scout/tsv.rb +27 -0
  38. data/lib/scout/work_queue/worker.rb +16 -11
  39. data/lib/scout/work_queue.rb +63 -21
  40. data/lib/scout/workflow/definition.rb +93 -4
  41. data/lib/scout/workflow/step/config.rb +18 -0
  42. data/lib/scout/workflow/step/dependencies.rb +40 -0
  43. data/lib/scout/workflow/step/file.rb +15 -0
  44. data/lib/scout/workflow/step/info.rb +33 -6
  45. data/lib/scout/workflow/step/provenance.rb +148 -0
  46. data/lib/scout/workflow/step.rb +70 -20
  47. data/lib/scout/workflow/task.rb +5 -4
  48. data/lib/scout/workflow/usage.rb +1 -1
  49. data/lib/scout/workflow.rb +11 -3
  50. data/lib/scout-gear.rb +1 -0
  51. data/lib/scout.rb +1 -0
  52. data/scout-gear.gemspec +38 -3
  53. data/scout_commands/find +1 -1
  54. data/scout_commands/workflow/task +16 -10
  55. data/share/software/install_helpers +523 -0
  56. data/test/scout/log/test_progress.rb +0 -2
  57. data/test/scout/misc/test_system.rb +21 -0
  58. data/test/scout/open/test_stream.rb +160 -1
  59. data/test/scout/path/test_find.rb +14 -7
  60. data/test/scout/resource/test_software.rb +24 -0
  61. data/test/scout/test_config.rb +66 -0
  62. data/test/scout/test_meta_extension.rb +10 -0
  63. data/test/scout/test_named_array.rb +19 -0
  64. data/test/scout/test_persist.rb +35 -0
  65. data/test/scout/test_semaphore.rb +1 -1
  66. data/test/scout/test_tmpfile.rb +2 -2
  67. data/test/scout/test_tsv.rb +74 -0
  68. data/test/scout/test_work_queue.rb +63 -8
  69. data/test/scout/tsv/persist/test_adapter.rb +34 -0
  70. data/test/scout/tsv/persist/test_tokyocabinet.rb +92 -0
  71. data/test/scout/tsv/test_dumper.rb +44 -0
  72. data/test/scout/tsv/test_index.rb +64 -0
  73. data/test/scout/tsv/test_parser.rb +173 -0
  74. data/test/scout/tsv/test_persist.rb +36 -0
  75. data/test/scout/tsv/test_traverse.rb +9 -0
  76. data/test/scout/tsv/test_util.rb +0 -0
  77. data/test/scout/work_queue/test_worker.rb +49 -1
  78. data/test/scout/workflow/step/test_dependencies.rb +25 -0
  79. data/test/scout/workflow/step/test_info.rb +15 -17
  80. data/test/scout/workflow/step/test_load.rb +16 -18
  81. data/test/scout/workflow/step/test_provenance.rb +25 -0
  82. data/test/scout/workflow/test_step.rb +206 -10
  83. data/test/scout/workflow/test_task.rb +0 -3
  84. data/test/test_helper.rb +6 -0
  85. metadata +37 -2
@@ -35,52 +35,94 @@ class WorkQueue
35
35
  end
36
36
 
37
37
  def remove_worker(pid)
38
- worker = @worker_mutex.synchronize do
39
- Log.debug "Remove #{pid}"
40
- @removed_workers.concat(@workers.delete_if{|w| w.pid == pid })
38
+ @worker_mutex.synchronize do
39
+ worker = @workers.index{|w| w.pid == pid}
40
+ if worker
41
+ Log.debug "Removed worker #{pid}"
42
+ @workers.delete_at(worker)
43
+ @removed_workers << pid
44
+ end
41
45
  end
42
46
  end
43
47
 
44
48
  def process(&callback)
45
- @workers.each do |w|
46
- w.process @input, @output, &@worker_proc
47
- end
48
- @reader = Thread.new do
49
+ @reader = Thread.new do |parent|
49
50
  begin
51
+ Thread.current.report_on_exception = false
52
+ Thread.current["name"] = "Output reader #{Process.pid}"
53
+ @done_workers ||= []
50
54
  while true
51
55
  obj = @output.read
52
56
  if DoneProcessing === obj
53
- remove_worker obj.pid if obj.pid
57
+
58
+ done = @worker_mutex.synchronize do
59
+ Log.low "Worker #{obj.pid} done"
60
+ @done_workers << obj.pid
61
+ @closed && @done_workers.length == @removed_workers.length + @workers.length
62
+ end
63
+
64
+ break if done
65
+ elsif Exception === obj
66
+ raise obj
54
67
  else
55
68
  callback.call obj if callback
56
69
  end
57
70
  end
71
+ rescue DoneProcessing
58
72
  rescue Aborted
73
+ rescue WorkerException
74
+ Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.message}"
75
+ self.abort
76
+ raise obj.worker_exception
77
+ rescue
78
+ Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
79
+ self.abort
80
+ raise $!
81
+ end
82
+ end
83
+
84
+ @workers.each do |w|
85
+ w.process @input, @output, &@worker_proc
86
+ end
87
+
88
+ Thread.pass until @reader["name"]
89
+
90
+ @waiter = Thread.new do
91
+ begin
92
+ Thread.current.report_on_exception = false
93
+ Thread.current["name"] = "Worker waiter #{Process.pid}"
94
+ while true
95
+ pid = Process.wait
96
+ remove_worker(pid)
97
+ break if @worker_mutex.synchronize{ @workers.empty? }
98
+ end
59
99
  end
60
- end if @output
100
+ end
101
+
102
+ Thread.pass until @worker_mutex.synchronize{ @workers.select{|w| w.pid.nil? }.empty? }
103
+ Thread.pass until @waiter["name"]
61
104
  end
62
105
 
63
106
  def write(obj)
64
107
  @input.write obj
65
108
  end
66
109
 
110
+ def abort
111
+ Log.low "Aborting #{@workers.length} workers in queue #{Process.pid}"
112
+ @worker_mutex.synchronize do
113
+ @workers.each{|w| w.abort }
114
+ end
115
+ end
116
+
67
117
  def close
68
- while @worker_mutex.synchronize{ @workers.length } > 0
69
- begin
70
- @input.write DoneProcessing.new
71
- pid = Process.wait
72
- status = $?
73
- worker = @worker_mutex.synchronize{ @removed_workers.delete_if{|w| w.pid == pid }.first }
74
- worker.exit $?.exitstatus if worker
75
- rescue Errno::ECHILD
76
- Thread.pass until @workers.length == 0
77
- break
78
- end
118
+ @closed = true
119
+ @worker_mutex.synchronize{ @workers.length }.times do
120
+ @input.write DoneProcessing.new()
79
121
  end
80
- @reader.raise Aborted if @reader
81
122
  end
82
123
 
83
124
  def join
125
+ @waiter.join if @waiter
84
126
  @reader.join if @reader
85
127
  end
86
128
  end
@@ -2,7 +2,7 @@ require_relative '../meta_extension'
2
2
 
3
3
  module Workflow
4
4
  extend MetaExtension
5
- extension_attr :name, :tasks
5
+ extension_attr :name, :tasks, :helpers
6
6
 
7
7
  class << self
8
8
  attr_accessor :directory
@@ -17,6 +17,32 @@ module Workflow
17
17
  @name ||= self.to_s
18
18
  end
19
19
 
20
+ def helpers
21
+ @helpers ||= {}
22
+ end
23
+
24
+ def helper(name, *args, &block)
25
+ if block_given?
26
+ helpers[name] = block
27
+ else
28
+ raise RbbtException, "helper #{name} unkown in #{self} workflow" unless helpers[name]
29
+ helpers[name].call(*args)
30
+ end
31
+ end
32
+
33
+ def step_module
34
+ @_m ||= begin
35
+ m = Module.new
36
+
37
+ helpers.each do |name,block|
38
+ m.send(:define_method, name, &block)
39
+ end
40
+
41
+ m
42
+ end
43
+ @_m
44
+ end
45
+
20
46
  attr_accessor :directory
21
47
  def directory
22
48
  @directory ||= Workflow.directory[name]
@@ -60,19 +86,82 @@ module Workflow
60
86
  annotate_next_task(:inputs, args)
61
87
  end
62
88
 
89
+ def desc(description)
90
+ annotate_next_task_single(:description, description)
91
+ end
92
+
93
+ def returns(type)
94
+ annotate_next_task_single(:returns, type)
95
+ end
96
+
97
+ def extension(extension)
98
+ annotate_next_task_single(:extension, extension)
99
+ end
100
+
63
101
  def task(name_and_type, &block)
64
102
  name, type = name_and_type.collect.first
65
103
  @tasks ||= IndiferentHash.setup({})
66
104
  begin
67
105
  @annotate_next_task ||= {}
68
- task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name]))
106
+ task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name], workflow: self))
69
107
  @tasks[name] = task
70
108
  ensure
71
109
  @annotate_next_task = {}
72
110
  end
73
111
  end
74
112
 
75
- def desc(description)
76
- annotate_next_task_single(:description, description)
113
+ def task_alias(name, workflow, oname, *rest, &block)
114
+ dep(workflow, oname, *rest, &block)
115
+ extension :dep_task unless @extension
116
+ returns workflow.tasks[oname].returns if workflow.tasks.include?(oname) unless @returns
117
+ task name => nil do
118
+ raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
119
+ Step.wait_for_jobs dependencies.select{|d| d.streaming? }
120
+ dep = dependencies.last
121
+ dep.join
122
+ raise dep.get_exception if dep.error?
123
+ raise Aborted, "Aborted dependency #{dep.path}" if dep.aborted?
124
+ set_info :result_type, dep.info[:result_type]
125
+ forget = config :forget_dep_tasks, "forget_dep_tasks", :default => FORGET_DEP_TASKS
126
+ if forget
127
+ remove = config :remove_dep_tasks, "remove_dep_tasks", :default => REMOVE_DEP_TASKS
128
+
129
+ self.archive_deps
130
+ self.copy_files_dir
131
+ self.dependencies = self.dependencies - [dep]
132
+ Open.rm_rf self.files_dir if Open.exist? self.files_dir
133
+ FileUtils.cp_r dep.files_dir, self.files_dir if Open.exist?(dep.files_dir)
134
+
135
+ if dep.overriden || ! Workflow.job_path?(dep.path)
136
+ Open.link dep.path, self.tmp_path
137
+ else
138
+ Open.ln_h dep.path, self.tmp_path
139
+
140
+ case remove.to_s
141
+ when 'true'
142
+ dep.clean
143
+ when 'recursive'
144
+ (dep.dependencies + dep.rec_dependencies).uniq.each do |d|
145
+ next if d.overriden
146
+ d.clean unless config(:remove_dep, d.task_signature, d.task_name, d.workflow.to_s, :default => true).to_s == 'false'
147
+ end
148
+ dep.clean unless config(:remove_dep, dep.task_signature, dep.task_name, dep.workflow.to_s, :default => true).to_s == 'false'
149
+ end
150
+ end
151
+ else
152
+ if Open.exists?(dep.files_dir)
153
+ Open.rm_rf self.files_dir
154
+ Open.link dep.files_dir, self.files_dir
155
+ end
156
+ if defined?(RemoteStep) && RemoteStep === dep
157
+ Open.write(self.tmp_path, Open.read(dep.path))
158
+ else
159
+ Open.link dep.path, self.path
160
+ end
161
+ end
162
+ nil
163
+ end
77
164
  end
165
+
166
+ alias dep_task task_alias
78
167
  end
@@ -0,0 +1,18 @@
1
+ require_relative '../../config'
2
+
3
+ class Step
4
+ def config(key, *tokens)
5
+ options = tokens.pop if Hash === tokens.last
6
+ options ||= {}
7
+
8
+ new_tokens = []
9
+ if workflow
10
+ workflow_name = workflow.to_s
11
+ new_tokens << ("workflow:" << workflow_name)
12
+ new_tokens << ("task:" << workflow_name << "#" << task_name.to_s)
13
+ end
14
+ new_tokens << ("task:" << task_name.to_s)
15
+
16
+ Scout::Config.get(key, tokens + new_tokens, options)
17
+ end
18
+ end
@@ -0,0 +1,40 @@
1
+ class Step
2
+ def recursive_inputs
3
+ dependencies.inject(@inputs.annotate(@inputs.dup)) do |acc,dep|
4
+ acc.concat(dep.inputs) if dep.inputs
5
+ acc
6
+ end
7
+ end
8
+
9
+ def input_dependencies
10
+ return [] unless inputs
11
+ inputs.select do |d|
12
+ Step === d
13
+ end
14
+ end
15
+
16
+ def prepare_dependencies
17
+ inverse_dep = {}
18
+ dependencies.each{|dep|
19
+ next if dep.done?
20
+ if dep.dependencies
21
+ dep.dependencies.each do |d|
22
+ inverse_dep[d] ||= []
23
+ inverse_dep[d] << dep
24
+ end
25
+ end
26
+ input_dependencies.each do |d|
27
+ inverse_dep[d] ||= []
28
+ inverse_dep[d] << dep
29
+ end
30
+ }
31
+ inverse_dep.each do |dep,list|
32
+ dep.tee_copies = list.length
33
+ end
34
+ end
35
+
36
+ def run_dependencies
37
+ dependencies.each{|dep| dep.run unless dep.running? || dep.done? }
38
+ end
39
+
40
+ end
@@ -0,0 +1,15 @@
1
+ class Step
2
+ def files_dir
3
+ @files_dir ||= begin
4
+ dir = @path + ".files"
5
+ @path.annotate(dir) if Path === @path
6
+ dir
7
+ end
8
+ end
9
+
10
+ def file(file)
11
+ dir = files_dir
12
+ Path.setup(dir) unless Path === dir
13
+ dir[file]
14
+ end
15
+ end
@@ -1,5 +1,5 @@
1
1
  class Step
2
- SERIALIZER = :json
2
+ SERIALIZER = :marshal
3
3
  def info_file
4
4
  @info_file ||= begin
5
5
  info_file = @path + ".info"
@@ -20,7 +20,11 @@ class Step
20
20
  end
21
21
 
22
22
  def info
23
- outdated = @info && Open.exists?(info_file) && @info_load_time && Open.mtime(info_file) > @info_load_time
23
+ outdated = begin
24
+ @info_load_time && (mtime = Open.mtime(info_file)) && mtime > @info_load_time
25
+ rescue
26
+ true
27
+ end
24
28
 
25
29
  if @info.nil? || outdated
26
30
  load_info
@@ -33,6 +37,19 @@ class Step
33
37
  info = self.info
34
38
  new_info.each do |key,value|
35
39
  report_status new_info[:status], new_info[:message] if key == :status
40
+ if Exception === value
41
+ begin
42
+ Marshal.dump(value)
43
+ rescue TypeError
44
+ if ScoutException === value
45
+ new = ScoutException.new value.message
46
+ else
47
+ new = Exception.new value.message
48
+ end
49
+ new.set_backtrace(value.backtrace)
50
+ value = new
51
+ end
52
+ end
36
53
  if info.include?(key)
37
54
  case info[key]
38
55
  when Array
@@ -55,14 +72,13 @@ class Step
55
72
 
56
73
  def report_status(status, message = nil)
57
74
  if message.nil?
58
- Log.info Log.color(status, status.to_s) + " " + Log.color(:path, path)
75
+ Log.info Log.color(:status, status, true) + " " + Log.color(:path, path)
59
76
  else
60
- Log.info Log.color(status, status.to_s) + " " + Log.color(:path, path) + " " + message
77
+ Log.info Log.color(:status, status, true) + " " + Log.color(:path, path) + " " + message
61
78
  end
62
79
  end
63
80
 
64
81
  def log(status, message = nil)
65
- report_status status, message
66
82
  if message
67
83
  merge_info :status => status, :messages => [message]
68
84
  else
@@ -71,7 +87,18 @@ class Step
71
87
  end
72
88
 
73
89
  def status
74
- info[:status]
90
+ info[:status].tap{|s| s.nil? ? s : s.to_sym }
91
+ end
92
+
93
+ def error?
94
+ status == :error
75
95
  end
76
96
 
97
+ def aborted?
98
+ status == :aborted
99
+ end
100
+
101
+ def running?
102
+ ! done? && (info[:pid] && Misc.pid_alive?(info[:pid]))
103
+ end
77
104
  end
@@ -0,0 +1,148 @@
1
+ class Step
2
+ def self.job_path?(path)
3
+ path.split("/")[-4] == "jobs"
4
+ end
5
+
6
+ def self.status_color(status)
7
+ case status.to_sym
8
+ when :error, :aborted, :missing, :dead, :unsync
9
+ :red
10
+ when :streaming, :started
11
+ :cyan
12
+ when :done, :noinfo
13
+ :green
14
+ when :dependencies, :waiting, :setup
15
+ :yellow
16
+ when :notfound, :cleaned
17
+ :blue
18
+ else
19
+ if status.to_s.index ">"
20
+ :cyan
21
+ else
22
+ :cyan
23
+ end
24
+ end
25
+ end
26
+
27
+ def self.prov_status_msg(status)
28
+ color = status_color(status)
29
+ Log.color(color, status.to_s)
30
+ end
31
+
32
+ def self.prov_report_msg(status, name, path, info, input = nil)
33
+ parts = path.sub(/\{.*/,'').split "/"
34
+
35
+ parts.pop
36
+
37
+ task = Log.color(:yellow, parts.pop)
38
+ workflow = Log.color(:magenta, parts.pop)
39
+
40
+ if ! Step.job_path?(path)
41
+ task, status, workflow = Log.color(:yellow, info[:task_name]), Log.color(:green, "file"), Log.color(:magenta, "-")
42
+ end
43
+
44
+ path_mtime = begin
45
+ Open.mtime(path)
46
+ rescue Exception
47
+ nil
48
+ end
49
+
50
+ if input.nil? || input.empty?
51
+ input_str = nil
52
+ else
53
+ input = input.reject{|dep,name| (input & dep.dependencies.collect{|d| [d,name]}).any? }
54
+ input = input.reject{|dep,name| (input & dep.input_dependencies.collect{|d| [d,name]}).any? }
55
+ input_str = Log.color(:magenta, "-> ") + input.collect{|dep,name| Log.color(:yellow, dep.task_name.to_s) + ":" + Log.color(:yellow, name) }.uniq * " "
56
+ end
57
+
58
+ str = if ! (Open.remote?(path) || Open.ssh?(path)) && (Open.exists?(path) && $main_mtime && path_mtime && ($main_mtime - path_mtime) < -2)
59
+ prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " " << " (#{Log.color(:red, "Mtime out of sync") })"
60
+ else
61
+ prov_status_msg(status.to_s) << " " << [workflow, task, path, input_str].compact * " "
62
+ end
63
+
64
+ if $inputs and $inputs.any?
65
+ job_inputs = Workflow.load_step(path).recursive_inputs.to_hash
66
+ IndiferentHash.setup(job_inputs)
67
+
68
+ $inputs.each do |input|
69
+ value = job_inputs[input]
70
+ next if value.nil?
71
+ value_str = Misc.fingerprint(value)
72
+ str << "\t#{Log.color :magenta, input}=#{value_str}"
73
+ end
74
+ end
75
+
76
+ if $info_fields and $info_fields.any?
77
+ $info_fields.each do |field|
78
+ IndiferentHash.setup(info)
79
+ value = info[field]
80
+ next if value.nil?
81
+ value_str = Misc.fingerprint(value)
82
+ str << "\t#{Log.color :magenta, field}=#{value_str}"
83
+ end
84
+ end
85
+
86
+ str << "\n"
87
+ end
88
+
89
+ def self.prov_report(step, offset = 0, task = nil, seen = [], expand_repeats = false, input = nil)
90
+ info = step.info || {}
91
+ info[:task_name] = task
92
+ path = step.path
93
+ status = info[:status] || :missing
94
+ status = "remote" if Open.remote?(path) || Open.ssh?(path)
95
+ name = info[:name] || File.basename(path)
96
+ status = :unsync if status == :done and not Open.exist?(path)
97
+ status = :notfound if status == :noinfo and not Open.exist?(path)
98
+
99
+
100
+ this_step_msg = prov_report_msg(status, name, path, info, input)
101
+
102
+ input_dependencies = {}
103
+ step.dependencies.each do |dep|
104
+ if dep.input_dependencies.any?
105
+ dep.input_dependencies.each do |id|
106
+ input_name, _dep = dep.recursive_inputs.fields.zip(dep.recursive_inputs).select{|f,d|
107
+ d == id || (String === d && d.start_with?(id.files_dir)) || (Array === d && d.include?(id))
108
+ }.last
109
+ if input_name
110
+ input_dependencies[id] ||= []
111
+ input_dependencies[id] << [dep, input_name]
112
+ end
113
+ end
114
+ end
115
+ end if step.dependencies
116
+
117
+ str = ""
118
+ str = " " * offset + this_step_msg if ENV["RBBT_ORIGINAL_STACK"] == 'true'
119
+
120
+ step.dependencies.dup.tap{|l|
121
+ l.reverse! if ENV["RBBT_ORIGINAL_STACK"] == 'true'
122
+ }.each do |dep|
123
+ path = dep.path
124
+ new = ! seen.include?(path)
125
+ if new
126
+ seen << path
127
+ str << prov_report(dep, offset + 1, task, seen, expand_repeats, input_dependencies[dep])
128
+ else
129
+ if expand_repeats
130
+ str << Log.color(Step.status_color(dep.status), Log.uncolor(prov_report(dep, offset+1, task)))
131
+ else
132
+ info = dep.info || {}
133
+ status = info[:status] || :missing
134
+ status = "remote" if Open.remote?(path) || Open.ssh?(path)
135
+ name = info[:name] || File.basename(path)
136
+ status = :unsync if status == :done and not Open.exist?(path)
137
+ status = :notfound if status == :noinfo and not Open.exist?(path)
138
+
139
+ str << Log.color(Step.status_color(status), " " * (offset + 1) + Log.uncolor(prov_report_msg(status, name, path, info, input_dependencies[dep])))
140
+ end
141
+ end
142
+ end if step.dependencies
143
+
144
+ str += (" " * offset) + this_step_msg unless ENV["RBBT_ORIGINAL_STACK"] == 'true'
145
+
146
+ str
147
+ end
148
+ end
@@ -2,15 +2,25 @@ require_relative '../path'
2
2
  require_relative '../persist'
3
3
  require_relative 'step/info'
4
4
  require_relative 'step/load'
5
+ require_relative 'step/file'
6
+ require_relative 'step/dependencies'
7
+ require_relative 'step/provenance'
8
+ require_relative 'step/config'
5
9
 
6
10
  class Step
7
11
 
8
- attr_accessor :path, :inputs, :dependencies, :task
12
+ attr_accessor :path, :inputs, :dependencies, :task, :tee_copies
9
13
  def initialize(path, inputs = nil, dependencies = nil, &task)
10
14
  @path = path
11
15
  @inputs = inputs
12
16
  @dependencies = dependencies
13
17
  @task = task
18
+ @mutex = Mutex.new
19
+ @tee_copies = 1
20
+ end
21
+
22
+ def synchronize(&block)
23
+ @mutex.synchronize(&block)
14
24
  end
15
25
 
16
26
  def inputs
@@ -48,30 +58,49 @@ class Step
48
58
  @task_name ||= @task.name if @task.respond_to?(:name)
49
59
  end
50
60
 
61
+ def workflow
62
+ @task.workflow if @task
63
+ end
64
+
51
65
  def exec
52
- self.instance_exec(*inputs, &task)
66
+ @result = self.instance_exec(*inputs, &task)
53
67
  end
54
68
 
55
69
  attr_reader :result
56
70
  def run
57
71
  return @result || self.load if done?
58
- dependencies.each{|dep| dep.run }
59
- @result = Persist.persist(name, type, :path => path) do
72
+ prepare_dependencies
73
+ run_dependencies
74
+ @result = Persist.persist(name, type, :path => path, :tee_copies => tee_copies) do
60
75
  begin
61
76
  merge_info :status => :start, :start => Time.now,
62
77
  :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"],
63
78
  :inputs => inputs, :type => type,
64
79
  :dependencies => dependencies.collect{|d| d.path }
65
80
 
66
- @result = exec
81
+ exec
82
+ rescue Exception => e
83
+ merge_info :status => :error, :exception => e
84
+ raise e
67
85
  ensure
68
- if streaming?
69
- ConcurrentStream.setup(@result) do
86
+ if ! (error? || aborted?)
87
+ if streaming?
88
+ ConcurrentStream.setup(@result) do
89
+ merge_info :status => :done, :end => Time.now
90
+ end
91
+
92
+ @result.abort_callback = proc do |exception|
93
+ if Aborted === exception || Interrupt === exception
94
+ merge_info :status => :aborted, :end => Time.now
95
+ else
96
+ merge_info :status => :error, :exception => exception, :end => Time.now
97
+ end
98
+ end
99
+
100
+ log :streaming
101
+ else
70
102
  merge_info :status => :done, :end => Time.now
71
103
  end
72
- log :streaming
73
- else
74
- merge_info :status => :done, :end => Time.now
75
104
  end
76
105
  end
77
106
  end
@@ -82,19 +111,34 @@ class Step
82
111
  end
83
112
 
84
113
  def streaming?
85
- IO === @result || StringIO === @result
114
+ @take_stream || IO === @result || StringIO === @result
86
115
  end
87
116
 
88
- def stream
89
- join
90
- streaming? ? @result : Open.open(path)
117
+ def get_stream
118
+ synchronize do
119
+ if streaming? && ! @result.nil?
120
+ if @result.next
121
+ Log.debug "Taking result #{Log.fingerprint @result} next #{Log.fingerprint @result.next}"
122
+ else
123
+ Log.debug "Taking result #{Log.fingerprint @result}"
124
+ end
125
+ @take_stream, @result = @result, @result.next
126
+ @take_stream
127
+ elsif done?
128
+ Open.open(self.path)
129
+ else
130
+ if running?
131
+ nil
132
+ else
133
+ exec
134
+ end
135
+ end
136
+ end
91
137
  end
92
138
 
93
139
  def join
94
- if streaming?
95
- Open.consume_stream(@result, false)
96
- @result = nil
97
- end
140
+ stream = get_stream if streaming?
141
+ Open.consume_stream(stream, false) if stream
98
142
  end
99
143
 
100
144
  def produce
@@ -109,7 +153,13 @@ class Step
109
153
  end
110
154
 
111
155
  def clean
112
- FileUtils.rm path.find if path.exist?
156
+ @take_stream = nil
157
+ @result = nil
158
+ @info = nil
159
+ @info_load_time = nil
160
+ Open.rm path if Open.exist?(path)
161
+ Open.rm info_file if Open.exist?(info_file)
162
+ Open.rm_rf files_dir if Open.exist?(files_dir)
113
163
  end
114
164
 
115
165
  def recursive_clean
@@ -127,6 +177,6 @@ class Step
127
177
  end
128
178
 
129
179
  def digest_str
130
- path
180
+ path.dup
131
181
  end
132
182
  end