scout-gear 7.3.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +20 -9
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +26 -23
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/log/color.rb +4 -1
  10. data/lib/scout/log/progress/report.rb +1 -1
  11. data/lib/scout/log/progress/util.rb +58 -54
  12. data/lib/scout/log/progress.rb +1 -1
  13. data/lib/scout/log/trap.rb +107 -0
  14. data/lib/scout/log.rb +56 -21
  15. data/lib/scout/meta_extension.rb +13 -6
  16. data/lib/scout/misc/digest.rb +1 -1
  17. data/lib/scout/misc/format.rb +12 -0
  18. data/lib/scout/misc/insist.rb +1 -1
  19. data/lib/scout/misc/monitor.rb +11 -0
  20. data/lib/scout/misc/system.rb +10 -0
  21. data/lib/scout/named_array.rb +65 -3
  22. data/lib/scout/open/lock/lockfile.rb +587 -0
  23. data/lib/scout/open/lock.rb +28 -2
  24. data/lib/scout/open/remote.rb +4 -0
  25. data/lib/scout/open/stream.rb +90 -15
  26. data/lib/scout/open/util.rb +13 -3
  27. data/lib/scout/path/find.rb +9 -1
  28. data/lib/scout/path/util.rb +35 -0
  29. data/lib/scout/persist/serialize.rb +18 -5
  30. data/lib/scout/persist.rb +28 -12
  31. data/lib/scout/resource/path.rb +53 -0
  32. data/lib/scout/resource/produce.rb +0 -8
  33. data/lib/scout/resource/util.rb +2 -1
  34. data/lib/scout/tmpfile.rb +7 -8
  35. data/lib/scout/tsv/attach.rb +177 -0
  36. data/lib/scout/tsv/change_id.rb +40 -0
  37. data/lib/scout/tsv/dumper.rb +72 -46
  38. data/lib/scout/tsv/index.rb +69 -13
  39. data/lib/scout/tsv/open.rb +138 -84
  40. data/lib/scout/tsv/parser.rb +135 -80
  41. data/lib/scout/tsv/path.rb +1 -2
  42. data/lib/scout/tsv/persist/adapter.rb +15 -45
  43. data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
  44. data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
  45. data/lib/scout/tsv/persist.rb +4 -0
  46. data/lib/scout/tsv/transformer.rb +141 -0
  47. data/lib/scout/tsv/traverse.rb +96 -92
  48. data/lib/scout/tsv/util/filter.rb +9 -0
  49. data/lib/scout/tsv/util/reorder.rb +81 -0
  50. data/lib/scout/tsv/util/select.rb +78 -33
  51. data/lib/scout/tsv/util/unzip.rb +86 -0
  52. data/lib/scout/tsv/util.rb +60 -11
  53. data/lib/scout/tsv.rb +26 -3
  54. data/lib/scout/work_queue/socket.rb +6 -1
  55. data/lib/scout/work_queue/worker.rb +5 -2
  56. data/lib/scout/work_queue.rb +15 -8
  57. data/lib/scout/workflow/definition.rb +21 -2
  58. data/lib/scout/workflow/step/dependencies.rb +24 -4
  59. data/lib/scout/workflow/step/info.rb +36 -5
  60. data/lib/scout/workflow/step/provenance.rb +8 -7
  61. data/lib/scout/workflow/step/status.rb +45 -0
  62. data/lib/scout/workflow/step.rb +100 -34
  63. data/lib/scout/workflow/task/inputs.rb +14 -20
  64. data/lib/scout/workflow/task.rb +81 -46
  65. data/lib/scout/workflow/usage.rb +8 -6
  66. data/scout-gear.gemspec +24 -20
  67. data/scout_commands/workflow/task +34 -7
  68. data/test/scout/open/test_stream.rb +60 -58
  69. data/test/scout/path/test_find.rb +10 -1
  70. data/test/scout/resource/test_produce.rb +15 -0
  71. data/test/scout/test_meta_extension.rb +25 -0
  72. data/test/scout/test_named_array.rb +18 -0
  73. data/test/scout/test_persist.rb +6 -0
  74. data/test/scout/test_tsv.rb +212 -2
  75. data/test/scout/test_work_queue.rb +21 -19
  76. data/test/scout/tsv/persist/test_adapter.rb +1 -1
  77. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  78. data/test/scout/tsv/test_attach.rb +227 -0
  79. data/test/scout/tsv/test_change_id.rb +98 -0
  80. data/test/scout/tsv/test_dumper.rb +1 -1
  81. data/test/scout/tsv/test_index.rb +35 -3
  82. data/test/scout/tsv/test_open.rb +160 -2
  83. data/test/scout/tsv/test_parser.rb +19 -2
  84. data/test/scout/tsv/test_persist.rb +2 -0
  85. data/test/scout/tsv/test_transformer.rb +108 -0
  86. data/test/scout/tsv/test_traverse.rb +88 -3
  87. data/test/scout/tsv/test_util.rb +1 -0
  88. data/test/scout/tsv/util/test_reorder.rb +94 -0
  89. data/test/scout/tsv/util/test_select.rb +25 -11
  90. data/test/scout/tsv/util/test_unzip.rb +112 -0
  91. data/test/scout/work_queue/test_socket.rb +0 -1
  92. data/test/scout/workflow/step/test_status.rb +31 -0
  93. data/test/scout/workflow/task/test_inputs.rb +14 -14
  94. data/test/scout/workflow/test_step.rb +3 -3
  95. data/test/scout/workflow/test_task.rb +168 -32
  96. data/test/scout/workflow/test_usage.rb +33 -6
  97. metadata +20 -6
@@ -1,4 +1,70 @@
1
1
  module TSV
2
+ def self.select(key, values, method, fields: nil, field: nil, invert: false, type: nil, sep: nil, &block)
3
+ return ! select(key, values, method, field: field, invert: false, type: type, sep: sep, &block) if invert
4
+
5
+ return yield(key, values) if method.nil? && block_given
6
+
7
+ if Hash === method
8
+ if method.include?(:invert)
9
+ method = method.dup
10
+ invert = method.delete(:invert)
11
+ return select(key, values, method, fields: fields, field: field, invert: invert, type: type, sep: sep, &block)
12
+ end
13
+ field = method.keys.first
14
+ value = method[field]
15
+ return select(key, values, value, fields: fields, field: field, invert: invert, type: type, sep: sep, &block)
16
+ end
17
+
18
+ if field
19
+ field = fields.index(field) if fields && String === field
20
+ set = field == :key ? [key] : (type == :double ? values[field].split(sep) : values[field])
21
+ else
22
+ set = [key, (type == :double ? values.collect{|v| v.split(sep) } : values)]
23
+ end
24
+
25
+ if Array === set
26
+ set.flatten!
27
+ else
28
+ set = [set]
29
+ end
30
+
31
+ case method
32
+ when Array
33
+ (method & set).any?
34
+ when Regexp
35
+ set.select{|v| v =~ method }.any?
36
+ when Symbol
37
+ set.first.send(method)
38
+ when Numeric
39
+ set.size > method
40
+ when String
41
+ if block_given?
42
+ field = method
43
+ field = fields.index?(field) if fields && String === field
44
+ case
45
+ when block.arity == 1
46
+ if (method == key_field or method == :key)
47
+ yield(key)
48
+ else
49
+ yield(values[method])
50
+ end
51
+ when block.arity == 2
52
+ if (method == key_field or method == :key)
53
+ yield(key, key)
54
+ else
55
+ yield(key, values[method])
56
+ end
57
+ end
58
+ elsif m = method.match(/^([<>]=?)(.*)/)
59
+ set.select{|v| v.to_f.send($1, $2.to_f) }.any?
60
+ else
61
+ set.select{|v| v == method }.any?
62
+ end
63
+ when Proc
64
+ set.select{|v| method.call(v) }.any?
65
+ end
66
+ end
67
+
2
68
  def select(method = nil, invert = false, &block)
3
69
  new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
4
70
 
@@ -33,7 +99,7 @@ module TSV
33
99
  new[key] = values if invert ^ ([key,values].flatten.select{|v| v =~ method}.any?)
34
100
  end
35
101
  end
36
- when (String === method || Symbol === method)
102
+ when ((String === method) || (Symbol === method))
37
103
  if block_given?
38
104
  case
39
105
  when block.arity == 1
@@ -88,11 +154,11 @@ module TSV
88
154
  key = method.keys.first
89
155
  method = method.values.first
90
156
  case
91
- when (Array === method and (key == :key or key_field == key))
157
+ when ((Array === method) and (key == :key or key_field == key))
92
158
  with_unnamed do
93
- Annotated.purge(method).each{|key|
94
- new[key] = self[key] if invert ^ (self.include? key)
95
- }
159
+ keys.each do |key|
160
+ new[key] = self[key] if invert ^ (method.include? key)
161
+ end
96
162
  end
97
163
  when Array === method
98
164
  with_unnamed do
@@ -125,7 +191,7 @@ module TSV
125
191
  end
126
192
  end
127
193
 
128
- when (String === method and method =~ /name:(.*)/)
194
+ when ((String === method) and (method =~ /name:(.*)/))
129
195
  name = $1
130
196
  old_unnamed = self.unnamed
131
197
  self.unnamed = false
@@ -187,34 +253,13 @@ module TSV
187
253
  new
188
254
  end
189
255
 
190
- def reorder(key_field = nil, fields = nil, merge: true, one2one: :fill)
191
- res = self.annotate({})
192
- key_field_name, field_names = through key_field, fields, one2one: one2one do |k,v|
193
- if @type == :double && merge && res.include?(k)
194
- current = res[k]
195
- if merge == :concat
196
- v.each_with_index do |new,i|
197
- next if new.empty?
198
- current[i].concat(new)
199
- end
200
- else
201
- merged = []
202
- v.each_with_index do |new,i|
203
- next if new.empty?
204
- merged[i] = current[i] + new
205
- end
206
- res[k] = merged
207
- end
208
- else
209
- res[k] = v
256
+ def subset(keys)
257
+ new = self.annotate({})
258
+ self.with_unnamed do
259
+ keys.each do |k|
260
+ new[k] = self[k] if self.include?(k)
210
261
  end
211
262
  end
212
- res.key_field = key_field_name
213
- res.fields = field_names
214
- res
215
- end
216
-
217
- def slice(fields)
218
- reorder :key, fields
263
+ new
219
264
  end
220
265
  end
@@ -0,0 +1,86 @@
1
+ module TSV
2
+
3
+ def self.unzip(source, field, target: nil, sep: ":", delete: true, type: :list, merge: false, one2one: true, bar: nil)
4
+ source = TSV::Parser.new source if String === source
5
+
6
+ field_pos = source.identify_field(field)
7
+ new_fields = source.fields.dup
8
+ field_name = new_fields[field_pos]
9
+ new_fields.delete_at(field_pos) if delete
10
+ new_key_field = [source.key_field, field_name] * sep
11
+ type = :double if merge
12
+
13
+ stream = target == :stream
14
+
15
+ target = case target
16
+ when :stream
17
+ TSV::Dumper.new(source.options.merge(sep: "\t"))
18
+ when nil
19
+ TSV.setup({})
20
+ else
21
+ target
22
+ end
23
+
24
+ target.fields = new_fields
25
+ target.key_field = new_key_field
26
+ target.type = type
27
+
28
+ transformer = TSV::Transformer.new source, target, unnamed: true
29
+
30
+ bar = "Unzip #{new_key_field}" if TrueClass === bar
31
+
32
+ transformer.traverse unnamed: true, one2one: one2one, bar: bar do |k,v|
33
+ if source.type == :double
34
+ if one2one
35
+ res = NamedArray.zip_fields(v).collect do |_v|
36
+ field_value = _v[field_pos]
37
+
38
+ if delete
39
+ new_values = _v.dup
40
+ new_values.delete_at field_pos
41
+ else
42
+ new_values = _v
43
+ end
44
+
45
+ new_key = [k,field_value] * sep
46
+ new_values = new_values.collect{|e| [e] } if transformer.type == :double
47
+ [new_key, new_values]
48
+ end
49
+ else
50
+ all_values = v.collect{|e| e.dup }
51
+ all_values.delete_at field_pos if delete
52
+ res = NamedArray.zip_fields(v).collect do |_v|
53
+ field_value = _v[field_pos]
54
+
55
+ new_key = [k,field_value] * sep
56
+ new_values = all_values if transformer.type == :double
57
+ [new_key, new_values]
58
+ end
59
+ end
60
+
61
+ MultipleResult.setup(res)
62
+ else
63
+ field_value = v[field_pos]
64
+
65
+ if delete
66
+ new_values = v.dup
67
+ new_values.delete_at field_pos
68
+ else
69
+ new_values = v
70
+ end
71
+
72
+ new_key = [k,field_value] * sep
73
+
74
+ new_values = new_values.collect{|e| [e] } if transformer.type == :double
75
+
76
+ [new_key, new_values]
77
+ end
78
+ end
79
+
80
+ stream ? transformer : transformer.tsv(merge: merge)
81
+ end
82
+
83
+ def unzip(*args, **kwargs)
84
+ TSV.unzip(self, *args, **kwargs)
85
+ end
86
+ end
@@ -1,21 +1,57 @@
1
1
  #require_relative '../../../modules/rbbt-util/lib/rbbt/tsv/manipulate'
2
2
  #Log.warn "USING OLD RBBT CODE: #{__FILE__}"
3
3
  require_relative 'traverse'
4
+ require_relative 'util/filter'
4
5
  require_relative 'util/process'
5
6
  require_relative 'util/select'
7
+ require_relative 'util/reorder'
8
+ require_relative 'util/unzip'
6
9
  module TSV
7
- def [](*args)
8
- v = super(*args)
9
- NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
10
+ def self.identify_field(key_field, fields, name, strict: nil)
11
+ return :key if name == :key || (! strict && NamedArray.field_match(key_field, name))
12
+ name.collect!{|n| key_field == n ? :key : n } if Array === name
13
+ NamedArray.identify_name(fields, name, strict: strict)
14
+ end
15
+
16
+ def identify_field(name, strict: nil)
17
+ TSV.identify_field(@key_field, @fields, name, strict: strict)
18
+ end
19
+
20
+ def [](key, *rest)
21
+ v = super(key, *rest)
22
+ NamedArray.setup(v, @fields, key) unless @unnamed || ! (Array === v)
10
23
  v
11
24
  end
12
25
 
26
+ def options
27
+ extension_attr_hash
28
+ end
29
+
30
+ def zip_new(key, values, insitu: :lax)
31
+ values = values.collect{|v| Array === v ? v : [v] } unless Array === values.first
32
+ if current_values = self[key]
33
+ if insitu == :lax
34
+ self[key] = NamedArray.add_zipped(current_values, values)
35
+ elsif insitu
36
+ NamedArray.add_zipped(current_values, values)
37
+ else
38
+ self[key] = NamedArray.add_zipped(current_values.dup, values)
39
+ end
40
+ else
41
+ if insitu && insitu != :lax
42
+ self[key] = values.dup
43
+ else
44
+ self[key] = values
45
+ end
46
+ end
47
+ end
48
+
13
49
  def each(*args, &block)
14
50
  if block_given?
15
51
  super(*args) do |k,v|
16
- NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
17
- block.call(k, v)
18
- end
52
+ NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
53
+ block.call(k, v)
54
+ end
19
55
  else
20
56
  super(*args)
21
57
  end
@@ -33,13 +69,13 @@ module TSV
33
69
  end
34
70
  end
35
71
 
36
- def with_unnamed
72
+ def with_unnamed(unnamed = true)
37
73
  begin
38
- old_unnamed = unnamed
39
- unnamed = true
74
+ old_unnamed = @unnamed
75
+ @unnamed = unnamed
40
76
  yield
41
77
  ensure
42
- unnamed = old_unnamed
78
+ @unnamed = old_unnamed
43
79
  end
44
80
  end
45
81
 
@@ -53,7 +89,7 @@ module TSV
53
89
  end
54
90
 
55
91
  filename = @filename
56
- filename = "No filename" if filename.nil? || filename.empty?
92
+ filename = "No filename" if filename.nil? || String === filename && filename.empty?
57
93
  filename.find if Path === filename
58
94
  filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
59
95
 
@@ -73,10 +109,23 @@ Example:
73
109
  end
74
110
 
75
111
  def all_fields
112
+ return [] if @fields.nil?
76
113
  [@key_field] + @fields
77
114
  end
78
115
 
116
+ def options
117
+ self.extension_attr_hash
118
+ end
119
+
79
120
  def fingerprint
80
121
  "TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
81
122
  end
123
+
124
+ def digest_str
125
+ fingerprint
126
+ end
127
+
128
+ def inspect
129
+ fingerprint
130
+ end
82
131
  end
data/lib/scout/tsv.rb CHANGED
@@ -2,23 +2,46 @@ require_relative 'meta_extension'
2
2
  require_relative 'tsv/util'
3
3
  require_relative 'tsv/parser'
4
4
  require_relative 'tsv/dumper'
5
+ require_relative 'tsv/transformer'
5
6
  require_relative 'tsv/persist'
6
7
  require_relative 'tsv/index'
7
8
  require_relative 'tsv/path'
8
9
  require_relative 'tsv/traverse'
9
10
  require_relative 'tsv/open'
11
+ require_relative 'tsv/attach'
12
+ require_relative 'tsv/change_id'
10
13
 
11
14
  module TSV
12
15
  extend MetaExtension
13
16
  extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed, :identifiers
14
17
 
18
+ def self.str2options(str)
19
+ field_options,_sep, rest = str.partition("#")
20
+ key, fields_str = field_options.split("~")
21
+
22
+ fields = fields_str.nil? ? [] : fields_str.split(/,\s*/)
23
+
24
+ rest = ":type=" << rest if rest =~ /^:?\w+$/
25
+ rest_options = rest.nil? ? {} : IndiferentHash.string2hash(rest)
26
+
27
+ {:key_field => key, :fields => fields}.merge(rest_options)
28
+ end
29
+
30
+ def self.str_setup(option_str, obj)
31
+ options = TSV.str2options(option_str)
32
+ setup(obj, options)
33
+ end
34
+
15
35
  def self.open(file, options = {})
16
- persist, type = IndiferentHash.process_options options, :persist, :persist_type, :persist => false, :persist_type => "HDB"
17
- Persist.persist(file, type, options.merge(:persist => persist)) do |filename|
36
+ persist, type, grep, invert_grep = IndiferentHash.process_options options, :persist, :persist_type, :grep, :invert_grep, :persist => false, :persist_type => "HDB"
37
+ type = type.to_sym if type
38
+ file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
39
+ Persist.persist(file, type, options.merge(:persist => persist, :prefix => "Tsv", :other_options => options)) do |filename|
18
40
  data = filename ? ScoutCabinet.open(filename, true, type) : nil
19
41
  options[:data] = data if data
20
42
  options[:filename] = file
21
- Open.open(file) do |f|
43
+ Log.debug "TSV open #{Log.fingerprint file}"
44
+ Open.open(file, grep: grep, invert_grep: invert_grep) do |f|
22
45
  TSV.parse(f, **options)
23
46
  end
24
47
  end
@@ -3,7 +3,7 @@ require 'scout/semaphore'
3
3
  require 'scout/exceptions'
4
4
  class WorkQueue
5
5
  class Socket
6
- attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned
6
+ attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned, :exception
7
7
  def initialize(serializer = nil)
8
8
  @sread, @swrite = Open.pipe
9
9
 
@@ -112,6 +112,11 @@ class WorkQueue
112
112
  end
113
113
  end
114
114
 
115
+ def abort(exception)
116
+ @exception = exception
117
+ @swrite.close unless closed_write?
118
+ end
119
+
115
120
  alias write push
116
121
 
117
122
  alias read pop
@@ -27,8 +27,10 @@ class WorkQueue
27
27
  rescue Interrupt
28
28
  rescue Exception
29
29
  output.write WorkerException.new($!, Process.pid)
30
- Kernel.exit -1
30
+ Process.exit! -1
31
+ ensure
31
32
  end
33
+ Process.exit! 0
32
34
  end
33
35
  end
34
36
 
@@ -36,7 +38,8 @@ class WorkQueue
36
38
  begin
37
39
  Log.debug "Aborting worker #{@pid}"
38
40
  Process.kill "INT", @pid
39
- rescue Errno::ECHILD
41
+ rescue Errno::ECHILD
42
+ rescue Errno::ESRCH
40
43
  end
41
44
  end
42
45
 
@@ -46,7 +46,11 @@ class WorkQueue
46
46
  end
47
47
 
48
48
  def process(&callback)
49
- @reader = Thread.new do |parent|
49
+ @workers.each do |w|
50
+ w.process @input, @output, &@worker_proc
51
+ end
52
+
53
+ @reader = Thread.new(Thread.current) do |parent|
50
54
  begin
51
55
  Thread.current.report_on_exception = false
52
56
  Thread.current["name"] = "Output reader #{Process.pid}"
@@ -71,8 +75,9 @@ class WorkQueue
71
75
  rescue DoneProcessing
72
76
  rescue Aborted
73
77
  rescue WorkerException
74
- Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.message}"
78
+ Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.worker_exception.message}"
75
79
  self.abort
80
+ @input.abort obj.worker_exception
76
81
  raise obj.worker_exception
77
82
  rescue
78
83
  Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
@@ -81,10 +86,6 @@ class WorkQueue
81
86
  end
82
87
  end
83
88
 
84
- @workers.each do |w|
85
- w.process @input, @output, &@worker_proc
86
- end
87
-
88
89
  Thread.pass until @reader["name"]
89
90
 
90
91
  @waiter = Thread.new do
@@ -104,7 +105,13 @@ class WorkQueue
104
105
  end
105
106
 
106
107
  def write(obj)
107
- @input.write obj
108
+ begin
109
+ @input.write obj
110
+ rescue Exception
111
+ raise $! unless @input.exception
112
+ ensure
113
+ raise @input.exception if @input.exception
114
+ end
108
115
  end
109
116
 
110
117
  def abort
@@ -117,7 +124,7 @@ class WorkQueue
117
124
  def close
118
125
  @closed = true
119
126
  @worker_mutex.synchronize{ @workers.length }.times do
120
- @input.write DoneProcessing.new()
127
+ @input.write DoneProcessing.new() unless @input.closed_write?
121
128
  end
122
129
  end
123
130
 
@@ -76,9 +76,11 @@ module Workflow
76
76
  end
77
77
  when 1
78
78
  task = args.first
79
+ options, task = task, nil if Hash === task
79
80
  end
80
81
  workflow = self if workflow.nil?
81
82
  options = {} if options.nil?
83
+ task = task.to_sym if task
82
84
  annotate_next_task :deps, [workflow, task, options, block, args]
83
85
  end
84
86
 
@@ -103,6 +105,20 @@ module Workflow
103
105
  @tasks ||= IndiferentHash.setup({})
104
106
  begin
105
107
  @annotate_next_task ||= {}
108
+ @annotate_next_task[:extension] ||=
109
+ case type
110
+ when :tsv
111
+ "tsv"
112
+ when :yaml
113
+ "yaml"
114
+ when :marshal
115
+ "marshal"
116
+ when :json
117
+ "json"
118
+ else
119
+ nil
120
+ end
121
+
106
122
  task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name], workflow: self))
107
123
  @tasks[name] = task
108
124
  ensure
@@ -110,11 +126,14 @@ module Workflow
110
126
  end
111
127
  end
112
128
 
129
+ FORGET_DEP_TASKS = ENV["SCOUT_FORGET_DEP_TASKS"] == "true"
130
+ REMOVE_DEP_TASKS = ENV["SCOUT_REMOVE_DEP_TASKS"] == "true"
113
131
  def task_alias(name, workflow, oname, *rest, &block)
114
132
  dep(workflow, oname, *rest, &block)
115
133
  extension :dep_task unless @extension
116
- returns workflow.tasks[oname].returns if workflow.tasks.include?(oname) unless @returns
117
- task name => nil do
134
+ returns workflow.tasks[oname].returns if @returns.nil?
135
+ type = workflow.tasks[oname].type
136
+ task name => type do
118
137
  raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
119
138
  Step.wait_for_jobs dependencies.select{|d| d.streaming? }
120
139
  dep = dependencies.last
@@ -1,8 +1,13 @@
1
1
  class Step
2
+ def rec_dependencies
3
+ rec_dependencies = dependencies.dup
4
+ dependencies.inject(rec_dependencies){|acc,d| acc.concat d.rec_dependencies }
5
+ end
6
+
2
7
  def recursive_inputs
3
- dependencies.inject(@inputs.annotate(@inputs.dup)) do |acc,dep|
4
- acc.concat(dep.inputs) if dep.inputs
5
- acc
8
+ recursive_inputs = @inputs.to_hash
9
+ dependencies.inject(recursive_inputs) do |acc,dep|
10
+ acc.merge(dep.recursive_inputs)
6
11
  end
7
12
  end
8
13
 
@@ -16,6 +21,10 @@ class Step
16
21
  def prepare_dependencies
17
22
  inverse_dep = {}
18
23
  dependencies.each{|dep|
24
+ if dep.present? && ! dep.updated?
25
+ Log.debug "Clean outdated #{dep.path}"
26
+ dep.clean
27
+ end
19
28
  next if dep.done?
20
29
  if dep.dependencies
21
30
  dep.dependencies.each do |d|
@@ -34,7 +43,18 @@ class Step
34
43
  end
35
44
 
36
45
  def run_dependencies
37
- dependencies.each{|dep| dep.run unless dep.running? || dep.done? }
46
+ dependencies.each{|dep| dep.run(true) unless dep.running? || dep.done? }
38
47
  end
39
48
 
49
+ def abort_dependencies
50
+ dependencies.each{|dep| dep.abort if dep.running? }
51
+ end
52
+
53
+ def self.wait_for_jobs(jobs)
54
+ threads = []
55
+ jobs.each do |job|
56
+ threads << job.join
57
+ end
58
+ threads.each do |t| t.join end
59
+ end
40
60
  end
@@ -1,6 +1,7 @@
1
1
  class Step
2
2
  SERIALIZER = :marshal
3
3
  def info_file
4
+ return nil if @path.nil?
4
5
  @info_file ||= begin
5
6
  info_file = @path + ".info"
6
7
  @path.annotate info_file if Path === @path
@@ -19,6 +20,10 @@ class Step
19
20
  @info_load_time = Time.now
20
21
  end
21
22
 
23
+ def clear_info
24
+ save_info(@info = {})
25
+ end
26
+
22
27
  def info
23
28
  outdated = begin
24
29
  @info_load_time && (mtime = Open.mtime(info_file)) && mtime > @info_load_time
@@ -36,7 +41,19 @@ class Step
36
41
  def merge_info(new_info)
37
42
  info = self.info
38
43
  new_info.each do |key,value|
39
- report_status new_info[:status], new_info[:message] if key == :status
44
+ if key == :status
45
+ message = new_info[:messages]
46
+ if message.nil? && value == :done || value == :error || value == :aborted
47
+ start = info[:start]
48
+ eend = new_info[:end]
49
+ if start && eend
50
+ time = eend - start
51
+ time_str = Misc.format_seconds_short(time)
52
+ message = Log.color(:time, time_str)
53
+ end
54
+ end
55
+ report_status value, message
56
+ end
40
57
  if Exception === value
41
58
  begin
42
59
  Marshal.dump(value)
@@ -72,15 +89,21 @@ class Step
72
89
 
73
90
  def report_status(status, message = nil)
74
91
  if message.nil?
75
- Log.info Log.color(:status, status, true) + " " + Log.color(:path, path)
92
+ Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
76
93
  else
77
- Log.info Log.color(:status, status, true) + " " + Log.color(:path, path) + " " + message
94
+ Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
78
95
  end
79
96
  end
80
97
 
81
- def log(status, message = nil)
98
+ def log(status, message = nil, &block)
99
+ if block_given?
100
+ time = Misc.exec_time &block
101
+ time_str = Misc.format_seconds_short time
102
+ message = message.nil? ? Log.color(:time, time_str) : "#{Log.color :time, time_str} - #{ message }"
103
+ end
104
+
82
105
  if message
83
- merge_info :status => status, :messages => [message]
106
+ merge_info :status => status, :messages => message
84
107
  else
85
108
  merge_info :status => status
86
109
  end
@@ -105,4 +128,12 @@ class Step
105
128
  def exception
106
129
  info[:exception]
107
130
  end
131
+
132
+ def marshal_dump
133
+ @path
134
+ end
135
+
136
+ def marshal_load(path)
137
+ Step.new path
138
+ end
108
139
  end