scout-gear 7.3.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +20 -9
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +26 -23
- data/lib/scout/config.rb +1 -1
- data/lib/scout/log/color.rb +4 -1
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +58 -54
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +11 -0
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +90 -15
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +28 -12
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +72 -46
- data/lib/scout/tsv/index.rb +69 -13
- data/lib/scout/tsv/open.rb +138 -84
- data/lib/scout/tsv/parser.rb +135 -80
- data/lib/scout/tsv/path.rb +1 -2
- data/lib/scout/tsv/persist/adapter.rb +15 -45
- data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
- data/lib/scout/tsv/persist.rb +4 -0
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +96 -92
- data/lib/scout/tsv/util/filter.rb +9 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +78 -33
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +60 -11
- data/lib/scout/tsv.rb +26 -3
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +21 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +36 -5
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +100 -34
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +81 -46
- data/lib/scout/workflow/usage.rb +8 -6
- data/scout-gear.gemspec +24 -20
- data/scout_commands/workflow/task +34 -7
- data/test/scout/open/test_stream.rb +60 -58
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +6 -0
- data/test/scout/test_tsv.rb +212 -2
- data/test/scout/test_work_queue.rb +21 -19
- data/test/scout/tsv/persist/test_adapter.rb +1 -1
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +35 -3
- data/test/scout/tsv/test_open.rb +160 -2
- data/test/scout/tsv/test_parser.rb +19 -2
- data/test/scout/tsv/test_persist.rb +2 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +88 -3
- data/test/scout/tsv/test_util.rb +1 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +25 -11
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +3 -3
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- metadata +20 -6
@@ -1,4 +1,70 @@
|
|
1
1
|
module TSV
|
2
|
+
def self.select(key, values, method, fields: nil, field: nil, invert: false, type: nil, sep: nil, &block)
|
3
|
+
return ! select(key, values, method, field: field, invert: false, type: type, sep: sep, &block) if invert
|
4
|
+
|
5
|
+
return yield(key, values) if method.nil? && block_given
|
6
|
+
|
7
|
+
if Hash === method
|
8
|
+
if method.include?(:invert)
|
9
|
+
method = method.dup
|
10
|
+
invert = method.delete(:invert)
|
11
|
+
return select(key, values, method, fields: fields, field: field, invert: invert, type: type, sep: sep, &block)
|
12
|
+
end
|
13
|
+
field = method.keys.first
|
14
|
+
value = method[field]
|
15
|
+
return select(key, values, value, fields: fields, field: field, invert: invert, type: type, sep: sep, &block)
|
16
|
+
end
|
17
|
+
|
18
|
+
if field
|
19
|
+
field = fields.index(field) if fields && String === field
|
20
|
+
set = field == :key ? [key] : (type == :double ? values[field].split(sep) : values[field])
|
21
|
+
else
|
22
|
+
set = [key, (type == :double ? values.collect{|v| v.split(sep) } : values)]
|
23
|
+
end
|
24
|
+
|
25
|
+
if Array === set
|
26
|
+
set.flatten!
|
27
|
+
else
|
28
|
+
set = [set]
|
29
|
+
end
|
30
|
+
|
31
|
+
case method
|
32
|
+
when Array
|
33
|
+
(method & set).any?
|
34
|
+
when Regexp
|
35
|
+
set.select{|v| v =~ method }.any?
|
36
|
+
when Symbol
|
37
|
+
set.first.send(method)
|
38
|
+
when Numeric
|
39
|
+
set.size > method
|
40
|
+
when String
|
41
|
+
if block_given?
|
42
|
+
field = method
|
43
|
+
field = fields.index?(field) if fields && String === field
|
44
|
+
case
|
45
|
+
when block.arity == 1
|
46
|
+
if (method == key_field or method == :key)
|
47
|
+
yield(key)
|
48
|
+
else
|
49
|
+
yield(values[method])
|
50
|
+
end
|
51
|
+
when block.arity == 2
|
52
|
+
if (method == key_field or method == :key)
|
53
|
+
yield(key, key)
|
54
|
+
else
|
55
|
+
yield(key, values[method])
|
56
|
+
end
|
57
|
+
end
|
58
|
+
elsif m = method.match(/^([<>]=?)(.*)/)
|
59
|
+
set.select{|v| v.to_f.send($1, $2.to_f) }.any?
|
60
|
+
else
|
61
|
+
set.select{|v| v == method }.any?
|
62
|
+
end
|
63
|
+
when Proc
|
64
|
+
set.select{|v| method.call(v) }.any?
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
2
68
|
def select(method = nil, invert = false, &block)
|
3
69
|
new = TSV.setup({}, :key_field => key_field, :fields => fields, :type => type, :filename => filename, :identifiers => identifiers)
|
4
70
|
|
@@ -33,7 +99,7 @@ module TSV
|
|
33
99
|
new[key] = values if invert ^ ([key,values].flatten.select{|v| v =~ method}.any?)
|
34
100
|
end
|
35
101
|
end
|
36
|
-
when (String === method || Symbol === method)
|
102
|
+
when ((String === method) || (Symbol === method))
|
37
103
|
if block_given?
|
38
104
|
case
|
39
105
|
when block.arity == 1
|
@@ -88,11 +154,11 @@ module TSV
|
|
88
154
|
key = method.keys.first
|
89
155
|
method = method.values.first
|
90
156
|
case
|
91
|
-
when (Array === method and (key == :key or key_field == key))
|
157
|
+
when ((Array === method) and (key == :key or key_field == key))
|
92
158
|
with_unnamed do
|
93
|
-
|
94
|
-
new[key] = self[key] if invert ^ (
|
95
|
-
|
159
|
+
keys.each do |key|
|
160
|
+
new[key] = self[key] if invert ^ (method.include? key)
|
161
|
+
end
|
96
162
|
end
|
97
163
|
when Array === method
|
98
164
|
with_unnamed do
|
@@ -125,7 +191,7 @@ module TSV
|
|
125
191
|
end
|
126
192
|
end
|
127
193
|
|
128
|
-
when (String === method and method =~ /name:(.*)/)
|
194
|
+
when ((String === method) and (method =~ /name:(.*)/))
|
129
195
|
name = $1
|
130
196
|
old_unnamed = self.unnamed
|
131
197
|
self.unnamed = false
|
@@ -187,34 +253,13 @@ module TSV
|
|
187
253
|
new
|
188
254
|
end
|
189
255
|
|
190
|
-
def
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
195
|
-
if merge == :concat
|
196
|
-
v.each_with_index do |new,i|
|
197
|
-
next if new.empty?
|
198
|
-
current[i].concat(new)
|
199
|
-
end
|
200
|
-
else
|
201
|
-
merged = []
|
202
|
-
v.each_with_index do |new,i|
|
203
|
-
next if new.empty?
|
204
|
-
merged[i] = current[i] + new
|
205
|
-
end
|
206
|
-
res[k] = merged
|
207
|
-
end
|
208
|
-
else
|
209
|
-
res[k] = v
|
256
|
+
def subset(keys)
|
257
|
+
new = self.annotate({})
|
258
|
+
self.with_unnamed do
|
259
|
+
keys.each do |k|
|
260
|
+
new[k] = self[k] if self.include?(k)
|
210
261
|
end
|
211
262
|
end
|
212
|
-
|
213
|
-
res.fields = field_names
|
214
|
-
res
|
215
|
-
end
|
216
|
-
|
217
|
-
def slice(fields)
|
218
|
-
reorder :key, fields
|
263
|
+
new
|
219
264
|
end
|
220
265
|
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
module TSV
|
2
|
+
|
3
|
+
def self.unzip(source, field, target: nil, sep: ":", delete: true, type: :list, merge: false, one2one: true, bar: nil)
|
4
|
+
source = TSV::Parser.new source if String === source
|
5
|
+
|
6
|
+
field_pos = source.identify_field(field)
|
7
|
+
new_fields = source.fields.dup
|
8
|
+
field_name = new_fields[field_pos]
|
9
|
+
new_fields.delete_at(field_pos) if delete
|
10
|
+
new_key_field = [source.key_field, field_name] * sep
|
11
|
+
type = :double if merge
|
12
|
+
|
13
|
+
stream = target == :stream
|
14
|
+
|
15
|
+
target = case target
|
16
|
+
when :stream
|
17
|
+
TSV::Dumper.new(source.options.merge(sep: "\t"))
|
18
|
+
when nil
|
19
|
+
TSV.setup({})
|
20
|
+
else
|
21
|
+
target
|
22
|
+
end
|
23
|
+
|
24
|
+
target.fields = new_fields
|
25
|
+
target.key_field = new_key_field
|
26
|
+
target.type = type
|
27
|
+
|
28
|
+
transformer = TSV::Transformer.new source, target, unnamed: true
|
29
|
+
|
30
|
+
bar = "Unzip #{new_key_field}" if TrueClass === bar
|
31
|
+
|
32
|
+
transformer.traverse unnamed: true, one2one: one2one, bar: bar do |k,v|
|
33
|
+
if source.type == :double
|
34
|
+
if one2one
|
35
|
+
res = NamedArray.zip_fields(v).collect do |_v|
|
36
|
+
field_value = _v[field_pos]
|
37
|
+
|
38
|
+
if delete
|
39
|
+
new_values = _v.dup
|
40
|
+
new_values.delete_at field_pos
|
41
|
+
else
|
42
|
+
new_values = _v
|
43
|
+
end
|
44
|
+
|
45
|
+
new_key = [k,field_value] * sep
|
46
|
+
new_values = new_values.collect{|e| [e] } if transformer.type == :double
|
47
|
+
[new_key, new_values]
|
48
|
+
end
|
49
|
+
else
|
50
|
+
all_values = v.collect{|e| e.dup }
|
51
|
+
all_values.delete_at field_pos if delete
|
52
|
+
res = NamedArray.zip_fields(v).collect do |_v|
|
53
|
+
field_value = _v[field_pos]
|
54
|
+
|
55
|
+
new_key = [k,field_value] * sep
|
56
|
+
new_values = all_values if transformer.type == :double
|
57
|
+
[new_key, new_values]
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
MultipleResult.setup(res)
|
62
|
+
else
|
63
|
+
field_value = v[field_pos]
|
64
|
+
|
65
|
+
if delete
|
66
|
+
new_values = v.dup
|
67
|
+
new_values.delete_at field_pos
|
68
|
+
else
|
69
|
+
new_values = v
|
70
|
+
end
|
71
|
+
|
72
|
+
new_key = [k,field_value] * sep
|
73
|
+
|
74
|
+
new_values = new_values.collect{|e| [e] } if transformer.type == :double
|
75
|
+
|
76
|
+
[new_key, new_values]
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
stream ? transformer : transformer.tsv(merge: merge)
|
81
|
+
end
|
82
|
+
|
83
|
+
def unzip(*args, **kwargs)
|
84
|
+
TSV.unzip(self, *args, **kwargs)
|
85
|
+
end
|
86
|
+
end
|
data/lib/scout/tsv/util.rb
CHANGED
@@ -1,21 +1,57 @@
|
|
1
1
|
#require_relative '../../../modules/rbbt-util/lib/rbbt/tsv/manipulate'
|
2
2
|
#Log.warn "USING OLD RBBT CODE: #{__FILE__}"
|
3
3
|
require_relative 'traverse'
|
4
|
+
require_relative 'util/filter'
|
4
5
|
require_relative 'util/process'
|
5
6
|
require_relative 'util/select'
|
7
|
+
require_relative 'util/reorder'
|
8
|
+
require_relative 'util/unzip'
|
6
9
|
module TSV
|
7
|
-
def
|
8
|
-
|
9
|
-
|
10
|
+
def self.identify_field(key_field, fields, name, strict: nil)
|
11
|
+
return :key if name == :key || (! strict && NamedArray.field_match(key_field, name))
|
12
|
+
name.collect!{|n| key_field == n ? :key : n } if Array === name
|
13
|
+
NamedArray.identify_name(fields, name, strict: strict)
|
14
|
+
end
|
15
|
+
|
16
|
+
def identify_field(name, strict: nil)
|
17
|
+
TSV.identify_field(@key_field, @fields, name, strict: strict)
|
18
|
+
end
|
19
|
+
|
20
|
+
def [](key, *rest)
|
21
|
+
v = super(key, *rest)
|
22
|
+
NamedArray.setup(v, @fields, key) unless @unnamed || ! (Array === v)
|
10
23
|
v
|
11
24
|
end
|
12
25
|
|
26
|
+
def options
|
27
|
+
extension_attr_hash
|
28
|
+
end
|
29
|
+
|
30
|
+
def zip_new(key, values, insitu: :lax)
|
31
|
+
values = values.collect{|v| Array === v ? v : [v] } unless Array === values.first
|
32
|
+
if current_values = self[key]
|
33
|
+
if insitu == :lax
|
34
|
+
self[key] = NamedArray.add_zipped(current_values, values)
|
35
|
+
elsif insitu
|
36
|
+
NamedArray.add_zipped(current_values, values)
|
37
|
+
else
|
38
|
+
self[key] = NamedArray.add_zipped(current_values.dup, values)
|
39
|
+
end
|
40
|
+
else
|
41
|
+
if insitu && insitu != :lax
|
42
|
+
self[key] = values.dup
|
43
|
+
else
|
44
|
+
self[key] = values
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
13
49
|
def each(*args, &block)
|
14
50
|
if block_given?
|
15
51
|
super(*args) do |k,v|
|
16
|
-
|
17
|
-
|
18
|
-
|
52
|
+
NamedArray.setup(v, @fields) unless @unnamed || ! (Array === v)
|
53
|
+
block.call(k, v)
|
54
|
+
end
|
19
55
|
else
|
20
56
|
super(*args)
|
21
57
|
end
|
@@ -33,13 +69,13 @@ module TSV
|
|
33
69
|
end
|
34
70
|
end
|
35
71
|
|
36
|
-
def with_unnamed
|
72
|
+
def with_unnamed(unnamed = true)
|
37
73
|
begin
|
38
|
-
old_unnamed = unnamed
|
39
|
-
unnamed =
|
74
|
+
old_unnamed = @unnamed
|
75
|
+
@unnamed = unnamed
|
40
76
|
yield
|
41
77
|
ensure
|
42
|
-
unnamed = old_unnamed
|
78
|
+
@unnamed = old_unnamed
|
43
79
|
end
|
44
80
|
end
|
45
81
|
|
@@ -53,7 +89,7 @@ module TSV
|
|
53
89
|
end
|
54
90
|
|
55
91
|
filename = @filename
|
56
|
-
filename = "No filename" if filename.nil? || filename.empty?
|
92
|
+
filename = "No filename" if filename.nil? || String === filename && filename.empty?
|
57
93
|
filename.find if Path === filename
|
58
94
|
filename = File.basename(filename) + " [" + File.basename(persistence_path) + "]" if respond_to?(:persistence_path) and persistence_path
|
59
95
|
|
@@ -73,10 +109,23 @@ Example:
|
|
73
109
|
end
|
74
110
|
|
75
111
|
def all_fields
|
112
|
+
return [] if @fields.nil?
|
76
113
|
[@key_field] + @fields
|
77
114
|
end
|
78
115
|
|
116
|
+
def options
|
117
|
+
self.extension_attr_hash
|
118
|
+
end
|
119
|
+
|
79
120
|
def fingerprint
|
80
121
|
"TSV:{"<< Log.fingerprint(self.all_fields|| []) << ";" << Log.fingerprint(self.keys) << "}"
|
81
122
|
end
|
123
|
+
|
124
|
+
def digest_str
|
125
|
+
fingerprint
|
126
|
+
end
|
127
|
+
|
128
|
+
def inspect
|
129
|
+
fingerprint
|
130
|
+
end
|
82
131
|
end
|
data/lib/scout/tsv.rb
CHANGED
@@ -2,23 +2,46 @@ require_relative 'meta_extension'
|
|
2
2
|
require_relative 'tsv/util'
|
3
3
|
require_relative 'tsv/parser'
|
4
4
|
require_relative 'tsv/dumper'
|
5
|
+
require_relative 'tsv/transformer'
|
5
6
|
require_relative 'tsv/persist'
|
6
7
|
require_relative 'tsv/index'
|
7
8
|
require_relative 'tsv/path'
|
8
9
|
require_relative 'tsv/traverse'
|
9
10
|
require_relative 'tsv/open'
|
11
|
+
require_relative 'tsv/attach'
|
12
|
+
require_relative 'tsv/change_id'
|
10
13
|
|
11
14
|
module TSV
|
12
15
|
extend MetaExtension
|
13
16
|
extension_attr :key_field, :fields, :type, :filename, :namespace, :unnamed, :identifiers
|
14
17
|
|
18
|
+
def self.str2options(str)
|
19
|
+
field_options,_sep, rest = str.partition("#")
|
20
|
+
key, fields_str = field_options.split("~")
|
21
|
+
|
22
|
+
fields = fields_str.nil? ? [] : fields_str.split(/,\s*/)
|
23
|
+
|
24
|
+
rest = ":type=" << rest if rest =~ /^:?\w+$/
|
25
|
+
rest_options = rest.nil? ? {} : IndiferentHash.string2hash(rest)
|
26
|
+
|
27
|
+
{:key_field => key, :fields => fields}.merge(rest_options)
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.str_setup(option_str, obj)
|
31
|
+
options = TSV.str2options(option_str)
|
32
|
+
setup(obj, options)
|
33
|
+
end
|
34
|
+
|
15
35
|
def self.open(file, options = {})
|
16
|
-
persist, type = IndiferentHash.process_options options, :persist, :persist_type, :persist => false, :persist_type => "HDB"
|
17
|
-
|
36
|
+
persist, type, grep, invert_grep = IndiferentHash.process_options options, :persist, :persist_type, :grep, :invert_grep, :persist => false, :persist_type => "HDB"
|
37
|
+
type = type.to_sym if type
|
38
|
+
file = StringIO.new file if String === file && ! (Path === file) && file.index("\n")
|
39
|
+
Persist.persist(file, type, options.merge(:persist => persist, :prefix => "Tsv", :other_options => options)) do |filename|
|
18
40
|
data = filename ? ScoutCabinet.open(filename, true, type) : nil
|
19
41
|
options[:data] = data if data
|
20
42
|
options[:filename] = file
|
21
|
-
|
43
|
+
Log.debug "TSV open #{Log.fingerprint file}"
|
44
|
+
Open.open(file, grep: grep, invert_grep: invert_grep) do |f|
|
22
45
|
TSV.parse(f, **options)
|
23
46
|
end
|
24
47
|
end
|
@@ -3,7 +3,7 @@ require 'scout/semaphore'
|
|
3
3
|
require 'scout/exceptions'
|
4
4
|
class WorkQueue
|
5
5
|
class Socket
|
6
|
-
attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned
|
6
|
+
attr_accessor :sread, :swrite, :write_sem, :read_sem, :cleaned, :exception
|
7
7
|
def initialize(serializer = nil)
|
8
8
|
@sread, @swrite = Open.pipe
|
9
9
|
|
@@ -112,6 +112,11 @@ class WorkQueue
|
|
112
112
|
end
|
113
113
|
end
|
114
114
|
|
115
|
+
def abort(exception)
|
116
|
+
@exception = exception
|
117
|
+
@swrite.close unless closed_write?
|
118
|
+
end
|
119
|
+
|
115
120
|
alias write push
|
116
121
|
|
117
122
|
alias read pop
|
@@ -27,8 +27,10 @@ class WorkQueue
|
|
27
27
|
rescue Interrupt
|
28
28
|
rescue Exception
|
29
29
|
output.write WorkerException.new($!, Process.pid)
|
30
|
-
|
30
|
+
Process.exit! -1
|
31
|
+
ensure
|
31
32
|
end
|
33
|
+
Process.exit! 0
|
32
34
|
end
|
33
35
|
end
|
34
36
|
|
@@ -36,7 +38,8 @@ class WorkQueue
|
|
36
38
|
begin
|
37
39
|
Log.debug "Aborting worker #{@pid}"
|
38
40
|
Process.kill "INT", @pid
|
39
|
-
rescue Errno::ECHILD
|
41
|
+
rescue Errno::ECHILD
|
42
|
+
rescue Errno::ESRCH
|
40
43
|
end
|
41
44
|
end
|
42
45
|
|
data/lib/scout/work_queue.rb
CHANGED
@@ -46,7 +46,11 @@ class WorkQueue
|
|
46
46
|
end
|
47
47
|
|
48
48
|
def process(&callback)
|
49
|
-
@
|
49
|
+
@workers.each do |w|
|
50
|
+
w.process @input, @output, &@worker_proc
|
51
|
+
end
|
52
|
+
|
53
|
+
@reader = Thread.new(Thread.current) do |parent|
|
50
54
|
begin
|
51
55
|
Thread.current.report_on_exception = false
|
52
56
|
Thread.current["name"] = "Output reader #{Process.pid}"
|
@@ -71,8 +75,9 @@ class WorkQueue
|
|
71
75
|
rescue DoneProcessing
|
72
76
|
rescue Aborted
|
73
77
|
rescue WorkerException
|
74
|
-
Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.message}"
|
78
|
+
Log.error "Exception in worker #{obj.pid} in queue #{Process.pid}: #{obj.worker_exception.message}"
|
75
79
|
self.abort
|
80
|
+
@input.abort obj.worker_exception
|
76
81
|
raise obj.worker_exception
|
77
82
|
rescue
|
78
83
|
Log.error "Exception processing output in queue #{Process.pid}: #{$!.message}"
|
@@ -81,10 +86,6 @@ class WorkQueue
|
|
81
86
|
end
|
82
87
|
end
|
83
88
|
|
84
|
-
@workers.each do |w|
|
85
|
-
w.process @input, @output, &@worker_proc
|
86
|
-
end
|
87
|
-
|
88
89
|
Thread.pass until @reader["name"]
|
89
90
|
|
90
91
|
@waiter = Thread.new do
|
@@ -104,7 +105,13 @@ class WorkQueue
|
|
104
105
|
end
|
105
106
|
|
106
107
|
def write(obj)
|
107
|
-
|
108
|
+
begin
|
109
|
+
@input.write obj
|
110
|
+
rescue Exception
|
111
|
+
raise $! unless @input.exception
|
112
|
+
ensure
|
113
|
+
raise @input.exception if @input.exception
|
114
|
+
end
|
108
115
|
end
|
109
116
|
|
110
117
|
def abort
|
@@ -117,7 +124,7 @@ class WorkQueue
|
|
117
124
|
def close
|
118
125
|
@closed = true
|
119
126
|
@worker_mutex.synchronize{ @workers.length }.times do
|
120
|
-
@input.write DoneProcessing.new()
|
127
|
+
@input.write DoneProcessing.new() unless @input.closed_write?
|
121
128
|
end
|
122
129
|
end
|
123
130
|
|
@@ -76,9 +76,11 @@ module Workflow
|
|
76
76
|
end
|
77
77
|
when 1
|
78
78
|
task = args.first
|
79
|
+
options, task = task, nil if Hash === task
|
79
80
|
end
|
80
81
|
workflow = self if workflow.nil?
|
81
82
|
options = {} if options.nil?
|
83
|
+
task = task.to_sym if task
|
82
84
|
annotate_next_task :deps, [workflow, task, options, block, args]
|
83
85
|
end
|
84
86
|
|
@@ -103,6 +105,20 @@ module Workflow
|
|
103
105
|
@tasks ||= IndiferentHash.setup({})
|
104
106
|
begin
|
105
107
|
@annotate_next_task ||= {}
|
108
|
+
@annotate_next_task[:extension] ||=
|
109
|
+
case type
|
110
|
+
when :tsv
|
111
|
+
"tsv"
|
112
|
+
when :yaml
|
113
|
+
"yaml"
|
114
|
+
when :marshal
|
115
|
+
"marshal"
|
116
|
+
when :json
|
117
|
+
"json"
|
118
|
+
else
|
119
|
+
nil
|
120
|
+
end
|
121
|
+
|
106
122
|
task = Task.setup(block, @annotate_next_task.merge(name: name, type: type, directory: directory[name], workflow: self))
|
107
123
|
@tasks[name] = task
|
108
124
|
ensure
|
@@ -110,11 +126,14 @@ module Workflow
|
|
110
126
|
end
|
111
127
|
end
|
112
128
|
|
129
|
+
FORGET_DEP_TASKS = ENV["SCOUT_FORGET_DEP_TASKS"] == "true"
|
130
|
+
REMOVE_DEP_TASKS = ENV["SCOUT_REMOVE_DEP_TASKS"] == "true"
|
113
131
|
def task_alias(name, workflow, oname, *rest, &block)
|
114
132
|
dep(workflow, oname, *rest, &block)
|
115
133
|
extension :dep_task unless @extension
|
116
|
-
returns workflow.tasks[oname].returns if
|
117
|
-
|
134
|
+
returns workflow.tasks[oname].returns if @returns.nil?
|
135
|
+
type = workflow.tasks[oname].type
|
136
|
+
task name => type do
|
118
137
|
raise RbbtException, "dep_task does not have any dependencies" if dependencies.empty?
|
119
138
|
Step.wait_for_jobs dependencies.select{|d| d.streaming? }
|
120
139
|
dep = dependencies.last
|
@@ -1,8 +1,13 @@
|
|
1
1
|
class Step
|
2
|
+
def rec_dependencies
|
3
|
+
rec_dependencies = dependencies.dup
|
4
|
+
dependencies.inject(rec_dependencies){|acc,d| acc.concat d.rec_dependencies }
|
5
|
+
end
|
6
|
+
|
2
7
|
def recursive_inputs
|
3
|
-
|
4
|
-
|
5
|
-
acc
|
8
|
+
recursive_inputs = @inputs.to_hash
|
9
|
+
dependencies.inject(recursive_inputs) do |acc,dep|
|
10
|
+
acc.merge(dep.recursive_inputs)
|
6
11
|
end
|
7
12
|
end
|
8
13
|
|
@@ -16,6 +21,10 @@ class Step
|
|
16
21
|
def prepare_dependencies
|
17
22
|
inverse_dep = {}
|
18
23
|
dependencies.each{|dep|
|
24
|
+
if dep.present? && ! dep.updated?
|
25
|
+
Log.debug "Clean outdated #{dep.path}"
|
26
|
+
dep.clean
|
27
|
+
end
|
19
28
|
next if dep.done?
|
20
29
|
if dep.dependencies
|
21
30
|
dep.dependencies.each do |d|
|
@@ -34,7 +43,18 @@ class Step
|
|
34
43
|
end
|
35
44
|
|
36
45
|
def run_dependencies
|
37
|
-
dependencies.each{|dep| dep.run unless dep.running? || dep.done? }
|
46
|
+
dependencies.each{|dep| dep.run(true) unless dep.running? || dep.done? }
|
38
47
|
end
|
39
48
|
|
49
|
+
def abort_dependencies
|
50
|
+
dependencies.each{|dep| dep.abort if dep.running? }
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.wait_for_jobs(jobs)
|
54
|
+
threads = []
|
55
|
+
jobs.each do |job|
|
56
|
+
threads << job.join
|
57
|
+
end
|
58
|
+
threads.each do |t| t.join end
|
59
|
+
end
|
40
60
|
end
|
@@ -1,6 +1,7 @@
|
|
1
1
|
class Step
|
2
2
|
SERIALIZER = :marshal
|
3
3
|
def info_file
|
4
|
+
return nil if @path.nil?
|
4
5
|
@info_file ||= begin
|
5
6
|
info_file = @path + ".info"
|
6
7
|
@path.annotate info_file if Path === @path
|
@@ -19,6 +20,10 @@ class Step
|
|
19
20
|
@info_load_time = Time.now
|
20
21
|
end
|
21
22
|
|
23
|
+
def clear_info
|
24
|
+
save_info(@info = {})
|
25
|
+
end
|
26
|
+
|
22
27
|
def info
|
23
28
|
outdated = begin
|
24
29
|
@info_load_time && (mtime = Open.mtime(info_file)) && mtime > @info_load_time
|
@@ -36,7 +41,19 @@ class Step
|
|
36
41
|
def merge_info(new_info)
|
37
42
|
info = self.info
|
38
43
|
new_info.each do |key,value|
|
39
|
-
|
44
|
+
if key == :status
|
45
|
+
message = new_info[:messages]
|
46
|
+
if message.nil? && value == :done || value == :error || value == :aborted
|
47
|
+
start = info[:start]
|
48
|
+
eend = new_info[:end]
|
49
|
+
if start && eend
|
50
|
+
time = eend - start
|
51
|
+
time_str = Misc.format_seconds_short(time)
|
52
|
+
message = Log.color(:time, time_str)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
report_status value, message
|
56
|
+
end
|
40
57
|
if Exception === value
|
41
58
|
begin
|
42
59
|
Marshal.dump(value)
|
@@ -72,15 +89,21 @@ class Step
|
|
72
89
|
|
73
90
|
def report_status(status, message = nil)
|
74
91
|
if message.nil?
|
75
|
-
Log.info Log.color(:status, status, true)
|
92
|
+
Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), Log.color(:path, path)] * " "
|
76
93
|
else
|
77
|
-
Log.info Log.color(:status, status, true)
|
94
|
+
Log.info [Log.color(:status, status, true), Log.color(:task, task_name, true), message, Log.color(:path, path)] * " "
|
78
95
|
end
|
79
96
|
end
|
80
97
|
|
81
|
-
def log(status, message = nil)
|
98
|
+
def log(status, message = nil, &block)
|
99
|
+
if block_given?
|
100
|
+
time = Misc.exec_time &block
|
101
|
+
time_str = Misc.format_seconds_short time
|
102
|
+
message = message.nil? ? Log.color(:time, time_str) : "#{Log.color :time, time_str} - #{ message }"
|
103
|
+
end
|
104
|
+
|
82
105
|
if message
|
83
|
-
merge_info :status => status, :messages =>
|
106
|
+
merge_info :status => status, :messages => message
|
84
107
|
else
|
85
108
|
merge_info :status => status
|
86
109
|
end
|
@@ -105,4 +128,12 @@ class Step
|
|
105
128
|
def exception
|
106
129
|
info[:exception]
|
107
130
|
end
|
131
|
+
|
132
|
+
def marshal_dump
|
133
|
+
@path
|
134
|
+
end
|
135
|
+
|
136
|
+
def marshal_load(path)
|
137
|
+
Step.new path
|
138
|
+
end
|
108
139
|
end
|