rbbt-util 5.27.4 → 5.27.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/entity.rb +1 -1
- data/lib/rbbt/fix_width_table.rb +6 -5
- data/lib/rbbt/persist.rb +2 -2
- data/lib/rbbt/persist/tsv/tokyocabinet.rb +2 -2
- data/lib/rbbt/resource.rb +8 -5
- data/lib/rbbt/resource/path.rb +3 -3
- data/lib/rbbt/tsv/attach.rb +34 -30
- data/lib/rbbt/tsv/attach/util.rb +17 -0
- data/lib/rbbt/tsv/stream.rb +1 -0
- data/lib/rbbt/util/misc/exceptions.rb +8 -0
- data/lib/rbbt/util/misc/format.rb +1 -1
- data/lib/rbbt/util/misc/pipes.rb +37 -7
- data/lib/rbbt/util/tmpfile.rb +4 -4
- data/lib/rbbt/workflow.rb +2 -5
- data/lib/rbbt/workflow/accessor.rb +4 -2
- data/lib/rbbt/workflow/definition.rb +6 -0
- data/lib/rbbt/workflow/remote_workflow/driver/rest.rb +12 -3
- data/lib/rbbt/workflow/remote_workflow/remote_step.rb +18 -4
- data/lib/rbbt/workflow/remote_workflow/remote_step/rest.rb +7 -1
- data/lib/rbbt/workflow/step/accessor.rb +4 -0
- data/lib/rbbt/workflow/step/dependencies.rb +11 -7
- data/lib/rbbt/workflow/step/run.rb +1 -1
- data/lib/rbbt/workflow/task.rb +1 -1
- data/lib/rbbt/workflow/usage.rb +1 -1
- data/lib/rbbt/workflow/util/archive.rb +30 -4
- data/share/rbbt_commands/system/clean +2 -2
- data/share/rbbt_commands/system/status +3 -2
- data/share/rbbt_commands/tsv/uncollapse +29 -0
- data/share/rbbt_commands/workflow/info +1 -1
- data/share/rbbt_commands/workflow/monitor +3 -3
- data/share/rbbt_commands/workflow/task +2 -0
- data/test/rbbt/tsv/test_attach.rb +3 -2
- data/test/rbbt/tsv/test_stream.rb +4 -3
- data/test/rbbt/util/misc/test_format.rb +10 -0
- data/test/rbbt/util/misc/test_pipes.rb +1 -1
- data/test/rbbt/workflow/step/test_dependencies.rb +106 -17
- data/test/rbbt/workflow/test_remote_workflow.rb +13 -1
- metadata +5 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 88040e654ed6cb7818930897c0d66763e7f7834e7ea530ac0ef45e2b318ed078
|
4
|
+
data.tar.gz: b4813ae7c39189b7a17af4bd958a15823b506a293a03234c7f4cd7af773d37a4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f940c0f1c33c7a9afa7463ee04ff972cf2ec39ccf400bb6e5bb5abb94b54e9dba5c6a1dc2c1651f25f5e44284d5af7a405a436acd8cd1a6d0f143b14a2c8688b
|
7
|
+
data.tar.gz: 0d17a7e040af5a666d21a8a6db880ede4a50a7491e59476fe0288e6f9c444e174e34a27eec09aa15c6bccfb81e7348e1bb17b2f5ed5b9c115ea88b1321cf8265
|
data/lib/rbbt/entity.rb
CHANGED
@@ -261,7 +261,7 @@ module Entity
|
|
261
261
|
|
262
262
|
define_method method_name do |*args|
|
263
263
|
id = self.id
|
264
|
-
persist_name = orig_method_name.to_s
|
264
|
+
persist_name = orig_method_name.to_s + ":" << (Array === id ? Misc.obj2digest(id) : id)
|
265
265
|
|
266
266
|
persist_options = options
|
267
267
|
persist_options = persist_options.merge(:other => {:args => args}) if args and args.any?
|
data/lib/rbbt/fix_width_table.rb
CHANGED
@@ -4,20 +4,21 @@ class FixWidthTable
|
|
4
4
|
def initialize(filename, value_size = nil, range = nil, update = false, in_memory = true)
|
5
5
|
@filename = filename
|
6
6
|
|
7
|
-
if update
|
7
|
+
if update || %w(memory stringio).include?(filename.to_s.downcase) || ! File.exist?(filename)
|
8
8
|
Log.debug "FixWidthTable create: #{ filename }"
|
9
9
|
@value_size = value_size
|
10
10
|
@range = range
|
11
11
|
@record_size = @value_size + (@range ? 16 : 8)
|
12
12
|
@write = true
|
13
13
|
|
14
|
-
if %w(memory stringio).include?
|
14
|
+
if %w(memory stringio).include?(filename.to_s.downcase)
|
15
15
|
@filename = :memory
|
16
16
|
@file = StringIO.new
|
17
17
|
else
|
18
18
|
FileUtils.rm @filename if File.exist? @filename
|
19
19
|
FileUtils.mkdir_p File.dirname(@filename) unless File.exist? @filename
|
20
|
-
|
20
|
+
#@file = File.open(@filename, 'wb')
|
21
|
+
@file = File.open(@filename, 'w:ASCII-8BIT')
|
21
22
|
end
|
22
23
|
|
23
24
|
@file.write [value_size].pack("L")
|
@@ -25,9 +26,9 @@ class FixWidthTable
|
|
25
26
|
|
26
27
|
@size = 0
|
27
28
|
else
|
28
|
-
Log.debug "FixWidthTable up-to-date: #{ filename }"
|
29
|
+
Log.debug "FixWidthTable up-to-date: #{ filename } - (in_memory:#{in_memory})"
|
29
30
|
if in_memory
|
30
|
-
@file =
|
31
|
+
@file = Open.open(@filename, :mode => 'r:ASCII-8BIT'){|f| StringIO.new f.read}
|
31
32
|
else
|
32
33
|
@file = File.open(@filename, 'r:ASCII-8BIT')
|
33
34
|
end
|
data/lib/rbbt/persist.rb
CHANGED
@@ -365,12 +365,12 @@ module Persist
|
|
365
365
|
type ||= :marshal
|
366
366
|
|
367
367
|
persist_options ||= {}
|
368
|
-
if type == :memory
|
368
|
+
if type == :memory && persist_options[:file] && persist_options[:persist]
|
369
369
|
repo = persist_options[:repo] || Persist::MEMORY
|
370
370
|
if persist_options[:persist] == :update || persist_options[:update]
|
371
371
|
repo.delete persist_options[:file]
|
372
372
|
end
|
373
|
-
return repo[persist_options[:file]] ||= yield
|
373
|
+
return repo[persist_options[:file]] ||= yield
|
374
374
|
end
|
375
375
|
|
376
376
|
if FalseClass === persist_options[:persist]
|
@@ -16,7 +16,7 @@ module Persist
|
|
16
16
|
end
|
17
17
|
|
18
18
|
dir = File.dirname(File.expand_path(path))
|
19
|
-
|
19
|
+
File.mkdir(dir) unless File.exists?(dir)
|
20
20
|
|
21
21
|
tokyocabinet_class = TokyoCabinet::HDB if tokyocabinet_class == "HDB" or tokyocabinet_class.nil?
|
22
22
|
tokyocabinet_class = TokyoCabinet::BDB if tokyocabinet_class == "BDB"
|
@@ -87,7 +87,7 @@ module Persist
|
|
87
87
|
def self.open_tokyocabinet(path, write, serializer = nil, tokyocabinet_class = TokyoCabinet::HDB)
|
88
88
|
write = true unless File.exist? path
|
89
89
|
|
90
|
-
FileUtils.mkdir_p File.dirname(path) unless
|
90
|
+
FileUtils.mkdir_p File.dirname(path) unless File.exist?(File.dirname(path))
|
91
91
|
|
92
92
|
database = Persist::TCAdapter.open(path, write, tokyocabinet_class)
|
93
93
|
|
data/lib/rbbt/resource.rb
CHANGED
@@ -170,10 +170,12 @@ module Resource
|
|
170
170
|
if type and not File.exist?(final_path) or force
|
171
171
|
Log.medium "Producing: #{ final_path }"
|
172
172
|
lock_filename = Persist.persistence_path(final_path, {:dir => Resource.lock_dir})
|
173
|
+
|
173
174
|
Misc.lock lock_filename do
|
174
175
|
FileUtils.rm_rf final_path if force and File.exist? final_path
|
175
|
-
|
176
|
-
|
176
|
+
|
177
|
+
if ! File.exist?(final_path) || force
|
178
|
+
|
177
179
|
begin
|
178
180
|
case type
|
179
181
|
when :string
|
@@ -291,7 +293,7 @@ url='#{url}'
|
|
291
293
|
rescue
|
292
294
|
FileUtils.rm_rf final_path if File.exist? final_path
|
293
295
|
raise $!
|
294
|
-
end
|
296
|
+
end unless (remote_server && get_from_server(path, final_path))
|
295
297
|
end
|
296
298
|
end
|
297
299
|
end
|
@@ -304,11 +306,12 @@ url='#{url}'
|
|
304
306
|
resource ||= Rbbt
|
305
307
|
(Path::STANDARD_SEARCH + resource.search_order + resource.search_paths.keys).uniq.each do |name|
|
306
308
|
pattern = resource.search_paths[name]
|
307
|
-
next if
|
309
|
+
next if pattern.nil?
|
310
|
+
pattern = pattern.sub('{PWD}', Dir.pwd)
|
308
311
|
if String === pattern and pattern.include?('{')
|
309
312
|
regexp = "^" + pattern.gsub(/{([^}]+)}/,'(?<\1>[^/]+)') + "(?:/(?<REST>.*))?/?$"
|
310
313
|
if m = path.match(regexp)
|
311
|
-
if m["PKGDIR"] == resource.pkgdir
|
314
|
+
if ! m.named_captures.include?("PKGDIR") || m["PKGDIR"] == resource.pkgdir
|
312
315
|
return self[m["TOPLEVEL"]][m["SUBPATH"]][m["REST"]]
|
313
316
|
end
|
314
317
|
end
|
data/lib/rbbt/resource/path.rb
CHANGED
@@ -199,7 +199,7 @@ module Path
|
|
199
199
|
next if res
|
200
200
|
next unless paths.include? w
|
201
201
|
path = find(w, caller_lib, paths)
|
202
|
-
res = path if File.exist?
|
202
|
+
res = path if File.exist?(path)
|
203
203
|
end if res.nil?
|
204
204
|
|
205
205
|
(paths.keys - STANDARD_SEARCH - search_order).each do |w|
|
@@ -241,8 +241,8 @@ module Path
|
|
241
241
|
sub('{REMOVE}/', '').
|
242
242
|
sub('{REMOVE}', '')
|
243
243
|
|
244
|
-
path = path + '.gz' if File.exist?
|
245
|
-
path = path + '.bgz' if File.exist?
|
244
|
+
path = path + '.gz' if File.exist?(path + '.gz')
|
245
|
+
path = path + '.bgz' if File.exist?(path + '.bgz')
|
246
246
|
|
247
247
|
self.annotate path
|
248
248
|
|
data/lib/rbbt/tsv/attach.rb
CHANGED
@@ -32,7 +32,8 @@ module TSV
|
|
32
32
|
done = false
|
33
33
|
Open.write(output) do |os|
|
34
34
|
options.delete :sep if options[:sep] == "\t"
|
35
|
-
|
35
|
+
header_lines = TSV.header_lines(key_field, fields, options)
|
36
|
+
os.puts header_lines unless header_lines.empty?
|
36
37
|
|
37
38
|
while line
|
38
39
|
key, *parts = line.sub("\n",'').split(sep, -1)
|
@@ -212,6 +213,35 @@ module TSV
|
|
212
213
|
other_filename = other.respond_to?(:filename) ? other.filename : other.inspect
|
213
214
|
Log.low("Attaching fields:#{Misc.fingerprint fields } from #{other_filename}.")
|
214
215
|
|
216
|
+
same_key = true
|
217
|
+
begin
|
218
|
+
case
|
219
|
+
when (Misc.match_fields(key_field, other.key_field) and same_key)
|
220
|
+
Log.debug "Attachment with same key: #{other.key_field}"
|
221
|
+
attach_same_key other, fields
|
222
|
+
when (not in_namespace and self.fields.select{|f| Misc.match_fields(f, other.key_field)}.any?)
|
223
|
+
Log.debug "Found other key field: #{other.key_field}"
|
224
|
+
attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
|
225
|
+
when (in_namespace and self.fields_in_namespace.select{|f| Misc.match_fields(f, other.key_field)}.any?)
|
226
|
+
Log.debug "Found other key field in #{in_namespace}: #{other.key_field}"
|
227
|
+
attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
|
228
|
+
else
|
229
|
+
index = TSV.find_traversal(self, other, options)
|
230
|
+
raise FieldNotFoundError, "Cannot traverse identifiers" if index.nil?
|
231
|
+
Log.debug "Attachment with index: #{other.key_field}"
|
232
|
+
attach_index other, index, fields
|
233
|
+
end
|
234
|
+
rescue Exception
|
235
|
+
if same_key
|
236
|
+
Log.warn "Could not translate identifiers with same_key"
|
237
|
+
same_key = false
|
238
|
+
retry
|
239
|
+
else
|
240
|
+
raise $!
|
241
|
+
end
|
242
|
+
end
|
243
|
+
Log.debug("Attachment of fields:#{Misc.fingerprint fields } from #{other.filename.inspect} finished.")
|
244
|
+
|
215
245
|
if complete
|
216
246
|
fill = TrueClass === complete ? nil : complete
|
217
247
|
field_length = self.fields.length
|
@@ -219,6 +249,9 @@ module TSV
|
|
219
249
|
other_common_pos = common_fields.collect{|f| other.fields.index f}
|
220
250
|
this_common_pos = common_fields.collect{|f| self.fields.index f}
|
221
251
|
missing = other.keys - self.keys
|
252
|
+
|
253
|
+
other = other.to_list if other.type == :single
|
254
|
+
|
222
255
|
case type
|
223
256
|
when :single
|
224
257
|
missing.each do |k|
|
@@ -249,35 +282,6 @@ module TSV
|
|
249
282
|
end
|
250
283
|
end
|
251
284
|
|
252
|
-
same_key = true
|
253
|
-
begin
|
254
|
-
case
|
255
|
-
when (Misc.match_fields(key_field, other.key_field) and same_key)
|
256
|
-
Log.debug "Attachment with same key: #{other.key_field}"
|
257
|
-
attach_same_key other, fields
|
258
|
-
when (not in_namespace and self.fields.select{|f| Misc.match_fields(f, other.key_field)}.any?)
|
259
|
-
Log.debug "Found other key field: #{other.key_field}"
|
260
|
-
attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
|
261
|
-
when (in_namespace and self.fields_in_namespace.select{|f| Misc.match_fields(f, other.key_field)}.any?)
|
262
|
-
Log.debug "Found other key field in #{in_namespace}: #{other.key_field}"
|
263
|
-
attach_source_key other, other.key_field, :fields => fields, :one2one => one2one
|
264
|
-
else
|
265
|
-
index = TSV.find_traversal(self, other, options)
|
266
|
-
raise FieldNotFoundError, "Cannot traverse identifiers" if index.nil?
|
267
|
-
Log.debug "Attachment with index: #{other.key_field}"
|
268
|
-
attach_index other, index, fields
|
269
|
-
end
|
270
|
-
rescue Exception
|
271
|
-
if same_key
|
272
|
-
Log.warn "Could not translate identifiers with same_key"
|
273
|
-
same_key = false
|
274
|
-
retry
|
275
|
-
else
|
276
|
-
raise $!
|
277
|
-
end
|
278
|
-
end
|
279
|
-
Log.debug("Attachment of fields:#{Misc.fingerprint fields } from #{other.filename.inspect} finished.")
|
280
|
-
|
281
285
|
self
|
282
286
|
end
|
283
287
|
|
data/lib/rbbt/tsv/attach/util.rb
CHANGED
@@ -4,9 +4,20 @@ module TSV
|
|
4
4
|
fields = other.fields - [key_field].concat(self.fields) if fields.nil?
|
5
5
|
|
6
6
|
fields = [fields].compact unless Array === fields
|
7
|
+
|
8
|
+
common_fields = self.fields & fields
|
9
|
+
|
10
|
+
fields = fields - common_fields
|
11
|
+
|
7
12
|
num_fields = fields.length
|
8
13
|
|
9
14
|
field_positions = fields.collect{|field| other.identify_field field}
|
15
|
+
|
16
|
+
if common_fields.any?
|
17
|
+
common_field_positions = common_fields.collect{|field| self.identify_field field}
|
18
|
+
common_field_positions_other = common_fields.collect{|field| other.identify_field field}
|
19
|
+
end
|
20
|
+
|
10
21
|
other.with_unnamed do
|
11
22
|
with_unnamed do
|
12
23
|
through do |key, values|
|
@@ -28,6 +39,12 @@ module TSV
|
|
28
39
|
new_values = field_positions.collect do |pos|
|
29
40
|
pos == :key ? key : other_values[pos]
|
30
41
|
end
|
42
|
+
|
43
|
+
if common_fields.any?
|
44
|
+
common_field_positions.zip(common_field_positions_other).each do |p1,p2|
|
45
|
+
current[p1] += other_values[p2]
|
46
|
+
end
|
47
|
+
end
|
31
48
|
end
|
32
49
|
|
33
50
|
new_values.collect!{|v| [v]} if type == :double and not (other.type == :double or other.type == :flat)
|
data/lib/rbbt/tsv/stream.rb
CHANGED
@@ -15,6 +15,14 @@ end
|
|
15
15
|
class Aborted < StandardError; end
|
16
16
|
|
17
17
|
class TryAgain < StandardError; end
|
18
|
+
|
19
|
+
class TryThis < StandardError
|
20
|
+
attr_accessor :payload
|
21
|
+
def initialize(payload = nil)
|
22
|
+
@payload = payload
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
18
26
|
class SemaphoreInterrupted < TryAgain; end
|
19
27
|
class LockInterrupted < TryAgain; end
|
20
28
|
|
@@ -163,7 +163,7 @@ module Misc
|
|
163
163
|
|
164
164
|
def self.fixutf8(string)
|
165
165
|
return nil if string.nil?
|
166
|
-
return string if string.respond_to?(:encoding) && string.encoding == "UTF-8" && (string.respond_to?(:valid_encoding?) && string.valid_encoding?) ||
|
166
|
+
return string if string.respond_to?(:encoding) && string.encoding.to_s == "UTF-8" && (string.respond_to?(:valid_encoding?) && string.valid_encoding?) ||
|
167
167
|
(string.respond_to?(:valid_encoding) && string.valid_encoding)
|
168
168
|
|
169
169
|
if string.respond_to?(:encode)
|
data/lib/rbbt/util/misc/pipes.rb
CHANGED
@@ -18,6 +18,8 @@ module Misc
|
|
18
18
|
|
19
19
|
BLOCK_SIZE=1024 * 8
|
20
20
|
|
21
|
+
SKIP_TAG="[SKIP TAG]"
|
22
|
+
|
21
23
|
PIPE_MUTEX = Mutex.new
|
22
24
|
|
23
25
|
OPEN_PIPE_IN = []
|
@@ -29,7 +31,7 @@ module Misc
|
|
29
31
|
|
30
32
|
[sout, sin]
|
31
33
|
end
|
32
|
-
Log.debug{"Creating pipe #{[res.last.inspect,res.first.inspect] * " => "}"}
|
34
|
+
Log.debug{"Creating pipe #{[res.last.inspect, res.first.inspect] * " => "}"}
|
33
35
|
res
|
34
36
|
end
|
35
37
|
|
@@ -255,6 +257,11 @@ module Misc
|
|
255
257
|
end
|
256
258
|
tee1, *rest = Misc.tee_stream stream_dup, num + 1
|
257
259
|
stream.reopen(tee1)
|
260
|
+
|
261
|
+
#ToDo: I can't explain why the @threads variable appears with the value of
|
262
|
+
# @filename
|
263
|
+
stream.instance_variable_set(:@threads, nil) if stream.instance_variables.include?(:@threads)
|
264
|
+
|
258
265
|
tee1.annotate(stream)
|
259
266
|
rest
|
260
267
|
end
|
@@ -537,18 +544,29 @@ module Misc
|
|
537
544
|
end
|
538
545
|
end
|
539
546
|
|
547
|
+
def self.buffer_stream(stream)
|
548
|
+
sout, sin = Misc.pipe
|
549
|
+
Misc.consume_stream(stream, true, sin)
|
550
|
+
sout
|
551
|
+
end
|
552
|
+
|
540
553
|
def self._paste_streams(streams, output, lines = nil, sep = "\t", header = nil, &block)
|
541
554
|
output.puts header if header
|
542
555
|
streams = streams.collect do |stream|
|
543
556
|
if defined? Step and Step === stream
|
544
|
-
stream.get_stream
|
557
|
+
io = stream.get_stream
|
558
|
+
if io
|
559
|
+
buffer_stream(io)
|
560
|
+
else
|
561
|
+
stream.join.path.open
|
562
|
+
end
|
545
563
|
else
|
546
564
|
stream
|
547
565
|
end
|
548
566
|
end
|
549
567
|
|
550
568
|
begin
|
551
|
-
|
569
|
+
|
552
570
|
lines ||= streams.collect{|s| s.gets }
|
553
571
|
keys = []
|
554
572
|
parts = []
|
@@ -564,6 +582,7 @@ module Misc
|
|
564
582
|
end
|
565
583
|
sizes = parts.collect{|p| p.nil? ? 0 : p.length }
|
566
584
|
last_min = nil
|
585
|
+
|
567
586
|
while lines.compact.any?
|
568
587
|
if block_given?
|
569
588
|
min = keys.compact.sort(&block).first
|
@@ -571,14 +590,23 @@ module Misc
|
|
571
590
|
min = keys.compact.sort.first
|
572
591
|
end
|
573
592
|
str = []
|
593
|
+
threads = []
|
574
594
|
keys.each_with_index do |key,i|
|
575
595
|
case key
|
576
596
|
when min
|
577
|
-
|
597
|
+
if parts[i] == [SKIP_TAG]
|
598
|
+
str << [sep * (sizes[i]-1)] if sizes[i] > 0
|
599
|
+
else
|
600
|
+
str << [parts[i] * sep]
|
601
|
+
end
|
602
|
+
|
578
603
|
line = lines[i] = streams[i].gets
|
579
|
-
|
604
|
+
|
605
|
+
if line.nil?
|
580
606
|
keys[i] = nil
|
581
607
|
parts[i] = nil
|
608
|
+
streams[i].close unless streams[i].closed?
|
609
|
+
streams[i].join if streams[i].respond_to?(:join)
|
582
610
|
else
|
583
611
|
k, *p = line.chomp.split(sep, -1)
|
584
612
|
keys[i] = k
|
@@ -589,10 +617,12 @@ module Misc
|
|
589
617
|
end
|
590
618
|
end
|
591
619
|
|
592
|
-
output.puts [min, str*sep] * sep
|
620
|
+
output.puts [min, str.flatten*sep] * sep
|
593
621
|
end
|
622
|
+
|
594
623
|
streams.each do |stream|
|
595
|
-
stream.
|
624
|
+
stream.close unless stream.closed?
|
625
|
+
stream.join if stream.respond_to?(:join)
|
596
626
|
end
|
597
627
|
rescue
|
598
628
|
Log.exception $!
|
data/lib/rbbt/util/tmpfile.rb
CHANGED
@@ -17,14 +17,14 @@ module TmpFile
|
|
17
17
|
|
18
18
|
# Creates a random file name, with the given suffix and a random number
|
19
19
|
# up to +max+
|
20
|
-
def self.random_name(s = "tmp-", max =
|
20
|
+
def self.random_name(s = "tmp-", max = 1_000_000_000)
|
21
21
|
n = rand(max)
|
22
22
|
s + n.to_s
|
23
23
|
end
|
24
24
|
|
25
25
|
# Creates a random filename in the temporary directory
|
26
|
-
def self.tmp_file(s = "tmp-", max=
|
27
|
-
File.expand_path(File.join(dir, random_name(s,max)))
|
26
|
+
def self.tmp_file(s = "tmp-", max=1_000_000_000, dir = TMPDIR)
|
27
|
+
File.expand_path(File.join(dir, random_name(s, max)))
|
28
28
|
end
|
29
29
|
|
30
30
|
def self.with_file(content = nil, erase = true, options = {})
|
@@ -33,7 +33,7 @@ module TmpFile
|
|
33
33
|
|
34
34
|
prefix = options[:prefix] || "tmp-"
|
35
35
|
tmpdir = options[:tmpdir] || TMPDIR
|
36
|
-
max = options[:max] ||
|
36
|
+
max = options[:max] || 1_000_000_000
|
37
37
|
tmpfile = tmp_file prefix, max, tmpdir
|
38
38
|
if options[:extension]
|
39
39
|
tmpfile += ".#{options[:extension]}"
|
data/lib/rbbt/workflow.rb
CHANGED
@@ -10,9 +10,6 @@ require 'rbbt/workflow/util/provenance'
|
|
10
10
|
|
11
11
|
module Workflow
|
12
12
|
|
13
|
-
STEP_CACHE = {}
|
14
|
-
LOAD_STEP_CACHE = {}
|
15
|
-
|
16
13
|
class TaskNotFoundException < Exception
|
17
14
|
def initialize(workflow, task = nil)
|
18
15
|
if task
|
@@ -264,11 +261,11 @@ module Workflow
|
|
264
261
|
end
|
265
262
|
|
266
263
|
def step_cache
|
267
|
-
|
264
|
+
Thread.current[:step_cache] ||= {}
|
268
265
|
end
|
269
266
|
|
270
267
|
def self.load_step_cache
|
271
|
-
|
268
|
+
Thread.current[:load_step_cache] ||= {}
|
272
269
|
end
|
273
270
|
|
274
271
|
|
@@ -35,6 +35,7 @@ module Workflow
|
|
35
35
|
input_use = rec_input_use(name)
|
36
36
|
input_defaults = rec_input_defaults(name)
|
37
37
|
input_options = rec_input_options(name)
|
38
|
+
extension = task.extension
|
38
39
|
export = case
|
39
40
|
when (synchronous_exports.include?(name.to_sym) or synchronous_exports.include?(name.to_s))
|
40
41
|
:synchronous
|
@@ -60,7 +61,8 @@ module Workflow
|
|
60
61
|
:input_use => input_use,
|
61
62
|
:result_type => result_type,
|
62
63
|
:result_description => result_description,
|
63
|
-
:dependencies => dependencies
|
64
|
+
:dependencies => dependencies,
|
65
|
+
:extension => extension
|
64
66
|
}
|
65
67
|
end
|
66
68
|
end
|
@@ -416,7 +418,7 @@ module Workflow
|
|
416
418
|
when :hash
|
417
419
|
clean_inputs = Annotated.purge(inputs)
|
418
420
|
clean_inputs = clean_inputs.collect{|i| Symbol === i ? i.to_s : i }
|
419
|
-
deps_str = dependencies.collect{|d| Step === d ? d.short_path : d }
|
421
|
+
deps_str = dependencies.collect{|d| (Step === d || (defined?(RemoteStep) && RemoteStep === Step)) ? "Step: " << d.short_path : d }
|
420
422
|
key_obj = {:inputs => clean_inputs, :dependencies => deps_str }
|
421
423
|
key_str = Misc.obj2str(key_obj)
|
422
424
|
hash_str = Misc.digest(key_str)
|
@@ -18,6 +18,7 @@ module Workflow
|
|
18
18
|
:description => "",
|
19
19
|
:result_type => nil,
|
20
20
|
:result_description => "",
|
21
|
+
:resumable => false,
|
21
22
|
:extension => nil)
|
22
23
|
|
23
24
|
|
@@ -33,6 +34,10 @@ module Workflow
|
|
33
34
|
@extension = extension
|
34
35
|
end
|
35
36
|
|
37
|
+
def resumable
|
38
|
+
@resumable = true
|
39
|
+
end
|
40
|
+
|
36
41
|
def returns(description)
|
37
42
|
@result_description = description
|
38
43
|
end
|
@@ -118,6 +123,7 @@ module Workflow
|
|
118
123
|
:input_descriptions => consume_input_descriptions,
|
119
124
|
:required_inputs => consume_required_inputs,
|
120
125
|
:extension => consume_extension,
|
126
|
+
:resumable => consume_resumable,
|
121
127
|
:input_options => consume_input_options
|
122
128
|
}
|
123
129
|
|
@@ -137,13 +137,17 @@ class RemoteWorkflow
|
|
137
137
|
|
138
138
|
post_thread = Thread.new(Thread.current) do |parent|
|
139
139
|
bl = lambda do |rok|
|
140
|
-
|
140
|
+
case rok
|
141
|
+
when Net::HTTPOK
|
141
142
|
_url = rok["RBBT-STREAMING-JOB-URL"]
|
142
143
|
@url = File.join(task_url, File.basename(_url)) if _url
|
143
144
|
rok.read_body do |c,_a, _b|
|
144
145
|
sin.write c
|
145
146
|
end
|
146
147
|
sin.close
|
148
|
+
when Net::HTTPRedirection, Net::HTTPAccepted
|
149
|
+
Thread.current.report_on_exception = false
|
150
|
+
raise TryThis.new(rok)
|
147
151
|
else
|
148
152
|
err = StringIO.new
|
149
153
|
rok.read_body do |c,_a, _b|
|
@@ -156,7 +160,7 @@ class RemoteWorkflow
|
|
156
160
|
err.rewind
|
157
161
|
err.read
|
158
162
|
end
|
159
|
-
ne =
|
163
|
+
ne = RemoteWorkflow.parse_exception text
|
160
164
|
case ne
|
161
165
|
when String
|
162
166
|
parent.raise e.class, ne
|
@@ -173,7 +177,12 @@ class RemoteWorkflow
|
|
173
177
|
end
|
174
178
|
|
175
179
|
Log.debug{ "RestClient execute: #{ task_url } - #{Misc.fingerprint task_params}" }
|
176
|
-
|
180
|
+
begin
|
181
|
+
RestClient::Request.execute(:method => :post, :url => task_url, :payload => task_params, :block_response => bl)
|
182
|
+
rescue TryThis
|
183
|
+
url = $!.payload["location"]
|
184
|
+
RestClient::Request.execute(:method => :get, :url => url, :block_response => bl)
|
185
|
+
end
|
177
186
|
end
|
178
187
|
|
179
188
|
# It seems like now response body are now decoded by Net::HTTP after 2.1
|
@@ -25,8 +25,13 @@ class RemoteStep < Step
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def cache_file
|
28
|
-
|
29
|
-
|
28
|
+
begin
|
29
|
+
digest = Misc.obj2digest([base_url, task.to_s, base_name, inputs])
|
30
|
+
Rbbt.var.cache.REST[task.to_s][[clean_name, digest].compact * "."].find
|
31
|
+
rescue
|
32
|
+
Log.exception $!
|
33
|
+
raise $!
|
34
|
+
end
|
30
35
|
end
|
31
36
|
|
32
37
|
def cache_files
|
@@ -62,7 +67,6 @@ class RemoteStep < Step
|
|
62
67
|
no_load ? Misc.add_GET_param(path, "_format", "raw") : @result
|
63
68
|
end
|
64
69
|
|
65
|
-
|
66
70
|
def self.get_streams(inputs, stream_input = nil)
|
67
71
|
new_inputs = {}
|
68
72
|
inputs.each do |k,v|
|
@@ -139,7 +143,11 @@ class RemoteStep < Step
|
|
139
143
|
@info = Persist.memory("RemoteSteps Info", :url => @url, :persist => true, :update => update) do
|
140
144
|
@last_info_time = Time.now
|
141
145
|
init_job unless @url
|
142
|
-
info =
|
146
|
+
info = begin
|
147
|
+
@adaptor.get_json(File.join(@url, 'info'))
|
148
|
+
rescue
|
149
|
+
{:status => :noinfo}
|
150
|
+
end
|
143
151
|
info = RemoteWorkflow.fix_hash(info)
|
144
152
|
info[:status] = info[:status].to_sym if String === info[:status]
|
145
153
|
info
|
@@ -239,7 +247,9 @@ class RemoteStep < Step
|
|
239
247
|
def join
|
240
248
|
return true if cache_files.any?
|
241
249
|
init_job unless @url
|
250
|
+
produce unless @started
|
242
251
|
Log.debug{ "Joining RemoteStep: #{path}" }
|
252
|
+
|
243
253
|
if IO === @result
|
244
254
|
res = @result
|
245
255
|
@result = nil
|
@@ -305,6 +315,10 @@ class RemoteStep < Step
|
|
305
315
|
end
|
306
316
|
end
|
307
317
|
|
318
|
+
def short_path
|
319
|
+
init_job unless @url
|
320
|
+
[@task, @name] * "/"
|
321
|
+
end
|
308
322
|
|
309
323
|
def input_checks
|
310
324
|
[]
|
@@ -129,7 +129,7 @@ class RemoteStep
|
|
129
129
|
end
|
130
130
|
end
|
131
131
|
|
132
|
-
def _run_job(cache_type = :
|
132
|
+
def _run_job(cache_type = :asynchronous)
|
133
133
|
get_streams
|
134
134
|
|
135
135
|
task_url = URI.encode(File.join(base_url, task.to_s))
|
@@ -142,7 +142,13 @@ class RemoteStep
|
|
142
142
|
else
|
143
143
|
@adaptor.execute_job(base_url, task, task_params, cache_type)
|
144
144
|
end
|
145
|
+
end
|
146
|
+
|
145
147
|
|
148
|
+
def produce(*args)
|
149
|
+
@started = true
|
150
|
+
_run_job
|
146
151
|
end
|
152
|
+
|
147
153
|
end
|
148
154
|
end
|
@@ -92,7 +92,7 @@ class Step
|
|
92
92
|
(job.done? && job.dirty?) || (job.error? && job.dirty?) ||
|
93
93
|
(!(job.noinfo? || job.done? || job.error? || job.aborted? || job.running?))
|
94
94
|
|
95
|
-
job.clean
|
95
|
+
job.clean unless job.resumable? && (job.updated? && ! job.dirty?)
|
96
96
|
job.set_info :status, :cleaned
|
97
97
|
end
|
98
98
|
|
@@ -144,14 +144,18 @@ class Step
|
|
144
144
|
|
145
145
|
dependency.status_lock.synchronize do
|
146
146
|
if dependency.aborted? || (dependency.error? && dependency.recoverable_error? && ! canfail_paths.include?(dependency.path) && ! already_failed.include?(dependency.path)) || (!Open.remote?(dependency.path) && dependency.missing?)
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
147
|
+
if dependency.resumable?
|
148
|
+
dependency.status = :resume
|
149
|
+
else
|
150
|
+
Log.warn "Cleaning dep. on exec #{Log.color :blue, dependency.path} (missing: #{dependency.missing?}; error #{dependency.error?})"
|
151
|
+
dependency.clean
|
152
|
+
already_failed << dependency.path
|
153
|
+
raise TryAgain
|
154
|
+
end
|
151
155
|
end
|
152
156
|
end
|
153
157
|
|
154
|
-
if ! (dependency.started? || dependency.error?)
|
158
|
+
if dependency.status == :resume || ! (dependency.started? || dependency.error?)
|
155
159
|
log_dependency_exec(dependency, :starting)
|
156
160
|
dependency.run(true)
|
157
161
|
raise TryAgain
|
@@ -362,7 +366,7 @@ class Step
|
|
362
366
|
(step.dependencies + step.input_dependencies).each do |step_dep|
|
363
367
|
next if step_dep.done? or step_dep.running? or (ComputeDependency === step_dep and (step_dep.compute == :nodup or step_dep.compute == :ignore))
|
364
368
|
dep_step[step_dep.path] ||= []
|
365
|
-
dep_step[step_dep.path] <<
|
369
|
+
dep_step[step_dep.path] << step
|
366
370
|
end
|
367
371
|
end
|
368
372
|
|
data/lib/rbbt/workflow/task.rb
CHANGED
@@ -2,7 +2,7 @@ require 'rbbt/util/misc'
|
|
2
2
|
require 'rbbt/persist'
|
3
3
|
|
4
4
|
module Task
|
5
|
-
attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow
|
5
|
+
attr_accessor :inputs, :input_types, :result_type, :input_defaults, :input_descriptions, :input_options, :required_inputs, :description, :name, :result_description, :extension, :workflow, :resumable
|
6
6
|
|
7
7
|
def self.setup(options = {}, &block)
|
8
8
|
block.extend Task
|
data/lib/rbbt/workflow/usage.rb
CHANGED
@@ -125,6 +125,7 @@ class Step
|
|
125
125
|
def self.migrate(path, search_path, options = {})
|
126
126
|
resource=Rbbt
|
127
127
|
|
128
|
+
orig_path = path
|
128
129
|
other_rsync_args = options[:rsync]
|
129
130
|
|
130
131
|
recursive = options[:recursive]
|
@@ -154,6 +155,7 @@ puts files * "\n"
|
|
154
155
|
else
|
155
156
|
if File.exists?(path)
|
156
157
|
path = resource.identify(path)
|
158
|
+
raise "Resource #{resource} could not identify #{orig_path}" if path.nil?
|
157
159
|
else
|
158
160
|
path = Path.setup(path)
|
159
161
|
end
|
@@ -178,7 +180,7 @@ puts resource[path].find(search_path)
|
|
178
180
|
subpath_files = {}
|
179
181
|
paths.sort.each do |path|
|
180
182
|
parts = path.split("/")
|
181
|
-
subpath = parts[0..-4] * "/"
|
183
|
+
subpath = parts[0..-4] * "/" + "/"
|
182
184
|
|
183
185
|
if subpath_files.keys.any? && subpath.start_with?(subpath_files.keys.last)
|
184
186
|
subpath = subpath_files.keys.last
|
@@ -190,6 +192,7 @@ puts resource[path].find(search_path)
|
|
190
192
|
subpath_files[subpath] << source
|
191
193
|
end
|
192
194
|
|
195
|
+
synced_files = []
|
193
196
|
subpath_files.each do |subpath, files|
|
194
197
|
if options[:target]
|
195
198
|
CMD.cmd("ssh #{options[:target]} mkdir -p '#{File.dirname(target)}'")
|
@@ -204,11 +207,15 @@ puts resource[path].find(search_path)
|
|
204
207
|
end
|
205
208
|
target = [options[:target], target] * ":" if options[:target]
|
206
209
|
|
210
|
+
next if File.exists?(source) && File.exists?(target) && File.expand_path(source) == File.expand_path(target)
|
211
|
+
|
207
212
|
files_and_dirs = Set.new( files )
|
208
213
|
files.each do |file|
|
214
|
+
synced_files << File.join(subpath, file)
|
215
|
+
|
209
216
|
parts = file.split("/")[0..-2].reject{|p| p.empty?}
|
210
217
|
while parts.any?
|
211
|
-
files_and_dirs <<
|
218
|
+
files_and_dirs << parts * "/"
|
212
219
|
parts.pop
|
213
220
|
end
|
214
221
|
end
|
@@ -218,8 +225,7 @@ puts resource[path].find(search_path)
|
|
218
225
|
|
219
226
|
cmd = "rsync #{MAIN_RSYNC_ARGS} --progress #{test_str} --files-from='#{tmp_include_file}' #{source}/ #{target}/ #{other_rsync_args}"
|
220
227
|
|
221
|
-
cmd << " && rm -Rf #{source}" if options[:delete]
|
222
|
-
|
228
|
+
#cmd << " && rm -Rf #{source}" if options[:delete]
|
223
229
|
if options[:print]
|
224
230
|
ppp Open.read(tmp_include_file)
|
225
231
|
puts cmd
|
@@ -228,6 +234,26 @@ puts resource[path].find(search_path)
|
|
228
234
|
end
|
229
235
|
end
|
230
236
|
end
|
237
|
+
|
238
|
+
if options[:delete] && synced_files.any?
|
239
|
+
puts Log.color :magenta, "About to erase these files:"
|
240
|
+
synced_files.each do |p|
|
241
|
+
puts Log.color :red, p
|
242
|
+
end
|
243
|
+
|
244
|
+
if options[:non_interactive]
|
245
|
+
response = 'yes'
|
246
|
+
else
|
247
|
+
puts Log.color :magenta, "Type 'yes' if you are sure:"
|
248
|
+
response = STDIN.gets.chomp
|
249
|
+
end
|
250
|
+
|
251
|
+
if response == 'yes'
|
252
|
+
synced_files.each do |p|
|
253
|
+
Open.rm p
|
254
|
+
end
|
255
|
+
end
|
256
|
+
end
|
231
257
|
end
|
232
258
|
|
233
259
|
def self.purge(path, recursive = false)
|
@@ -97,7 +97,7 @@ TSV.traverse jobs do |file,i|
|
|
97
97
|
else
|
98
98
|
info = begin
|
99
99
|
Open.open(i[:info_file]) do |f|
|
100
|
-
Step::
|
100
|
+
Step::INFO_SERIALIZER.load(f)
|
101
101
|
end
|
102
102
|
rescue
|
103
103
|
{:status => :noinfo}
|
@@ -134,7 +134,7 @@ TSV.traverse jobs do |file,i|
|
|
134
134
|
end
|
135
135
|
end
|
136
136
|
|
137
|
-
if (force
|
137
|
+
if (force && status !~ /done/) or
|
138
138
|
status =~ /\b(old|dirty|nopid|error|missing|aborted|dead|sync)$/ or
|
139
139
|
(status == "noinfo" and not done) or
|
140
140
|
status == ""
|
@@ -187,7 +187,7 @@ workflows.sort.each do |workflow,tasks|
|
|
187
187
|
if info_fields and info_fields.any?
|
188
188
|
info = begin
|
189
189
|
Open.open(i[:info_file]) do |f|
|
190
|
-
Step::
|
190
|
+
Step::INFO_SERIALIZER.load(f)
|
191
191
|
end
|
192
192
|
rescue
|
193
193
|
{:status => :noinfo}
|
@@ -198,9 +198,10 @@ workflows.sort.each do |workflow,tasks|
|
|
198
198
|
else
|
199
199
|
info = begin
|
200
200
|
Open.open(i[:info_file]) do |f|
|
201
|
-
Step::
|
201
|
+
Step::INFO_SERIALIZER.load(f)
|
202
202
|
end
|
203
203
|
rescue
|
204
|
+
Log.exception $!
|
204
205
|
{:status => :noinfo}
|
205
206
|
end
|
206
207
|
IndiferentHash.setup(info)
|
@@ -0,0 +1,29 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'rbbt-util'
|
4
|
+
require 'rbbt/util/simpleopt'
|
5
|
+
|
6
|
+
options = SOPT.setup <<EOF
|
7
|
+
Read a TSV file and dump it
|
8
|
+
|
9
|
+
$ rbbt tsv read [options] <filename.tsv|->
|
10
|
+
|
11
|
+
Use - to read from STDIN
|
12
|
+
|
13
|
+
-k--key_field* Key field
|
14
|
+
-f--fields* Fields
|
15
|
+
-t--type* Type
|
16
|
+
-m--merge* Merge from multiple rows
|
17
|
+
-h--help Print this help
|
18
|
+
|
19
|
+
EOF
|
20
|
+
rbbt_usage and exit 0 if options[:help]
|
21
|
+
|
22
|
+
file = ARGV.shift
|
23
|
+
|
24
|
+
file = STDIN if file == '-'
|
25
|
+
|
26
|
+
tsv = TSV.open file, :merge => true, :type => :double
|
27
|
+
|
28
|
+
|
29
|
+
puts tsv.to_unmerged_expanded_s
|
@@ -153,7 +153,7 @@ if recursive
|
|
153
153
|
dep = deps.shift
|
154
154
|
inputs = {} if inputs.nil?
|
155
155
|
inputs = inputs.merge(dep.info[:inputs] || {})
|
156
|
-
deps = deps.concat dep.dependencies
|
156
|
+
deps = deps.concat dep.dependencies if dep.dependencies
|
157
157
|
end
|
158
158
|
|
159
159
|
inputs = inputs.merge step.archived_inputs
|
@@ -41,7 +41,7 @@ def list_jobs(options)
|
|
41
41
|
clean_file = file.sub('.info','')
|
42
42
|
begin
|
43
43
|
next if File.exist? clean_file and $quick
|
44
|
-
info = Step::
|
44
|
+
info = Step::INFO_SERIALIZER.load(Open.read(file, :mode => 'rb'))
|
45
45
|
next if (File.exist?(clean_file) or info[:status] == :done) and (info[:children_pids].nil? or info[:children_done] or info[:children_pids].select{|pid| Misc.pid_exists? pid}.empty?)
|
46
46
|
rescue Exception
|
47
47
|
puts "Error parsing info file: #{ file }"
|
@@ -86,7 +86,7 @@ def clean_jobs(options)
|
|
86
86
|
info = nil
|
87
87
|
next if File.exist? clean_file
|
88
88
|
begin
|
89
|
-
info = Step::
|
89
|
+
info = Step::INFO_SERIALIZER.load(Open.read(file, :mode => 'rb'))
|
90
90
|
rescue Exception
|
91
91
|
Log.debug "Error process #{ file }"
|
92
92
|
remove_job file if options[:errors]
|
@@ -117,7 +117,7 @@ begin
|
|
117
117
|
end
|
118
118
|
else
|
119
119
|
if options[:file]
|
120
|
-
info = Step::
|
120
|
+
info = Step::INFO_SERIALIZER.load(Open.read(options[:file], :mode => 'rb'))
|
121
121
|
print_job options[:file], info
|
122
122
|
else
|
123
123
|
list_jobs options
|
@@ -213,8 +213,9 @@ row2 E
|
|
213
213
|
|
214
214
|
tsv1 = tsv2 = nil
|
215
215
|
|
216
|
-
tsv1 = Rbbt.tmp.test.test1.data.tsv :double, :sep => /\s+/
|
217
|
-
tsv2 = Rbbt.tmp.test.test2.data.tsv :double, :sep => /\s+/
|
216
|
+
tsv1 = Rbbt.tmp.test.test1.data.produce(true).tsv :double, :sep => /\s+/
|
217
|
+
tsv2 = Rbbt.tmp.test.test2.data.produce(true).tsv :double, :sep => /\s+/
|
218
|
+
Log.tsv tsv2
|
218
219
|
|
219
220
|
tsv2.identifiers = Rbbt.tmp.test.test2.identifiers.produce.find #.to_s
|
220
221
|
|
@@ -4,6 +4,7 @@ require 'rbbt/tsv/stream'
|
|
4
4
|
require 'rbbt'
|
5
5
|
|
6
6
|
class TestStream < Test::Unit::TestCase
|
7
|
+
|
7
8
|
def test_collapse_stream
|
8
9
|
text=<<-EOF
|
9
10
|
#: :sep=" "
|
@@ -65,7 +66,7 @@ row3 AAA BBB CCC
|
|
65
66
|
|
66
67
|
text2=<<-EOF
|
67
68
|
#: :sep=" "
|
68
|
-
#Row Labela Labelb
|
69
|
+
#Row Labela Labelb
|
69
70
|
row1 a b
|
70
71
|
row3 aaa bbb
|
71
72
|
row2 aa bb
|
@@ -100,7 +101,7 @@ row1 A B C
|
|
100
101
|
|
101
102
|
text2=<<-EOF
|
102
103
|
#: :sep=" "
|
103
|
-
#Row Labela Labelb
|
104
|
+
#Row Labela Labelb
|
104
105
|
row2 aa bb
|
105
106
|
EOF
|
106
107
|
|
@@ -132,7 +133,7 @@ row1 A B C
|
|
132
133
|
|
133
134
|
text2=<<-EOF
|
134
135
|
#: :sep=" "
|
135
|
-
#Row Labela Labelb
|
136
|
+
#Row Labela Labelb
|
136
137
|
row2 aa bb
|
137
138
|
EOF
|
138
139
|
|
@@ -4,6 +4,27 @@ require 'rbbt/workflow'
|
|
4
4
|
module DepWorkflow
|
5
5
|
extend Workflow
|
6
6
|
|
7
|
+
input :input_file, :file, "Input file", nil, :stream => true
|
8
|
+
task :s1 => :array do |input_file|
|
9
|
+
TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
|
10
|
+
line + "\t" << "Task1"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
dep :s1
|
15
|
+
task :s2 => :array do |input_file|
|
16
|
+
TSV.traverse step(:s1), :type => :array, :into => :stream, :bar => "Task2" do |line|
|
17
|
+
next [line.split("\t").first, Misc::SKIP_TAG] * "\t" if rand < 0.9
|
18
|
+
line + "\t" << "Task2"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
dep :s1
|
23
|
+
dep :s2
|
24
|
+
task :s3 => :array do |input_file|
|
25
|
+
Misc.paste_streams(dependencies.reverse)
|
26
|
+
end
|
27
|
+
|
7
28
|
input :input_file, :file, "Input file", nil, :stream => true
|
8
29
|
task :task1 => :array do |input_file|
|
9
30
|
TSV.traverse input_file, :type => :array, :into => :stream, :bar => "Task1" do |line|
|
@@ -28,9 +49,7 @@ module DepWorkflow
|
|
28
49
|
dep :task2
|
29
50
|
dep :task3
|
30
51
|
task :task4 => :array do
|
31
|
-
|
32
|
-
s2 = TSV.get_stream step(:task3)
|
33
|
-
Misc.paste_streams([s1, s2])
|
52
|
+
Misc.paste_streams(dependencies)
|
34
53
|
end
|
35
54
|
|
36
55
|
dep :task4
|
@@ -43,9 +62,7 @@ module DepWorkflow
|
|
43
62
|
dep :task2
|
44
63
|
dep :task5
|
45
64
|
task :task6 => :array do
|
46
|
-
|
47
|
-
s2 = TSV.get_stream step(:task5)
|
48
|
-
Misc.paste_streams([s1, s2])
|
65
|
+
Misc.paste_streams(dependencies)
|
49
66
|
end
|
50
67
|
|
51
68
|
input :stream_file, :file, "Streamed file", nil, :stream => true
|
@@ -81,6 +98,26 @@ module ComputeWorkflow
|
|
81
98
|
|
82
99
|
end
|
83
100
|
|
101
|
+
module ResumeWorkflow
|
102
|
+
extend Workflow
|
103
|
+
|
104
|
+
resumable
|
105
|
+
task :resume => :string do
|
106
|
+
if file('foo').exists?
|
107
|
+
'done'
|
108
|
+
else
|
109
|
+
Open.mkdir files_dir
|
110
|
+
Open.touch(file('foo'))
|
111
|
+
raise
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
dep :resume
|
116
|
+
task :reverse => :string do
|
117
|
+
step(:resume).load.reverse
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
84
121
|
class TestWorkflowDependency < Test::Unit::TestCase
|
85
122
|
def test_task1
|
86
123
|
size = 10000
|
@@ -114,8 +151,25 @@ class TestWorkflowDependency < Test::Unit::TestCase
|
|
114
151
|
end
|
115
152
|
end
|
116
153
|
|
154
|
+
def test_task3
|
155
|
+
size = 100000
|
156
|
+
content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
|
157
|
+
TmpFile.with_file(content) do |input_file|
|
158
|
+
job = DepWorkflow.job(:task3, "TEST", :input_file => input_file)
|
159
|
+
io = TSV.get_stream job.run(:stream)
|
160
|
+
last_line = nil
|
161
|
+
while line = io.gets
|
162
|
+
last_line = line.strip
|
163
|
+
end
|
164
|
+
io.join
|
165
|
+
|
166
|
+
assert_equal "Line #{size}\tTask1\tTask3", last_line
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
117
170
|
def test_task4
|
118
|
-
size =
|
171
|
+
size = 100000
|
172
|
+
Log.severity = 0
|
119
173
|
content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
|
120
174
|
last_line = nil
|
121
175
|
TmpFile.with_file(content) do |input_file|
|
@@ -131,7 +185,7 @@ class TestWorkflowDependency < Test::Unit::TestCase
|
|
131
185
|
end
|
132
186
|
|
133
187
|
def test_task5
|
134
|
-
size =
|
188
|
+
size = 10000
|
135
189
|
content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
|
136
190
|
last_line = nil
|
137
191
|
TmpFile.with_file(content) do |input_file|
|
@@ -145,20 +199,44 @@ class TestWorkflowDependency < Test::Unit::TestCase
|
|
145
199
|
assert_equal "Line #{size}\tTask1\tTask2\tTask1\tTask3\tTask5", last_line
|
146
200
|
end
|
147
201
|
|
148
|
-
def
|
202
|
+
def test_s3
|
149
203
|
size = 100000
|
150
|
-
content = (
|
204
|
+
content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
|
151
205
|
last_line = nil
|
152
206
|
Log.severity = 0
|
153
207
|
TmpFile.with_file(content) do |input_file|
|
154
208
|
begin
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
209
|
+
job = DepWorkflow.job(:s3, "TEST", :input_file => input_file)
|
210
|
+
job.recursive_clean
|
211
|
+
job.run(:stream)
|
212
|
+
io = TSV.get_stream job
|
213
|
+
while line = io.gets
|
214
|
+
last_line = line.strip
|
215
|
+
end
|
216
|
+
io.join if io.respond_to? :join
|
217
|
+
rescue Exception
|
218
|
+
job.abort
|
219
|
+
raise $!
|
160
220
|
end
|
161
|
-
|
221
|
+
end
|
222
|
+
assert last_line.include? "Line #{size}"
|
223
|
+
end
|
224
|
+
|
225
|
+
def test_task6
|
226
|
+
size = 100000
|
227
|
+
content = (1..size).to_a.collect{|num| "Line #{num}" } * "\n"
|
228
|
+
last_line = nil
|
229
|
+
Log.severity = 0
|
230
|
+
TmpFile.with_file(content) do |input_file|
|
231
|
+
begin
|
232
|
+
job = DepWorkflow.job(:task6, "TEST", :input_file => input_file)
|
233
|
+
job.recursive_clean
|
234
|
+
job.run(:stream)
|
235
|
+
io = TSV.get_stream job
|
236
|
+
while line = io.gets
|
237
|
+
last_line = line.strip
|
238
|
+
end
|
239
|
+
io.join
|
162
240
|
rescue Exception
|
163
241
|
job.abort
|
164
242
|
raise $!
|
@@ -168,7 +246,7 @@ class TestWorkflowDependency < Test::Unit::TestCase
|
|
168
246
|
end
|
169
247
|
|
170
248
|
def test_task8
|
171
|
-
size =
|
249
|
+
size = 10000
|
172
250
|
content = (0..size).to_a.collect{|num| "Line #{num}" } * "\n"
|
173
251
|
last_line = nil
|
174
252
|
Log.severity = 0
|
@@ -204,5 +282,16 @@ class TestWorkflowDependency < Test::Unit::TestCase
|
|
204
282
|
assert_equal "Line #{size}\tTask1\tTask2", last_line
|
205
283
|
end
|
206
284
|
end
|
285
|
+
|
286
|
+
def test_resume
|
287
|
+
Log.severity = 0
|
288
|
+
job = ResumeWorkflow.job(:reverse)
|
289
|
+
job.recursive_clean
|
290
|
+
assert_raise do
|
291
|
+
job.run
|
292
|
+
end
|
293
|
+
assert job.dependencies.first.file('foo').exists?
|
294
|
+
assert_equal 'done'.reverse, job.run
|
295
|
+
end
|
207
296
|
end
|
208
297
|
|
@@ -61,6 +61,18 @@ class TestRemoteWorkflow < Test::Unit::TestCase
|
|
61
61
|
def test_rest
|
62
62
|
Log.with_severity 0 do
|
63
63
|
|
64
|
+
remote_workflow_server(TestWFRest) do |client|
|
65
|
+
job = client.job(:hi, nil, {})
|
66
|
+
job.clean
|
67
|
+
job = client.job(:hi, nil, {})
|
68
|
+
assert ! job.done?
|
69
|
+
job.run
|
70
|
+
job.produce
|
71
|
+
job = client.job(:hi, nil, {})
|
72
|
+
assert job.done?
|
73
|
+
sleep 1
|
74
|
+
end
|
75
|
+
|
64
76
|
remote_workflow_server(TestWFRest) do |client|
|
65
77
|
assert_equal "Hello World", client.job(:hi, nil, {}).run.chomp
|
66
78
|
assert_equal "Hello Miguel", client.job(:hi, nil, {:name => :Miguel}).run.chomp
|
@@ -78,7 +90,7 @@ class TestRemoteWorkflow < Test::Unit::TestCase
|
|
78
90
|
|
79
91
|
def _test_ssh
|
80
92
|
Log.severity = 0
|
81
|
-
client = RemoteWorkflow.new "ssh
|
93
|
+
client = RemoteWorkflow.new "ssh://#{ENV["HOSTNAME"]}:Translation", "Translation"
|
82
94
|
job = client.job("translate", "SSH-TEST-1", :genes => ["TP53","KRAS"])
|
83
95
|
assert_equal 2, job.run.select{|l| l =~ /ENSG/}.length
|
84
96
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.27.
|
4
|
+
version: 5.27.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-07-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -403,6 +403,7 @@ files:
|
|
403
403
|
- share/rbbt_commands/tsv/sort
|
404
404
|
- share/rbbt_commands/tsv/subset
|
405
405
|
- share/rbbt_commands/tsv/transpose
|
406
|
+
- share/rbbt_commands/tsv/uncollapse
|
406
407
|
- share/rbbt_commands/tsv/unzip
|
407
408
|
- share/rbbt_commands/tsv/values
|
408
409
|
- share/rbbt_commands/tsv/write_excel
|
@@ -487,6 +488,7 @@ files:
|
|
487
488
|
- test/rbbt/util/concurrency/test_threads.rb
|
488
489
|
- test/rbbt/util/log/test_progress.rb
|
489
490
|
- test/rbbt/util/misc/test_bgzf.rb
|
491
|
+
- test/rbbt/util/misc/test_format.rb
|
490
492
|
- test/rbbt/util/misc/test_lock.rb
|
491
493
|
- test/rbbt/util/misc/test_multipart_payload.rb
|
492
494
|
- test/rbbt/util/misc/test_omics.rb
|
@@ -552,6 +554,7 @@ test_files:
|
|
552
554
|
- test/rbbt/util/test_colorize.rb
|
553
555
|
- test/rbbt/util/misc/test_omics.rb
|
554
556
|
- test/rbbt/util/misc/test_pipes.rb
|
557
|
+
- test/rbbt/util/misc/test_format.rb
|
555
558
|
- test/rbbt/util/misc/test_lock.rb
|
556
559
|
- test/rbbt/util/misc/test_multipart_payload.rb
|
557
560
|
- test/rbbt/util/misc/test_bgzf.rb
|