rbbt-util 5.20.22 → 5.20.23
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/rbbt/persist.rb +7 -2
- data/lib/rbbt/tsv/parallel/traverse.rb +50 -48
- data/lib/rbbt/tsv/util.rb +17 -0
- data/lib/rbbt/util/cmd.rb +2 -76
- data/lib/rbbt/util/log/progress.rb +1 -0
- data/lib/rbbt/util/misc/concurrent_stream.rb +59 -25
- data/lib/rbbt/util/misc/pipes.rb +23 -8
- data/lib/rbbt/util/open.rb +3 -3
- data/lib/rbbt/workflow/accessor.rb +3 -2
- data/lib/rbbt/workflow/step/dependencies.rb +6 -3
- data/lib/rbbt/workflow/step/run.rb +44 -15
- metadata +159 -181
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: f2f08ffb6c3f5394b0349460d1c6bd9232bcaff2
|
4
|
+
data.tar.gz: 32688697499d69a24888527b903ebedebed94245
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 7edf42547f74103bf6df9abbb50f7371488adb0d4e4103e70983f791d0278b8cb4a8cfa8f46fcb12436705ac8ceffa5dd3a4454508fa5f06593f3a49f326ab7d
|
7
|
+
data.tar.gz: 31318b7facb960e7a0c6cd149afb3534674d424f9103afbf7b00452ec86a1c3e891a2bbf4c7da3303e20b14b229d4c07f85405757a3dc813beb024b9933888b2
|
data/lib/rbbt/persist.rb
CHANGED
@@ -217,9 +217,14 @@ module Persist
|
|
217
217
|
end
|
218
218
|
end
|
219
219
|
ConcurrentStream.setup(out, :threads => saver_thread, :filename => path)
|
220
|
+
|
220
221
|
out.callback = callback
|
221
222
|
out.abort_callback = abort_callback
|
222
223
|
out.lockfile = stream.lockfile if stream.respond_to? :lockfile and stream.lockfile
|
224
|
+
|
225
|
+
stream.callback = callback
|
226
|
+
stream.abort_callback = abort_callback
|
227
|
+
|
223
228
|
out
|
224
229
|
end
|
225
230
|
|
@@ -239,8 +244,8 @@ module Persist
|
|
239
244
|
|
240
245
|
# This is to avoid calling the callbacks twice, since they have been
|
241
246
|
# moved to the new 'res' stream
|
242
|
-
stream.callback = nil
|
243
|
-
stream.abort_callback = nil
|
247
|
+
#stream.callback = nil
|
248
|
+
#stream.abort_callback = nil
|
244
249
|
|
245
250
|
res = tee_stream(stream, path, type, callback, abort_callback, lockfile)
|
246
251
|
|
@@ -76,6 +76,9 @@ module TSV
|
|
76
76
|
tsv.through options[:key_field], options[:fields] do |k,v|
|
77
77
|
begin
|
78
78
|
callback.call yield(k,v)
|
79
|
+
rescue Exception
|
80
|
+
Log.exception $!
|
81
|
+
raise $!
|
79
82
|
ensure
|
80
83
|
bar.tick if bar
|
81
84
|
end
|
@@ -202,12 +205,19 @@ module TSV
|
|
202
205
|
|
203
206
|
if callback
|
204
207
|
bar.init if bar
|
208
|
+
exception = nil
|
209
|
+
begin
|
205
210
|
TSV::Parser.traverse(io, options) do |k,v|
|
206
211
|
begin
|
207
212
|
callback.call yield k, v
|
208
|
-
|
209
|
-
|
213
|
+
rescue Exception
|
214
|
+
exception = $!
|
215
|
+
raise $!
|
210
216
|
end
|
217
|
+
bar.tick if bar
|
218
|
+
end
|
219
|
+
ensure
|
220
|
+
raise exception if exception
|
211
221
|
end
|
212
222
|
else
|
213
223
|
options[:monitor] = bar
|
@@ -291,36 +301,24 @@ module TSV
|
|
291
301
|
end
|
292
302
|
rescue IOError
|
293
303
|
Log.low{"IOError traversing #{stream_name(obj)}: #{$!.message}"}
|
294
|
-
|
295
|
-
|
296
|
-
stream = obj_stream(options[:into])
|
297
|
-
stream.abort if stream.respond_to? :abort
|
304
|
+
abort_stream obj
|
305
|
+
abort_stream options[:into], $!
|
298
306
|
raise $!
|
299
307
|
rescue Errno::EPIPE
|
300
308
|
Log.low{"Pipe closed while traversing #{stream_name(obj)}: #{$!.message}"}
|
301
|
-
|
302
|
-
|
303
|
-
stream = obj_stream(options[:into])
|
304
|
-
stream.abort if stream.respond_to? :abort
|
309
|
+
abort_stream obj
|
310
|
+
abort_stream options[:into], $!
|
305
311
|
raise $!
|
306
312
|
rescue Aborted
|
307
313
|
Log.low{"Aborted traversing #{stream_name(obj)}"}
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
stream.abort if stream.respond_to? :abort
|
312
|
-
Log.low{"Aborted traversing 2 #{stream_name(obj)}"}
|
314
|
+
abort_stream obj
|
315
|
+
abort_stream options[:into], $!
|
316
|
+
raise $!
|
313
317
|
rescue Exception
|
314
318
|
Log.low{"Exception traversing #{stream_name(obj)}"}
|
315
|
-
|
316
|
-
|
317
|
-
|
318
|
-
stream = obj_stream(options[:into])
|
319
|
-
stream.abort if stream.respond_to? :abort
|
320
|
-
rescue Exception
|
321
|
-
ensure
|
322
|
-
raise $!
|
323
|
-
end
|
319
|
+
abort_stream obj
|
320
|
+
abort_stream options[:into], $!
|
321
|
+
raise $!
|
324
322
|
end
|
325
323
|
end
|
326
324
|
|
@@ -437,12 +435,11 @@ module TSV
|
|
437
435
|
true
|
438
436
|
rescue Aborted, Interrupt
|
439
437
|
Log.low "Aborted storing into #{Misc.fingerprint store}"
|
440
|
-
|
441
|
-
|
438
|
+
abort_stream(store, $!)
|
439
|
+
raise $!
|
442
440
|
rescue Exception
|
443
441
|
Log.low "Exception storing into #{Misc.fingerprint store}: #{$!.message}"
|
444
|
-
|
445
|
-
stream.abort if stream.respond_to? :abort
|
442
|
+
abort_stream(store, $!)
|
446
443
|
raise $!
|
447
444
|
end
|
448
445
|
end
|
@@ -503,20 +500,19 @@ module TSV
|
|
503
500
|
end
|
504
501
|
|
505
502
|
def self.traverse_stream(obj, threads = nil, cpus = nil, options = {}, &block)
|
506
|
-
into = options[:into]
|
503
|
+
into = options[:into]
|
504
|
+
|
507
505
|
thread = Thread.new(Thread.current) do |parent|
|
508
506
|
begin
|
509
507
|
traverse_run(obj, threads, cpus, options, &block)
|
510
|
-
into.close if into.respond_to?(:close) and not (into.respond_to? :closed? and into.closed?)
|
508
|
+
into.close if into.respond_to?(:close) and not (into.respond_to? :closed? and into.closed?)
|
511
509
|
rescue Exception
|
512
|
-
|
513
|
-
stream.abort if stream and stream.respond_to? :abort
|
514
|
-
stream = obj_stream(into)
|
515
|
-
stream.abort if stream and stream.respond_to? :abort
|
510
|
+
abort_stream obj
|
516
511
|
parent.raise $!
|
517
512
|
raise $!
|
518
513
|
end
|
519
514
|
end
|
515
|
+
|
520
516
|
ConcurrentStream.setup(obj_stream(into), :threads => thread)
|
521
517
|
end
|
522
518
|
|
@@ -529,8 +525,13 @@ module TSV
|
|
529
525
|
begin
|
530
526
|
traverse(obj, options.merge(:into => sin), &block)
|
531
527
|
rescue Exception
|
532
|
-
|
533
|
-
|
528
|
+
Log.exception $!
|
529
|
+
begin
|
530
|
+
sout.abort if sout.respond_to? :abort
|
531
|
+
sout.join if sout.respond_to? :join
|
532
|
+
ensure
|
533
|
+
raise $!
|
534
|
+
end
|
534
535
|
end
|
535
536
|
end
|
536
537
|
return sout
|
@@ -598,15 +599,11 @@ module TSV
|
|
598
599
|
store_into into, e
|
599
600
|
rescue Aborted
|
600
601
|
Log.low "Aborted callback #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
|
601
|
-
|
602
|
-
stream = get_stream obj
|
603
|
-
stream.abort if stream.respond_to? :abort
|
602
|
+
abort_stream(into, $!)
|
604
603
|
raise $!
|
605
604
|
rescue Exception
|
606
605
|
Log.low "Exception callback #{stream_name(obj)} #{Log.color :green, "->"} #{stream_name(options[:into])}"
|
607
|
-
|
608
|
-
stream = get_stream obj
|
609
|
-
stream.abort if stream.respond_to? :abort
|
606
|
+
abort_stream(into, $!)
|
610
607
|
raise $!
|
611
608
|
ensure
|
612
609
|
bar.tick if bar
|
@@ -614,12 +611,17 @@ module TSV
|
|
614
611
|
end
|
615
612
|
|
616
613
|
bar.init if bar
|
617
|
-
|
618
|
-
|
619
|
-
|
620
|
-
|
621
|
-
|
622
|
-
|
614
|
+
begin
|
615
|
+
case into
|
616
|
+
when TSV::Dumper, IO
|
617
|
+
traverse_stream(obj, threads, cpus, options, &block)
|
618
|
+
else
|
619
|
+
traverse_run(obj, threads, cpus, options, &block)
|
620
|
+
end
|
621
|
+
rescue Exception
|
622
|
+
Log.exception $!
|
623
|
+
abort_stream(into, $!)
|
624
|
+
raise $!
|
623
625
|
end
|
624
626
|
|
625
627
|
into
|
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -58,6 +58,23 @@ module TSV
|
|
58
58
|
end
|
59
59
|
end
|
60
60
|
|
61
|
+
def self.abort_stream(file, exception = nil)
|
62
|
+
return if file.nil?
|
63
|
+
if Step === file
|
64
|
+
if exception
|
65
|
+
file.exception exception
|
66
|
+
else
|
67
|
+
if not (file.aborted? or file.done?)
|
68
|
+
file.abort
|
69
|
+
end
|
70
|
+
end
|
71
|
+
else
|
72
|
+
stream = get_stream(file)
|
73
|
+
stream.abort(exception) if stream.respond_to? :abort
|
74
|
+
stream.extend AbortedStream
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
61
78
|
def self.get_stream(file, open_options = {})
|
62
79
|
case file
|
63
80
|
when Zlib::GzipReader
|
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -3,80 +3,6 @@ require 'stringio'
|
|
3
3
|
|
4
4
|
module CMD
|
5
5
|
|
6
|
-
#module SmartIO
|
7
|
-
# attr_accessor :pid, :cmd, :post, :in, :out, :err, :log
|
8
|
-
# def self.tie(io, pid = nil, cmd = "", post = nil, sin = nil, out = nil, err = nil, log = true)
|
9
|
-
# io.extend SmartIO
|
10
|
-
# io.pid = pid
|
11
|
-
# io.cmd = cmd
|
12
|
-
# io.in = sin
|
13
|
-
# io.out = out
|
14
|
-
# io.err = err
|
15
|
-
# io.post = post
|
16
|
-
# io.log = log
|
17
|
-
|
18
|
-
# io.class.send(:alias_method, :original_close, :close)
|
19
|
-
# io.class.send(:alias_method, :original_read, :read)
|
20
|
-
# io
|
21
|
-
# end
|
22
|
-
|
23
|
-
# def wait_and_status
|
24
|
-
# if @pid
|
25
|
-
# begin
|
26
|
-
# Process.waitpid(@pid)
|
27
|
-
# rescue
|
28
|
-
# end
|
29
|
-
|
30
|
-
# Log.debug{"Process #{ cmd } succeded" if $? and $?.success? and log}
|
31
|
-
|
32
|
-
# if $? and not $?.success?
|
33
|
-
# Log.debug{"Raising exception" if log}
|
34
|
-
# exception = ProcessFailed.new "Command [#{@pid}] '#{@cmd}' failed with error status #{$?.exitstatus}"
|
35
|
-
# begin
|
36
|
-
# original_close
|
37
|
-
# ensure
|
38
|
-
# raise exception
|
39
|
-
# end
|
40
|
-
# end
|
41
|
-
# end
|
42
|
-
# end
|
43
|
-
|
44
|
-
# def close
|
45
|
-
# self.original_read unless self.closed? or self.eof?
|
46
|
-
|
47
|
-
# wait_and_status
|
48
|
-
|
49
|
-
# @post.call if @post
|
50
|
-
|
51
|
-
# original_close unless self.closed?
|
52
|
-
# end
|
53
|
-
|
54
|
-
# def force_close
|
55
|
-
# if @pid
|
56
|
-
# Log.debug{"Forcing close by killing '#{@pid}'" if log}
|
57
|
-
# begin
|
58
|
-
# Process.kill("KILL", @pid)
|
59
|
-
# Process.waitpid(@pid)
|
60
|
-
# rescue
|
61
|
-
# Log.low{"Exception in forcing close of command [#{ @pid }, #{cmd}]: #{$!.message}"}
|
62
|
-
# end
|
63
|
-
# end
|
64
|
-
|
65
|
-
# @post.call if @post
|
66
|
-
|
67
|
-
# original_close unless self.closed?
|
68
|
-
# end
|
69
|
-
|
70
|
-
# def read(*args)
|
71
|
-
# data = original_read(*args) unless self.closed? #or self.eof?
|
72
|
-
|
73
|
-
# self.close if not self.closed? and self.eof?
|
74
|
-
|
75
|
-
# data #|| ""
|
76
|
-
# end
|
77
|
-
#end
|
78
|
-
|
79
|
-
|
80
6
|
def self.process_cmd_options(options = {})
|
81
7
|
string = ""
|
82
8
|
options.each do |option, value|
|
@@ -201,7 +127,7 @@ module CMD
|
|
201
127
|
end
|
202
128
|
|
203
129
|
#SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
|
204
|
-
ConcurrentStream.setup sout, :pids => [pid], :autojoin =>
|
130
|
+
ConcurrentStream.setup sout, :pids => [pid], :autojoin => no_wait, :no_fail => no_fail
|
205
131
|
|
206
132
|
sout
|
207
133
|
else
|
@@ -213,7 +139,7 @@ module CMD
|
|
213
139
|
serr.close
|
214
140
|
end
|
215
141
|
|
216
|
-
ConcurrentStream.setup sout, :pids => [pid], :autojoin =>
|
142
|
+
ConcurrentStream.setup sout, :pids => [pid], :autojoin => no_wait, :no_fail => no_fail
|
217
143
|
out = StringIO.new sout.read
|
218
144
|
sout.close unless sout.closed?
|
219
145
|
|
@@ -1,8 +1,17 @@
|
|
1
|
+
module AbortedStream
|
2
|
+
attr_accessor :exception
|
3
|
+
def self.setup(obj, exception = nil)
|
4
|
+
obj.extend AbortedStream
|
5
|
+
obj.exception = exception
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
1
9
|
module ConcurrentStream
|
2
|
-
attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail
|
10
|
+
attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :aborted, :autojoin, :lockfile, :no_fail, :pair, :thread
|
3
11
|
|
4
12
|
def self.setup(stream, options = {}, &block)
|
5
|
-
|
13
|
+
|
14
|
+
threads, pids, callback, abort_callback, filename, autojoin, lockfile, no_fail, pair = Misc.process_options options, :threads, :pids, :callback, :abort_callback, :filename, :autojoin, :lockfile, :no_fail, :pair
|
6
15
|
stream.extend ConcurrentStream unless ConcurrentStream === stream
|
7
16
|
|
8
17
|
stream.threads ||= []
|
@@ -12,21 +21,39 @@ module ConcurrentStream
|
|
12
21
|
stream.autojoin = autojoin
|
13
22
|
stream.no_fail = no_fail
|
14
23
|
|
24
|
+
stream.pair = pair if pair
|
25
|
+
|
15
26
|
callback = block if block_given?
|
16
|
-
if
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
27
|
+
if callback
|
28
|
+
if stream.callback
|
29
|
+
old_callback = stream.callback
|
30
|
+
stream.callback = Proc.new do
|
31
|
+
old_callback.call
|
32
|
+
callback.call
|
33
|
+
end
|
34
|
+
else
|
35
|
+
stream.callback = callback
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
if abort_callback
|
40
|
+
if stream.abort_callback
|
41
|
+
old_abort_callback = stream.abort_callback
|
42
|
+
stream.abort_callback = Proc.new do
|
43
|
+
old_abort_callback.call
|
44
|
+
abort_callback.call
|
45
|
+
end
|
46
|
+
else
|
47
|
+
stream.abort_callback = abort_callback
|
21
48
|
end
|
22
|
-
else
|
23
|
-
stream.callback = callback
|
24
49
|
end
|
25
50
|
|
26
51
|
stream.filename = filename unless filename.nil?
|
27
52
|
|
28
53
|
stream.lockfile = lockfile if lockfile
|
29
54
|
|
55
|
+
stream.aborted = false
|
56
|
+
|
30
57
|
stream
|
31
58
|
end
|
32
59
|
|
@@ -52,7 +79,6 @@ module ConcurrentStream
|
|
52
79
|
@threads.each do |t|
|
53
80
|
next if t == Thread.current
|
54
81
|
begin
|
55
|
-
#t.join if t.status == 'run'
|
56
82
|
t.join unless FalseClass === t.status
|
57
83
|
rescue Exception
|
58
84
|
Log.warn "Exception joining thread in ConcurrenStream: #{filename}"
|
@@ -97,12 +123,11 @@ module ConcurrentStream
|
|
97
123
|
end
|
98
124
|
|
99
125
|
def abort_threads(exception)
|
100
|
-
Log.
|
126
|
+
Log.low "Aborting threads (#{Thread.current.inspect}) #{@threads.collect{|t| t.inspect } * ", "}"
|
101
127
|
|
102
128
|
@threads.each do |t|
|
103
|
-
@aborted = false if t == Thread.current
|
104
129
|
next if t == Thread.current
|
105
|
-
Log.
|
130
|
+
Log.low "Aborting thread #{t.inspect}"
|
106
131
|
t.raise exception ? exception : Aborted.new
|
107
132
|
end if @threads
|
108
133
|
|
@@ -112,18 +137,17 @@ module ConcurrentStream
|
|
112
137
|
if t.alive?
|
113
138
|
sleep 1 unless sleeped
|
114
139
|
sleeped = true
|
115
|
-
Log.
|
140
|
+
Log.low "Kill thread #{t.inspect}"
|
116
141
|
t.kill
|
117
142
|
end
|
118
143
|
begin
|
119
|
-
Log.
|
144
|
+
Log.low "Join thread #{t.inspect}"
|
120
145
|
t.join unless t == Thread.current
|
121
146
|
rescue Aborted
|
122
147
|
rescue Exception
|
123
|
-
Log.exception
|
148
|
+
Log.warn "Thread exception: #{$!.message}"
|
124
149
|
end
|
125
150
|
end
|
126
|
-
Log.medium "Aborted threads (#{Thread.current.inspect}) #{@threads.collect{|t| t.inspect } * ", "}"
|
127
151
|
end
|
128
152
|
|
129
153
|
def abort_pids
|
@@ -137,21 +161,31 @@ module ConcurrentStream
|
|
137
161
|
end
|
138
162
|
|
139
163
|
def abort(exception = nil)
|
140
|
-
|
141
|
-
|
164
|
+
if @aborted
|
165
|
+
Log.medium "YET aborted stream #{Misc.fingerprint self} [#{@aborted}]"
|
166
|
+
return
|
167
|
+
else
|
168
|
+
Log.medium "Aborting stream #{Misc.fingerprint self} [#{@aborted}]"
|
169
|
+
end
|
170
|
+
AbortedStream.setup(self, exception)
|
142
171
|
@aborted = true
|
143
172
|
begin
|
144
|
-
@
|
145
|
-
@abort_callback.call if @abort_callback
|
146
|
-
@abort_callback = nil
|
147
|
-
close unless closed?
|
173
|
+
@abort_callback.call exception if @abort_callback
|
148
174
|
|
149
175
|
abort_threads(exception)
|
150
176
|
abort_pids
|
177
|
+
|
178
|
+
@callback = nil
|
179
|
+
@abort_callback = nil
|
180
|
+
|
181
|
+
@pair.abort exception if @pair
|
182
|
+
|
183
|
+
close unless closed?
|
151
184
|
ensure
|
152
|
-
|
185
|
+
if lockfile and lockfile.locked?
|
186
|
+
lockfile.unlock
|
187
|
+
end
|
153
188
|
end
|
154
|
-
Log.medium "Aborted stream #{Misc.fingerprint self} -- #{@abort_callback} [#{@aborted}]"
|
155
189
|
end
|
156
190
|
|
157
191
|
def super(*args)
|
data/lib/rbbt/util/misc/pipes.rb
CHANGED
@@ -74,22 +74,30 @@ module Misc
|
|
74
74
|
|
75
75
|
ConcurrentStream.setup sout, :pids => [pid]
|
76
76
|
else
|
77
|
+
|
78
|
+
|
79
|
+
ConcurrentStream.setup sin, :pair => sout
|
80
|
+
|
77
81
|
thread = Thread.new(Thread.current) do |parent|
|
78
82
|
begin
|
79
83
|
|
80
84
|
yield sin
|
81
|
-
|
85
|
+
|
86
|
+
sin.close if close and not sin.closed? and not sin.aborted?
|
82
87
|
|
83
88
|
rescue Aborted
|
84
89
|
Log.medium "Aborted open_pipe: #{$!.message}"
|
85
90
|
rescue Exception
|
86
91
|
Log.medium "Exception in open_pipe: #{$!.message}"
|
92
|
+
Log.exception $!
|
87
93
|
parent.raise $!
|
88
94
|
raise $!
|
89
95
|
end
|
90
96
|
end
|
91
97
|
|
92
|
-
|
98
|
+
|
99
|
+
sin.threads = [thread]
|
100
|
+
ConcurrentStream.setup sout, :threads => [thread], :pair => sin
|
93
101
|
end
|
94
102
|
|
95
103
|
sout
|
@@ -171,7 +179,7 @@ module Misc
|
|
171
179
|
skip[i] = true
|
172
180
|
end unless skip[i]
|
173
181
|
end
|
174
|
-
|
182
|
+
end
|
175
183
|
rescue IOError
|
176
184
|
end
|
177
185
|
|
@@ -201,6 +209,12 @@ module Misc
|
|
201
209
|
ConcurrentStream.setup sout, :threads => splitter_thread, :filename => filename
|
202
210
|
end
|
203
211
|
|
212
|
+
abort_callback = Proc.new do
|
213
|
+
out_pipes.each do |s|
|
214
|
+
s.abort if s.respond_to? :abort
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
204
218
|
out_pipes
|
205
219
|
end
|
206
220
|
|
@@ -261,7 +275,7 @@ module Misc
|
|
261
275
|
|
262
276
|
begin
|
263
277
|
into = into.find if Path === into
|
264
|
-
into = Open.open(into, :mode => 'w') if String === into
|
278
|
+
into_path, into = into, Open.open(into, :mode => 'w') if String === into
|
265
279
|
into.sync = true if IO === into
|
266
280
|
into_close = false unless into.respond_to? :close
|
267
281
|
io.sync = true
|
@@ -281,11 +295,13 @@ module Misc
|
|
281
295
|
rescue Aborted
|
282
296
|
Log.medium "Consume stream aborted #{Misc.fingerprint io}"
|
283
297
|
io.abort if io.respond_to? :abort
|
284
|
-
io.close unless io.closed?
|
298
|
+
#io.close unless io.closed?
|
299
|
+
FileUtils.rm into_path if into_path and File.exists? into_path
|
285
300
|
rescue Exception
|
286
301
|
Log.medium "Exception consuming stream: #{Misc.fingerprint io}: #{$!.message}"
|
287
|
-
io.abort if io.respond_to? :abort
|
288
|
-
io.close unless io.closed?
|
302
|
+
io.abort $! if io.respond_to? :abort
|
303
|
+
#io.close unless io.closed?
|
304
|
+
FileUtils.rm into_path if into_path and File.exists? into_path
|
289
305
|
raise $!
|
290
306
|
end
|
291
307
|
end
|
@@ -372,7 +388,6 @@ module Misc
|
|
372
388
|
Open.rm path if File.exist? path
|
373
389
|
rescue Exception
|
374
390
|
Log.medium "Exception in sensiblewrite: [#{Process.pid}] #{$!.message} -- #{ Log.color :blue, path }"
|
375
|
-
Log.exception $!
|
376
391
|
content.abort if content.respond_to? :abort
|
377
392
|
Open.rm path if File.exist? path
|
378
393
|
raise $!
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -126,13 +126,14 @@ module Open
|
|
126
126
|
# Grep
|
127
127
|
|
128
128
|
def self.grep(stream, grep, invert = false)
|
129
|
+
grep_cmd = 'egrep'
|
129
130
|
case
|
130
131
|
when Array === grep
|
131
132
|
TmpFile.with_file(grep * "\n", false) do |f|
|
132
|
-
CMD.cmd("
|
133
|
+
CMD.cmd("#{grep_cmd} #{invert ? '-v' : ''}", "-w" => true, "-f" => f, :in => stream, :pipe => true, :post => proc{FileUtils.rm f})
|
133
134
|
end
|
134
135
|
else
|
135
|
-
CMD.cmd("
|
136
|
+
CMD.cmd("#{grep_cmd} #{invert ? '-v ' : ''} '#{grep}' -", :in => stream, :pipe => true, :post => proc{begin stream.force_close; rescue Exception; end if stream.respond_to?(:force_close)})
|
136
137
|
end
|
137
138
|
end
|
138
139
|
|
@@ -293,7 +294,6 @@ module Open
|
|
293
294
|
end
|
294
295
|
|
295
296
|
def self.gunzip(stream)
|
296
|
-
#Zlib::GzipReader.new(stream)
|
297
297
|
CMD.cmd('zcat', :in => stream, :pipe => true, :no_fail => true, :no_wait => true)
|
298
298
|
end
|
299
299
|
|
@@ -304,7 +304,6 @@ class Step
|
|
304
304
|
end
|
305
305
|
|
306
306
|
def exception(ex, msg = nil)
|
307
|
-
self._abort
|
308
307
|
ex_class = ex.class.to_s
|
309
308
|
set_info :backtrace, ex.backtrace
|
310
309
|
set_info :exception, {:class => ex_class, :message => ex.message, :backtrace => ex.backtrace}
|
@@ -313,6 +312,7 @@ class Step
|
|
313
312
|
else
|
314
313
|
log :error, "#{msg} -- #{ex.message}"
|
315
314
|
end
|
315
|
+
self._abort
|
316
316
|
end
|
317
317
|
|
318
318
|
def started?
|
@@ -678,7 +678,8 @@ module Workflow
|
|
678
678
|
if rec_dependency.streaming? and rec_dependency.running?
|
679
679
|
_inputs[i] = rec_dependency.join.load
|
680
680
|
else
|
681
|
-
rec_dependency.run(true)
|
681
|
+
rec_dependency.run(true)
|
682
|
+
rec_dependency.join
|
682
683
|
_inputs[i] = rec_dependency.load
|
683
684
|
end
|
684
685
|
end
|
@@ -202,10 +202,13 @@ class Step
|
|
202
202
|
Misc.bootstrap(list, cpus, :bar => "Bootstrapping dependencies for #{path}", :respawn => :always) do |dep|
|
203
203
|
Misc.insist do
|
204
204
|
begin
|
205
|
-
dep.produce
|
206
|
-
rescue
|
205
|
+
dep.produce
|
206
|
+
rescue Aborted
|
207
207
|
dep.abort
|
208
208
|
raise $!
|
209
|
+
rescue Exception
|
210
|
+
dep.exception $!
|
211
|
+
raise $!
|
209
212
|
end
|
210
213
|
end
|
211
214
|
nil
|
@@ -315,7 +318,7 @@ class Step
|
|
315
318
|
|
316
319
|
def stop_dependencies
|
317
320
|
dependencies.each do |dep|
|
318
|
-
dep.abort
|
321
|
+
dep.abort unless dep.done? or dep.aborted?
|
319
322
|
end
|
320
323
|
kill_children
|
321
324
|
end
|
@@ -159,7 +159,15 @@ class Step
|
|
159
159
|
|
160
160
|
if stream
|
161
161
|
log :streaming, "Streaming step #{Log.color :yellow, task.name.to_s || ""}"
|
162
|
-
|
162
|
+
|
163
|
+
callback = Proc.new do
|
164
|
+
if AbortedStream === stream
|
165
|
+
if stream.exception
|
166
|
+
raise stream.exception
|
167
|
+
else
|
168
|
+
raise Aborted
|
169
|
+
end
|
170
|
+
end
|
163
171
|
begin
|
164
172
|
if status != :done
|
165
173
|
Misc.insist do
|
@@ -177,15 +185,30 @@ class Step
|
|
177
185
|
FileUtils.rm pid_file if File.exist?(pid_file)
|
178
186
|
end
|
179
187
|
end
|
180
|
-
|
188
|
+
|
189
|
+
abort_callback = Proc.new do |exception|
|
181
190
|
begin
|
182
|
-
|
191
|
+
if exception
|
192
|
+
self.exception exception
|
193
|
+
else
|
194
|
+
log :aborted, "#{Log.color :red, "Aborted"} step #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
|
195
|
+
end
|
196
|
+
_clean_finished
|
183
197
|
rescue
|
184
198
|
Log.exception $!
|
185
199
|
stop_dependencies
|
186
200
|
FileUtils.rm pid_file if File.exist?(pid_file)
|
187
201
|
end
|
188
202
|
end
|
203
|
+
|
204
|
+
ConcurrentStream.setup stream, :callback => callback, :abort_callback => abort_callback
|
205
|
+
|
206
|
+
if AbortedStream === stream
|
207
|
+
exception = stream.exception || Aborted
|
208
|
+
self.exception exception
|
209
|
+
_clean_finished
|
210
|
+
raise exception
|
211
|
+
end
|
189
212
|
else
|
190
213
|
set_info :done, (done_time = Time.now)
|
191
214
|
set_info :total_time_elapsed, (total_time_elapsed = done_time - issue_time)
|
@@ -314,8 +337,7 @@ class Step
|
|
314
337
|
end
|
315
338
|
|
316
339
|
def abort_stream
|
317
|
-
stream =
|
318
|
-
stream ||= @saved_stream
|
340
|
+
stream = @result if IO === @result
|
319
341
|
@saved_stream = nil
|
320
342
|
if stream and stream.respond_to? :abort and not stream.aborted?
|
321
343
|
begin
|
@@ -330,6 +352,17 @@ class Step
|
|
330
352
|
end
|
331
353
|
end
|
332
354
|
|
355
|
+
def _clean_finished
|
356
|
+
if Open.exists? path
|
357
|
+
Log.warn "Aborted job had finished. Removing result -- #{ path }"
|
358
|
+
begin
|
359
|
+
Open.rm path
|
360
|
+
rescue Exception
|
361
|
+
Log.warn "Exception removing result of aborted job: #{$!.message}"
|
362
|
+
end
|
363
|
+
end
|
364
|
+
end
|
365
|
+
|
333
366
|
def _abort
|
334
367
|
return if @aborted
|
335
368
|
@aborted = true
|
@@ -346,14 +379,7 @@ class Step
|
|
346
379
|
Log.exception $!
|
347
380
|
retry
|
348
381
|
ensure
|
349
|
-
|
350
|
-
Log.warn "Aborted job had finished. Removing result -- #{ path }"
|
351
|
-
begin
|
352
|
-
Open.rm path
|
353
|
-
rescue Exception
|
354
|
-
Log.warn "Exception removing result of aborted job: #{$!.message}"
|
355
|
-
end
|
356
|
-
end
|
382
|
+
_clean_finished
|
357
383
|
end
|
358
384
|
end
|
359
385
|
|
@@ -371,9 +397,8 @@ class Step
|
|
371
397
|
Misc.consume_stream stream
|
372
398
|
stream.join if stream.respond_to? :join
|
373
399
|
rescue Exception
|
374
|
-
stream.abort
|
400
|
+
stream.abort $!
|
375
401
|
self._abort
|
376
|
-
raise $!
|
377
402
|
end
|
378
403
|
end
|
379
404
|
end
|
@@ -403,6 +428,7 @@ class Step
|
|
403
428
|
return self if not Open.exists? info_file
|
404
429
|
|
405
430
|
return self if info[:joined]
|
431
|
+
|
406
432
|
pid = @pid
|
407
433
|
|
408
434
|
Misc.insist [0.1, 0.2, 0.5, 1] do
|
@@ -410,6 +436,7 @@ class Step
|
|
410
436
|
end
|
411
437
|
|
412
438
|
begin
|
439
|
+
|
413
440
|
if pid.nil? or Process.pid == pid
|
414
441
|
dependencies.each{|dep| dep.join }
|
415
442
|
else
|
@@ -422,7 +449,9 @@ class Step
|
|
422
449
|
pid = nil
|
423
450
|
dependencies.each{|dep| dep.join }
|
424
451
|
end
|
452
|
+
|
425
453
|
sleep 1 until path.exists? or error? or aborted?
|
454
|
+
|
426
455
|
self
|
427
456
|
ensure
|
428
457
|
set_info :joined, true
|
metadata
CHANGED
@@ -1,174 +1,153 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.20.
|
5
|
-
prerelease:
|
4
|
+
version: 5.20.23
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Miguel Vazquez
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2016-07-
|
11
|
+
date: 2016-07-18 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: rake
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - ">="
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - ">="
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: '0'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: progress-monitor
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - ">="
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - ">="
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: lockfile
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - ">="
|
52
46
|
- !ruby/object:Gem::Version
|
53
47
|
version: '0'
|
54
48
|
type: :runtime
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - ">="
|
60
53
|
- !ruby/object:Gem::Version
|
61
54
|
version: '0'
|
62
55
|
- !ruby/object:Gem::Dependency
|
63
56
|
name: spreadsheet
|
64
57
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
58
|
requirements:
|
67
|
-
- -
|
59
|
+
- - ">="
|
68
60
|
- !ruby/object:Gem::Version
|
69
61
|
version: '0'
|
70
62
|
type: :runtime
|
71
63
|
prerelease: false
|
72
64
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
65
|
requirements:
|
75
|
-
- -
|
66
|
+
- - ">="
|
76
67
|
- !ruby/object:Gem::Version
|
77
68
|
version: '0'
|
78
69
|
- !ruby/object:Gem::Dependency
|
79
70
|
name: highline
|
80
71
|
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
72
|
requirements:
|
83
|
-
- -
|
73
|
+
- - ">="
|
84
74
|
- !ruby/object:Gem::Version
|
85
75
|
version: '0'
|
86
76
|
type: :runtime
|
87
77
|
prerelease: false
|
88
78
|
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
79
|
requirements:
|
91
|
-
- -
|
80
|
+
- - ">="
|
92
81
|
- !ruby/object:Gem::Version
|
93
82
|
version: '0'
|
94
83
|
- !ruby/object:Gem::Dependency
|
95
84
|
name: bio-bgzf
|
96
85
|
requirement: !ruby/object:Gem::Requirement
|
97
|
-
none: false
|
98
86
|
requirements:
|
99
|
-
- -
|
87
|
+
- - ">="
|
100
88
|
- !ruby/object:Gem::Version
|
101
89
|
version: '0'
|
102
90
|
type: :runtime
|
103
91
|
prerelease: false
|
104
92
|
version_requirements: !ruby/object:Gem::Requirement
|
105
|
-
none: false
|
106
93
|
requirements:
|
107
|
-
- -
|
94
|
+
- - ">="
|
108
95
|
- !ruby/object:Gem::Version
|
109
96
|
version: '0'
|
110
97
|
- !ruby/object:Gem::Dependency
|
111
98
|
name: term-ansicolor
|
112
99
|
requirement: !ruby/object:Gem::Requirement
|
113
|
-
none: false
|
114
100
|
requirements:
|
115
|
-
- -
|
101
|
+
- - ">="
|
116
102
|
- !ruby/object:Gem::Version
|
117
103
|
version: '0'
|
118
104
|
type: :runtime
|
119
105
|
prerelease: false
|
120
106
|
version_requirements: !ruby/object:Gem::Requirement
|
121
|
-
none: false
|
122
107
|
requirements:
|
123
|
-
- -
|
108
|
+
- - ">="
|
124
109
|
- !ruby/object:Gem::Version
|
125
110
|
version: '0'
|
126
111
|
- !ruby/object:Gem::Dependency
|
127
112
|
name: rest-client
|
128
113
|
requirement: !ruby/object:Gem::Requirement
|
129
|
-
none: false
|
130
114
|
requirements:
|
131
|
-
- -
|
115
|
+
- - ">="
|
132
116
|
- !ruby/object:Gem::Version
|
133
117
|
version: '0'
|
134
118
|
type: :runtime
|
135
119
|
prerelease: false
|
136
120
|
version_requirements: !ruby/object:Gem::Requirement
|
137
|
-
none: false
|
138
121
|
requirements:
|
139
|
-
- -
|
122
|
+
- - ">="
|
140
123
|
- !ruby/object:Gem::Version
|
141
124
|
version: '0'
|
142
125
|
- !ruby/object:Gem::Dependency
|
143
126
|
name: to_regexp
|
144
127
|
requirement: !ruby/object:Gem::Requirement
|
145
|
-
none: false
|
146
128
|
requirements:
|
147
|
-
- -
|
129
|
+
- - ">="
|
148
130
|
- !ruby/object:Gem::Version
|
149
131
|
version: '0'
|
150
132
|
type: :runtime
|
151
133
|
prerelease: false
|
152
134
|
version_requirements: !ruby/object:Gem::Requirement
|
153
|
-
none: false
|
154
135
|
requirements:
|
155
|
-
- -
|
136
|
+
- - ">="
|
156
137
|
- !ruby/object:Gem::Version
|
157
138
|
version: '0'
|
158
139
|
- !ruby/object:Gem::Dependency
|
159
140
|
name: nakayoshi_fork
|
160
141
|
requirement: !ruby/object:Gem::Requirement
|
161
|
-
none: false
|
162
142
|
requirements:
|
163
|
-
- -
|
143
|
+
- - ">="
|
164
144
|
- !ruby/object:Gem::Version
|
165
145
|
version: '0'
|
166
146
|
type: :runtime
|
167
147
|
prerelease: false
|
168
148
|
version_requirements: !ruby/object:Gem::Requirement
|
169
|
-
none: false
|
170
149
|
requirements:
|
171
|
-
- -
|
150
|
+
- - ">="
|
172
151
|
- !ruby/object:Gem::Version
|
173
152
|
version: '0'
|
174
153
|
description: Utilities for handling tsv files, caches, etc
|
@@ -185,6 +164,12 @@ extra_rdoc_files:
|
|
185
164
|
- README.rdoc
|
186
165
|
files:
|
187
166
|
- LICENSE
|
167
|
+
- README.rdoc
|
168
|
+
- bin/rbbt
|
169
|
+
- bin/rbbt_Rutil.rb
|
170
|
+
- bin/rbbt_dangling_locks.rb
|
171
|
+
- bin/rbbt_exec.rb
|
172
|
+
- bin/rbbt_query.rb
|
188
173
|
- etc/app.d/base.rb
|
189
174
|
- etc/app.d/entities.rb
|
190
175
|
- etc/app.d/finder.rb
|
@@ -392,193 +377,186 @@ files:
|
|
392
377
|
- share/rbbt_commands/workflow/task
|
393
378
|
- share/unicorn.rb
|
394
379
|
- share/workflow_config.ru
|
395
|
-
-
|
396
|
-
- test/test_helper.rb
|
397
|
-
- test/rbbt/resource/test_path.rb
|
398
|
-
- test/rbbt/association/test_item.rb
|
380
|
+
- test/rbbt/annotations/test_util.rb
|
399
381
|
- test/rbbt/association/test_database.rb
|
400
|
-
- test/rbbt/association/test_open.rb
|
401
382
|
- test/rbbt/association/test_index.rb
|
383
|
+
- test/rbbt/association/test_item.rb
|
384
|
+
- test/rbbt/association/test_open.rb
|
402
385
|
- test/rbbt/association/test_util.rb
|
403
|
-
- test/rbbt/
|
404
|
-
- test/rbbt/
|
405
|
-
- test/rbbt/
|
406
|
-
- test/rbbt/
|
407
|
-
- test/rbbt/
|
408
|
-
- test/rbbt/
|
409
|
-
- test/rbbt/
|
410
|
-
- test/rbbt/
|
411
|
-
- test/rbbt/
|
412
|
-
- test/rbbt/
|
413
|
-
- test/rbbt/
|
414
|
-
- test/rbbt/
|
415
|
-
- test/rbbt/
|
416
|
-
- test/rbbt/
|
417
|
-
- test/rbbt/
|
386
|
+
- test/rbbt/entity/test_identifiers.rb
|
387
|
+
- test/rbbt/knowledge_base/test_enrichment.rb
|
388
|
+
- test/rbbt/knowledge_base/test_entity.rb
|
389
|
+
- test/rbbt/knowledge_base/test_query.rb
|
390
|
+
- test/rbbt/knowledge_base/test_registry.rb
|
391
|
+
- test/rbbt/knowledge_base/test_syndicate.rb
|
392
|
+
- test/rbbt/knowledge_base/test_traverse.rb
|
393
|
+
- test/rbbt/persist/test_tsv.rb
|
394
|
+
- test/rbbt/persist/tsv/test_cdb.rb
|
395
|
+
- test/rbbt/persist/tsv/test_kyotocabinet.rb
|
396
|
+
- test/rbbt/persist/tsv/test_leveldb.rb
|
397
|
+
- test/rbbt/persist/tsv/test_lmdb.rb
|
398
|
+
- test/rbbt/persist/tsv/test_sharder.rb
|
399
|
+
- test/rbbt/persist/tsv/test_tokyocabinet.rb
|
400
|
+
- test/rbbt/resource/test_path.rb
|
401
|
+
- test/rbbt/test_annotations.rb
|
402
|
+
- test/rbbt/test_association.rb
|
403
|
+
- test/rbbt/test_entity.rb
|
404
|
+
- test/rbbt/test_fix_width_table.rb
|
405
|
+
- test/rbbt/test_knowledge_base.rb
|
406
|
+
- test/rbbt/test_monitor.rb
|
407
|
+
- test/rbbt/test_packed_index.rb
|
408
|
+
- test/rbbt/test_persist.rb
|
409
|
+
- test/rbbt/test_resource.rb
|
410
|
+
- test/rbbt/test_tsv.rb
|
411
|
+
- test/rbbt/test_workflow.rb
|
412
|
+
- test/rbbt/tsv/parallel/test_through.rb
|
413
|
+
- test/rbbt/tsv/parallel/test_traverse.rb
|
414
|
+
- test/rbbt/tsv/test_accessor.rb
|
415
|
+
- test/rbbt/tsv/test_attach.rb
|
416
|
+
- test/rbbt/tsv/test_change_id.rb
|
417
|
+
- test/rbbt/tsv/test_field_index.rb
|
418
|
+
- test/rbbt/tsv/test_filter.rb
|
419
|
+
- test/rbbt/tsv/test_index.rb
|
420
|
+
- test/rbbt/tsv/test_manipulate.rb
|
421
|
+
- test/rbbt/tsv/test_matrix.rb
|
422
|
+
- test/rbbt/tsv/test_parallel.rb
|
423
|
+
- test/rbbt/tsv/test_parser.rb
|
424
|
+
- test/rbbt/tsv/test_stream.rb
|
425
|
+
- test/rbbt/tsv/test_util.rb
|
418
426
|
- test/rbbt/util/R/test_eval.rb
|
419
427
|
- test/rbbt/util/R/test_model.rb
|
420
|
-
- test/rbbt/util/
|
428
|
+
- test/rbbt/util/concurrency/processes/test_socket.rb
|
429
|
+
- test/rbbt/util/concurrency/test_processes.rb
|
430
|
+
- test/rbbt/util/concurrency/test_threads.rb
|
421
431
|
- test/rbbt/util/log/test_progress.rb
|
422
|
-
- test/rbbt/util/test_colorize.rb
|
423
|
-
- test/rbbt/util/test_R.rb
|
424
|
-
- test/rbbt/util/misc/test_lock.rb
|
425
|
-
- test/rbbt/util/misc/test_pipes.rb
|
426
432
|
- test/rbbt/util/misc/test_bgzf.rb
|
427
|
-
- test/rbbt/util/misc/
|
433
|
+
- test/rbbt/util/misc/test_lock.rb
|
428
434
|
- test/rbbt/util/misc/test_multipart_payload.rb
|
435
|
+
- test/rbbt/util/misc/test_omics.rb
|
436
|
+
- test/rbbt/util/misc/test_pipes.rb
|
437
|
+
- test/rbbt/util/simpleopt/test_get.rb
|
438
|
+
- test/rbbt/util/simpleopt/test_parse.rb
|
439
|
+
- test/rbbt/util/simpleopt/test_setup.rb
|
440
|
+
- test/rbbt/util/test_R.rb
|
441
|
+
- test/rbbt/util/test_chain_methods.rb
|
442
|
+
- test/rbbt/util/test_cmd.rb
|
443
|
+
- test/rbbt/util/test_colorize.rb
|
444
|
+
- test/rbbt/util/test_concurrency.rb
|
429
445
|
- test/rbbt/util/test_excel2tsv.rb
|
446
|
+
- test/rbbt/util/test_filecache.rb
|
447
|
+
- test/rbbt/util/test_log.rb
|
430
448
|
- test/rbbt/util/test_misc.rb
|
431
|
-
- test/rbbt/
|
449
|
+
- test/rbbt/util/test_open.rb
|
450
|
+
- test/rbbt/util/test_semaphore.rb
|
451
|
+
- test/rbbt/util/test_simpleDSL.rb
|
452
|
+
- test/rbbt/util/test_simpleopt.rb
|
453
|
+
- test/rbbt/util/test_tmpfile.rb
|
432
454
|
- test/rbbt/workflow/step/test_dependencies.rb
|
433
455
|
- test/rbbt/workflow/test_doc.rb
|
434
456
|
- test/rbbt/workflow/test_step.rb
|
435
457
|
- test/rbbt/workflow/test_task.rb
|
436
|
-
- test/
|
437
|
-
- test/rbbt/test_knowledge_base.rb
|
438
|
-
- test/rbbt/tsv/parallel/test_traverse.rb
|
439
|
-
- test/rbbt/tsv/parallel/test_through.rb
|
440
|
-
- test/rbbt/tsv/test_parallel.rb
|
441
|
-
- test/rbbt/tsv/test_accessor.rb
|
442
|
-
- test/rbbt/tsv/test_change_id.rb
|
443
|
-
- test/rbbt/tsv/test_stream.rb
|
444
|
-
- test/rbbt/tsv/test_filter.rb
|
445
|
-
- test/rbbt/tsv/test_matrix.rb
|
446
|
-
- test/rbbt/tsv/test_attach.rb
|
447
|
-
- test/rbbt/tsv/test_manipulate.rb
|
448
|
-
- test/rbbt/tsv/test_field_index.rb
|
449
|
-
- test/rbbt/tsv/test_index.rb
|
450
|
-
- test/rbbt/tsv/test_util.rb
|
451
|
-
- test/rbbt/tsv/test_parser.rb
|
452
|
-
- test/rbbt/test_packed_index.rb
|
453
|
-
- test/rbbt/test_persist.rb
|
454
|
-
- test/rbbt/test_fix_width_table.rb
|
455
|
-
- test/rbbt/knowledge_base/test_traverse.rb
|
456
|
-
- test/rbbt/knowledge_base/test_entity.rb
|
457
|
-
- test/rbbt/knowledge_base/test_query.rb
|
458
|
-
- test/rbbt/knowledge_base/test_enrichment.rb
|
459
|
-
- test/rbbt/knowledge_base/test_syndicate.rb
|
460
|
-
- test/rbbt/knowledge_base/test_registry.rb
|
461
|
-
- test/rbbt/entity/test_identifiers.rb
|
462
|
-
- test/rbbt/test_monitor.rb
|
463
|
-
- test/rbbt/test_workflow.rb
|
464
|
-
- test/rbbt/test_annotations.rb
|
465
|
-
- test/rbbt/annotations/test_util.rb
|
466
|
-
- test/rbbt/test_resource.rb
|
467
|
-
- test/rbbt/persist/tsv/test_tokyocabinet.rb
|
468
|
-
- test/rbbt/persist/tsv/test_kyotocabinet.rb
|
469
|
-
- test/rbbt/persist/tsv/test_lmdb.rb
|
470
|
-
- test/rbbt/persist/tsv/test_leveldb.rb
|
471
|
-
- test/rbbt/persist/tsv/test_cdb.rb
|
472
|
-
- test/rbbt/persist/tsv/test_sharder.rb
|
473
|
-
- test/rbbt/persist/test_tsv.rb
|
474
|
-
- test/rbbt/test_tsv.rb
|
475
|
-
- bin/rbbt_query.rb
|
476
|
-
- bin/rbbt_exec.rb
|
477
|
-
- bin/rbbt_Rutil.rb
|
478
|
-
- bin/rbbt
|
479
|
-
- bin/rbbt_dangling_locks.rb
|
458
|
+
- test/test_helper.rb
|
480
459
|
homepage: http://github.com/mikisvaz/rbbt-util
|
481
460
|
licenses:
|
482
461
|
- MIT
|
462
|
+
metadata: {}
|
483
463
|
post_install_message:
|
484
464
|
rdoc_options: []
|
485
465
|
require_paths:
|
486
466
|
- lib
|
487
467
|
required_ruby_version: !ruby/object:Gem::Requirement
|
488
|
-
none: false
|
489
468
|
requirements:
|
490
|
-
- -
|
469
|
+
- - ">="
|
491
470
|
- !ruby/object:Gem::Version
|
492
471
|
version: '0'
|
493
472
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
494
|
-
none: false
|
495
473
|
requirements:
|
496
|
-
- -
|
474
|
+
- - ">="
|
497
475
|
- !ruby/object:Gem::Version
|
498
476
|
version: '0'
|
499
477
|
requirements: []
|
500
478
|
rubyforge_project:
|
501
|
-
rubygems_version:
|
479
|
+
rubygems_version: 2.4.6
|
502
480
|
signing_key:
|
503
|
-
specification_version:
|
481
|
+
specification_version: 4
|
504
482
|
summary: Utilities for the Ruby Bioinformatics Toolkit (rbbt)
|
505
483
|
test_files:
|
506
|
-
- test/
|
484
|
+
- test/rbbt/test_workflow.rb
|
507
485
|
- test/rbbt/resource/test_path.rb
|
508
|
-
- test/rbbt/association/test_item.rb
|
509
|
-
- test/rbbt/association/test_database.rb
|
510
|
-
- test/rbbt/association/test_open.rb
|
511
|
-
- test/rbbt/association/test_index.rb
|
512
|
-
- test/rbbt/association/test_util.rb
|
513
|
-
- test/rbbt/util/test_concurrency.rb
|
514
|
-
- test/rbbt/util/test_log.rb
|
515
|
-
- test/rbbt/util/test_chain_methods.rb
|
516
|
-
- test/rbbt/util/test_simpleopt.rb
|
517
|
-
- test/rbbt/util/simpleopt/test_parse.rb
|
518
|
-
- test/rbbt/util/simpleopt/test_get.rb
|
519
|
-
- test/rbbt/util/simpleopt/test_setup.rb
|
520
486
|
- test/rbbt/util/test_cmd.rb
|
521
|
-
- test/rbbt/util/
|
522
|
-
- test/rbbt/util/
|
523
|
-
- test/rbbt/util/
|
524
|
-
- test/rbbt/util/
|
525
|
-
- test/rbbt/util/test_tmpfile.rb
|
526
|
-
- test/rbbt/util/test_open.rb
|
527
|
-
- test/rbbt/util/test_filecache.rb
|
528
|
-
- test/rbbt/util/R/test_eval.rb
|
529
|
-
- test/rbbt/util/R/test_model.rb
|
487
|
+
- test/rbbt/util/simpleopt/test_setup.rb
|
488
|
+
- test/rbbt/util/simpleopt/test_get.rb
|
489
|
+
- test/rbbt/util/simpleopt/test_parse.rb
|
490
|
+
- test/rbbt/util/test_chain_methods.rb
|
530
491
|
- test/rbbt/util/test_simpleDSL.rb
|
531
|
-
- test/rbbt/util/
|
532
|
-
- test/rbbt/util/
|
533
|
-
- test/rbbt/util/test_R.rb
|
492
|
+
- test/rbbt/util/test_log.rb
|
493
|
+
- test/rbbt/util/test_open.rb
|
534
494
|
- test/rbbt/util/misc/test_lock.rb
|
535
|
-
- test/rbbt/util/misc/
|
495
|
+
- test/rbbt/util/misc/test_multipart_payload.rb
|
536
496
|
- test/rbbt/util/misc/test_bgzf.rb
|
497
|
+
- test/rbbt/util/misc/test_pipes.rb
|
537
498
|
- test/rbbt/util/misc/test_omics.rb
|
538
|
-
- test/rbbt/util/
|
499
|
+
- test/rbbt/util/test_concurrency.rb
|
500
|
+
- test/rbbt/util/test_R.rb
|
501
|
+
- test/rbbt/util/log/test_progress.rb
|
502
|
+
- test/rbbt/util/test_colorize.rb
|
503
|
+
- test/rbbt/util/test_simpleopt.rb
|
539
504
|
- test/rbbt/util/test_excel2tsv.rb
|
505
|
+
- test/rbbt/util/test_filecache.rb
|
506
|
+
- test/rbbt/util/concurrency/test_processes.rb
|
507
|
+
- test/rbbt/util/concurrency/test_threads.rb
|
508
|
+
- test/rbbt/util/concurrency/processes/test_socket.rb
|
509
|
+
- test/rbbt/util/test_semaphore.rb
|
540
510
|
- test/rbbt/util/test_misc.rb
|
511
|
+
- test/rbbt/util/test_tmpfile.rb
|
512
|
+
- test/rbbt/util/R/test_model.rb
|
513
|
+
- test/rbbt/util/R/test_eval.rb
|
514
|
+
- test/rbbt/test_packed_index.rb
|
515
|
+
- test/rbbt/entity/test_identifiers.rb
|
516
|
+
- test/rbbt/test_association.rb
|
517
|
+
- test/rbbt/knowledge_base/test_traverse.rb
|
518
|
+
- test/rbbt/knowledge_base/test_registry.rb
|
519
|
+
- test/rbbt/knowledge_base/test_entity.rb
|
520
|
+
- test/rbbt/knowledge_base/test_enrichment.rb
|
521
|
+
- test/rbbt/knowledge_base/test_syndicate.rb
|
522
|
+
- test/rbbt/knowledge_base/test_query.rb
|
523
|
+
- test/rbbt/test_resource.rb
|
541
524
|
- test/rbbt/test_entity.rb
|
525
|
+
- test/rbbt/test_knowledge_base.rb
|
526
|
+
- test/rbbt/annotations/test_util.rb
|
527
|
+
- test/rbbt/association/test_index.rb
|
528
|
+
- test/rbbt/association/test_item.rb
|
529
|
+
- test/rbbt/association/test_open.rb
|
530
|
+
- test/rbbt/association/test_util.rb
|
531
|
+
- test/rbbt/association/test_database.rb
|
532
|
+
- test/rbbt/test_tsv.rb
|
542
533
|
- test/rbbt/workflow/step/test_dependencies.rb
|
543
|
-
- test/rbbt/workflow/test_doc.rb
|
544
|
-
- test/rbbt/workflow/test_step.rb
|
545
534
|
- test/rbbt/workflow/test_task.rb
|
546
|
-
- test/rbbt/
|
547
|
-
- test/rbbt/
|
548
|
-
- test/rbbt/
|
549
|
-
- test/rbbt/
|
535
|
+
- test/rbbt/workflow/test_step.rb
|
536
|
+
- test/rbbt/workflow/test_doc.rb
|
537
|
+
- test/rbbt/test_monitor.rb
|
538
|
+
- test/rbbt/test_persist.rb
|
539
|
+
- test/rbbt/test_annotations.rb
|
540
|
+
- test/rbbt/persist/test_tsv.rb
|
541
|
+
- test/rbbt/persist/tsv/test_lmdb.rb
|
542
|
+
- test/rbbt/persist/tsv/test_kyotocabinet.rb
|
543
|
+
- test/rbbt/persist/tsv/test_sharder.rb
|
544
|
+
- test/rbbt/persist/tsv/test_cdb.rb
|
545
|
+
- test/rbbt/persist/tsv/test_tokyocabinet.rb
|
546
|
+
- test/rbbt/persist/tsv/test_leveldb.rb
|
547
|
+
- test/rbbt/tsv/test_field_index.rb
|
550
548
|
- test/rbbt/tsv/test_parallel.rb
|
551
|
-
- test/rbbt/tsv/
|
549
|
+
- test/rbbt/tsv/test_index.rb
|
550
|
+
- test/rbbt/tsv/test_matrix.rb
|
552
551
|
- test/rbbt/tsv/test_change_id.rb
|
552
|
+
- test/rbbt/tsv/test_parser.rb
|
553
553
|
- test/rbbt/tsv/test_stream.rb
|
554
|
+
- test/rbbt/tsv/test_util.rb
|
555
|
+
- test/rbbt/tsv/test_accessor.rb
|
554
556
|
- test/rbbt/tsv/test_filter.rb
|
555
|
-
- test/rbbt/tsv/test_matrix.rb
|
556
557
|
- test/rbbt/tsv/test_attach.rb
|
557
558
|
- test/rbbt/tsv/test_manipulate.rb
|
558
|
-
- test/rbbt/tsv/
|
559
|
-
- test/rbbt/tsv/
|
560
|
-
- test/rbbt/tsv/test_util.rb
|
561
|
-
- test/rbbt/tsv/test_parser.rb
|
562
|
-
- test/rbbt/test_packed_index.rb
|
563
|
-
- test/rbbt/test_persist.rb
|
559
|
+
- test/rbbt/tsv/parallel/test_through.rb
|
560
|
+
- test/rbbt/tsv/parallel/test_traverse.rb
|
564
561
|
- test/rbbt/test_fix_width_table.rb
|
565
|
-
- test/
|
566
|
-
- test/rbbt/knowledge_base/test_entity.rb
|
567
|
-
- test/rbbt/knowledge_base/test_query.rb
|
568
|
-
- test/rbbt/knowledge_base/test_enrichment.rb
|
569
|
-
- test/rbbt/knowledge_base/test_syndicate.rb
|
570
|
-
- test/rbbt/knowledge_base/test_registry.rb
|
571
|
-
- test/rbbt/entity/test_identifiers.rb
|
572
|
-
- test/rbbt/test_monitor.rb
|
573
|
-
- test/rbbt/test_workflow.rb
|
574
|
-
- test/rbbt/test_annotations.rb
|
575
|
-
- test/rbbt/annotations/test_util.rb
|
576
|
-
- test/rbbt/test_resource.rb
|
577
|
-
- test/rbbt/persist/tsv/test_tokyocabinet.rb
|
578
|
-
- test/rbbt/persist/tsv/test_kyotocabinet.rb
|
579
|
-
- test/rbbt/persist/tsv/test_lmdb.rb
|
580
|
-
- test/rbbt/persist/tsv/test_leveldb.rb
|
581
|
-
- test/rbbt/persist/tsv/test_cdb.rb
|
582
|
-
- test/rbbt/persist/tsv/test_sharder.rb
|
583
|
-
- test/rbbt/persist/test_tsv.rb
|
584
|
-
- test/rbbt/test_tsv.rb
|
562
|
+
- test/test_helper.rb
|