rbbt-util 5.12.3 → 5.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/persist.rb +127 -109
- data/lib/rbbt/tsv/dumper.rb +0 -1
- data/lib/rbbt/tsv/parallel/traverse.rb +101 -36
- data/lib/rbbt/util/concurrency/processes.rb +5 -2
- data/lib/rbbt/util/concurrency/threads.rb +6 -4
- data/lib/rbbt/util/log.rb +1 -0
- data/lib/rbbt/util/log/progress.rb +163 -0
- data/lib/rbbt/util/misc/options.rb +1 -0
- data/lib/rbbt/util/misc/pipes.rb +63 -50
- data/lib/rbbt/util/misc/progress.rb +0 -0
- data/lib/rbbt/util/open.rb +45 -11
- data/lib/rbbt/util/simpleopt/get.rb +1 -1
- data/lib/rbbt/workflow/accessor.rb +59 -38
- data/lib/rbbt/workflow/step/run.rb +2 -3
- data/share/rbbt_commands/workflow/task +14 -4
- data/test/rbbt/tsv/parallel/test_traverse.rb +35 -1
- data/test/rbbt/util/log/test_progress.rb +49 -0
- data/test/rbbt/util/misc/test_pipes.rb +4 -4
- data/test/rbbt/util/test_open.rb +0 -3
- metadata +6 -2
@@ -26,7 +26,7 @@ module SOPT
|
|
26
26
|
values[input] = value
|
27
27
|
else
|
28
28
|
if value.nil? and %w(F false FALSE no).include?(args[i])
|
29
|
-
Log.warn "Boolean values must are best specified as #{current}[
|
29
|
+
Log.warn "Boolean values must are best specified as #{current}=[true|false], not #{ current } [true|false]. Token '#{args[i]}' following '#{current}' automatically assigned as value"
|
30
30
|
value = args.delete_at(i)
|
31
31
|
end
|
32
32
|
values[input] = %w(F false FALSE no).include?(value)? false : true
|
@@ -104,9 +104,41 @@ class Step
|
|
104
104
|
set_info(:messages, (messages || []) << message)
|
105
105
|
end
|
106
106
|
|
107
|
-
def self.
|
108
|
-
|
109
|
-
|
107
|
+
def self.log_block(staus, message, path, &block)
|
108
|
+
start = Time.now
|
109
|
+
status = status.to_s
|
110
|
+
status_color = case status
|
111
|
+
when "starting"
|
112
|
+
:yellow
|
113
|
+
when "error"
|
114
|
+
:red
|
115
|
+
when "done"
|
116
|
+
:green
|
117
|
+
else
|
118
|
+
:cyan
|
119
|
+
end
|
120
|
+
Log.info do
|
121
|
+
now = Time.now
|
122
|
+
str = Log.color :reset
|
123
|
+
str << "#{ Log.color status_color, status}"
|
124
|
+
str << ": #{ message }" if message
|
125
|
+
str << " -- #{Log.color :blue, path.to_s}" if path
|
126
|
+
str
|
127
|
+
end
|
128
|
+
res = yield
|
129
|
+
eend = Time.now
|
130
|
+
Log.info do
|
131
|
+
now = Time.now
|
132
|
+
str = "#{ Log.color :cyan, status.to_s } +#{Log.color :green, "%.1g" % (eend - start)}"
|
133
|
+
str << " -- #{Log.color :blue, path.to_s}" if path
|
134
|
+
str
|
135
|
+
end
|
136
|
+
res
|
137
|
+
end
|
138
|
+
|
139
|
+
def self.log_string(status, message, path)
|
140
|
+
Log.info do
|
141
|
+
|
110
142
|
status = status.to_s
|
111
143
|
status_color = case status
|
112
144
|
when "starting"
|
@@ -118,43 +150,32 @@ class Step
|
|
118
150
|
else
|
119
151
|
:cyan
|
120
152
|
end
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
|
153
|
+
|
154
|
+
str = Log.color :reset
|
155
|
+
str << "#{ Log.color status_color, status}"
|
156
|
+
str << ": #{ message }" if message
|
157
|
+
str << " -- #{Log.color :blue, path.to_s}" if path
|
158
|
+
str
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
def self.log_progress(status, options, path, &block)
|
163
|
+
options = Misc.add_defaults options, :severity => Log::INFO
|
164
|
+
max = Misc.process_options options, :max
|
165
|
+
Log::ProgressBar.with_bar(max, options) do |bar|
|
166
|
+
yield bar
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
def self.log(status, message, path, &block)
|
171
|
+
if block_given?
|
172
|
+
if Hash === message
|
173
|
+
log_progress(status, message, path, &block)
|
174
|
+
else
|
175
|
+
log_block(status, message, path, &block)
|
136
176
|
end
|
137
|
-
res
|
138
177
|
else
|
139
|
-
status
|
140
|
-
status_color = case status
|
141
|
-
when "starting"
|
142
|
-
:yellow
|
143
|
-
when "error"
|
144
|
-
:red
|
145
|
-
when "done"
|
146
|
-
:green
|
147
|
-
else
|
148
|
-
:cyan
|
149
|
-
end
|
150
|
-
Log.info do
|
151
|
-
now = Time.now
|
152
|
-
str = Log.color :reset
|
153
|
-
str << "#{ Log.color status_color, status}"
|
154
|
-
str << ": #{ message }" if message
|
155
|
-
str << " -- #{Log.color :blue, path.to_s}" if path
|
156
|
-
str
|
157
|
-
end
|
178
|
+
log_string(status, message, path)
|
158
179
|
end
|
159
180
|
end
|
160
181
|
|
@@ -61,7 +61,6 @@ class Step
|
|
61
61
|
|
62
62
|
def _exec
|
63
63
|
@exec = true if @exec.nil?
|
64
|
-
#@task.exec_in((bindings ? bindings : self), *@inputs)
|
65
64
|
@task.exec_in((bindings ? bindings : self), *dup_inputs)
|
66
65
|
end
|
67
66
|
|
@@ -242,7 +241,7 @@ class Step
|
|
242
241
|
RbbtSemaphore.wait_semaphore(semaphore) if semaphore
|
243
242
|
FileUtils.mkdir_p File.dirname(path) unless Open.exists? File.dirname(path)
|
244
243
|
begin
|
245
|
-
res = run
|
244
|
+
res = run true
|
246
245
|
rescue Aborted
|
247
246
|
Log.debug{"Forked process aborted: #{path}"}
|
248
247
|
log :aborted, "Aborted"
|
@@ -251,7 +250,7 @@ class Step
|
|
251
250
|
Log.debug("Exception '#{$!.message}' caught on forked process: #{path}")
|
252
251
|
raise $!
|
253
252
|
ensure
|
254
|
-
|
253
|
+
join_stream
|
255
254
|
end
|
256
255
|
|
257
256
|
begin
|
@@ -337,8 +337,12 @@ begin
|
|
337
337
|
end
|
338
338
|
|
339
339
|
if options.delete(:printpath)
|
340
|
-
job.join
|
341
|
-
|
340
|
+
job.join
|
341
|
+
if Open.remote? job.path
|
342
|
+
puts job.path + Log.color(:blue, "?_format=raw")
|
343
|
+
else
|
344
|
+
puts job.path
|
345
|
+
end
|
342
346
|
exit 0
|
343
347
|
end
|
344
348
|
|
@@ -378,7 +382,13 @@ begin
|
|
378
382
|
space.times do
|
379
383
|
Log.clear_line
|
380
384
|
end
|
381
|
-
|
385
|
+
|
386
|
+
if Open.remote? job.path
|
387
|
+
out.puts job.path + Log.color(:blue, "?_format=raw")
|
388
|
+
else
|
389
|
+
out.puts job.path
|
390
|
+
end
|
391
|
+
|
382
392
|
exit 0
|
383
393
|
end
|
384
394
|
rescue ParameterException
|
@@ -412,7 +422,7 @@ when Step
|
|
412
422
|
Thread.pass while IO.select([io]).nil?
|
413
423
|
while block = io.read(2048) do
|
414
424
|
out.write block
|
415
|
-
end unless io.closed?
|
425
|
+
end #unless io.closed?
|
416
426
|
io.join if io.respond_to? :join
|
417
427
|
rescue Exception
|
418
428
|
Log.exception $!
|
@@ -283,7 +283,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
283
283
|
require 'rbbt/sources/organism'
|
284
284
|
|
285
285
|
head = 10_000
|
286
|
-
cpus =
|
286
|
+
cpus = 4
|
287
287
|
|
288
288
|
stream = Organism.identifiers("Hsa").open
|
289
289
|
dumper = TSV::Dumper.new Organism.identifiers("Hsa").tsv_options
|
@@ -337,4 +337,38 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
337
337
|
end
|
338
338
|
assert_equal size, stream.read.split("\n").length
|
339
339
|
end
|
340
|
+
|
341
|
+
def test_traverse_progress
|
342
|
+
size = 1000
|
343
|
+
array = (1..size).to_a.collect{|n| n.to_s}
|
344
|
+
stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
|
345
|
+
sleep 0.001
|
346
|
+
e
|
347
|
+
end
|
348
|
+
assert_equal size, stream.read.split("\n").length
|
349
|
+
|
350
|
+
size = 1000
|
351
|
+
array = (1..size).to_a.collect{|n| n.to_s}
|
352
|
+
stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
|
353
|
+
sleep 0.001
|
354
|
+
e
|
355
|
+
end
|
356
|
+
assert_equal size, stream.read.split("\n").length
|
357
|
+
|
358
|
+
size = 1000
|
359
|
+
array = (1..size).to_a.collect{|n| n.to_s}
|
360
|
+
stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
|
361
|
+
sleep 0.001
|
362
|
+
e
|
363
|
+
end
|
364
|
+
assert_equal size, stream.read.split("\n").length
|
365
|
+
|
366
|
+
size = 1000
|
367
|
+
array = (1..size).to_a.collect{|n| n.to_s}
|
368
|
+
stream = TSV.traverse array, :bar => {:max => size, :desc => "Array"}, :cpus => 5, :into => :stream do |e|
|
369
|
+
sleep 0.01
|
370
|
+
e
|
371
|
+
end
|
372
|
+
assert_equal size, stream.read.split("\n").length
|
373
|
+
end
|
340
374
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require File.join(File.expand_path(File.dirname(__FILE__)), '../../..', 'test_helper.rb')
|
2
|
+
require 'rbbt/util/log/progress'
|
3
|
+
|
4
|
+
class TestProgress < Test::Unit::TestCase
|
5
|
+
def _test_bar
|
6
|
+
t1 = Thread.new do
|
7
|
+
Log::ProgressBar.with_bar(20) do |bar|
|
8
|
+
20.times do
|
9
|
+
bar.tick
|
10
|
+
sleep 0.3
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
t2 = Thread.new do
|
16
|
+
Log::ProgressBar.with_bar(20) do |bar|
|
17
|
+
20.times do
|
18
|
+
bar.tick
|
19
|
+
sleep 0.2
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
t1.join
|
24
|
+
t2.join
|
25
|
+
end
|
26
|
+
|
27
|
+
def test_bar_no_size
|
28
|
+
t1 = Thread.new do
|
29
|
+
Log::ProgressBar.with_bar(nil) do |bar|
|
30
|
+
20.times do
|
31
|
+
bar.tick
|
32
|
+
sleep 0.3
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
t2 = Thread.new do
|
38
|
+
Log::ProgressBar.with_bar(nil) do |bar|
|
39
|
+
20.times do
|
40
|
+
bar.tick
|
41
|
+
sleep 0.2
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
t1.join
|
46
|
+
t2.join
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
@@ -5,7 +5,7 @@ require 'rbbt/util/misc'
|
|
5
5
|
|
6
6
|
class TestMiscPipes < Test::Unit::TestCase
|
7
7
|
|
8
|
-
def
|
8
|
+
def test_collapse_stream
|
9
9
|
text=<<-EOF
|
10
10
|
row1 A B C
|
11
11
|
row1 a b c
|
@@ -19,7 +19,7 @@ row2 aa bb cc
|
|
19
19
|
assert_equal ["BB", "bb"], tsv["row2"][1]
|
20
20
|
end
|
21
21
|
|
22
|
-
def
|
22
|
+
def test_paste_stream
|
23
23
|
text1=<<-EOF
|
24
24
|
row1 A B C
|
25
25
|
row2 AA BB CC
|
@@ -46,7 +46,7 @@ row3 ccc
|
|
46
46
|
assert_equal ["AAA", "BBB", "CCC", "", "", "ccc"], tsv["row3"]
|
47
47
|
end
|
48
48
|
|
49
|
-
def
|
49
|
+
def test_sort_stream
|
50
50
|
text =<<-EOF
|
51
51
|
#: :sep=" "
|
52
52
|
#Row LabelA LabelB LabelC
|
@@ -60,7 +60,7 @@ row1 A B C
|
|
60
60
|
assert_equal %w(#: #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
|
61
61
|
end
|
62
62
|
|
63
|
-
def
|
63
|
+
def test_dup_stream
|
64
64
|
text =<<-EOF
|
65
65
|
#: :sep=" "
|
66
66
|
#Row LabelA LabelB LabelC
|
data/test/rbbt/util/test_open.rb
CHANGED
@@ -122,7 +122,6 @@ class TestOpen < Test::Unit::TestCase
|
|
122
122
|
end
|
123
123
|
end
|
124
124
|
|
125
|
-
|
126
125
|
Misc.benchmark(100) do
|
127
126
|
filename = "file" << (rand * 100).to_i.to_s
|
128
127
|
Open.write(File.join(repo, filename), file2)
|
@@ -130,8 +129,6 @@ class TestOpen < Test::Unit::TestCase
|
|
130
129
|
Open.read(File.join(repo, filename))
|
131
130
|
end
|
132
131
|
end
|
133
|
-
|
134
|
-
|
135
132
|
end
|
136
133
|
end
|
137
134
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.13.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|
@@ -174,6 +174,7 @@ files:
|
|
174
174
|
- lib/rbbt/util/excel2tsv.rb
|
175
175
|
- lib/rbbt/util/filecache.rb
|
176
176
|
- lib/rbbt/util/log.rb
|
177
|
+
- lib/rbbt/util/log/progress.rb
|
177
178
|
- lib/rbbt/util/misc.rb
|
178
179
|
- lib/rbbt/util/misc/concurrent_stream.rb
|
179
180
|
- lib/rbbt/util/misc/development.rb
|
@@ -188,6 +189,7 @@ files:
|
|
188
189
|
- lib/rbbt/util/misc/omics.rb
|
189
190
|
- lib/rbbt/util/misc/options.rb
|
190
191
|
- lib/rbbt/util/misc/pipes.rb
|
192
|
+
- lib/rbbt/util/misc/progress.rb
|
191
193
|
- lib/rbbt/util/misc/system.rb
|
192
194
|
- lib/rbbt/util/named_array.rb
|
193
195
|
- lib/rbbt/util/open.rb
|
@@ -297,6 +299,7 @@ files:
|
|
297
299
|
- test/rbbt/util/concurrency/processes/test_socket.rb
|
298
300
|
- test/rbbt/util/concurrency/test_processes.rb
|
299
301
|
- test/rbbt/util/concurrency/test_threads.rb
|
302
|
+
- test/rbbt/util/log/test_progress.rb
|
300
303
|
- test/rbbt/util/misc/test_pipes.rb
|
301
304
|
- test/rbbt/util/simpleopt/test_get.rb
|
302
305
|
- test/rbbt/util/simpleopt/test_parse.rb
|
@@ -357,6 +360,7 @@ test_files:
|
|
357
360
|
- test/rbbt/util/misc/test_pipes.rb
|
358
361
|
- test/rbbt/util/test_concurrency.rb
|
359
362
|
- test/rbbt/util/test_R.rb
|
363
|
+
- test/rbbt/util/log/test_progress.rb
|
360
364
|
- test/rbbt/util/test_colorize.rb
|
361
365
|
- test/rbbt/util/test_simpleopt.rb
|
362
366
|
- test/rbbt/util/test_excel2tsv.rb
|