rbbt-util 5.13.19 → 5.13.20
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/tsv/stream.rb +2 -1
- data/lib/rbbt/util/concurrency/processes/socket.rb +2 -2
- data/lib/rbbt/util/log.rb +21 -3
- data/lib/rbbt/workflow/step/run.rb +17 -38
- data/share/rbbt_commands/benchmark/throughput +1 -1
- data/share/rbbt_commands/workflow/cmd +3 -2
- data/test/rbbt/tsv/test_stream.rb +33 -0
- data/test/rbbt/util/test_log.rb +4 -2
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 66e0264c3104ab53b278845aaab51e2ae75b5d40
|
4
|
+
data.tar.gz: 7abc522ee044b2527b0a6a40acff4a8fde3c7c90
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 925b812e86aed64d95cb08b6f56491518640953f8284168bbefd21e5500989b9a6a8755df667d552115f908a5e4913b4d9c5fac3b9fba2214c66e0162c492c1e
|
7
|
+
data.tar.gz: 0af9ab272db85087c7c2244d894a0af2c68867c93117d6d1e594bb676b59978bdb1fa5790e915204be8a2423187713b344b02d56af19abcd9ac19d146bc61b6c
|
data/lib/rbbt/tsv/stream.rb
CHANGED
@@ -74,6 +74,7 @@ module TSV
|
|
74
74
|
|
75
75
|
lines = []
|
76
76
|
fields = []
|
77
|
+
sizes = []
|
77
78
|
key_fields = []
|
78
79
|
input_options = []
|
79
80
|
empty = []
|
@@ -84,6 +85,7 @@ module TSV
|
|
84
85
|
empty << stream if parser.first_line.nil?
|
85
86
|
key_fields << parser.key_field
|
86
87
|
fields << parser.fields
|
88
|
+
sizes << parser.fields.length
|
87
89
|
input_options << parser.options
|
88
90
|
|
89
91
|
parser.stream
|
@@ -113,7 +115,6 @@ module TSV
|
|
113
115
|
keys[i] = key
|
114
116
|
parts[i] = p
|
115
117
|
end
|
116
|
-
sizes = parts.collect{|p| p.length }
|
117
118
|
last_min = nil
|
118
119
|
while lines.compact.any?
|
119
120
|
min = keys.compact.sort.first
|
@@ -12,7 +12,7 @@ class RbbtProcessQueue
|
|
12
12
|
key = "/" << rand(100000000).to_s;
|
13
13
|
@write_sem = key + '.in'
|
14
14
|
@read_sem = key + '.out'
|
15
|
-
Log.
|
15
|
+
Log.low "Creating socket semaphores: #{key}"
|
16
16
|
RbbtSemaphore.create_semaphore(@write_sem,1)
|
17
17
|
RbbtSemaphore.create_semaphore(@read_sem,1)
|
18
18
|
end
|
@@ -20,7 +20,7 @@ class RbbtProcessQueue
|
|
20
20
|
def clean
|
21
21
|
@sread.close unless @sread.closed?
|
22
22
|
@swrite.close unless @swrite.closed?
|
23
|
-
Log.
|
23
|
+
Log.low "Destroying socket semaphores"
|
24
24
|
RbbtSemaphore.delete_semaphore(@write_sem)
|
25
25
|
RbbtSemaphore.delete_semaphore(@read_sem)
|
26
26
|
end
|
data/lib/rbbt/util/log.rb
CHANGED
@@ -6,6 +6,8 @@ module Log
|
|
6
6
|
extend Term::ANSIColor
|
7
7
|
|
8
8
|
|
9
|
+
LOG_MUTEX = Mutex.new
|
10
|
+
|
9
11
|
SEVERITY_NAMES ||= begin
|
10
12
|
names = %w(DEBUG LOW MEDIUM HIGH INFO WARN ERROR )
|
11
13
|
names.each_with_index do |name,i|
|
@@ -14,6 +16,20 @@ module Log
|
|
14
16
|
names
|
15
17
|
end
|
16
18
|
|
19
|
+
def self.ignore_stderr
|
20
|
+
backup_stderr = STDERR.dup
|
21
|
+
LOG_MUTEX.synchronize do
|
22
|
+
begin
|
23
|
+
File.open('/dev/null', 'w') do |f|
|
24
|
+
STDERR.reopen(f)
|
25
|
+
yield
|
26
|
+
end
|
27
|
+
ensure
|
28
|
+
STDERR.reopen backup_stderr
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
|
17
33
|
def self.get_level(level)
|
18
34
|
case level
|
19
35
|
when Fixnum
|
@@ -35,8 +51,11 @@ module Log
|
|
35
51
|
|
36
52
|
self.nocolor = ENV["RBBT_NOCOLOR"] == 'true'
|
37
53
|
|
38
|
-
|
39
|
-
|
54
|
+
self.ignore_stderr do
|
55
|
+
require 'nokogiri'
|
56
|
+
require "highline/system_extensions.rb"
|
57
|
+
self.tty_size = HighLine::SystemExtensions.terminal_size.first
|
58
|
+
end
|
40
59
|
|
41
60
|
def self.with_severity(level)
|
42
61
|
orig = Log.severity
|
@@ -95,7 +114,6 @@ module Log
|
|
95
114
|
end
|
96
115
|
end
|
97
116
|
|
98
|
-
LOG_MUTEX = Mutex.new
|
99
117
|
LAST = "log"
|
100
118
|
def self.log(message = nil, severity = MEDIUM, &block)
|
101
119
|
return if severity < self.severity
|
@@ -21,11 +21,6 @@ class Step
|
|
21
21
|
else
|
22
22
|
Log.medium "Duplicating file #{ Misc.fingerprint(current) } #{current.inspect}"
|
23
23
|
Misc.dup_stream(current)
|
24
|
-
#STREAM_CACHE[stream] = new
|
25
|
-
#new
|
26
|
-
#new = current.dup
|
27
|
-
#new.rewind
|
28
|
-
#new
|
29
24
|
end
|
30
25
|
|
31
26
|
else
|
@@ -145,7 +140,7 @@ class Step
|
|
145
140
|
next if dependency.running?
|
146
141
|
dependency.clean
|
147
142
|
else
|
148
|
-
dependency.clean if (dependency.error? or dependency.aborted? or not dependency.done?)
|
143
|
+
dependency.clean if (dependency.error? or dependency.aborted? or dependency.status.nil?)# or not dependency.done?)
|
149
144
|
end
|
150
145
|
|
151
146
|
unless dependency.result or dependency.done?
|
@@ -178,7 +173,7 @@ class Step
|
|
178
173
|
|
179
174
|
Open.rm info_file if Open.exists? info_file
|
180
175
|
|
181
|
-
log :setup, "#{Log.color :
|
176
|
+
log :setup, "#{Log.color :cyan, "Setup"} #{Log.color :yellow, task.name.to_s || ""}"
|
182
177
|
|
183
178
|
merge_info({
|
184
179
|
:pid => Process.pid,
|
@@ -199,13 +194,13 @@ class Step
|
|
199
194
|
set_info :inputs, Misc.remove_long_items(Misc.zip2hash(task.inputs, @inputs)) unless task.inputs.nil?
|
200
195
|
|
201
196
|
set_info :started, (start_time = Time.now)
|
202
|
-
log :started, "#{Log.color :
|
197
|
+
log :started, "#{Log.color :magenta, "Starting"} #{Log.color :yellow, task.name.to_s || ""}"
|
203
198
|
|
204
199
|
begin
|
205
200
|
result = _exec
|
206
201
|
rescue Aborted
|
207
202
|
stop_dependencies
|
208
|
-
log(:
|
203
|
+
log(:aborted, "Aborted")
|
209
204
|
raise $!
|
210
205
|
rescue Exception
|
211
206
|
backtrace = $!.backtrace
|
@@ -224,16 +219,21 @@ class Step
|
|
224
219
|
result = prepare_result result.stream, @task.description, info if TSV::Dumper === result
|
225
220
|
end
|
226
221
|
|
227
|
-
case result
|
228
|
-
|
222
|
+
stream = case result
|
223
|
+
when IO
|
224
|
+
result
|
225
|
+
when TSV::Dumper
|
226
|
+
result.stream
|
227
|
+
end
|
229
228
|
|
230
|
-
|
231
|
-
|
229
|
+
if stream
|
230
|
+
log :streaming, "#{Log.color :magenta, "Streaming"} #{Log.color :yellow, task.name.to_s || ""}"
|
231
|
+
ConcurrentStream.setup stream do
|
232
232
|
begin
|
233
233
|
if status != :done
|
234
234
|
set_info :done, (done_time = Time.now)
|
235
235
|
set_info :time_elapsed, (time_elapsed = done_time - start_time)
|
236
|
-
log :done, "#{Log.color :
|
236
|
+
log :done, "#{Log.color :magenta, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
|
237
237
|
end
|
238
238
|
rescue
|
239
239
|
Log.exception $!
|
@@ -241,7 +241,7 @@ class Step
|
|
241
241
|
join
|
242
242
|
end
|
243
243
|
end
|
244
|
-
|
244
|
+
stream.abort_callback = Proc.new do
|
245
245
|
begin
|
246
246
|
log :aborted, "#{Log.color :red, "Aborted"} #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
|
247
247
|
rescue
|
@@ -250,32 +250,10 @@ class Step
|
|
250
250
|
join
|
251
251
|
end
|
252
252
|
end
|
253
|
-
when TSV::Dumper
|
254
|
-
log :streaming, "#{Log.color :magenta, "Streaming TSV::Dumper"} #{Log.color :yellow, task.name.to_s || ""}"
|
255
|
-
ConcurrentStream.setup result.stream do
|
256
|
-
begin
|
257
|
-
if status != :done
|
258
|
-
set_info :done, (done_time = Time.now)
|
259
|
-
set_info :time_elapsed, (time_elapsed = done_time - start_time)
|
260
|
-
log :done, "#{Log.color :red, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
|
261
|
-
end
|
262
|
-
rescue
|
263
|
-
Log.exception $!
|
264
|
-
ensure
|
265
|
-
join
|
266
|
-
end
|
267
|
-
end
|
268
|
-
result.stream.abort_callback = Proc.new do
|
269
|
-
begin
|
270
|
-
log :aborted, "#{Log.color :red, "Aborted"} #{Log.color :yellow, task.name.to_s || ""}" if status == :streaming
|
271
|
-
rescue Exception
|
272
|
-
Log.exception $!
|
273
|
-
end
|
274
|
-
end
|
275
253
|
else
|
276
254
|
set_info :done, (done_time = Time.now)
|
277
255
|
set_info :time_elapsed, (time_elapsed = done_time - start_time)
|
278
|
-
log :done, "#{Log.color :
|
256
|
+
log :done, "#{Log.color :magenta, "Completed"} #{Log.color :yellow, task.name.to_s || ""} in #{time_elapsed.to_i} sec."
|
279
257
|
end
|
280
258
|
|
281
259
|
result
|
@@ -470,6 +448,7 @@ class Step
|
|
470
448
|
dependencies.each{|dep| dep.join }
|
471
449
|
end
|
472
450
|
self
|
451
|
+
sleep 1 until path.exists?
|
473
452
|
ensure
|
474
453
|
set_info :joined, true
|
475
454
|
end
|
@@ -6,8 +6,9 @@ require 'rbbt/workflow'
|
|
6
6
|
|
7
7
|
workflow = ARGV.shift
|
8
8
|
|
9
|
-
Workflow.require_workflow workflow
|
10
|
-
dir = $command_dir =
|
9
|
+
wf = Workflow.require_workflow workflow
|
10
|
+
dir = $command_dir = wf.libdir.share.rbbt_commands
|
11
|
+
|
11
12
|
def commands(prev)
|
12
13
|
rbbt_command_dir = $command_dir
|
13
14
|
|
@@ -89,4 +89,37 @@ row2 cc
|
|
89
89
|
assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
|
90
90
|
assert_equal ["AAA", "BBB", "CCC", "aaa", "bbb", "ccc"], tsv["row3"]
|
91
91
|
end
|
92
|
+
|
93
|
+
def test_paste_stream_missing
|
94
|
+
text1=<<-EOF
|
95
|
+
#: :sep=" "
|
96
|
+
#Row LabelA LabelB LabelC
|
97
|
+
row2 AA BB CC
|
98
|
+
row1 A B C
|
99
|
+
EOF
|
100
|
+
|
101
|
+
text2=<<-EOF
|
102
|
+
#: :sep=" "
|
103
|
+
#Row Labela Labelb
|
104
|
+
row1 a b
|
105
|
+
row2 aa bb
|
106
|
+
EOF
|
107
|
+
|
108
|
+
text3=<<-EOF
|
109
|
+
#: :sep=" "
|
110
|
+
#Row Labelc
|
111
|
+
row3 ccc
|
112
|
+
row2 cc
|
113
|
+
EOF
|
114
|
+
|
115
|
+
s1 = StringIO.new text1
|
116
|
+
s2 = StringIO.new text2
|
117
|
+
s3 = StringIO.new text3
|
118
|
+
tsv = TSV.open TSV.paste_streams([s1,s2,s3], :sep => " ", :type => :list, :sort => true)
|
119
|
+
assert_equal "Row", tsv.key_field
|
120
|
+
assert_equal %w(LabelA LabelB LabelC Labela Labelb Labelc), tsv.fields
|
121
|
+
assert_equal ["A", "B", "C", "a", "b", ""], tsv["row1"]
|
122
|
+
assert_equal ["AA", "BB", "CC", "aa", "bb", "cc"], tsv["row2"]
|
123
|
+
assert_equal ["", "", "", "", "", "ccc"], tsv["row3"]
|
124
|
+
end
|
92
125
|
end
|
data/test/rbbt/util/test_log.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.13.
|
4
|
+
version: 5.13.20
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-05-
|
11
|
+
date: 2014-05-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|