rbbt-util 5.13.34 → 5.13.35
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/tsv/stream.rb +9 -7
- data/lib/rbbt/util/misc/concurrent_stream.rb +2 -1
- data/lib/rbbt/util/misc/pipes.rb +15 -19
- data/lib/rbbt/util/open.rb +4 -1
- data/lib/rbbt/workflow/accessor.rb +11 -0
- data/lib/rbbt/workflow/step/run.rb +1 -5
- data/test/rbbt/tsv/parallel/test_traverse.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 81458a4ac4aaf570ca085039bc9edef474939dcf
|
4
|
+
data.tar.gz: 0347d3cdb338145d6633126373eaad1430aaea79
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f3eed434127735a837685ffd0ceb170c4e05988cdb02cab6626d63b1c7f4968f62ceff494bf50a18ad10baef182a9cea2b4f135b2fef21814d472aaf7b7ffd73
|
7
|
+
data.tar.gz: 6c212fb6f2cd6a28ff2e45a3fcceffccc571a02f802f45627c0b7299b2df38461eb8181434ac7a21caf83b9c1855a2d51d1fdb9f7a9b27501bcc5b84b3481232
|
data/lib/rbbt/tsv/stream.rb
CHANGED
@@ -62,6 +62,7 @@ module TSV
|
|
62
62
|
|
63
63
|
streams = streams.collect do |stream|
|
64
64
|
if defined? Step and Step === stream
|
65
|
+
stream.grace
|
65
66
|
stream.get_stream || stream.join.path.open
|
66
67
|
else
|
67
68
|
stream
|
@@ -69,7 +70,10 @@ module TSV
|
|
69
70
|
end
|
70
71
|
|
71
72
|
streams = streams.collect do |stream|
|
72
|
-
Misc.sort_stream(stream)
|
73
|
+
sorted = Misc.sort_stream(stream)
|
74
|
+
stream.annotate sorted if stream.respond_to? :annotate
|
75
|
+
stream.clear if stream.respond_to? :annotate
|
76
|
+
sorted
|
73
77
|
end if sort
|
74
78
|
|
75
79
|
lines = []
|
@@ -123,6 +127,7 @@ module TSV
|
|
123
127
|
case key
|
124
128
|
when min
|
125
129
|
str << [parts[i] * sep]
|
130
|
+
|
126
131
|
line = lines[i] = begin
|
127
132
|
streams[i].gets
|
128
133
|
rescue
|
@@ -131,7 +136,6 @@ module TSV
|
|
131
136
|
end
|
132
137
|
if line.nil?
|
133
138
|
stream = streams[i]
|
134
|
-
stream.join if stream.respond_to? :join
|
135
139
|
keys[i] = nil
|
136
140
|
parts[i] = nil
|
137
141
|
else
|
@@ -150,12 +154,10 @@ module TSV
|
|
150
154
|
stream.join if stream.respond_to? :join
|
151
155
|
end
|
152
156
|
rescue Exception
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
end
|
157
|
+
Log.error "Exception pasting streams #{streams.inspect}: #{$!.message}"
|
158
|
+
streams.each do |stream|
|
159
|
+
stream.abort
|
157
160
|
end
|
158
|
-
ts.each do |t| t.join end
|
159
161
|
raise $!
|
160
162
|
end
|
161
163
|
end
|
@@ -36,10 +36,11 @@ module ConcurrentStream
|
|
36
36
|
stream.filename = filename
|
37
37
|
stream.autojoin = autojoin
|
38
38
|
stream.joined = joined
|
39
|
+
stream
|
39
40
|
end
|
40
41
|
|
41
42
|
def clear
|
42
|
-
threads, pids, callback, abort_callback = nil
|
43
|
+
threads, pids, callback, abort_callback, joined = nil
|
43
44
|
end
|
44
45
|
|
45
46
|
def joined?
|
data/lib/rbbt/util/misc/pipes.rb
CHANGED
@@ -90,6 +90,10 @@ module Misc
|
|
90
90
|
stream_out1, stream_in1 = Misc.pipe
|
91
91
|
stream_out2, stream_in2 = Misc.pipe
|
92
92
|
|
93
|
+
if ConcurrentStream === stream
|
94
|
+
stream.annotate stream_out1
|
95
|
+
end
|
96
|
+
|
93
97
|
splitter_thread = Thread.new(Thread.current) do |parent|
|
94
98
|
begin
|
95
99
|
skip1 = skip2 = false
|
@@ -109,20 +113,19 @@ module Misc
|
|
109
113
|
end unless skip2
|
110
114
|
end
|
111
115
|
stream_in1.close unless stream_in1.closed?
|
112
|
-
stream_in2.close unless stream_in2.closed?
|
113
116
|
stream.join if stream.respond_to? :join
|
117
|
+
stream_in2.close unless stream_in2.closed?
|
114
118
|
rescue Aborted, Interrupt
|
115
|
-
Log.medium "Tee aborting #{Misc.fingerprint stream}"
|
116
|
-
stream.abort if stream.respond_to? :abort
|
117
119
|
stream_out1.abort if stream_out1.respond_to? :abort
|
120
|
+
stream.abort if stream.respond_to? :abort
|
118
121
|
stream_out2.abort if stream_out2.respond_to? :abort
|
119
|
-
Log.medium "
|
122
|
+
Log.medium "Tee aborting #{Misc.fingerprint stream}"
|
123
|
+
raise $!
|
120
124
|
rescue Exception
|
121
|
-
stream.abort if stream.respond_to? :abort
|
122
125
|
stream_out1.abort if stream_out1.respond_to? :abort
|
126
|
+
stream.abort if stream.respond_to? :abort
|
123
127
|
stream_out2.abort if stream_out2.respond_to? :abort
|
124
|
-
|
125
|
-
Log.medium "Exception in tee_stream_thread: #{$!.message}"
|
128
|
+
Log.medium "Tee exception #{Misc.fingerprint stream}"
|
126
129
|
raise $!
|
127
130
|
end
|
128
131
|
end
|
@@ -228,19 +231,12 @@ module Misc
|
|
228
231
|
File.open(tmp_path, 'wb') do |f| f.write content end
|
229
232
|
when (IO === content or StringIO === content or File === content)
|
230
233
|
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
234
|
+
Open.write(tmp_path) do |f|
|
235
|
+
f.sync = true
|
236
|
+
while block = content.read(2048)
|
237
|
+
f.write block
|
238
|
+
end
|
236
239
|
end
|
237
|
-
|
238
|
-
while block = content.read(2048);
|
239
|
-
out.write block
|
240
|
-
end
|
241
|
-
|
242
|
-
content.close
|
243
|
-
out.close
|
244
240
|
else
|
245
241
|
File.open(tmp_path, 'wb') do |f| end
|
246
242
|
end
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -237,7 +237,10 @@ module Open
|
|
237
237
|
dir_sub_path_source = find_repo_dir(source)
|
238
238
|
dir_sub_path_target = find_repo_dir(target)
|
239
239
|
|
240
|
-
|
240
|
+
if dir_sub_path_source.nil? and dir_sub_path_target.nil?
|
241
|
+
FileUtils.mkdir_p File.dirname(target) unless File.exists? File.dirname(target)
|
242
|
+
return FileUtils.mv source, target
|
243
|
+
end
|
241
244
|
|
242
245
|
if dir_sub_path_source.nil?
|
243
246
|
save_content_in_repo(dir_sub_path_target[0], dir_sub_path_target[1], Open.read(source))
|
@@ -29,6 +29,17 @@ class Step
|
|
29
29
|
path.nil? ? nil : path + '.info'
|
30
30
|
end
|
31
31
|
|
32
|
+
def self.step_info(path)
|
33
|
+
begin
|
34
|
+
Open.open(info_file(path)) do |f|
|
35
|
+
INFO_SERIALIAZER.load(f)
|
36
|
+
end
|
37
|
+
rescue Exception
|
38
|
+
Log.exception $!
|
39
|
+
{}
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
32
43
|
def self.job_name_for_info_file(info_file, extension = nil)
|
33
44
|
if extension and not extension.empty?
|
34
45
|
info_file.sub(/\.#{extension}\.info$/,'')
|
@@ -400,8 +400,7 @@ class Step
|
|
400
400
|
stream = get_stream if @result
|
401
401
|
if stream
|
402
402
|
begin
|
403
|
-
|
404
|
-
Misc.consume_stream stream, false, @stream_data
|
403
|
+
Misc.consume_stream stream
|
405
404
|
rescue Exception
|
406
405
|
self._abort
|
407
406
|
raise $!
|
@@ -434,9 +433,6 @@ class Step
|
|
434
433
|
begin
|
435
434
|
if pid.nil? or Process.pid == pid
|
436
435
|
dependencies.each{|dep| dep.join }
|
437
|
-
while not done?
|
438
|
-
sleep 1
|
439
|
-
end
|
440
436
|
else
|
441
437
|
begin
|
442
438
|
Log.debug{"Waiting for pid: #{pid}"}
|
@@ -339,7 +339,7 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
339
339
|
assert_raise ProcessFailed do
|
340
340
|
TSV.traverse stream, :head => head, :cpus => cpus, :into => dumper do |k,v|
|
341
341
|
k = k.first
|
342
|
-
raise
|
342
|
+
raise ProcessFailed if rand(100) < 20
|
343
343
|
[k,v]
|
344
344
|
end
|
345
345
|
dumper.stream.join
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.13.
|
4
|
+
version: 5.13.35
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-06-
|
11
|
+
date: 2014-06-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|