rbbt-util 5.12.2 → 5.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/entity.rb +1 -1
- data/lib/rbbt/persist.rb +1 -1
- data/lib/rbbt/tsv/attach.rb +63 -54
- data/lib/rbbt/tsv/dumper.rb +1 -0
- data/lib/rbbt/tsv/parallel/traverse.rb +1 -1
- data/lib/rbbt/tsv/util.rb +1 -3
- data/lib/rbbt/util/cmd.rb +7 -8
- data/lib/rbbt/util/concurrency/processes/worker.rb +2 -0
- data/lib/rbbt/util/concurrency/processes.rb +5 -3
- data/lib/rbbt/util/misc/concurrent_stream.rb +20 -3
- data/lib/rbbt/util/misc/exceptions.rb +1 -0
- data/lib/rbbt/util/open.rb +8 -3
- data/lib/rbbt/workflow/step/run.rb +4 -4
- data/lib/rbbt/workflow/task.rb +1 -5
- data/test/rbbt/test_entity.rb +2 -2
- data/test/rbbt/tsv/parallel/test_traverse.rb +5 -1
- data/test/rbbt/tsv/test_util.rb +1 -1
- data/test/rbbt/util/misc/test_pipes.rb +4 -4
- data/test/rbbt/util/test_R.rb +2 -2
- data/test/rbbt/util/test_cmd.rb +8 -11
- data/test/rbbt/util/test_misc.rb +0 -9
- data/test/rbbt/util/test_open.rb +0 -1
- data/test/rbbt/util/test_tmpfile.rb +0 -1
- data/test/rbbt/workflow/test_step.rb +10 -8
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b7e5eb68b9212f47cb33b251ea8d4d0f9c878df1
|
4
|
+
data.tar.gz: 39ecf6d22cde5921db4011ca849d3b5c03ba2f13
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ca4f1549b877b62b6ae97d29b214c6bff957d626819ff14147f0eadaf63b9d394d3bb3e2db31c08f77329ed823d61afba9f44704dc975767a23f5f7fad7be793
|
7
|
+
data.tar.gz: 9e477fc7917df197279869c4bb4a754bf6e60be2d1695ca9f281f5406a1c51d29f311cebd8153430c8349abec29c529cac9e4481283387ef2929630cd5d9c914
|
data/lib/rbbt/entity.rb
CHANGED
@@ -139,7 +139,7 @@ module Entity
|
|
139
139
|
persist_options = options
|
140
140
|
persist_options = persist_options.merge(:other => {:args => args}) if args.any?
|
141
141
|
|
142
|
-
Persist.persist(persist_name, type, persist_options) do
|
142
|
+
Persist.persist(persist_name, type, persist_options.merge(:persist => true)) do
|
143
143
|
self.send(orig_name, *args)
|
144
144
|
end
|
145
145
|
end
|
data/lib/rbbt/persist.rb
CHANGED
@@ -398,7 +398,7 @@ module Persist
|
|
398
398
|
begin
|
399
399
|
|
400
400
|
lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
|
401
|
-
Misc.lock lock_filename
|
401
|
+
Misc.lock lock_filename do |lockfile|
|
402
402
|
|
403
403
|
if is_persisted?(path, persist_options)
|
404
404
|
Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"
|
data/lib/rbbt/tsv/attach.rb
CHANGED
@@ -88,75 +88,84 @@ module TSV
|
|
88
88
|
file2 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file2.to_s(:sort, true), :pipe => true)
|
89
89
|
end
|
90
90
|
|
91
|
-
|
91
|
+
begin
|
92
|
+
output = File.open(output, 'w') if String === output
|
92
93
|
|
93
|
-
|
94
|
-
|
94
|
+
cols1 = nil
|
95
|
+
cols2 = nil
|
95
96
|
|
96
|
-
|
97
|
-
|
97
|
+
done1 = false
|
98
|
+
done2 = false
|
98
99
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
100
|
+
key1 = key2 = nil
|
101
|
+
while key1.nil?
|
102
|
+
while (line1 = file1.gets) =~ /^#/
|
103
|
+
key_field1, *fields1 = line1.strip.sub('#','').split(sep)
|
104
|
+
end
|
105
|
+
key1, *parts1 = line1.sub("\n",'').split(sep, -1)
|
106
|
+
cols1 = parts1.length
|
103
107
|
end
|
104
|
-
key1, *parts1 = line1.sub("\n",'').split(sep, -1)
|
105
|
-
cols1 = parts1.length
|
106
|
-
end
|
107
108
|
|
108
|
-
|
109
|
-
|
110
|
-
|
109
|
+
while key2.nil?
|
110
|
+
while (line2 = file2.gets) =~ /^#/
|
111
|
+
key_field2, *fields2 = line2.strip.sub('#','').split(sep)
|
112
|
+
end
|
113
|
+
key2, *parts2 = line2.sub("\n",'').split(sep, -1)
|
114
|
+
cols2 = parts2.length
|
111
115
|
end
|
112
|
-
key2, *parts2 = line2.sub("\n",'').split(sep, -1)
|
113
|
-
cols2 = parts2.length
|
114
|
-
end
|
115
116
|
|
116
|
-
|
117
|
+
progress_monitor = Progress::Bar.new(size, 0, 100, "Merging fields") if monitor
|
117
118
|
|
118
|
-
|
119
|
-
|
120
|
-
|
119
|
+
entry_hash = options
|
120
|
+
entry_hash.delete :sep if entry_hash[:sep] == "\t"
|
121
|
+
output.puts TSV.header_lines key_field1, fields1 + fields2, entry_hash if key_field1 and fields1 and fields2
|
121
122
|
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
end
|
135
|
-
while (not done2 and key2 == key)
|
136
|
-
parts2.each_with_index do |part, i|
|
137
|
-
i += cols1
|
138
|
-
parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
|
123
|
+
key = key1 < key2 ? key1 : key2
|
124
|
+
parts = [""] * (cols1 + cols2)
|
125
|
+
while not (done1 and done2)
|
126
|
+
while (not done1 and key1 == key)
|
127
|
+
parts1.each_with_index do |part, i|
|
128
|
+
parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
|
129
|
+
end
|
130
|
+
key1 = nil
|
131
|
+
while key1.nil? and not done1
|
132
|
+
if file1.eof?; done1 = true; else key1, *parts1 = file1.gets.sub("\n",'').split(sep, -1) end
|
133
|
+
end
|
134
|
+
progress_monitor.tick if monitor
|
139
135
|
end
|
140
|
-
key2
|
141
|
-
|
142
|
-
|
136
|
+
while (not done2 and key2 == key)
|
137
|
+
parts2.each_with_index do |part, i|
|
138
|
+
i += cols1
|
139
|
+
parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
|
140
|
+
end
|
141
|
+
key2 = nil
|
142
|
+
while key2.nil? and not done2
|
143
|
+
if file2.eof?; done2 = true; else key2, *parts2 = file2.gets.sub("\n",'').split(sep, -1) end
|
144
|
+
end
|
143
145
|
end
|
144
|
-
end
|
145
146
|
|
146
|
-
|
147
|
-
|
147
|
+
output.puts [key, parts].flatten * sep
|
148
|
+
parts = [""] * (cols1 + cols2)
|
148
149
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
150
|
+
case
|
151
|
+
when done1
|
152
|
+
key = key2
|
153
|
+
when done2
|
154
|
+
key = key1
|
155
|
+
else
|
156
|
+
key = key1 < key2 ? key1 : key2
|
157
|
+
end
|
156
158
|
end
|
157
|
-
end
|
158
159
|
|
159
|
-
|
160
|
+
output.close
|
161
|
+
file1.join if file1.respond_to? :join
|
162
|
+
file2.join if file2.respond_to? :join
|
163
|
+
rescue
|
164
|
+
file1.abort if file1.respond_to? :abort
|
165
|
+
file2.abort if file2.respond_to? :abort
|
166
|
+
file1.join if file1.respond_to? :join
|
167
|
+
file2.join if file2.respond_to? :join
|
168
|
+
end
|
160
169
|
end
|
161
170
|
|
162
171
|
# Merge columns from different files
|
data/lib/rbbt/tsv/dumper.rb
CHANGED
@@ -283,7 +283,7 @@ module TSV
|
|
283
283
|
end
|
284
284
|
|
285
285
|
def self.traverse_run(obj, threads, cpus, options = {}, &block)
|
286
|
-
if threads.nil? and cpus.nil?
|
286
|
+
if ENV["RBBT_NO_MAP_REDUCE"] == "true" or (threads.nil? and cpus.nil?)
|
287
287
|
traverse_obj obj, options, &block
|
288
288
|
else
|
289
289
|
if threads
|
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -40,7 +40,7 @@ module TSV
|
|
40
40
|
persist_options = Misc.pull_keys options, :persist
|
41
41
|
|
42
42
|
filename = TSV === file ? file.filename : file
|
43
|
-
|
43
|
+
path = Persist.persist filename, :string, persist_options.merge(:no_load => true) do
|
44
44
|
tsv = TSV === file ? file : TSV.open(file)
|
45
45
|
|
46
46
|
text = ""
|
@@ -55,13 +55,11 @@ module TSV
|
|
55
55
|
text
|
56
56
|
end
|
57
57
|
|
58
|
-
path = Persist.persistence_path(filename, persist_options)
|
59
58
|
TmpFile.with_file(values.uniq * "\n") do |value_file|
|
60
59
|
cmd = "cat '#{ path }' | sed 's/\\t/\\tHEADERNOMATCH/' | grep -w -F -f '#{ value_file }' |cut -f 2 | sed 's/HEADERNOMATCH//' | sort|uniq -c|sed 's/^ *//;s/ /\t/'"
|
61
60
|
begin
|
62
61
|
TSV.open(CMD.cmd(cmd), :key_field => 1, :type => :single, :cast => :to_i)
|
63
62
|
rescue
|
64
|
-
Log.exception $!
|
65
63
|
TSV.setup({}, :type => :single, :cast => :to_i)
|
66
64
|
end
|
67
65
|
end
|
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -3,8 +3,6 @@ require 'stringio'
|
|
3
3
|
|
4
4
|
module CMD
|
5
5
|
|
6
|
-
class CMDError < StandardError; end
|
7
|
-
|
8
6
|
module SmartIO
|
9
7
|
attr_accessor :pid, :cmd, :post, :in, :out, :err, :log
|
10
8
|
def self.tie(io, pid = nil, cmd = "", post = nil, sin = nil, out = nil, err = nil, log = true)
|
@@ -33,7 +31,7 @@ module CMD
|
|
33
31
|
|
34
32
|
if $? and not $?.success?
|
35
33
|
Log.debug{"Raising exception" if log}
|
36
|
-
exception =
|
34
|
+
exception = ProcessFailed.new "Command [#{@pid}] '#{@cmd}' failed with error status #{$?.exitstatus}"
|
37
35
|
begin
|
38
36
|
original_close
|
39
37
|
ensure
|
@@ -160,9 +158,9 @@ module CMD
|
|
160
158
|
|
161
159
|
exit(-1)
|
162
160
|
rescue Exception
|
163
|
-
Log.debug{ "
|
161
|
+
Log.debug{ "ProcessFailed: #{$!.message}" } if log
|
164
162
|
Log.debug{ "Backtrace: \n" + $!.backtrace * "\n" } if log
|
165
|
-
raise
|
163
|
+
raise ProcessFailed, $!.message
|
166
164
|
end
|
167
165
|
}
|
168
166
|
|
@@ -207,7 +205,8 @@ module CMD
|
|
207
205
|
Thread.exit
|
208
206
|
end
|
209
207
|
|
210
|
-
SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
|
208
|
+
#SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
|
209
|
+
ConcurrentStream.setup sout, :pids => [pid], :autojoin => true
|
211
210
|
|
212
211
|
sout
|
213
212
|
else
|
@@ -222,12 +221,12 @@ module CMD
|
|
222
221
|
|
223
222
|
out = StringIO.new sout.read
|
224
223
|
sout.close unless sout.closed?
|
225
|
-
SmartIO.tie out, pid, cmd, post, in_content, sin, serr
|
224
|
+
#SmartIO.tie out, pid, cmd, post, in_content, sin, serr
|
226
225
|
|
227
226
|
Process.waitpid pid
|
228
227
|
|
229
228
|
if not $?.success?
|
230
|
-
exception =
|
229
|
+
exception = ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}.\n#{err}"
|
231
230
|
raise exception
|
232
231
|
else
|
233
232
|
Log.log err, stderr if Integer === stderr and log
|
@@ -18,6 +18,7 @@ class RbbtProcessQueue
|
|
18
18
|
end
|
19
19
|
|
20
20
|
Signal.trap(:INT){ raise Aborted; }
|
21
|
+
|
21
22
|
loop do
|
22
23
|
p = @queue.pop
|
23
24
|
next if p.nil?
|
@@ -32,6 +33,7 @@ class RbbtProcessQueue
|
|
32
33
|
Log.error "Worker #{Process.pid} aborted"
|
33
34
|
Kernel.exit! -1
|
34
35
|
rescue Exception
|
36
|
+
Log.exception $!
|
35
37
|
@callback_queue.push($!) if @callback_queue
|
36
38
|
Kernel.exit! -1
|
37
39
|
ensure
|
@@ -56,6 +56,7 @@ class RbbtProcessQueue
|
|
56
56
|
end
|
57
57
|
rescue Aborted
|
58
58
|
@processes.each{|p| p.abort }
|
59
|
+
@processes.each{|p| p.join }
|
59
60
|
Log.error "Process monitor aborted"
|
60
61
|
rescue Exception
|
61
62
|
Log.error "Process monitor exception: #{$!.message}"
|
@@ -78,7 +79,8 @@ class RbbtProcessQueue
|
|
78
79
|
def join
|
79
80
|
@processes.length.times do
|
80
81
|
@queue.push ClosedStream.new
|
81
|
-
end
|
82
|
+
end if @process_monitor.alive?
|
83
|
+
|
82
84
|
begin
|
83
85
|
@process_monitor.join
|
84
86
|
close_callback if @callback
|
@@ -104,8 +106,8 @@ class RbbtProcessQueue
|
|
104
106
|
end
|
105
107
|
|
106
108
|
def abort
|
107
|
-
@process_monitor.raise
|
108
|
-
@callback_thread.raise
|
109
|
+
@process_monitor.raise(Aborted.new); @process_monitor.join if @process_monitor and @process_monitor.alive?
|
110
|
+
@callback_thread.raise(Aborted.new); @callback_thread.join if @callback_thread and @callback_thread.alive?
|
109
111
|
end
|
110
112
|
|
111
113
|
def process(*e)
|
@@ -1,14 +1,15 @@
|
|
1
1
|
module ConcurrentStream
|
2
|
-
attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined
|
2
|
+
attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :autojoin
|
3
3
|
|
4
4
|
def self.setup(stream, options = {}, &block)
|
5
|
-
threads, pids, callback, filename = Misc.process_options options, :threads, :pids, :callback, :filename
|
5
|
+
threads, pids, callback, filename, autojoin = Misc.process_options options, :threads, :pids, :callback, :filename, :autojoin
|
6
6
|
stream.extend ConcurrentStream unless ConcurrentStream === stream
|
7
7
|
|
8
8
|
stream.threads ||= []
|
9
9
|
stream.pids ||= []
|
10
10
|
stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
|
11
11
|
stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
|
12
|
+
stream.autojoin = autojoin
|
12
13
|
|
13
14
|
callback = block if block_given?
|
14
15
|
if stream.callback and callback
|
@@ -33,6 +34,7 @@ module ConcurrentStream
|
|
33
34
|
stream.callback = callback
|
34
35
|
stream.abort_callback = abort_callback
|
35
36
|
stream.filename = filename
|
37
|
+
stream.autojoin = autojoin
|
36
38
|
stream.joined = joined
|
37
39
|
end
|
38
40
|
|
@@ -61,7 +63,7 @@ module ConcurrentStream
|
|
61
63
|
@pids.each do |pid|
|
62
64
|
begin
|
63
65
|
Process.waitpid(pid, Process::WUNTRACED)
|
64
|
-
raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
|
66
|
+
raise ProcessFailed.new "Error joining process #{pid} in #{self.inspect}" unless $?.success?
|
65
67
|
rescue Errno::ECHILD
|
66
68
|
end
|
67
69
|
end
|
@@ -102,4 +104,19 @@ module ConcurrentStream
|
|
102
104
|
@callback = nil
|
103
105
|
end
|
104
106
|
|
107
|
+
def read(*args)
|
108
|
+
if autojoin
|
109
|
+
begin
|
110
|
+
super(*args)
|
111
|
+
rescue
|
112
|
+
self.abort
|
113
|
+
self.join
|
114
|
+
ensure
|
115
|
+
self.join if self.closed? or self.eof?
|
116
|
+
end
|
117
|
+
else
|
118
|
+
super(*args)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
105
122
|
end
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -341,9 +341,14 @@ module Open
|
|
341
341
|
io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]
|
342
342
|
|
343
343
|
if block_given?
|
344
|
-
|
345
|
-
|
346
|
-
|
344
|
+
begin
|
345
|
+
return yield(io)
|
346
|
+
rescue
|
347
|
+
io.abort if io.respond_to? :abort
|
348
|
+
io.join if io.respond_to? :join
|
349
|
+
ensure
|
350
|
+
io.join if io.respond_to? :join
|
351
|
+
end
|
347
352
|
else
|
348
353
|
io
|
349
354
|
end
|
@@ -100,7 +100,6 @@ class Step
|
|
100
100
|
dependency.relay_log self
|
101
101
|
dependency.clean if not dependency.done? and (dependency.error? or dependency.aborted?)
|
102
102
|
dependency.clean if dependency.streaming? and not dependency.running?
|
103
|
-
#dependency.run_dependencies(seen)
|
104
103
|
dependency.run(ENV["RBBT_NO_STREAM"] != 'true') unless dependency.result or dependency.done?
|
105
104
|
seen << dependency.path
|
106
105
|
seen.concat dependency.rec_dependencies.collect{|d| d.path}
|
@@ -159,12 +158,13 @@ class Step
|
|
159
158
|
raise $!
|
160
159
|
end
|
161
160
|
|
162
|
-
|
163
|
-
|
161
|
+
if not no_load or ENV["RBBT_NO_STREAM"] == "true"
|
162
|
+
result = prepare_result result, @task.description, info if IO === result
|
163
|
+
result = prepare_result result.stream, @task.description, info if TSV::Dumper === result
|
164
|
+
end
|
164
165
|
|
165
166
|
case result
|
166
167
|
when IO
|
167
|
-
result = Misc.read_stream(result) if ENV["RBBT_NO_STREAM"]
|
168
168
|
|
169
169
|
log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
|
170
170
|
ConcurrentStream.setup result do
|
data/lib/rbbt/workflow/task.rb
CHANGED
@@ -44,11 +44,7 @@ module Task
|
|
44
44
|
def exec(*args)
|
45
45
|
case
|
46
46
|
when (args.length == 1 and not inputs.nil? and inputs.length > 1 and Hash === args.first)
|
47
|
-
|
48
|
-
self.call *take_input_values(IndiferentHash.setup(args.first))
|
49
|
-
ensure
|
50
|
-
purge_stream_cache
|
51
|
-
end
|
47
|
+
self.call *take_input_values(IndiferentHash.setup(args.first))
|
52
48
|
else
|
53
49
|
self.call *args
|
54
50
|
end
|
data/test/rbbt/test_entity.rb
CHANGED
@@ -27,7 +27,7 @@ module ReversableString
|
|
27
27
|
self.reverse
|
28
28
|
end
|
29
29
|
|
30
|
-
property :reverse_text_ary_p => :
|
30
|
+
property :reverse_text_ary_p => :array2single do
|
31
31
|
$count += 1
|
32
32
|
self.collect{|s| s.reverse}
|
33
33
|
end
|
@@ -103,7 +103,7 @@ class TestEntity < Test::Unit::TestCase
|
|
103
103
|
$count = 0
|
104
104
|
|
105
105
|
assert_equal "2gnirtS", a.reverse_text_ary_p.last
|
106
|
-
assert_equal "2gnirtS", a[1]
|
106
|
+
assert_equal "2gnirtS", a.collect{|e| e.reverse_text_ary_p }[1]
|
107
107
|
assert_equal 1, $count
|
108
108
|
end
|
109
109
|
|
@@ -317,9 +317,13 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
317
317
|
raise "STOP" if rand(100) < 1
|
318
318
|
[k,v]
|
319
319
|
end
|
320
|
-
dumper.stream
|
320
|
+
stream = dumper.stream
|
321
|
+
stream.read
|
322
|
+
stream.join
|
321
323
|
rescue Exception
|
322
324
|
Log.exception $!
|
325
|
+
stream.abort if stream.respond_to? :abort
|
326
|
+
stream.join if stream.respond_to? :join
|
323
327
|
raise $!
|
324
328
|
end
|
325
329
|
end
|
data/test/rbbt/tsv/test_util.rb
CHANGED
@@ -5,7 +5,7 @@ require 'rbbt/util/misc'
|
|
5
5
|
|
6
6
|
class TestMiscPipes < Test::Unit::TestCase
|
7
7
|
|
8
|
-
def
|
8
|
+
def _test_collapse_stream
|
9
9
|
text=<<-EOF
|
10
10
|
row1 A B C
|
11
11
|
row1 a b c
|
@@ -19,7 +19,7 @@ row2 aa bb cc
|
|
19
19
|
assert_equal ["BB", "bb"], tsv["row2"][1]
|
20
20
|
end
|
21
21
|
|
22
|
-
def
|
22
|
+
def _test_paste_stream
|
23
23
|
text1=<<-EOF
|
24
24
|
row1 A B C
|
25
25
|
row2 AA BB CC
|
@@ -46,7 +46,7 @@ row3 ccc
|
|
46
46
|
assert_equal ["AAA", "BBB", "CCC", "", "", "ccc"], tsv["row3"]
|
47
47
|
end
|
48
48
|
|
49
|
-
def
|
49
|
+
def _test_sort_stream
|
50
50
|
text =<<-EOF
|
51
51
|
#: :sep=" "
|
52
52
|
#Row LabelA LabelB LabelC
|
@@ -60,7 +60,7 @@ row1 A B C
|
|
60
60
|
assert_equal %w(#: #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
|
61
61
|
end
|
62
62
|
|
63
|
-
def
|
63
|
+
def _test_dup_stream
|
64
64
|
text =<<-EOF
|
65
65
|
#: :sep=" "
|
66
66
|
#Row LabelA LabelB LabelC
|
data/test/rbbt/util/test_R.rb
CHANGED
@@ -2,11 +2,11 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe
|
|
2
2
|
require 'rbbt/util/R'
|
3
3
|
|
4
4
|
class TestR < Test::Unit::TestCase
|
5
|
-
def
|
5
|
+
def test_sum
|
6
6
|
assert_equal "6", R.run('cat(3+3)').read.split(/\n/).last
|
7
7
|
end
|
8
8
|
|
9
|
-
def
|
9
|
+
def test_tsv_R
|
10
10
|
tsv = TSV.setup({:a => 1, :b => 2})
|
11
11
|
tsv2 = tsv.R <<-EOF
|
12
12
|
data = data + 1
|
data/test/rbbt/util/test_cmd.rb
CHANGED
@@ -29,17 +29,17 @@ class TestCmd < Test::Unit::TestCase
|
|
29
29
|
end
|
30
30
|
|
31
31
|
def test_error
|
32
|
-
assert_raise
|
33
|
-
assert_raise
|
32
|
+
assert_raise ProcessFailed do CMD.cmd('fake-command') end
|
33
|
+
assert_raise ProcessFailed do CMD.cmd('ls -fake_option') end
|
34
34
|
|
35
|
-
assert_raise
|
36
|
-
assert_raise
|
35
|
+
assert_raise ProcessFailed do CMD.cmd('fake-command', :stderr => true) end
|
36
|
+
assert_raise ProcessFailed do CMD.cmd('ls -fake_option', :stderr => true) end
|
37
37
|
|
38
|
-
assert_nothing_raised
|
39
|
-
assert_nothing_raised
|
38
|
+
assert_nothing_raised ProcessFailed do CMD.cmd('fake-command', :stderr => false, :pipe => true) end
|
39
|
+
assert_nothing_raised ProcessFailed do CMD.cmd('ls -fake_option', :stderr => false, :pipe => true) end
|
40
40
|
|
41
|
-
assert_raise
|
42
|
-
assert_raise
|
41
|
+
assert_raise ProcessFailed do CMD.cmd('fake-command', :stderr => true, :pipe => true).join end
|
42
|
+
assert_raise ProcessFailed do CMD.cmd('ls -fake_option', :stderr => true, :pipe => true).join end
|
43
43
|
end
|
44
44
|
|
45
45
|
def test_pipes
|
@@ -58,10 +58,7 @@ line33
|
|
58
58
|
gz = CMD.cmd("gunzip", :in => File.open(file + '.gz'), :pipe => true)
|
59
59
|
io = CMD.cmd('tail -n 10', :in => gz, :pipe => true)
|
60
60
|
assert_equal 10, io.read.split(/\n/).length
|
61
|
-
|
62
|
-
|
63
61
|
end
|
64
|
-
|
65
62
|
end
|
66
63
|
|
67
64
|
end
|
data/test/rbbt/util/test_misc.rb
CHANGED
@@ -115,15 +115,6 @@ eum fugiat quo voluptas nulla pariatur?"
|
|
115
115
|
assert_equal 4, Misc.process_to_hash(list){|l| l.collect{|e| e * 2}}[2]
|
116
116
|
end
|
117
117
|
|
118
|
-
def test_pipe
|
119
|
-
sout, sin = Misc.pipe
|
120
|
-
assert_equal 1, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
|
121
|
-
sin.close
|
122
|
-
assert sout.eof?
|
123
|
-
Misc.purge_pipes
|
124
|
-
assert_equal 0, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
|
125
|
-
end
|
126
|
-
|
127
118
|
def test_pipe_fork
|
128
119
|
sout, sin = Misc.pipe
|
129
120
|
pid = Process.fork do
|
data/test/rbbt/util/test_open.rb
CHANGED
@@ -62,7 +62,7 @@ class TestStep < Test::Unit::TestCase
|
|
62
62
|
step2 = Step.new tmpfile + 'step2', task2, [], [step1]
|
63
63
|
|
64
64
|
step2.run
|
65
|
-
assert step2.messages.include? "
|
65
|
+
assert step2.messages.include? "Starting Task1"
|
66
66
|
end
|
67
67
|
end
|
68
68
|
|
@@ -83,7 +83,8 @@ class TestStep < Test::Unit::TestCase
|
|
83
83
|
|
84
84
|
Step.log_relay_step = step2
|
85
85
|
step2.run
|
86
|
-
|
86
|
+
|
87
|
+
assert step2.messages.include? "Starting Task1"
|
87
88
|
end
|
88
89
|
end
|
89
90
|
|
@@ -99,7 +100,7 @@ class TestStep < Test::Unit::TestCase
|
|
99
100
|
end
|
100
101
|
|
101
102
|
|
102
|
-
def
|
103
|
+
def __test_fork
|
103
104
|
TmpFile.with_file do |lock|
|
104
105
|
task = Task.setup do while not File.exists?(lock) do sleep 1; end; "TEST" end
|
105
106
|
TmpFile.with_file do |tmp|
|
@@ -115,19 +116,20 @@ class TestStep < Test::Unit::TestCase
|
|
115
116
|
end
|
116
117
|
end
|
117
118
|
|
118
|
-
def
|
119
|
+
def __test_abort
|
119
120
|
TmpFile.with_file do |lock|
|
120
121
|
task = Task.setup do while not File.exists?(lock) do sleep 1; end; "TEST" end
|
121
122
|
TmpFile.with_file do |tmp|
|
122
123
|
step = Step.new tmp, task
|
123
124
|
job = step.fork
|
124
125
|
assert !job.done?
|
125
|
-
|
126
|
-
|
127
|
-
|
126
|
+
step.clean.fork
|
127
|
+
job.abort
|
128
|
+
assert_equal :aborted, job.status
|
128
129
|
Open.write(lock, "open")
|
130
|
+
job.clean.fork
|
129
131
|
job.join
|
130
|
-
assert job.
|
132
|
+
assert job.done?
|
131
133
|
end
|
132
134
|
end
|
133
135
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.12.
|
4
|
+
version: 5.12.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|