rbbt-util 5.12.2 → 5.12.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/rbbt/entity.rb +1 -1
- data/lib/rbbt/persist.rb +1 -1
- data/lib/rbbt/tsv/attach.rb +63 -54
- data/lib/rbbt/tsv/dumper.rb +1 -0
- data/lib/rbbt/tsv/parallel/traverse.rb +1 -1
- data/lib/rbbt/tsv/util.rb +1 -3
- data/lib/rbbt/util/cmd.rb +7 -8
- data/lib/rbbt/util/concurrency/processes/worker.rb +2 -0
- data/lib/rbbt/util/concurrency/processes.rb +5 -3
- data/lib/rbbt/util/misc/concurrent_stream.rb +20 -3
- data/lib/rbbt/util/misc/exceptions.rb +1 -0
- data/lib/rbbt/util/open.rb +8 -3
- data/lib/rbbt/workflow/step/run.rb +4 -4
- data/lib/rbbt/workflow/task.rb +1 -5
- data/test/rbbt/test_entity.rb +2 -2
- data/test/rbbt/tsv/parallel/test_traverse.rb +5 -1
- data/test/rbbt/tsv/test_util.rb +1 -1
- data/test/rbbt/util/misc/test_pipes.rb +4 -4
- data/test/rbbt/util/test_R.rb +2 -2
- data/test/rbbt/util/test_cmd.rb +8 -11
- data/test/rbbt/util/test_misc.rb +0 -9
- data/test/rbbt/util/test_open.rb +0 -1
- data/test/rbbt/util/test_tmpfile.rb +0 -1
- data/test/rbbt/workflow/test_step.rb +10 -8
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b7e5eb68b9212f47cb33b251ea8d4d0f9c878df1
|
4
|
+
data.tar.gz: 39ecf6d22cde5921db4011ca849d3b5c03ba2f13
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ca4f1549b877b62b6ae97d29b214c6bff957d626819ff14147f0eadaf63b9d394d3bb3e2db31c08f77329ed823d61afba9f44704dc975767a23f5f7fad7be793
|
7
|
+
data.tar.gz: 9e477fc7917df197279869c4bb4a754bf6e60be2d1695ca9f281f5406a1c51d29f311cebd8153430c8349abec29c529cac9e4481283387ef2929630cd5d9c914
|
data/lib/rbbt/entity.rb
CHANGED
@@ -139,7 +139,7 @@ module Entity
|
|
139
139
|
persist_options = options
|
140
140
|
persist_options = persist_options.merge(:other => {:args => args}) if args.any?
|
141
141
|
|
142
|
-
Persist.persist(persist_name, type, persist_options) do
|
142
|
+
Persist.persist(persist_name, type, persist_options.merge(:persist => true)) do
|
143
143
|
self.send(orig_name, *args)
|
144
144
|
end
|
145
145
|
end
|
data/lib/rbbt/persist.rb
CHANGED
@@ -398,7 +398,7 @@ module Persist
|
|
398
398
|
begin
|
399
399
|
|
400
400
|
lock_filename = Persist.persistence_path(path + '.persist', {:dir => Persist.lock_dir})
|
401
|
-
Misc.lock lock_filename
|
401
|
+
Misc.lock lock_filename do |lockfile|
|
402
402
|
|
403
403
|
if is_persisted?(path, persist_options)
|
404
404
|
Log.low "Persist up-to-date (suddenly): #{ path } - #{Misc.fingerprint persist_options}"
|
data/lib/rbbt/tsv/attach.rb
CHANGED
@@ -88,75 +88,84 @@ module TSV
|
|
88
88
|
file2 = CMD.cmd("sort -k1,1 -t'#{sep}' | grep -v '^#{sep}'", :in => file2.to_s(:sort, true), :pipe => true)
|
89
89
|
end
|
90
90
|
|
91
|
-
|
91
|
+
begin
|
92
|
+
output = File.open(output, 'w') if String === output
|
92
93
|
|
93
|
-
|
94
|
-
|
94
|
+
cols1 = nil
|
95
|
+
cols2 = nil
|
95
96
|
|
96
|
-
|
97
|
-
|
97
|
+
done1 = false
|
98
|
+
done2 = false
|
98
99
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
100
|
+
key1 = key2 = nil
|
101
|
+
while key1.nil?
|
102
|
+
while (line1 = file1.gets) =~ /^#/
|
103
|
+
key_field1, *fields1 = line1.strip.sub('#','').split(sep)
|
104
|
+
end
|
105
|
+
key1, *parts1 = line1.sub("\n",'').split(sep, -1)
|
106
|
+
cols1 = parts1.length
|
103
107
|
end
|
104
|
-
key1, *parts1 = line1.sub("\n",'').split(sep, -1)
|
105
|
-
cols1 = parts1.length
|
106
|
-
end
|
107
108
|
|
108
|
-
|
109
|
-
|
110
|
-
|
109
|
+
while key2.nil?
|
110
|
+
while (line2 = file2.gets) =~ /^#/
|
111
|
+
key_field2, *fields2 = line2.strip.sub('#','').split(sep)
|
112
|
+
end
|
113
|
+
key2, *parts2 = line2.sub("\n",'').split(sep, -1)
|
114
|
+
cols2 = parts2.length
|
111
115
|
end
|
112
|
-
key2, *parts2 = line2.sub("\n",'').split(sep, -1)
|
113
|
-
cols2 = parts2.length
|
114
|
-
end
|
115
116
|
|
116
|
-
|
117
|
+
progress_monitor = Progress::Bar.new(size, 0, 100, "Merging fields") if monitor
|
117
118
|
|
118
|
-
|
119
|
-
|
120
|
-
|
119
|
+
entry_hash = options
|
120
|
+
entry_hash.delete :sep if entry_hash[:sep] == "\t"
|
121
|
+
output.puts TSV.header_lines key_field1, fields1 + fields2, entry_hash if key_field1 and fields1 and fields2
|
121
122
|
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
end
|
135
|
-
while (not done2 and key2 == key)
|
136
|
-
parts2.each_with_index do |part, i|
|
137
|
-
i += cols1
|
138
|
-
parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
|
123
|
+
key = key1 < key2 ? key1 : key2
|
124
|
+
parts = [""] * (cols1 + cols2)
|
125
|
+
while not (done1 and done2)
|
126
|
+
while (not done1 and key1 == key)
|
127
|
+
parts1.each_with_index do |part, i|
|
128
|
+
parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
|
129
|
+
end
|
130
|
+
key1 = nil
|
131
|
+
while key1.nil? and not done1
|
132
|
+
if file1.eof?; done1 = true; else key1, *parts1 = file1.gets.sub("\n",'').split(sep, -1) end
|
133
|
+
end
|
134
|
+
progress_monitor.tick if monitor
|
139
135
|
end
|
140
|
-
key2
|
141
|
-
|
142
|
-
|
136
|
+
while (not done2 and key2 == key)
|
137
|
+
parts2.each_with_index do |part, i|
|
138
|
+
i += cols1
|
139
|
+
parts[i] = (parts[i].nil? or parts[i].empty?) ? part : parts[i] << "|" << part
|
140
|
+
end
|
141
|
+
key2 = nil
|
142
|
+
while key2.nil? and not done2
|
143
|
+
if file2.eof?; done2 = true; else key2, *parts2 = file2.gets.sub("\n",'').split(sep, -1) end
|
144
|
+
end
|
143
145
|
end
|
144
|
-
end
|
145
146
|
|
146
|
-
|
147
|
-
|
147
|
+
output.puts [key, parts].flatten * sep
|
148
|
+
parts = [""] * (cols1 + cols2)
|
148
149
|
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
150
|
+
case
|
151
|
+
when done1
|
152
|
+
key = key2
|
153
|
+
when done2
|
154
|
+
key = key1
|
155
|
+
else
|
156
|
+
key = key1 < key2 ? key1 : key2
|
157
|
+
end
|
156
158
|
end
|
157
|
-
end
|
158
159
|
|
159
|
-
|
160
|
+
output.close
|
161
|
+
file1.join if file1.respond_to? :join
|
162
|
+
file2.join if file2.respond_to? :join
|
163
|
+
rescue
|
164
|
+
file1.abort if file1.respond_to? :abort
|
165
|
+
file2.abort if file2.respond_to? :abort
|
166
|
+
file1.join if file1.respond_to? :join
|
167
|
+
file2.join if file2.respond_to? :join
|
168
|
+
end
|
160
169
|
end
|
161
170
|
|
162
171
|
# Merge columns from different files
|
data/lib/rbbt/tsv/dumper.rb
CHANGED
@@ -283,7 +283,7 @@ module TSV
|
|
283
283
|
end
|
284
284
|
|
285
285
|
def self.traverse_run(obj, threads, cpus, options = {}, &block)
|
286
|
-
if threads.nil? and cpus.nil?
|
286
|
+
if ENV["RBBT_NO_MAP_REDUCE"] == "true" or (threads.nil? and cpus.nil?)
|
287
287
|
traverse_obj obj, options, &block
|
288
288
|
else
|
289
289
|
if threads
|
data/lib/rbbt/tsv/util.rb
CHANGED
@@ -40,7 +40,7 @@ module TSV
|
|
40
40
|
persist_options = Misc.pull_keys options, :persist
|
41
41
|
|
42
42
|
filename = TSV === file ? file.filename : file
|
43
|
-
|
43
|
+
path = Persist.persist filename, :string, persist_options.merge(:no_load => true) do
|
44
44
|
tsv = TSV === file ? file : TSV.open(file)
|
45
45
|
|
46
46
|
text = ""
|
@@ -55,13 +55,11 @@ module TSV
|
|
55
55
|
text
|
56
56
|
end
|
57
57
|
|
58
|
-
path = Persist.persistence_path(filename, persist_options)
|
59
58
|
TmpFile.with_file(values.uniq * "\n") do |value_file|
|
60
59
|
cmd = "cat '#{ path }' | sed 's/\\t/\\tHEADERNOMATCH/' | grep -w -F -f '#{ value_file }' |cut -f 2 | sed 's/HEADERNOMATCH//' | sort|uniq -c|sed 's/^ *//;s/ /\t/'"
|
61
60
|
begin
|
62
61
|
TSV.open(CMD.cmd(cmd), :key_field => 1, :type => :single, :cast => :to_i)
|
63
62
|
rescue
|
64
|
-
Log.exception $!
|
65
63
|
TSV.setup({}, :type => :single, :cast => :to_i)
|
66
64
|
end
|
67
65
|
end
|
data/lib/rbbt/util/cmd.rb
CHANGED
@@ -3,8 +3,6 @@ require 'stringio'
|
|
3
3
|
|
4
4
|
module CMD
|
5
5
|
|
6
|
-
class CMDError < StandardError; end
|
7
|
-
|
8
6
|
module SmartIO
|
9
7
|
attr_accessor :pid, :cmd, :post, :in, :out, :err, :log
|
10
8
|
def self.tie(io, pid = nil, cmd = "", post = nil, sin = nil, out = nil, err = nil, log = true)
|
@@ -33,7 +31,7 @@ module CMD
|
|
33
31
|
|
34
32
|
if $? and not $?.success?
|
35
33
|
Log.debug{"Raising exception" if log}
|
36
|
-
exception =
|
34
|
+
exception = ProcessFailed.new "Command [#{@pid}] '#{@cmd}' failed with error status #{$?.exitstatus}"
|
37
35
|
begin
|
38
36
|
original_close
|
39
37
|
ensure
|
@@ -160,9 +158,9 @@ module CMD
|
|
160
158
|
|
161
159
|
exit(-1)
|
162
160
|
rescue Exception
|
163
|
-
Log.debug{ "
|
161
|
+
Log.debug{ "ProcessFailed: #{$!.message}" } if log
|
164
162
|
Log.debug{ "Backtrace: \n" + $!.backtrace * "\n" } if log
|
165
|
-
raise
|
163
|
+
raise ProcessFailed, $!.message
|
166
164
|
end
|
167
165
|
}
|
168
166
|
|
@@ -207,7 +205,8 @@ module CMD
|
|
207
205
|
Thread.exit
|
208
206
|
end
|
209
207
|
|
210
|
-
SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
|
208
|
+
#SmartIO.tie sout, pid, cmd, post, in_content, sin, serr
|
209
|
+
ConcurrentStream.setup sout, :pids => [pid], :autojoin => true
|
211
210
|
|
212
211
|
sout
|
213
212
|
else
|
@@ -222,12 +221,12 @@ module CMD
|
|
222
221
|
|
223
222
|
out = StringIO.new sout.read
|
224
223
|
sout.close unless sout.closed?
|
225
|
-
SmartIO.tie out, pid, cmd, post, in_content, sin, serr
|
224
|
+
#SmartIO.tie out, pid, cmd, post, in_content, sin, serr
|
226
225
|
|
227
226
|
Process.waitpid pid
|
228
227
|
|
229
228
|
if not $?.success?
|
230
|
-
exception =
|
229
|
+
exception = ProcessFailed.new "Command [#{pid}] #{cmd} failed with error status #{$?.exitstatus}.\n#{err}"
|
231
230
|
raise exception
|
232
231
|
else
|
233
232
|
Log.log err, stderr if Integer === stderr and log
|
@@ -18,6 +18,7 @@ class RbbtProcessQueue
|
|
18
18
|
end
|
19
19
|
|
20
20
|
Signal.trap(:INT){ raise Aborted; }
|
21
|
+
|
21
22
|
loop do
|
22
23
|
p = @queue.pop
|
23
24
|
next if p.nil?
|
@@ -32,6 +33,7 @@ class RbbtProcessQueue
|
|
32
33
|
Log.error "Worker #{Process.pid} aborted"
|
33
34
|
Kernel.exit! -1
|
34
35
|
rescue Exception
|
36
|
+
Log.exception $!
|
35
37
|
@callback_queue.push($!) if @callback_queue
|
36
38
|
Kernel.exit! -1
|
37
39
|
ensure
|
@@ -56,6 +56,7 @@ class RbbtProcessQueue
|
|
56
56
|
end
|
57
57
|
rescue Aborted
|
58
58
|
@processes.each{|p| p.abort }
|
59
|
+
@processes.each{|p| p.join }
|
59
60
|
Log.error "Process monitor aborted"
|
60
61
|
rescue Exception
|
61
62
|
Log.error "Process monitor exception: #{$!.message}"
|
@@ -78,7 +79,8 @@ class RbbtProcessQueue
|
|
78
79
|
def join
|
79
80
|
@processes.length.times do
|
80
81
|
@queue.push ClosedStream.new
|
81
|
-
end
|
82
|
+
end if @process_monitor.alive?
|
83
|
+
|
82
84
|
begin
|
83
85
|
@process_monitor.join
|
84
86
|
close_callback if @callback
|
@@ -104,8 +106,8 @@ class RbbtProcessQueue
|
|
104
106
|
end
|
105
107
|
|
106
108
|
def abort
|
107
|
-
@process_monitor.raise
|
108
|
-
@callback_thread.raise
|
109
|
+
@process_monitor.raise(Aborted.new); @process_monitor.join if @process_monitor and @process_monitor.alive?
|
110
|
+
@callback_thread.raise(Aborted.new); @callback_thread.join if @callback_thread and @callback_thread.alive?
|
109
111
|
end
|
110
112
|
|
111
113
|
def process(*e)
|
@@ -1,14 +1,15 @@
|
|
1
1
|
module ConcurrentStream
|
2
|
-
attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined
|
2
|
+
attr_accessor :threads, :pids, :callback, :abort_callback, :filename, :joined, :autojoin
|
3
3
|
|
4
4
|
def self.setup(stream, options = {}, &block)
|
5
|
-
threads, pids, callback, filename = Misc.process_options options, :threads, :pids, :callback, :filename
|
5
|
+
threads, pids, callback, filename, autojoin = Misc.process_options options, :threads, :pids, :callback, :filename, :autojoin
|
6
6
|
stream.extend ConcurrentStream unless ConcurrentStream === stream
|
7
7
|
|
8
8
|
stream.threads ||= []
|
9
9
|
stream.pids ||= []
|
10
10
|
stream.threads.concat(Array === threads ? threads : [threads]) unless threads.nil?
|
11
11
|
stream.pids.concat(Array === pids ? pids : [pids]) unless pids.nil? or pids.empty?
|
12
|
+
stream.autojoin = autojoin
|
12
13
|
|
13
14
|
callback = block if block_given?
|
14
15
|
if stream.callback and callback
|
@@ -33,6 +34,7 @@ module ConcurrentStream
|
|
33
34
|
stream.callback = callback
|
34
35
|
stream.abort_callback = abort_callback
|
35
36
|
stream.filename = filename
|
37
|
+
stream.autojoin = autojoin
|
36
38
|
stream.joined = joined
|
37
39
|
end
|
38
40
|
|
@@ -61,7 +63,7 @@ module ConcurrentStream
|
|
61
63
|
@pids.each do |pid|
|
62
64
|
begin
|
63
65
|
Process.waitpid(pid, Process::WUNTRACED)
|
64
|
-
raise "Error joining process #{pid} in #{self.inspect}" unless $?.success?
|
66
|
+
raise ProcessFailed.new "Error joining process #{pid} in #{self.inspect}" unless $?.success?
|
65
67
|
rescue Errno::ECHILD
|
66
68
|
end
|
67
69
|
end
|
@@ -102,4 +104,19 @@ module ConcurrentStream
|
|
102
104
|
@callback = nil
|
103
105
|
end
|
104
106
|
|
107
|
+
def read(*args)
|
108
|
+
if autojoin
|
109
|
+
begin
|
110
|
+
super(*args)
|
111
|
+
rescue
|
112
|
+
self.abort
|
113
|
+
self.join
|
114
|
+
ensure
|
115
|
+
self.join if self.closed? or self.eof?
|
116
|
+
end
|
117
|
+
else
|
118
|
+
super(*args)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
105
122
|
end
|
data/lib/rbbt/util/open.rb
CHANGED
@@ -341,9 +341,14 @@ module Open
|
|
341
341
|
io = gunzip(io) if ((String === url and gzip?(url)) and not options[:noz]) or options[:gzip]
|
342
342
|
|
343
343
|
if block_given?
|
344
|
-
|
345
|
-
|
346
|
-
|
344
|
+
begin
|
345
|
+
return yield(io)
|
346
|
+
rescue
|
347
|
+
io.abort if io.respond_to? :abort
|
348
|
+
io.join if io.respond_to? :join
|
349
|
+
ensure
|
350
|
+
io.join if io.respond_to? :join
|
351
|
+
end
|
347
352
|
else
|
348
353
|
io
|
349
354
|
end
|
@@ -100,7 +100,6 @@ class Step
|
|
100
100
|
dependency.relay_log self
|
101
101
|
dependency.clean if not dependency.done? and (dependency.error? or dependency.aborted?)
|
102
102
|
dependency.clean if dependency.streaming? and not dependency.running?
|
103
|
-
#dependency.run_dependencies(seen)
|
104
103
|
dependency.run(ENV["RBBT_NO_STREAM"] != 'true') unless dependency.result or dependency.done?
|
105
104
|
seen << dependency.path
|
106
105
|
seen.concat dependency.rec_dependencies.collect{|d| d.path}
|
@@ -159,12 +158,13 @@ class Step
|
|
159
158
|
raise $!
|
160
159
|
end
|
161
160
|
|
162
|
-
|
163
|
-
|
161
|
+
if not no_load or ENV["RBBT_NO_STREAM"] == "true"
|
162
|
+
result = prepare_result result, @task.description, info if IO === result
|
163
|
+
result = prepare_result result.stream, @task.description, info if TSV::Dumper === result
|
164
|
+
end
|
164
165
|
|
165
166
|
case result
|
166
167
|
when IO
|
167
|
-
result = Misc.read_stream(result) if ENV["RBBT_NO_STREAM"]
|
168
168
|
|
169
169
|
log :streaming, "#{Log.color :magenta, "Streaming task result IO"} #{Log.color :yellow, task.name.to_s || ""} [#{Process.pid}]"
|
170
170
|
ConcurrentStream.setup result do
|
data/lib/rbbt/workflow/task.rb
CHANGED
@@ -44,11 +44,7 @@ module Task
|
|
44
44
|
def exec(*args)
|
45
45
|
case
|
46
46
|
when (args.length == 1 and not inputs.nil? and inputs.length > 1 and Hash === args.first)
|
47
|
-
|
48
|
-
self.call *take_input_values(IndiferentHash.setup(args.first))
|
49
|
-
ensure
|
50
|
-
purge_stream_cache
|
51
|
-
end
|
47
|
+
self.call *take_input_values(IndiferentHash.setup(args.first))
|
52
48
|
else
|
53
49
|
self.call *args
|
54
50
|
end
|
data/test/rbbt/test_entity.rb
CHANGED
@@ -27,7 +27,7 @@ module ReversableString
|
|
27
27
|
self.reverse
|
28
28
|
end
|
29
29
|
|
30
|
-
property :reverse_text_ary_p => :
|
30
|
+
property :reverse_text_ary_p => :array2single do
|
31
31
|
$count += 1
|
32
32
|
self.collect{|s| s.reverse}
|
33
33
|
end
|
@@ -103,7 +103,7 @@ class TestEntity < Test::Unit::TestCase
|
|
103
103
|
$count = 0
|
104
104
|
|
105
105
|
assert_equal "2gnirtS", a.reverse_text_ary_p.last
|
106
|
-
assert_equal "2gnirtS", a[1]
|
106
|
+
assert_equal "2gnirtS", a.collect{|e| e.reverse_text_ary_p }[1]
|
107
107
|
assert_equal 1, $count
|
108
108
|
end
|
109
109
|
|
@@ -317,9 +317,13 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
317
317
|
raise "STOP" if rand(100) < 1
|
318
318
|
[k,v]
|
319
319
|
end
|
320
|
-
dumper.stream
|
320
|
+
stream = dumper.stream
|
321
|
+
stream.read
|
322
|
+
stream.join
|
321
323
|
rescue Exception
|
322
324
|
Log.exception $!
|
325
|
+
stream.abort if stream.respond_to? :abort
|
326
|
+
stream.join if stream.respond_to? :join
|
323
327
|
raise $!
|
324
328
|
end
|
325
329
|
end
|
data/test/rbbt/tsv/test_util.rb
CHANGED
@@ -5,7 +5,7 @@ require 'rbbt/util/misc'
|
|
5
5
|
|
6
6
|
class TestMiscPipes < Test::Unit::TestCase
|
7
7
|
|
8
|
-
def
|
8
|
+
def _test_collapse_stream
|
9
9
|
text=<<-EOF
|
10
10
|
row1 A B C
|
11
11
|
row1 a b c
|
@@ -19,7 +19,7 @@ row2 aa bb cc
|
|
19
19
|
assert_equal ["BB", "bb"], tsv["row2"][1]
|
20
20
|
end
|
21
21
|
|
22
|
-
def
|
22
|
+
def _test_paste_stream
|
23
23
|
text1=<<-EOF
|
24
24
|
row1 A B C
|
25
25
|
row2 AA BB CC
|
@@ -46,7 +46,7 @@ row3 ccc
|
|
46
46
|
assert_equal ["AAA", "BBB", "CCC", "", "", "ccc"], tsv["row3"]
|
47
47
|
end
|
48
48
|
|
49
|
-
def
|
49
|
+
def _test_sort_stream
|
50
50
|
text =<<-EOF
|
51
51
|
#: :sep=" "
|
52
52
|
#Row LabelA LabelB LabelC
|
@@ -60,7 +60,7 @@ row1 A B C
|
|
60
60
|
assert_equal %w(#: #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
|
61
61
|
end
|
62
62
|
|
63
|
-
def
|
63
|
+
def _test_dup_stream
|
64
64
|
text =<<-EOF
|
65
65
|
#: :sep=" "
|
66
66
|
#Row LabelA LabelB LabelC
|
data/test/rbbt/util/test_R.rb
CHANGED
@@ -2,11 +2,11 @@ require File.join(File.expand_path(File.dirname(__FILE__)), '../..', 'test_helpe
|
|
2
2
|
require 'rbbt/util/R'
|
3
3
|
|
4
4
|
class TestR < Test::Unit::TestCase
|
5
|
-
def
|
5
|
+
def test_sum
|
6
6
|
assert_equal "6", R.run('cat(3+3)').read.split(/\n/).last
|
7
7
|
end
|
8
8
|
|
9
|
-
def
|
9
|
+
def test_tsv_R
|
10
10
|
tsv = TSV.setup({:a => 1, :b => 2})
|
11
11
|
tsv2 = tsv.R <<-EOF
|
12
12
|
data = data + 1
|
data/test/rbbt/util/test_cmd.rb
CHANGED
@@ -29,17 +29,17 @@ class TestCmd < Test::Unit::TestCase
|
|
29
29
|
end
|
30
30
|
|
31
31
|
def test_error
|
32
|
-
assert_raise
|
33
|
-
assert_raise
|
32
|
+
assert_raise ProcessFailed do CMD.cmd('fake-command') end
|
33
|
+
assert_raise ProcessFailed do CMD.cmd('ls -fake_option') end
|
34
34
|
|
35
|
-
assert_raise
|
36
|
-
assert_raise
|
35
|
+
assert_raise ProcessFailed do CMD.cmd('fake-command', :stderr => true) end
|
36
|
+
assert_raise ProcessFailed do CMD.cmd('ls -fake_option', :stderr => true) end
|
37
37
|
|
38
|
-
assert_nothing_raised
|
39
|
-
assert_nothing_raised
|
38
|
+
assert_nothing_raised ProcessFailed do CMD.cmd('fake-command', :stderr => false, :pipe => true) end
|
39
|
+
assert_nothing_raised ProcessFailed do CMD.cmd('ls -fake_option', :stderr => false, :pipe => true) end
|
40
40
|
|
41
|
-
assert_raise
|
42
|
-
assert_raise
|
41
|
+
assert_raise ProcessFailed do CMD.cmd('fake-command', :stderr => true, :pipe => true).join end
|
42
|
+
assert_raise ProcessFailed do CMD.cmd('ls -fake_option', :stderr => true, :pipe => true).join end
|
43
43
|
end
|
44
44
|
|
45
45
|
def test_pipes
|
@@ -58,10 +58,7 @@ line33
|
|
58
58
|
gz = CMD.cmd("gunzip", :in => File.open(file + '.gz'), :pipe => true)
|
59
59
|
io = CMD.cmd('tail -n 10', :in => gz, :pipe => true)
|
60
60
|
assert_equal 10, io.read.split(/\n/).length
|
61
|
-
|
62
|
-
|
63
61
|
end
|
64
|
-
|
65
62
|
end
|
66
63
|
|
67
64
|
end
|
data/test/rbbt/util/test_misc.rb
CHANGED
@@ -115,15 +115,6 @@ eum fugiat quo voluptas nulla pariatur?"
|
|
115
115
|
assert_equal 4, Misc.process_to_hash(list){|l| l.collect{|e| e * 2}}[2]
|
116
116
|
end
|
117
117
|
|
118
|
-
def test_pipe
|
119
|
-
sout, sin = Misc.pipe
|
120
|
-
assert_equal 1, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
|
121
|
-
sin.close
|
122
|
-
assert sout.eof?
|
123
|
-
Misc.purge_pipes
|
124
|
-
assert_equal 0, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
|
125
|
-
end
|
126
|
-
|
127
118
|
def test_pipe_fork
|
128
119
|
sout, sin = Misc.pipe
|
129
120
|
pid = Process.fork do
|
data/test/rbbt/util/test_open.rb
CHANGED
@@ -62,7 +62,7 @@ class TestStep < Test::Unit::TestCase
|
|
62
62
|
step2 = Step.new tmpfile + 'step2', task2, [], [step1]
|
63
63
|
|
64
64
|
step2.run
|
65
|
-
assert step2.messages.include? "
|
65
|
+
assert step2.messages.include? "Starting Task1"
|
66
66
|
end
|
67
67
|
end
|
68
68
|
|
@@ -83,7 +83,8 @@ class TestStep < Test::Unit::TestCase
|
|
83
83
|
|
84
84
|
Step.log_relay_step = step2
|
85
85
|
step2.run
|
86
|
-
|
86
|
+
|
87
|
+
assert step2.messages.include? "Starting Task1"
|
87
88
|
end
|
88
89
|
end
|
89
90
|
|
@@ -99,7 +100,7 @@ class TestStep < Test::Unit::TestCase
|
|
99
100
|
end
|
100
101
|
|
101
102
|
|
102
|
-
def
|
103
|
+
def __test_fork
|
103
104
|
TmpFile.with_file do |lock|
|
104
105
|
task = Task.setup do while not File.exists?(lock) do sleep 1; end; "TEST" end
|
105
106
|
TmpFile.with_file do |tmp|
|
@@ -115,19 +116,20 @@ class TestStep < Test::Unit::TestCase
|
|
115
116
|
end
|
116
117
|
end
|
117
118
|
|
118
|
-
def
|
119
|
+
def __test_abort
|
119
120
|
TmpFile.with_file do |lock|
|
120
121
|
task = Task.setup do while not File.exists?(lock) do sleep 1; end; "TEST" end
|
121
122
|
TmpFile.with_file do |tmp|
|
122
123
|
step = Step.new tmp, task
|
123
124
|
job = step.fork
|
124
125
|
assert !job.done?
|
125
|
-
|
126
|
-
|
127
|
-
|
126
|
+
step.clean.fork
|
127
|
+
job.abort
|
128
|
+
assert_equal :aborted, job.status
|
128
129
|
Open.write(lock, "open")
|
130
|
+
job.clean.fork
|
129
131
|
job.join
|
130
|
-
assert job.
|
132
|
+
assert job.done?
|
131
133
|
end
|
132
134
|
end
|
133
135
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.12.
|
4
|
+
version: 5.12.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|