rbbt-util 5.10.2 → 5.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/rbbt/persist.rb +60 -20
- data/lib/rbbt/tsv/dumper.rb +1 -1
- data/lib/rbbt/tsv/parallel/traverse.rb +83 -65
- data/lib/rbbt/tsv/parser.rb +23 -8
- data/lib/rbbt/util/concurrency/processes.rb +23 -10
- data/lib/rbbt/util/concurrency/processes/worker.rb +3 -1
- data/lib/rbbt/util/log.rb +5 -10
- data/lib/rbbt/util/misc.rb +85 -52
- data/lib/rbbt/workflow.rb +2 -1
- data/lib/rbbt/workflow/step.rb +79 -29
- data/share/rbbt_commands/benchmark/throughput +5 -4
- data/share/rbbt_commands/tsv/slice +13 -4
- data/share/rbbt_commands/workflow/task +13 -7
- data/test/rbbt/test_persist.rb +21 -4
- data/test/rbbt/tsv/parallel/test_traverse.rb +9 -0
- data/test/rbbt/util/concurrency/processes/test_socket.rb +3 -1
- data/test/rbbt/util/concurrency/test_processes.rb +29 -0
- data/test/rbbt/util/test_misc.rb +6 -6
- data/test/rbbt/workflow/test_step.rb +1 -1
- metadata +2 -2
@@ -17,7 +17,7 @@ EOF
|
|
17
17
|
|
18
18
|
SOPT.usage if options[:help]
|
19
19
|
|
20
|
-
|
20
|
+
start = Time.now
|
21
21
|
count = 0
|
22
22
|
max = 0
|
23
23
|
avg = 0
|
@@ -25,11 +25,12 @@ all = []
|
|
25
25
|
scale = 5
|
26
26
|
while line = STDIN.gets
|
27
27
|
count += 1
|
28
|
+
last ||= Time.now
|
28
29
|
if Time.now - last >= 1.0 / scale
|
29
30
|
Log.clear_line
|
30
|
-
puts "#{ count*scale } per second. Max #{max*scale}. Average #{avg*scale}"
|
31
|
-
last = Time.now
|
31
|
+
puts "Reading #{ Log.color :blue, count*scale } per second. Max #{ Log.color :blue, max*scale }. Average #{Log.color :blue, avg*scale}"
|
32
32
|
max = count > max ? count : max
|
33
|
+
last = Time.now
|
33
34
|
all << count
|
34
35
|
avg = Misc.mean(all).to_i if all.length > 3
|
35
36
|
count = 0
|
@@ -38,4 +39,4 @@ end
|
|
38
39
|
|
39
40
|
all << count
|
40
41
|
|
41
|
-
puts "Total #{Misc.sum(all).to_i} in #{(Time.now - start).to_i} seconds -- #{(Misc.sum(all).to_f / (Time.now - start)).to_i } per second. Max #{max*scale}. Average #{avg*scale}"
|
42
|
+
puts "Total #{Log.color :blue, Misc.sum(all).to_i} in #{Log.color :blue, (Time.now - start).to_i} seconds -- #{Log.color :green, (Misc.sum(all).to_f / (Time.now - start)).to_i } per second. Max #{Log.color :blue, max*scale}. Average #{Log.color :green, avg*scale}"
|
@@ -23,9 +23,14 @@ SOPT.usage if options[:help]
|
|
23
23
|
|
24
24
|
file = ARGV.shift
|
25
25
|
|
26
|
-
file =
|
27
|
-
|
28
|
-
|
26
|
+
file = case file
|
27
|
+
when "-"
|
28
|
+
STDIN
|
29
|
+
when String
|
30
|
+
Open.open(file)
|
31
|
+
else
|
32
|
+
raise ParameterException, "Please specify the tsv file as argument" if file.nil?
|
33
|
+
end
|
29
34
|
|
30
35
|
fields = options[:fields]
|
31
36
|
raise ParameterException, "Please specify the fields to slice" if fields.nil?
|
@@ -40,7 +45,11 @@ when options[:tokyocabinet_bd]
|
|
40
45
|
tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
|
41
46
|
puts tsv.summary
|
42
47
|
else
|
43
|
-
|
48
|
+
stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :keys => fields, :unnamed => true) do |*p|
|
49
|
+
p * "\t"
|
50
|
+
end
|
51
|
+
puts stream.read
|
52
|
+
exit 0
|
44
53
|
end
|
45
54
|
|
46
55
|
puts tsv.slice(fields.split(","))
|
@@ -333,7 +333,7 @@ begin
|
|
333
333
|
end
|
334
334
|
|
335
335
|
if options.delete(:printpath)
|
336
|
-
job.join
|
336
|
+
job.join.load
|
337
337
|
puts job.path
|
338
338
|
exit 0
|
339
339
|
end
|
@@ -403,12 +403,18 @@ when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
|
|
403
403
|
out.puts res.load
|
404
404
|
when Step
|
405
405
|
if IO === res.result
|
406
|
-
|
407
|
-
|
408
|
-
|
409
|
-
|
410
|
-
|
411
|
-
|
406
|
+
begin
|
407
|
+
io = res.result
|
408
|
+
Thread.pass while IO.select([io]).nil?
|
409
|
+
while block = io.read(2048) do
|
410
|
+
out.write block
|
411
|
+
end unless io.closed?
|
412
|
+
io.join if io.respond_to? :join
|
413
|
+
rescue Exception
|
414
|
+
io.abort if io.respond_to? :abort
|
415
|
+
ensure
|
416
|
+
io.join if io.respond_to? :join
|
417
|
+
end
|
412
418
|
else
|
413
419
|
res.join
|
414
420
|
out.puts Open.read(res.path) if File.exists? res.path
|
data/test/rbbt/test_persist.rb
CHANGED
@@ -12,7 +12,7 @@ end
|
|
12
12
|
|
13
13
|
class TestPersist < Test::Unit::TestCase
|
14
14
|
|
15
|
-
def
|
15
|
+
def test_array_persist
|
16
16
|
TmpFile.with_file do |tmp|
|
17
17
|
10.times do
|
18
18
|
assert_equal ["1", "2"],(Persist.persist("Test", :array, :file => tmp) do
|
@@ -40,7 +40,7 @@ class TestPersist < Test::Unit::TestCase
|
|
40
40
|
|
41
41
|
def test_tsv_dumper
|
42
42
|
TmpFile.with_file do |tmpdir|
|
43
|
-
|
43
|
+
tsv = Persist.persist("Dumper", :tsv, :dir => tmpdir) do
|
44
44
|
dumper = TSV::Dumper.new :key_field => "Field 1", :fields => ["Field 2"], :type => :single
|
45
45
|
|
46
46
|
dumper.init
|
@@ -54,10 +54,27 @@ class TestPersist < Test::Unit::TestCase
|
|
54
54
|
dumper
|
55
55
|
end
|
56
56
|
|
57
|
-
|
58
|
-
|
57
|
+
assert_equal 10, tsv.size
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def test_tsv_dumper_stream
|
62
|
+
TmpFile.with_file do |tmpdir|
|
63
|
+
stream = Persist.persist("Dumper", :tsv, :dir => tmpdir, :no_load => :stream) do
|
64
|
+
dumper = TSV::Dumper.new :key_field => "Field 1", :fields => ["Field 2"], :type => :single
|
65
|
+
|
66
|
+
Thread.new do
|
67
|
+
10.times do |i|
|
68
|
+
key = i.to_s
|
69
|
+
dumper.add key, key + " - 2"
|
70
|
+
end
|
71
|
+
dumper.close
|
72
|
+
end
|
73
|
+
|
74
|
+
dumper
|
59
75
|
end
|
60
76
|
|
77
|
+
assert_equal 10, stream.read.split("\n").length
|
61
78
|
end
|
62
79
|
end
|
63
80
|
end
|
@@ -324,4 +324,13 @@ class TestTSVParallelThrough < Test::Unit::TestCase
|
|
324
324
|
end
|
325
325
|
end
|
326
326
|
end
|
327
|
+
|
328
|
+
def test_traverse_into_stream
|
329
|
+
size = 100
|
330
|
+
array = (1..size).to_a.collect{|n| n.to_s}
|
331
|
+
stream = TSV.traverse array, :into => :stream do |e|
|
332
|
+
e
|
333
|
+
end
|
334
|
+
assert_equal size, stream.read.split("\n").length
|
335
|
+
end
|
327
336
|
end
|
@@ -44,6 +44,35 @@ class TestConcurrencyProcess < Test::Unit::TestCase
|
|
44
44
|
assert_equal times, Dir.glob(File.join(dir, '*')).length
|
45
45
|
end
|
46
46
|
end
|
47
|
+
|
48
|
+
def test_process_abort
|
49
|
+
assert_raise Aborted do
|
50
|
+
q = RbbtProcessQueue.new 10
|
51
|
+
|
52
|
+
res = []
|
53
|
+
|
54
|
+
q.callback do |v|
|
55
|
+
res << v
|
56
|
+
end
|
57
|
+
|
58
|
+
q.init do |i|
|
59
|
+
sleep 1 while true
|
60
|
+
end
|
61
|
+
|
62
|
+
times = 500
|
63
|
+
t = TSV.setup({"a" => 1}, :type => :single)
|
64
|
+
|
65
|
+
times.times do |i|
|
66
|
+
q.process i
|
67
|
+
end
|
68
|
+
|
69
|
+
sleep 2
|
70
|
+
q.clean
|
71
|
+
|
72
|
+
q.join
|
73
|
+
|
74
|
+
end
|
75
|
+
end
|
47
76
|
end
|
48
77
|
|
49
78
|
|
data/test/rbbt/util/test_misc.rb
CHANGED
@@ -117,11 +117,11 @@ eum fugiat quo voluptas nulla pariatur?"
|
|
117
117
|
|
118
118
|
def test_pipe
|
119
119
|
sout, sin = Misc.pipe
|
120
|
-
assert_equal 1, Misc::OPEN_PIPE_IN.length
|
120
|
+
assert_equal 1, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
|
121
121
|
sin.close
|
122
122
|
assert sout.eof?
|
123
123
|
Misc.purge_pipes
|
124
|
-
assert_equal 0, Misc::OPEN_PIPE_IN.length
|
124
|
+
assert_equal 0, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
|
125
125
|
end
|
126
126
|
|
127
127
|
def test_pipe_fork
|
@@ -325,10 +325,10 @@ eum fugiat quo voluptas nulla pariatur?"
|
|
325
325
|
pids = []
|
326
326
|
4.times do |i|
|
327
327
|
pids << Process.fork do
|
328
|
-
|
329
|
-
|
330
|
-
Open.write(
|
331
|
-
sleep rand *
|
328
|
+
status = Misc.lock(tmpfile) do
|
329
|
+
pid = Process.pid.to_s
|
330
|
+
Open.write(tmpfile, pid)
|
331
|
+
sleep rand * 1
|
332
332
|
if pid == Open.read(tmpfile)
|
333
333
|
0
|
334
334
|
else
|
@@ -122,7 +122,7 @@ class TestStep < Test::Unit::TestCase
|
|
122
122
|
step = Step.new tmp, task
|
123
123
|
job = step.fork
|
124
124
|
assert !job.done?
|
125
|
-
assert_raise RuntimeError do step.fork end
|
125
|
+
assert_raise RuntimeError do step.clean.fork end
|
126
126
|
sleep 1
|
127
127
|
while not job.abort do sleep 1 end
|
128
128
|
Open.write(lock, "open")
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rbbt-util
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.11.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Miguel Vazquez
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|