rbbt-util 5.10.2 → 5.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -17,7 +17,7 @@ EOF
17
17
 
18
18
  SOPT.usage if options[:help]
19
19
 
20
- last = start = Time.now
20
+ start = Time.now
21
21
  count = 0
22
22
  max = 0
23
23
  avg = 0
@@ -25,11 +25,12 @@ all = []
25
25
  scale = 5
26
26
  while line = STDIN.gets
27
27
  count += 1
28
+ last ||= Time.now
28
29
  if Time.now - last >= 1.0 / scale
29
30
  Log.clear_line
30
- puts "#{ count*scale } per second. Max #{max*scale}. Average #{avg*scale}"
31
- last = Time.now
31
+ puts "Reading #{ Log.color :blue, count*scale } per second. Max #{ Log.color :blue, max*scale }. Average #{Log.color :blue, avg*scale}"
32
32
  max = count > max ? count : max
33
+ last = Time.now
33
34
  all << count
34
35
  avg = Misc.mean(all).to_i if all.length > 3
35
36
  count = 0
@@ -38,4 +39,4 @@ end
38
39
 
39
40
  all << count
40
41
 
41
- puts "Total #{Misc.sum(all).to_i} in #{(Time.now - start).to_i} seconds -- #{(Misc.sum(all).to_f / (Time.now - start)).to_i } per second. Max #{max*scale}. Average #{avg*scale}"
42
+ puts "Total #{Log.color :blue, Misc.sum(all).to_i} in #{Log.color :blue, (Time.now - start).to_i} seconds -- #{Log.color :green, (Misc.sum(all).to_f / (Time.now - start)).to_i } per second. Max #{Log.color :blue, max*scale}. Average #{Log.color :green, avg*scale}"
@@ -23,9 +23,14 @@ SOPT.usage if options[:help]
23
23
 
24
24
  file = ARGV.shift
25
25
 
26
- file = STDIN if file == '-'
27
-
28
- raise ParameterException, "Please specify the tsv file as argument" if file.nil?
26
+ file = case file
27
+ when "-"
28
+ STDIN
29
+ when String
30
+ Open.open(file)
31
+ else
32
+ raise ParameterException, "Please specify the tsv file as argument" if file.nil?
33
+ end
29
34
 
30
35
  fields = options[:fields]
31
36
  raise ParameterException, "Please specify the fields to slice" if fields.nil?
@@ -40,7 +45,11 @@ when options[:tokyocabinet_bd]
40
45
  tsv = Persist.open_tokyocabinet(file, false, nil, TokyoCabinet::BDB)
41
46
  puts tsv.summary
42
47
  else
43
- tsv = TSV.open(file, options)
48
+ stream = TSV.traverse file, options.merge(:into => :stream, :type => :list, :keys => fields, :unnamed => true) do |*p|
49
+ p * "\t"
50
+ end
51
+ puts stream.read
52
+ exit 0
44
53
  end
45
54
 
46
55
  puts tsv.slice(fields.split(","))
@@ -333,7 +333,7 @@ begin
333
333
  end
334
334
 
335
335
  if options.delete(:printpath)
336
- job.join
336
+ job.join.load
337
337
  puts job.path
338
338
  exit 0
339
339
  end
@@ -403,12 +403,18 @@ when (defined?(WorkflowRESTClient) and WorkflowRESTClient::RemoteStep)
403
403
  out.puts res.load
404
404
  when Step
405
405
  if IO === res.result
406
- io = res.result
407
- Thread.pass while IO.select([io]).nil?
408
- while block = io.read(2048) do
409
- out.write block
410
- end unless io.closed?
411
- io.join if io.respond_to? :join
406
+ begin
407
+ io = res.result
408
+ Thread.pass while IO.select([io]).nil?
409
+ while block = io.read(2048) do
410
+ out.write block
411
+ end unless io.closed?
412
+ io.join if io.respond_to? :join
413
+ rescue Exception
414
+ io.abort if io.respond_to? :abort
415
+ ensure
416
+ io.join if io.respond_to? :join
417
+ end
412
418
  else
413
419
  res.join
414
420
  out.puts Open.read(res.path) if File.exists? res.path
@@ -12,7 +12,7 @@ end
12
12
 
13
13
  class TestPersist < Test::Unit::TestCase
14
14
 
15
- def _test_array_persist
15
+ def test_array_persist
16
16
  TmpFile.with_file do |tmp|
17
17
  10.times do
18
18
  assert_equal ["1", "2"],(Persist.persist("Test", :array, :file => tmp) do
@@ -40,7 +40,7 @@ class TestPersist < Test::Unit::TestCase
40
40
 
41
41
  def test_tsv_dumper
42
42
  TmpFile.with_file do |tmpdir|
43
- stream = Persist.persist("Dumper", :tsv, :dir => tmpdir) do
43
+ tsv = Persist.persist("Dumper", :tsv, :dir => tmpdir) do
44
44
  dumper = TSV::Dumper.new :key_field => "Field 1", :fields => ["Field 2"], :type => :single
45
45
 
46
46
  dumper.init
@@ -54,10 +54,27 @@ class TestPersist < Test::Unit::TestCase
54
54
  dumper
55
55
  end
56
56
 
57
- while line = stream.gets do
58
- puts line
57
+ assert_equal 10, tsv.size
58
+ end
59
+ end
60
+
61
+ def test_tsv_dumper_stream
62
+ TmpFile.with_file do |tmpdir|
63
+ stream = Persist.persist("Dumper", :tsv, :dir => tmpdir, :no_load => :stream) do
64
+ dumper = TSV::Dumper.new :key_field => "Field 1", :fields => ["Field 2"], :type => :single
65
+
66
+ Thread.new do
67
+ 10.times do |i|
68
+ key = i.to_s
69
+ dumper.add key, key + " - 2"
70
+ end
71
+ dumper.close
72
+ end
73
+
74
+ dumper
59
75
  end
60
76
 
77
+ assert_equal 10, stream.read.split("\n").length
61
78
  end
62
79
  end
63
80
  end
@@ -324,4 +324,13 @@ class TestTSVParallelThrough < Test::Unit::TestCase
324
324
  end
325
325
  end
326
326
  end
327
+
328
+ def test_traverse_into_stream
329
+ size = 100
330
+ array = (1..size).to_a.collect{|n| n.to_s}
331
+ stream = TSV.traverse array, :into => :stream do |e|
332
+ e
333
+ end
334
+ assert_equal size, stream.read.split("\n").length
335
+ end
327
336
  end
@@ -28,7 +28,9 @@ class TestConcurrency < Test::Unit::TestCase
28
28
  end
29
29
 
30
30
  socket.swrite.close
31
- assert ClosedStream === socket.pop
31
+ assert_raise ClosedStream do
32
+ socket.pop
33
+ end
32
34
 
33
35
  socket.clean
34
36
  end
@@ -44,6 +44,35 @@ class TestConcurrencyProcess < Test::Unit::TestCase
44
44
  assert_equal times, Dir.glob(File.join(dir, '*')).length
45
45
  end
46
46
  end
47
+
48
+ def test_process_abort
49
+ assert_raise Aborted do
50
+ q = RbbtProcessQueue.new 10
51
+
52
+ res = []
53
+
54
+ q.callback do |v|
55
+ res << v
56
+ end
57
+
58
+ q.init do |i|
59
+ sleep 1 while true
60
+ end
61
+
62
+ times = 500
63
+ t = TSV.setup({"a" => 1}, :type => :single)
64
+
65
+ times.times do |i|
66
+ q.process i
67
+ end
68
+
69
+ sleep 2
70
+ q.clean
71
+
72
+ q.join
73
+
74
+ end
75
+ end
47
76
  end
48
77
 
49
78
 
@@ -117,11 +117,11 @@ eum fugiat quo voluptas nulla pariatur?"
117
117
 
118
118
  def test_pipe
119
119
  sout, sin = Misc.pipe
120
- assert_equal 1, Misc::OPEN_PIPE_IN.length
120
+ assert_equal 1, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
121
121
  sin.close
122
122
  assert sout.eof?
123
123
  Misc.purge_pipes
124
- assert_equal 0, Misc::OPEN_PIPE_IN.length
124
+ assert_equal 0, Misc::OPEN_PIPE_IN.reject{|p| p.closed? }.length
125
125
  end
126
126
 
127
127
  def test_pipe_fork
@@ -325,10 +325,10 @@ eum fugiat quo voluptas nulla pariatur?"
325
325
  pids = []
326
326
  4.times do |i|
327
327
  pids << Process.fork do
328
- pid = Process.pid().to_s
329
- status = Misc.lock(tmpfile, pid) do |f, val|
330
- Open.write(f, val)
331
- sleep rand * 2
328
+ status = Misc.lock(tmpfile) do
329
+ pid = Process.pid.to_s
330
+ Open.write(tmpfile, pid)
331
+ sleep rand * 1
332
332
  if pid == Open.read(tmpfile)
333
333
  0
334
334
  else
@@ -122,7 +122,7 @@ class TestStep < Test::Unit::TestCase
122
122
  step = Step.new tmp, task
123
123
  job = step.fork
124
124
  assert !job.done?
125
- assert_raise RuntimeError do step.fork end
125
+ assert_raise RuntimeError do step.clean.fork end
126
126
  sleep 1
127
127
  while not job.abort do sleep 1 end
128
128
  Open.write(lock, "open")
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rbbt-util
3
3
  version: !ruby/object:Gem::Version
4
- version: 5.10.2
4
+ version: 5.11.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-04-03 00:00:00.000000000 Z
11
+ date: 2014-04-05 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: rake