scout-gear 6.0.0 → 7.2.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +465 -432
  3. data/VERSION +1 -1
  4. data/bin/scout +5 -1
  5. data/lib/rbbt-scout.rb +5 -0
  6. data/lib/scout/concurrent_stream.rb +6 -2
  7. data/lib/scout/config.rb +168 -0
  8. data/lib/scout/exceptions.rb +9 -0
  9. data/lib/scout/indiferent_hash/options.rb +1 -0
  10. data/lib/scout/indiferent_hash.rb +4 -2
  11. data/lib/scout/log/color.rb +31 -2
  12. data/lib/scout/log/progress/report.rb +1 -0
  13. data/lib/scout/log/progress/util.rb +3 -1
  14. data/lib/scout/log/progress.rb +7 -3
  15. data/lib/scout/log.rb +8 -3
  16. data/lib/scout/misc/digest.rb +1 -3
  17. data/lib/scout/misc/monitor.rb +3 -0
  18. data/lib/scout/misc/system.rb +15 -0
  19. data/lib/scout/misc.rb +1 -0
  20. data/lib/scout/named_array.rb +68 -0
  21. data/lib/scout/open/stream.rb +58 -26
  22. data/lib/scout/path/find.rb +27 -3
  23. data/lib/scout/path/util.rb +7 -4
  24. data/lib/scout/persist/serialize.rb +7 -14
  25. data/lib/scout/persist.rb +21 -1
  26. data/lib/scout/resource/produce.rb +7 -94
  27. data/lib/scout/resource/software.rb +176 -0
  28. data/lib/scout/tsv/dumper.rb +107 -0
  29. data/lib/scout/tsv/index.rb +49 -0
  30. data/lib/scout/tsv/parser.rb +317 -0
  31. data/lib/scout/tsv/path.rb +13 -0
  32. data/lib/scout/tsv/persist/adapter.rb +348 -0
  33. data/lib/scout/tsv/persist/tokyocabinet.rb +113 -0
  34. data/lib/scout/tsv/persist.rb +15 -0
  35. data/lib/scout/tsv/traverse.rb +48 -0
  36. data/lib/scout/tsv/util.rb +24 -0
  37. data/lib/scout/tsv.rb +27 -0
  38. data/lib/scout/work_queue/worker.rb +16 -11
  39. data/lib/scout/work_queue.rb +63 -21
  40. data/lib/scout/workflow/definition.rb +93 -4
  41. data/lib/scout/workflow/step/config.rb +18 -0
  42. data/lib/scout/workflow/step/dependencies.rb +40 -0
  43. data/lib/scout/workflow/step/file.rb +15 -0
  44. data/lib/scout/workflow/step/info.rb +33 -6
  45. data/lib/scout/workflow/step/provenance.rb +148 -0
  46. data/lib/scout/workflow/step.rb +70 -20
  47. data/lib/scout/workflow/task.rb +5 -4
  48. data/lib/scout/workflow/usage.rb +1 -1
  49. data/lib/scout/workflow.rb +11 -3
  50. data/lib/scout-gear.rb +1 -0
  51. data/lib/scout.rb +1 -0
  52. data/scout-gear.gemspec +38 -3
  53. data/scout_commands/find +1 -1
  54. data/scout_commands/workflow/task +16 -10
  55. data/share/software/install_helpers +523 -0
  56. data/test/scout/log/test_progress.rb +0 -2
  57. data/test/scout/misc/test_system.rb +21 -0
  58. data/test/scout/open/test_stream.rb +160 -1
  59. data/test/scout/path/test_find.rb +14 -7
  60. data/test/scout/resource/test_software.rb +24 -0
  61. data/test/scout/test_config.rb +66 -0
  62. data/test/scout/test_meta_extension.rb +10 -0
  63. data/test/scout/test_named_array.rb +19 -0
  64. data/test/scout/test_persist.rb +35 -0
  65. data/test/scout/test_semaphore.rb +1 -1
  66. data/test/scout/test_tmpfile.rb +2 -2
  67. data/test/scout/test_tsv.rb +74 -0
  68. data/test/scout/test_work_queue.rb +63 -8
  69. data/test/scout/tsv/persist/test_adapter.rb +34 -0
  70. data/test/scout/tsv/persist/test_tokyocabinet.rb +92 -0
  71. data/test/scout/tsv/test_dumper.rb +44 -0
  72. data/test/scout/tsv/test_index.rb +64 -0
  73. data/test/scout/tsv/test_parser.rb +173 -0
  74. data/test/scout/tsv/test_persist.rb +36 -0
  75. data/test/scout/tsv/test_traverse.rb +9 -0
  76. data/test/scout/tsv/test_util.rb +0 -0
  77. data/test/scout/work_queue/test_worker.rb +49 -1
  78. data/test/scout/workflow/step/test_dependencies.rb +25 -0
  79. data/test/scout/workflow/step/test_info.rb +15 -17
  80. data/test/scout/workflow/step/test_load.rb +16 -18
  81. data/test/scout/workflow/step/test_provenance.rb +25 -0
  82. data/test/scout/workflow/test_step.rb +206 -10
  83. data/test/scout/workflow/test_task.rb +0 -3
  84. data/test/test_helper.rb +6 -0
  85. metadata +37 -2
@@ -99,6 +99,32 @@ class TestOpenStream < Test::Unit::TestCase
99
99
  end
100
100
  end
101
101
 
102
+ def test_tee_stream_multiple
103
+ num = 2000
104
+ sout = Open.open_pipe do |sin|
105
+ num.times do |i|
106
+ sin.puts "line #{i} - #{rand(100000).to_s * 100}"
107
+ end
108
+ end
109
+
110
+ TmpFile.with_file do |tmp|
111
+ Path.setup tmp
112
+
113
+ s1, s2, s3 = Open.tee_stream_thread_multiple sout, 3
114
+
115
+ t1 = Open.consume_stream(s1, true, tmp.file1)
116
+ t2 = Open.consume_stream(s2, true, tmp.file2)
117
+ t3 = Open.consume_stream(s3, true, tmp.file3)
118
+ t1.join
119
+ t2.join
120
+ t3.join
121
+ assert_equal num, Open.read(tmp.file1).split("\n").length
122
+ assert_equal num, Open.read(tmp.file2).split("\n").length
123
+ assert_equal num, Open.read(tmp.file3).split("\n").length
124
+ end
125
+ end
126
+
127
+
102
128
  def test_tee_stream_source_error
103
129
  5.times do |i|
104
130
  num = 2000
@@ -137,9 +163,50 @@ class TestOpenStream < Test::Unit::TestCase
137
163
  end
138
164
  end
139
165
 
166
+ def test_tee_stream_source_error_multiple
167
+ 5.times do |i|
168
+ num = 2000
169
+ sout = Open.open_pipe do |sin|
170
+ num.times do |i|
171
+ sin.puts "line #{i} - #{rand(100000).to_s * 100}"
172
+ end
173
+ raise ScoutException
174
+ end
175
+
176
+ TmpFile.with_file do |tmp|
177
+ Path.setup tmp
178
+
179
+ s1, s2, s3 = Open.tee_stream_thread_multiple sout, 3
180
+
181
+ t1 = Open.consume_stream(s1, true, tmp.file1)
182
+ t2 = Open.consume_stream(s2, true, tmp.file2)
183
+ t3 = Open.consume_stream(s3, true, tmp.file3)
184
+ assert_raise ScoutException do
185
+ begin
186
+ t1.join
187
+ t2.join
188
+ t3.join
189
+ rescue
190
+ raise
191
+ ensure
192
+ [t1, t2, t2].each do |t|
193
+ begin
194
+ t.join if t.alive?
195
+ rescue ScoutException
196
+ end
197
+ end
198
+ end
199
+ end
200
+ refute Open.exist?(tmp.file1)
201
+ refute Open.exist?(tmp.file2)
202
+ refute Open.exist?(tmp.file3)
203
+ end
204
+ end
205
+ end
206
+
140
207
  def test_tee_stream_save_error
141
208
  Log.with_severity 6 do
142
- 5.times do |i|
209
+ 50.times do |i|
143
210
  TmpFile.with_file do |tmp|
144
211
  Path.setup tmp
145
212
  assert_raise ScoutException do
@@ -175,6 +242,98 @@ class TestOpenStream < Test::Unit::TestCase
175
242
  end
176
243
  end
177
244
 
245
+ def test_tee_stream_save_error_multiple
246
+ Log.with_severity 6 do
247
+ 50.times do |i|
248
+ TmpFile.with_file do |tmp|
249
+ Path.setup tmp
250
+ assert_raise ScoutException do
251
+ num = 2000
252
+ begin
253
+ sout = Open.open_pipe do |sin|
254
+ num.times do |i|
255
+ sin.puts "line #{i} - #{rand(100000).to_s * 100}"
256
+ end
257
+ end
258
+
259
+ begin
260
+ s1, s2, s3 = Open.tee_stream_thread_multiple sout, 3
261
+
262
+ s2.abort ScoutException.new
263
+ t1 = Open.consume_stream(s1, true, tmp.file1)
264
+ t2 = Open.consume_stream(s2, true, tmp.file2)
265
+
266
+ t1.join
267
+ t2.join
268
+ ensure
269
+ s1.close if s1.respond_to?(:close) && ! s1.closed?
270
+ s1.join if s1.respond_to?(:join) && ! s1.joined?
271
+ s2.close if s2.respond_to?(:close) && ! s2.closed?
272
+ s2.join if s2.respond_to?(:join) && ! s2.joined?
273
+ s3.close if s3.respond_to?(:close) && ! s3.closed?
274
+ s3.join if s3.respond_to?(:join) && ! s3.joined?
275
+ end
276
+ ensure
277
+ sout.close if sout.respond_to?(:close) && ! sout.closed?
278
+ sout.join if sout.respond_to?(:join) && ! sout.joined?
279
+ end
280
+ end
281
+ refute Open.exist?(tmp.file1)
282
+ refute Open.exist?(tmp.file2)
283
+ end
284
+ end
285
+ end
286
+ end
287
+
288
+ def test_sort_stream
289
+ text =<<-EOF
290
+ ##
291
+ ##
292
+ ##
293
+ #Row LabelA LabelB LabelC
294
+ row2 AA BB CC
295
+ row3 AAA BBB CCC
296
+ row1 A B C
297
+ EOF
298
+ s = StringIO.new text
299
+ sorted = Open.sort_stream(s)
300
+ assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
301
+ assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
302
+ end
303
+
304
+ def test_sort_long_stream
305
+ text =<<-EOF
306
+ ##
307
+ ##
308
+ ##
309
+ #Row LabelA LabelB LabelC
310
+ row2 AA BB CC
311
+ row3 AAA BBB CCC
312
+ row1 A B C
313
+ EOF
314
+
315
+ s = StringIO.new text + (text.split("\n")[-3..-1] * "\n" + "\n") * 10000
316
+ sorted = Open.sort_stream(s)
317
+ assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
318
+ assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
319
+ end
320
+
321
+ def test_sort_stream2
322
+ text =<<-EOF
323
+ ##
324
+ ##
325
+ ##
326
+ #Row LabelA LabelB LabelC
327
+ row2 AA BB CC
328
+ row3 AAA BBB CCC
329
+ row1 A B C
330
+ EOF
331
+ s = StringIO.new text
332
+ sorted = Open.sort_stream(Open.sort_stream(s))
333
+ assert_equal %w(## ## ## #Row row2 row3 row1), text.split("\n").collect{|l| l.split(" ").first}
334
+ assert_equal %w(## ## ## #Row row1 row2 row3), sorted.read.split("\n").collect{|l| l.split(" ").first}
335
+ end
336
+
178
337
  # def test_collapse_stream
179
338
  # text=<<-EOF
180
339
  #row1 A B C
@@ -29,7 +29,7 @@ class TestPathFind < Test::Unit::TestCase
29
29
  def test_current
30
30
  path = Path.setup("share/data/some_file", 'scout')
31
31
  TmpFile.in_dir do |tmpdir|
32
- assert_equal File.join(tmpdir,"share/data/some_file"), path.find(:current)
32
+ assert_equal_path File.join(tmpdir,"share/data/some_file"), path.find(:current)
33
33
  end
34
34
  end
35
35
 
@@ -38,7 +38,7 @@ class TestPathFind < Test::Unit::TestCase
38
38
  TmpFile.in_dir do |tmpdir|
39
39
  FileUtils.mkdir_p(File.dirname(File.join(tmpdir, path)))
40
40
  File.write(File.join(tmpdir, path), 'string')
41
- assert_equal File.join(tmpdir,"share/data/some_file"), path.find
41
+ assert_equal_path File.join(tmpdir,"share/data/some_file"), path.find
42
42
  assert_equal :current, path.find.where
43
43
  assert_equal "share/data/some_file", path.find.original
44
44
  end
@@ -67,7 +67,7 @@ class TestPathFind < Test::Unit::TestCase
67
67
 
68
68
  p = Path.setup("/tmp/foo/bar")
69
69
  assert p.located?
70
- assert_equal p, p.find
70
+ assert_equal_path p, p.find
71
71
 
72
72
  end
73
73
 
@@ -75,10 +75,10 @@ class TestPathFind < Test::Unit::TestCase
75
75
  path = Path.setup("share/data/some_file", 'scout')
76
76
  TmpFile.with_file do |tmpdir|
77
77
  path.path_maps[:custom] = [tmpdir, '{PATH}'] * "/"
78
- assert_equal File.join(tmpdir,"share/data/some_file"), path.find(:custom)
78
+ assert_equal_path File.join(tmpdir,"share/data/some_file"), path.find(:custom)
79
79
 
80
80
  path.path_maps[:custom] = [tmpdir, '{TOPLEVEL}/{PKGDIR}/{SUBPATH}'] * "/"
81
- assert_equal File.join(tmpdir,"share/scout/data/some_file"), path.find(:custom)
81
+ assert_equal_path File.join(tmpdir,"share/scout/data/some_file"), path.find(:custom)
82
82
  end
83
83
  end
84
84
 
@@ -87,7 +87,7 @@ class TestPathFind < Test::Unit::TestCase
87
87
  TmpFile.with_file do |tmpdir|
88
88
  path.pkgdir = 'scout_alt'
89
89
  path.path_maps[:custom] = [tmpdir, '{TOPLEVEL}/{PKGDIR}/{SUBPATH}'] * "/"
90
- assert_equal File.join(tmpdir,"share/scout_alt/data/some_file"), path.find(:custom)
90
+ assert_equal_path File.join(tmpdir,"share/scout_alt/data/some_file"), path.find(:custom)
91
91
  end
92
92
  end
93
93
 
@@ -98,6 +98,13 @@ class TestPathFind < Test::Unit::TestCase
98
98
  assert_equal "/some_dir/scout_commands/find", Path.follow(path, '/some_dir/{PATH/bin\/scout/scout_commands}')
99
99
  end
100
100
 
101
-
101
+ def test_gz
102
+ TmpFile.with_file do |tmpdir|
103
+ Path.setup(tmpdir)
104
+ Open.write(tmpdir.somefile + '.gz', "FOO")
105
+ assert_equal_path tmpdir.somefile + '.gz', tmpdir.somefile.find
106
+ assert Open.exist?(tmpdir.somefile)
107
+ end
108
+ end
102
109
  end
103
110
 
@@ -0,0 +1,24 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ class TestResourceSoftware < Test::Unit::TestCase
5
+ module TestResource
6
+ extend Resource
7
+
8
+ self.subdir = Path.setup('tmp/test-resource')
9
+ end
10
+
11
+ def test_install
12
+ Resource.install nil, "scout_install_example", tmpdir.software do
13
+ <<-EOF
14
+ echo "#!/bin/bash\necho WORKING" > $OPT_BIN_DIR/scout_install_example
15
+ chmod +x $OPT_BIN_DIR/scout_install_example
16
+ EOF
17
+ end
18
+ assert_nothing_raised do
19
+ CMD.cmd(tmpdir.software.opt.bin.scout_install_example).read
20
+ end
21
+ assert_equal "WORKING", CMD.cmd('scout_install_example').read.strip
22
+ end
23
+ end
24
+
@@ -0,0 +1,66 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ class TestConfig < Test::Unit::TestCase
5
+ setup do
6
+ Scout::Config.set({:cpus => 30}, :test_config, :test)
7
+ Scout::Config.set(:cpus, 5, "slow::2", :test)
8
+ Scout::Config.set({:token => "token"}, "token")
9
+ Scout::Config.set(:notoken, "no_token")
10
+ Scout::Config.set({:emptytoken => "empty"})
11
+ end
12
+
13
+ def test_simple
14
+ assert_equal 30, Scout::Config.get(:cpus, :test_config)
15
+ end
16
+
17
+ def test_match
18
+ assert_equal({20 => ["token"]}, Scout::Config.match({["key:token"] => "token"}, "key:token"))
19
+ end
20
+
21
+ def test_simple_no_token
22
+ assert_equal "token", Scout::Config.get("token", "token")
23
+ assert_equal "no_token", Scout::Config.get("notoken", "key:notoken")
24
+ assert_equal 'token', Scout::Config.get("token", "key:token")
25
+ assert_equal 'token', Scout::Config.get("token")
26
+ assert_equal nil, Scout::Config.get("token", "someotherthing")
27
+ assert_equal "default_token", Scout::Config.get("token", 'unknown', :default => 'default_token')
28
+ assert_equal 'empty', Scout::Config.get("emptytoken", 'key:emptytoken')
29
+ end
30
+
31
+ def test_prio
32
+ assert_equal 5, Scout::Config.get(:cpus, :slow, :test)
33
+ end
34
+
35
+ def test_with_config
36
+ Scout::Config.add_entry 'key', 'valueA', 'token'
37
+ assert_equal "valueA", Scout::Config.get('key', 'token')
38
+ assert_equal "default", Scout::Config.get('key2', 'token', :default => 'default')
39
+
40
+ Scout::Config.with_config do
41
+ Scout::Config.add_entry 'key', 'valueB', 'token'
42
+ Scout::Config.add_entry 'key2', 'valueB2', 'token'
43
+ assert_equal "valueB", Scout::Config.get('key', 'token')
44
+ assert_equal "valueB2", Scout::Config.get('key2', 'token', :default => 'default')
45
+ end
46
+
47
+ assert_equal "valueA", Scout::Config.get('key', 'token')
48
+ assert_equal "default", Scout::Config.get('key2', 'token', :default => 'default')
49
+ end
50
+
51
+ def test_order
52
+ Scout::Config.add_entry 'key', 'V1', 'token1'
53
+ Scout::Config.add_entry 'key', 'V2', 'token2'
54
+ Scout::Config.add_entry 'key', 'V3', 'token2'
55
+
56
+ assert_equal "V3", Scout::Config.get('key', 'token2')
57
+ assert_equal "V1", Scout::Config.get('key', 'token1')
58
+ assert_equal "V3", Scout::Config.get('key', 'token2', 'token1')
59
+ assert_equal "V1", Scout::Config.get('key', 'token1', 'token2')
60
+ end
61
+
62
+ def test_default
63
+ Scout::Config.add_entry 'key', 'V1', 'token1'
64
+ assert_equal "V3", Scout::Config.get('key', 'token2', :default => 'V3')
65
+ end
66
+ end
@@ -19,6 +19,16 @@ class TestMetaExtension < Test::Unit::TestCase
19
19
  assert_equal :code, str2.code
20
20
  end
21
21
 
22
+ def test_marshal
23
+ str = "String"
24
+ ExtensionClass.setup(str, :code)
25
+ assert ExtensionClass === str
26
+ assert_equal :code, str.code
27
+
28
+ str2 = Marshal.load(Marshal.dump(str))
29
+ assert_equal :code, str2.code
30
+ end
31
+
22
32
  def test_setup_alternatives
23
33
  str = "String"
24
34
 
@@ -0,0 +1,19 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ class TestNamedArray < Test::Unit::TestCase
5
+ def test_identify_names
6
+ names =<<-EOF.split("\n")
7
+ ValueA
8
+ ValueB (Entity type)
9
+ 15
10
+ EOF
11
+ assert_equal 0, NamedArray.identify_name(names, "ValueA")
12
+ assert_equal :key, NamedArray.identify_name(names, :key)
13
+ assert_equal 0, NamedArray.identify_name(names, nil)
14
+ assert_equal 1, NamedArray.identify_name(names, "ValueB (Entity type)")
15
+ assert_equal 1, NamedArray.identify_name(names, "ValueB")
16
+ assert_equal 1, NamedArray.identify_name(names, 1)
17
+ end
18
+ end
19
+
@@ -56,6 +56,21 @@ class TestPersist < Test::Unit::TestCase
56
56
  end
57
57
  end
58
58
 
59
+ def test_stream_multiple
60
+ TmpFile.with_file do |tmpfile|
61
+ Path.setup(tmpfile)
62
+ obj = "TEST\nTEST"
63
+ stream = StringIO.new obj
64
+ stream.rewind
65
+ res1 = Persist.persist(tmpfile, :string, :dir => tmpdir.persist, :tee_copies => 2){ stream }
66
+ res2 = res1.next
67
+ assert IO === res1
68
+ assert_equal obj, res1.read
69
+ assert_equal obj, res2.read
70
+ assert_equal obj, Persist.persist(tmpfile, :string, :dir => tmpdir.persist){ raise ScoutException }
71
+ end
72
+ end
73
+
59
74
  def test_update_time
60
75
  TmpFile.with_file do |dir|
61
76
  Path.setup(dir)
@@ -79,5 +94,25 @@ class TestPersist < Test::Unit::TestCase
79
94
  assert_equal "TEST2", Persist.persist(dir.cache, type, :dir => tmpdir.persist, :update => dir.file3){ Open.read(dir.file3) }
80
95
  end
81
96
  end
97
+
98
+ def __test_speed
99
+ times = 100_000
100
+ TmpFile.with_file do |tmpfile|
101
+ sout = Persist.persist(tmpfile, :string, :path => tmpfile) do
102
+ Open.open_pipe do |sin|
103
+ times.times do |i|
104
+ sin.puts "line-#{i}"
105
+ end
106
+ end
107
+ end
108
+
109
+ Log::ProgressBar.with_bar do |bar|
110
+ while l = sout.gets
111
+ bar.tick
112
+ end
113
+ end
114
+ end
115
+ end
116
+
82
117
  end
83
118
 
@@ -4,7 +4,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
4
4
  require 'scout/work_queue/worker'
5
5
  class TestSemaphore < Test::Unit::TestCase
6
6
 
7
- def _test_simple
7
+ def test_simple
8
8
  ScoutSemaphore.with_semaphore 1 do |sem|
9
9
  10.times do
10
10
  ScoutSemaphore.synchronize(sem) do
@@ -10,7 +10,7 @@ class TestTmpFile < Test::Unit::TestCase
10
10
  def test_do_tmp_file
11
11
  content = "Hello World!"
12
12
  TmpFile.with_file(content) do |file|
13
- assert_equal content, File.open(file).read
13
+ assert_equal_path content, File.open(file).read
14
14
  end
15
15
  end
16
16
 
@@ -45,7 +45,7 @@ class TestTmpFile < Test::Unit::TestCase
45
45
 
46
46
  def test_in_dir
47
47
  TmpFile.in_dir do |dir|
48
- assert_equal dir, FileUtils.pwd
48
+ assert_equal_path dir, FileUtils.pwd
49
49
  end
50
50
  end
51
51
 
@@ -0,0 +1,74 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ class TestTSV < Test::Unit::TestCase
5
+ def test_open_with_data
6
+ content =<<-'EOF'
7
+ #: :sep=/\s+/#:type=:double#:merge=:concat
8
+ #Id ValueA ValueB OtherID
9
+ row1 a|aa|aaa b Id1|Id2
10
+ row2 A B Id3
11
+ row2 a a id3
12
+ EOF
13
+
14
+ content2 =<<-'EOF'
15
+ #: :sep=/\s+/#:type=:double#:merge=:concat
16
+ #Id ValueA ValueB OtherID
17
+ row3 a|aa|aaa b Id1|Id2
18
+ row4 A B Id3
19
+ row4 a a id3
20
+ EOF
21
+
22
+ tsv = TmpFile.with_file(content) do |filename|
23
+ TSV.open(filename, :persist => false)
24
+ end
25
+
26
+ TmpFile.with_file(content2) do |filename|
27
+ TSV.open(filename, :data => tsv)
28
+ end
29
+
30
+ assert_include tsv.keys, 'row4'
31
+ assert_include tsv.keys, 'row1'
32
+ end
33
+
34
+ def test_open_persist
35
+ content =<<-'EOF'
36
+ #: :sep=/\s+/#:type=:double#:merge=:concat
37
+ #Id ValueA ValueB OtherID
38
+ row1 a|aa|aaa b Id1|Id2
39
+ row2 A B Id3
40
+ row2 a a id3
41
+ EOF
42
+
43
+ tsv = TmpFile.with_file(content) do |filename|
44
+ TSV.open(filename, :persist => true)
45
+ end
46
+
47
+ assert tsv.respond_to?(:persistence_class)
48
+ assert_equal TokyoCabinet::HDB, tsv.persistence_class
49
+
50
+ assert_include tsv.keys, 'row1'
51
+ assert_include tsv.keys, 'row2'
52
+ end
53
+
54
+ def test_open_persist_in_situ
55
+ content =<<-'EOF'
56
+ #: :sep=/\s+/#:type=:double#:merge=:concat
57
+ #Id ValueA ValueB OtherID
58
+ row1 a|aa|aaa b Id1|Id2
59
+ row2 A B Id3
60
+ row2 a a id3
61
+ EOF
62
+
63
+ tsv = TmpFile.with_file(content) do |filename|
64
+ TSV.open(filename, :persist => true)
65
+ end
66
+
67
+ assert tsv.respond_to?(:persistence_class)
68
+ assert_equal TokyoCabinet::HDB, tsv.persistence_class
69
+
70
+ assert_include tsv.keys, 'row1'
71
+ assert_include tsv.keys, 'row2'
72
+ end
73
+ end
74
+
@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
3
3
 
4
4
  require 'scout/log'
5
5
  class TestWorkQueue < Test::Unit::TestCase
6
- def test_queue_remove_workers
6
+ def test_a_queue_remove_workers
7
7
  num = 10
8
8
  reps = 10_000
9
9
  q = WorkQueue.new num do |obj|
@@ -19,7 +19,9 @@ class TestWorkQueue < Test::Unit::TestCase
19
19
  q.write i
20
20
  end
21
21
 
22
- num.times do q.remove_one_worker end
22
+ (num - 1).times do q.remove_one_worker end
23
+
24
+ Thread.pass until q.workers.length == 1
23
25
 
24
26
  w = q.add_worker do |obj|
25
27
  "HEY"
@@ -38,7 +40,7 @@ class TestWorkQueue < Test::Unit::TestCase
38
40
 
39
41
  def test_queue
40
42
  num = 10
41
- reps = 10_000
43
+ reps = 1_000
42
44
  q = WorkQueue.new num do |obj|
43
45
  [Process.pid.to_s, obj.to_s] * " "
44
46
  end
@@ -55,8 +57,8 @@ class TestWorkQueue < Test::Unit::TestCase
55
57
  end
56
58
 
57
59
  Process.wait pid
58
-
59
60
  q.close
61
+
60
62
  q.join
61
63
 
62
64
  assert_equal reps, res.length
@@ -67,6 +69,7 @@ class TestWorkQueue < Test::Unit::TestCase
67
69
  reps = 10_000
68
70
  q = WorkQueue.new num do |obj|
69
71
  [Process.pid.to_s, obj.to_s] * " "
72
+ :ignore
70
73
  end
71
74
 
72
75
  q.ignore_ouput
@@ -76,18 +79,70 @@ class TestWorkQueue < Test::Unit::TestCase
76
79
  res << out
77
80
  end
78
81
 
82
+ reps.times do |i|
83
+ q.write i
84
+ end
85
+
86
+ q.close
87
+ q.join
88
+
89
+ assert_equal 0, res.length
90
+ end
91
+
92
+ def test_queue_error
93
+ num = 100
94
+ reps = 10_000
95
+
96
+ q = WorkQueue.new num do |obj|
97
+ raise ScoutException if rand < 0.1
98
+ [Process.pid.to_s, obj.to_s] * " "
99
+ end
100
+
101
+ res = []
102
+ q.process do |out|
103
+ res << out
104
+ end
105
+
79
106
  pid = Process.fork do
80
107
  reps.times do |i|
81
108
  q.write i
82
109
  end
83
110
  end
84
111
 
85
- Process.wait pid
86
-
112
+ Process.waitpid pid
87
113
  q.close
88
- q.join
89
114
 
90
- assert_equal 0, res.length
115
+ assert_raise ScoutException do
116
+ q.join
117
+ t.join
118
+ end
119
+ end
120
+
121
+ def test_queue_error_in_input
122
+ num = 100
123
+ reps = 10_000
124
+
125
+ q = WorkQueue.new num do |obj|
126
+ [Process.pid.to_s, obj.to_s] * " "
127
+ end
128
+
129
+ res = []
130
+ q.process do |out|
131
+ raise ScoutException if rand < 0.01
132
+ res << out
133
+ end
134
+
135
+ pid = Process.fork do
136
+ reps.times do |i|
137
+ q.write i
138
+ end
139
+ q.close
140
+ end
141
+
142
+ assert_raise ScoutException do
143
+ q.join
144
+ t.join
145
+ end
91
146
  end
92
147
  end
93
148
 
@@ -0,0 +1,34 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+ class TestTSVAdapter < Test::Unit::TestCase
6
+ def test_get_set
7
+ tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
8
+ tsv.type = :list
9
+ tsv.extend TSVAdapter
10
+ tsv["a"] = %w(1 2 3)
11
+
12
+ assert_equal %w(1 2 3) * "\t", tsv.dup["a"]
13
+ assert_equal %w(a), tsv.keys
14
+ assert_equal [%w(1 2 3)], tsv.collect{|k,v| v }
15
+ assert_equal [%w(1 2 3)], tsv.values
16
+
17
+ json = tsv.to_json
18
+ new = JSON.parse(json)
19
+ tsv.annotate(new)
20
+ new.extend TSVAdapter
21
+
22
+ tsv = new
23
+ assert_equal %w(1 2 3) * "\t", tsv.dup["a"]
24
+ assert_equal %w(a), tsv.keys
25
+ assert_equal [%w(1 2 3)], tsv.collect{|k,v| v }
26
+ assert_equal [%w(1 2 3)], tsv.values
27
+ assert_equal [["a", %w(1 2 3)]], tsv.sort
28
+
29
+ tsv["b"] = %w(11 22 33)
30
+ assert_equal [["a", %w(1 2 3)], ["b", %w(11 22 33)]], tsv.sort
31
+ assert_equal [["b", %w(11 22 33)], ["a", %w(1 2 3)]], tsv.sort_by{|k,v| -v[0].to_i }
32
+ end
33
+ end
34
+