scout-gear 10.4.0 → 10.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (143) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +100 -656
  3. data/Rakefile +1 -0
  4. data/VERSION +1 -1
  5. data/bin/scout +1 -3
  6. data/lib/scout/association/fields.rb +170 -0
  7. data/lib/scout/association/index.rb +229 -0
  8. data/lib/scout/association/item.rb +227 -0
  9. data/lib/scout/association/util.rb +7 -0
  10. data/lib/scout/association.rb +100 -0
  11. data/lib/scout/entity/format.rb +62 -0
  12. data/lib/scout/entity/identifiers.rb +111 -0
  13. data/lib/scout/entity/object.rb +20 -0
  14. data/lib/scout/entity/property.rb +165 -0
  15. data/lib/scout/entity.rb +41 -0
  16. data/lib/scout/offsite/step.rb +2 -2
  17. data/lib/scout/{tsv/persist → persist/engine}/fix_width_table.rb +25 -33
  18. data/lib/scout/persist/engine/packed_index.rb +100 -0
  19. data/lib/scout/persist/engine/sharder.rb +219 -0
  20. data/lib/scout/{tsv/persist → persist/engine}/tkrzw.rb +0 -17
  21. data/lib/scout/{tsv/persist → persist/engine}/tokyocabinet.rb +55 -31
  22. data/lib/scout/persist/engine.rb +4 -0
  23. data/lib/scout/{tsv/persist/adapter.rb → persist/tsv/adapter/base.rb} +80 -51
  24. data/lib/scout/persist/tsv/adapter/fix_width_table.rb +106 -0
  25. data/lib/scout/persist/tsv/adapter/packed_index.rb +95 -0
  26. data/lib/scout/persist/tsv/adapter/sharder.rb +54 -0
  27. data/lib/scout/persist/tsv/adapter/tkrzw.rb +18 -0
  28. data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +65 -0
  29. data/lib/scout/persist/tsv/adapter.rb +6 -0
  30. data/lib/scout/{tsv/persist → persist/tsv}/serialize.rb +5 -0
  31. data/lib/scout/persist/tsv.rb +107 -0
  32. data/lib/scout/tsv/annotation/repo.rb +87 -0
  33. data/lib/scout/tsv/annotation.rb +169 -0
  34. data/lib/scout/tsv/attach.rb +97 -21
  35. data/lib/scout/tsv/change_id/translate.rb +148 -0
  36. data/lib/scout/tsv/change_id.rb +3 -0
  37. data/lib/scout/tsv/csv.rb +85 -0
  38. data/lib/scout/tsv/dumper.rb +113 -25
  39. data/lib/scout/tsv/index.rb +88 -36
  40. data/lib/scout/tsv/open.rb +21 -8
  41. data/lib/scout/tsv/parser.rb +153 -90
  42. data/lib/scout/tsv/path.rb +7 -2
  43. data/lib/scout/tsv/stream.rb +48 -6
  44. data/lib/scout/tsv/transformer.rb +5 -3
  45. data/lib/scout/tsv/traverse.rb +28 -19
  46. data/lib/scout/tsv/util/process.rb +7 -0
  47. data/lib/scout/tsv/util/reorder.rb +25 -15
  48. data/lib/scout/tsv/util/select.rb +9 -1
  49. data/lib/scout/tsv/util/sort.rb +90 -2
  50. data/lib/scout/tsv/util/unzip.rb +56 -0
  51. data/lib/scout/tsv/util.rb +52 -5
  52. data/lib/scout/tsv.rb +42 -27
  53. data/lib/scout/work_queue/socket.rb +8 -0
  54. data/lib/scout/work_queue/worker.rb +22 -5
  55. data/lib/scout/work_queue.rb +41 -24
  56. data/lib/scout/workflow/definition.rb +15 -12
  57. data/lib/scout/workflow/deployment/orchestrator.rb +21 -3
  58. data/lib/scout/workflow/deployment/trace.rb +205 -0
  59. data/lib/scout/workflow/deployment.rb +1 -0
  60. data/lib/scout/workflow/documentation.rb +1 -1
  61. data/lib/scout/workflow/step/archive.rb +42 -0
  62. data/lib/scout/workflow/step/children.rb +51 -0
  63. data/lib/scout/workflow/step/config.rb +1 -1
  64. data/lib/scout/workflow/step/dependencies.rb +25 -8
  65. data/lib/scout/workflow/step/file.rb +19 -0
  66. data/lib/scout/workflow/step/info.rb +37 -9
  67. data/lib/scout/workflow/step/progress.rb +11 -2
  68. data/lib/scout/workflow/step/status.rb +9 -1
  69. data/lib/scout/workflow/step.rb +80 -25
  70. data/lib/scout/workflow/task/dependencies.rb +5 -2
  71. data/lib/scout/workflow/task/inputs.rb +91 -41
  72. data/lib/scout/workflow/task.rb +54 -57
  73. data/lib/scout/workflow/usage.rb +1 -1
  74. data/lib/scout/workflow/util.rb +4 -0
  75. data/lib/scout/workflow.rb +110 -13
  76. data/lib/scout-gear.rb +2 -0
  77. data/lib/scout.rb +0 -1
  78. data/scout-gear.gemspec +78 -23
  79. data/scout_commands/rbbt +2 -0
  80. data/test/data/person/brothers +4 -0
  81. data/test/data/person/identifiers +10 -0
  82. data/test/data/person/marriages +3 -0
  83. data/test/data/person/parents +6 -0
  84. data/test/scout/association/test_fields.rb +105 -0
  85. data/test/scout/association/test_index.rb +70 -0
  86. data/test/scout/association/test_item.rb +21 -0
  87. data/test/scout/entity/test_format.rb +19 -0
  88. data/test/scout/entity/test_identifiers.rb +58 -0
  89. data/test/scout/entity/test_object.rb +0 -0
  90. data/test/scout/entity/test_property.rb +345 -0
  91. data/test/scout/{tsv/persist → persist/engine}/test_fix_width_table.rb +0 -1
  92. data/test/scout/persist/engine/test_packed_index.rb +99 -0
  93. data/test/scout/persist/engine/test_sharder.rb +31 -0
  94. data/test/scout/persist/engine/test_tkrzw.rb +0 -0
  95. data/test/scout/persist/engine/test_tokyocabinet.rb +17 -0
  96. data/test/scout/persist/test_tsv.rb +146 -0
  97. data/test/scout/{tsv/persist/test_adapter.rb → persist/tsv/adapter/test_base.rb} +3 -4
  98. data/test/scout/persist/tsv/adapter/test_fix_width_table.rb +46 -0
  99. data/test/scout/persist/tsv/adapter/test_packed_index.rb +37 -0
  100. data/test/scout/persist/tsv/adapter/test_serialize.rb +0 -0
  101. data/test/scout/persist/tsv/adapter/test_sharder.rb +290 -0
  102. data/test/scout/{tsv/persist → persist/tsv/adapter}/test_tkrzw.rb +3 -6
  103. data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +282 -0
  104. data/test/scout/persist/tsv/test_serialize.rb +12 -0
  105. data/test/scout/test_association.rb +51 -0
  106. data/test/scout/test_entity.rb +40 -0
  107. data/test/scout/test_tsv.rb +33 -4
  108. data/test/scout/test_work_queue.rb +5 -2
  109. data/test/scout/test_workflow.rb +31 -14
  110. data/test/scout/tsv/annotation/test_repo.rb +150 -0
  111. data/test/scout/tsv/change_id/test_translate.rb +178 -0
  112. data/test/scout/tsv/test_annotation.rb +52 -0
  113. data/test/scout/tsv/test_attach.rb +255 -1
  114. data/test/scout/tsv/test_change_id.rb +25 -0
  115. data/test/scout/tsv/test_csv.rb +50 -0
  116. data/test/scout/tsv/test_dumper.rb +38 -0
  117. data/test/scout/tsv/test_index.rb +82 -0
  118. data/test/scout/tsv/test_open.rb +44 -0
  119. data/test/scout/tsv/test_parser.rb +70 -0
  120. data/test/scout/tsv/test_stream.rb +22 -0
  121. data/test/scout/tsv/test_transformer.rb +27 -3
  122. data/test/scout/tsv/test_traverse.rb +78 -0
  123. data/test/scout/tsv/util/test_process.rb +16 -0
  124. data/test/scout/tsv/util/test_reorder.rb +67 -0
  125. data/test/scout/tsv/util/test_sort.rb +28 -1
  126. data/test/scout/tsv/util/test_unzip.rb +32 -0
  127. data/test/scout/work_queue/test_socket.rb +4 -1
  128. data/test/scout/workflow/deployment/test_orchestrator.rb +17 -26
  129. data/test/scout/workflow/deployment/test_trace.rb +25 -0
  130. data/test/scout/workflow/step/test_archive.rb +28 -0
  131. data/test/scout/workflow/step/test_children.rb +25 -0
  132. data/test/scout/workflow/step/test_info.rb +16 -0
  133. data/test/scout/workflow/task/test_dependencies.rb +16 -16
  134. data/test/scout/workflow/task/test_inputs.rb +45 -1
  135. data/test/scout/workflow/test_definition.rb +52 -0
  136. data/test/scout/workflow/test_step.rb +57 -0
  137. data/test/scout/workflow/test_task.rb +26 -1
  138. data/test/scout/workflow/test_usage.rb +4 -4
  139. data/test/test_helper.rb +23 -1
  140. metadata +69 -14
  141. data/lib/scout/tsv/persist.rb +0 -27
  142. data/test/scout/tsv/persist/test_tokyocabinet.rb +0 -120
  143. data/test/scout/tsv/test_persist.rb +0 -45
@@ -31,6 +31,31 @@ row3 B Id3
31
31
  end
32
32
  end
33
33
 
34
+ def test_attach_single
35
+ content1 =<<-EOF
36
+ #: :sep=","
37
+ #ID,ValueA
38
+ row1,a
39
+ row2,A
40
+ EOF
41
+
42
+ content2 =<<-EOF
43
+ #: :sep=","
44
+ #ID,ValueB
45
+ row1,b
46
+ row3,B
47
+ EOF
48
+
49
+ TmpFile.with_file(content1) do |filename1|
50
+ TmpFile.with_file(content2) do |filename2|
51
+ tsv = TSV.open(filename1, type: :single)
52
+ other = TSV.open(filename2, type: :single)
53
+ tsv = tsv.attach other, :complete => true
54
+ assert_equal 'b', tsv["row1"]["ValueB"]
55
+ end
56
+ end
57
+ end
58
+
34
59
  def test_attach_by_key
35
60
  content1 =<<-EOF
36
61
  #: :sep=" "
@@ -162,7 +187,7 @@ B Id3
162
187
  tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
163
188
  end
164
189
 
165
- tsv1.attach tsv2, bar: true
190
+ tsv1 = tsv1.attach tsv2, bar: true
166
191
 
167
192
  assert_equal %w(ValueA ValueB OtherID), tsv1.fields
168
193
  assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
@@ -224,4 +249,233 @@ row3 Id3
224
249
  end
225
250
  end
226
251
  end
252
+
253
+ def test_attach_index
254
+ content1 =<<-EOF
255
+ #Id ValueA ValueB
256
+ row1 a|aa|aaa b
257
+ row2 A B
258
+ EOF
259
+
260
+ content2 =<<-EOF
261
+ #ValueE OtherID
262
+ e Id1|Id2
263
+ E Id3
264
+ EOF
265
+
266
+ content_index =<<-EOF
267
+ #Id ValueE
268
+ row1 e
269
+ row2 E
270
+ EOF
271
+
272
+ tsv1 = tsv2 = index = nil
273
+ TmpFile.with_file(content1) do |filename|
274
+ tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
275
+ end
276
+
277
+ TmpFile.with_file(content2) do |filename|
278
+ tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
279
+ end
280
+
281
+ TmpFile.with_file(content_index) do |filename|
282
+ index = TSV.open(File.open(filename), type: :flat, :sep => /\s+/)
283
+ end
284
+
285
+ tsv1.attach tsv2, index: index
286
+
287
+ assert_equal %w(ValueA ValueB OtherID), tsv1.fields
288
+ assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
289
+
290
+ TmpFile.with_file(content1) do |filename|
291
+ tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
292
+ end
293
+
294
+ tsv1.attach tsv2, index: index
295
+
296
+ assert_equal %w(ValueA ValueB OtherID), tsv1.fields
297
+ assert_equal "Id1", tsv1["row1"]["OtherID"]
298
+ end
299
+
300
+ def test_attach_complete_identifiers
301
+ content1 =<<-EOF
302
+ #: :sep=/\\s+/
303
+ #Id ValueA
304
+ row1 a|aa|aaa
305
+ row2 A
306
+ EOF
307
+
308
+ content2 =<<-EOF
309
+ #: :sep=/\\s+/
310
+ #Id2 ValueB
311
+ ROW_1 b
312
+ ROW_2 C
313
+ EOF
314
+
315
+ identifiers =<<-EOF
316
+ #: :sep=/\\s+/
317
+ #Id Id2
318
+ row1 ROW_1
319
+ row2 ROW_2
320
+ row3 ROW_3
321
+ EOF
322
+ Scout.claim Scout.tmp.test_tmpdir.test1.data, :string, content1
323
+ Scout.claim Scout.tmp.test_tmpdir.test2.data, :string, content2
324
+ Scout.claim Scout.tmp.test_tmpdir.identifiers.data, :string, identifiers
325
+
326
+ tsv1 = tsv2 = nil
327
+
328
+ tsv1 = Scout.tmp.test_tmpdir.test1.data.produce(true).tsv type: :double, :sep => /\s+/
329
+ tsv2 = Scout.tmp.test_tmpdir.test2.data.produce(true).tsv type: :double, :sep => /\s+/
330
+ ids = Scout.tmp.test_tmpdir.identifiers.data.produce(true).tsv type: :double, :sep => /\s+/
331
+
332
+ tsv1.identifiers = ids
333
+
334
+ tsv1 = tsv1.attach tsv2
335
+
336
+ assert_equal [["A"], ["C"]], tsv1["row2"]
337
+
338
+ tsv1 = Scout.tmp.test_tmpdir.test1.data.produce(true).tsv type: :double, :sep => /\s+/
339
+ tsv2 = Scout.tmp.test_tmpdir.test2.data.produce(true).tsv type: :double, :sep => /\s+/
340
+ ids = Scout.tmp.test_tmpdir.identifiers.data.produce(true).tsv type: :double, :sep => /\s+/
341
+
342
+ tsv1.identifiers = ids
343
+
344
+ tsv1 = tsv1.attach tsv2, :complete => true
345
+ assert_equal [["A"], ["C"]], tsv1["row2"]
346
+ end
347
+
348
+ def test_attach_index_both_non_key
349
+ content1 =<<-EOF
350
+ #: :sep=/\\s+/
351
+ #Id ValueA ValueB
352
+ row1 a|aa|aaa b
353
+ row2 A B
354
+ EOF
355
+
356
+ content2 =<<-EOF
357
+ #: :sep=/\\s+/
358
+ #ValueE OtherID
359
+ e Id1|Id2
360
+ E Id3
361
+ EOF
362
+
363
+ content_index =<<-EOF
364
+ #: :sep=/\\s+/
365
+ #ValueA OtherID
366
+ a Id1
367
+ A Id3
368
+ EOF
369
+
370
+ tmpdir = Scout.tmp.test_tmp
371
+
372
+ Scout.claim tmpdir.test1.data, :string, content1
373
+ Scout.claim tmpdir.test2.data, :string, content2
374
+ Scout.claim tmpdir.test2.identifiers, :string, content_index
375
+
376
+ tsv1 = tsv2 = nil
377
+
378
+ tsv1 = tmpdir.test1.data.produce(true).tsv type: :double, :sep => /\s+/
379
+ tsv2 = tmpdir.test2.data.produce(true).tsv type: :double, :sep => /\s+/
380
+
381
+ tsv2.identifiers = tmpdir.test2.identifiers.produce(true).produce.find #.to_s
382
+
383
+ tsv1.attach tsv2, :fields => ["ValueE"] #, :persist_input => true
384
+ assert_equal [["a", "aa", "aaa"], ["b"], ["e"]], tsv1["row1"]
385
+ end
386
+
387
+ def test_attach_single_nils
388
+ content1 =<<-EOF
389
+ #Id,ValueA
390
+ row1,
391
+ row2,AA
392
+ EOF
393
+ content2 =<<-EOF
394
+ #Id,ValueB
395
+ row1,B
396
+ row2,BB
397
+ EOF
398
+ content3 =<<-EOF
399
+ #Id,ValueC
400
+ row1,
401
+ row2,CC
402
+ EOF
403
+
404
+ tsv1 = tsv2 = tsv3 = tsv4 = index = nil
405
+ TmpFile.with_file(content1) do |filename|
406
+ tsv1 = TSV.open(File.open(filename), :sep => ',', :type => :single)
407
+ tsv1.keys.each{|k| tsv1[k] = nil if tsv1[k] == ""}
408
+ end
409
+
410
+ TmpFile.with_file(content2) do |filename|
411
+ tsv2 = TSV.open(File.open(filename), :sep => ',', :type => :single)
412
+ tsv2.keys.each{|k| tsv2[k] = nil if tsv2[k] == ""}
413
+ end
414
+
415
+ TmpFile.with_file(content3) do |filename|
416
+ tsv3 = TSV.open(File.open(filename), :sep => ',', :type => :single)
417
+ tsv3.keys.each{|k| tsv3[k] = nil if tsv3[k] == ""}
418
+ end
419
+
420
+ tmp = tsv1.attach(tsv2, :complete => true)
421
+ tmp = tmp.attach(tsv3, :complete => true)
422
+ assert_equal [nil, "B", nil], tsv1.attach(tsv2, :complete => true).attach(tsv3, :complete => true)["row1"]
423
+ assert_equal [nil, "B", nil], tsv1.attach(tsv2, :complete => true).attach(tsv3, :complete => true)["row1"]
424
+ end
425
+
426
+ def test_attach_flat
427
+ content1 =<<-EOF
428
+ #Id ValueA ValueB
429
+ row1 a|aa|aaa b
430
+ row2 A B
431
+ EOF
432
+
433
+ content2 =<<-EOF
434
+ #ValueA OtherID
435
+ a Id1|Id2
436
+ A Id3
437
+ EOF
438
+
439
+ tsv1 = tsv2 = index = nil
440
+ TmpFile.with_file(content1) do |filename|
441
+ tsv1 = TSV.open(File.open(filename), type: :flat, fields: ["ValueA"], sep: /\s+/)
442
+ end
443
+
444
+ TmpFile.with_file(content2) do |filename|
445
+ tsv2 = TSV.open(File.open(filename), type: :double, sep: /\s+/)
446
+ end
447
+
448
+ res = tsv1.attach tsv2, :fields => ["OtherID"]
449
+ assert res["row2"].include? "Id3"
450
+ assert ! res["row2"].include?("b")
451
+ end
452
+
453
+ def test_attach_list_to_list_with_complete
454
+ content1 =<<-EOF
455
+ #Id ValueA ValueB
456
+ row1 a b
457
+ row2 A B
458
+ EOF
459
+
460
+ content2 =<<-EOF
461
+ #Id ValueC
462
+ row1 c
463
+ row2 C
464
+ row3 CC
465
+ EOF
466
+
467
+ tsv1 = tsv2 = index = nil
468
+ TmpFile.with_file(content1) do |filename|
469
+ tsv1 = TSV.open(File.open(filename), type: :list, fields: ["ValueA"], sep: /\s+/)
470
+ end
471
+
472
+ TmpFile.with_file(content2) do |filename|
473
+ tsv2 = TSV.open(File.open(filename), type: :list, sep: /\s+/)
474
+ end
475
+
476
+ res = tsv1.attach tsv2, :fields => ["ValueC"], complete: true
477
+ assert res["row2"].include?("C")
478
+ refute res["row2"].include?("b")
479
+ assert res["row3"].include?("CC")
480
+ end
227
481
  end
@@ -94,5 +94,30 @@ row2 C2|C22 D2|D22
94
94
  assert_equal ["C1","C11"], res["row1"]["ValueC"]
95
95
  assert_equal ["C2","C22"], res["row2"]["ValueC"]
96
96
  end
97
+
98
+ def test_translate
99
+
100
+ f1=<<-EOF
101
+ #: :sep=' '
102
+ #A B C
103
+ a b c
104
+ aa bb cc
105
+ EOF
106
+
107
+ identifiers=<<-EOF
108
+ #: :sep=' '
109
+ #A X
110
+ a x
111
+ aa xx
112
+ EOF
113
+
114
+ TmpFile.with_file(f1) do |tf1|
115
+ TmpFile.with_file(identifiers) do |ti|
116
+ tsv = TSV.open tf1, :identifiers => ti
117
+
118
+ assert TSV.change_key(tsv, "X").include? "x"
119
+ end
120
+ end
121
+ end
97
122
  end
98
123
 
@@ -0,0 +1,50 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+ require 'scout/tsv'
4
+
5
+ class TestTSVCSV < Test::Unit::TestCase
6
+ def test_csv
7
+ text =<<-EOF
8
+ Key,FieldA,FieldB
9
+ k1,a,b
10
+ k2,aa,bb
11
+ EOF
12
+
13
+ tsv = TSV.csv(text)
14
+ assert_equal 'bb', tsv['k2']['FieldB']
15
+ end
16
+
17
+ def test_csv_key_field
18
+ text =<<-EOF
19
+ Key,FieldA,FieldB
20
+ k1,a,b
21
+ k2,aa,bb
22
+ EOF
23
+
24
+ tsv = TSV.csv(text, :key_field => 'FieldA', :type => :list)
25
+ assert_equal 'bb', tsv['aa']['FieldB']
26
+ end
27
+
28
+ def test_csv_double
29
+ text =<<-EOF
30
+ Key,FieldA,FieldB
31
+ k1,a,b
32
+ k2,aa,bb
33
+ EOF
34
+
35
+ tsv = TSV.csv(text, :key_field => 'FieldA', :type => :double)
36
+ assert_equal ['bb'], tsv['aa']['FieldB']
37
+ end
38
+
39
+ def test_csv_noheader
40
+ text =<<-EOF
41
+ k1,a,b
42
+ k2,aa,bb
43
+ EOF
44
+
45
+ tsv = TSV.csv(text, :headers => false)
46
+ assert_equal %w(k2 aa bb), tsv['row-1']
47
+ end
48
+
49
+ end
50
+
@@ -17,6 +17,22 @@ a\t1|11\t2|22
17
17
  assert_equal txt, dumper.stream.read
18
18
  end
19
19
 
20
+ def test_set_stream
21
+ io = StringIO.new
22
+ dumper = TSV::Dumper.new :key_field => "Key", :fields => %w(Field1 Field2), :type => :double
23
+ dumper.set_stream io
24
+ dumper.init
25
+ dumper.add "a", [["1", "11"], ["2", "22"]]
26
+ txt=<<-EOF
27
+ #: :type=:double
28
+ #Key\tField1\tField2
29
+ a\t1|11\t2|22
30
+ EOF
31
+ dumper.close
32
+ io.rewind
33
+ assert_equal txt, io.read
34
+ end
35
+
20
36
  def test_to_s
21
37
  tsv = TSV.setup({}, :key_field => "Key", :fields => %w(Field1 Field2), :type => :double)
22
38
  tsv["a"] = [["1", "11"], ["2", "22"]]
@@ -40,5 +56,27 @@ a\t1|11\t2|22
40
56
  TSV.open(dumper.stream, bar: true)
41
57
  end
42
58
  end
59
+
60
+ def test_to_s_sort
61
+ tsv = TSV.setup({}, :key_field => "Key", :fields => %w(Field1 Field2), :type => :double)
62
+ tsv["b"] = [["2", "22"], ["3", "33"]]
63
+ tsv["a"] = [["1", "11"], ["2", "22"]]
64
+ txt=<<-EOF
65
+ #: :type=:double
66
+ #Key\tField1\tField2
67
+ a\t1|11\t2|22
68
+ b\t2|22\t3|33
69
+ EOF
70
+ assert_equal txt, tsv.to_s(keys: tsv.keys.sort)
71
+ end
72
+
73
+ def test_filename
74
+ tsv = datadir_test.person.marriages.tsv
75
+ assert tsv.filename
76
+
77
+ tsv2 = TSV.open(tsv.dumper_stream)
78
+ assert tsv2.filename
79
+ end
80
+
43
81
  end
44
82
 
@@ -198,5 +198,87 @@ row2 a b id3
198
198
  end
199
199
  end
200
200
  end
201
+
202
+ def test_index_fields
203
+ content =<<-EOF
204
+ #Id ValueA ValueB OtherID
205
+ row1 a|aa|aaa b Id1|Id2
206
+ row2 A B Id3
207
+ EOF
208
+
209
+ TmpFile.with_file(content) do |filename|
210
+ tsv = TSV.open(File.open(filename), :sep => /\s+/, :key_field => "OtherID", :persist => false)
211
+ index = tsv.index(:persist => true, :persist_update => true)
212
+ assert index["row1"].include? "Id1"
213
+ assert_equal "OtherID", index.fields.first
214
+ end
215
+ end
216
+
217
+
218
+ def test_simple_index_key_field
219
+ text=<<-EOF
220
+ #: :sep=' '
221
+ #Y X
222
+ y x
223
+ yy xx
224
+ EOF
225
+
226
+ TmpFile.with_file(text) do |tmp|
227
+ assert_equal "Y", TSV.open(tmp).index(:target => "X", :fields => ["Y"]).key_field
228
+ assert_equal "Y", TSV.index(tmp, :target => "X", :fields => ["Y"]).key_field
229
+ end
230
+ end
231
+
232
+ def test_pos_and_range_index
233
+ content =<<-EOF
234
+ #Id ValueA ValueB Pos1 Pos2
235
+ row1 a|aa|aaa b 0|10 10|30
236
+ row2 A B 30 35
237
+ EOF
238
+
239
+ TmpFile.with_file(content) do |filename|
240
+ tsv = TSV.open(File.open(filename), type: :double, sep: /\s+/)
241
+ index = tsv.pos_index("Pos1")
242
+ assert_equal ["row1"], index[10]
243
+
244
+ index = tsv.range_index("Pos1", "Pos2")
245
+ assert_equal ["row1"], index[20]
246
+ end
247
+ end
248
+
249
+
250
+ def test_index_static_persist
251
+ content =<<-EOF
252
+ #Id ValueA ValueB OtherID
253
+ row1 a|aa|aaa b|A Id1
254
+ row2 A a|B Id3
255
+ row3 A a|B Id4
256
+ EOF
257
+
258
+ TmpFile.with_file(content) do |filename|
259
+ index = TSV.index(filename, :target => "OtherID", :sep => /\s+/, :order => true, :persist => false)
260
+ assert_equal "Id1", index['a']
261
+ assert_equal "Id3", index['A']
262
+ assert_equal "OtherID", index.fields.first
263
+
264
+ index = TSV.index(filename, :target => "OtherID", :sep => /\s+/, :order => true, :persist => true)
265
+ assert_equal "Id1", index['a']
266
+ assert_equal "Id3", index['A']
267
+ assert_equal "OtherID", index.fields.first
268
+
269
+ Open.write(filename, Open.read(filename).sub(/row1.*Id1\n/,''))
270
+
271
+ index = TSV.index(filename, :target => "OtherID", :sep => /\s+/, :order => true, :persist => true)
272
+ assert_equal "Id1", index['a']
273
+ assert_equal "Id3", index['A']
274
+ assert_equal "OtherID", index.fields.first
275
+ assert index.include?('aaa')
276
+
277
+ index = TSV.index(filename, :target => "OtherID", :sep => /\s+/, :order => true, :persist => false)
278
+ assert ! index.include?('aaa')
279
+ end
280
+ end
281
+
282
+
201
283
  end
202
284
 
@@ -163,5 +163,49 @@ row2 aa bb cc
163
163
 
164
164
  assert_equal size, step.run.length
165
165
  end
166
+
167
+ def test_traverse_priority
168
+ require 'fc'
169
+
170
+ queue = FastContainers::PriorityQueue.new(:min)
171
+
172
+ array = []
173
+ 100.times do e = rand(1000).to_i; array << e; queue.push(e,e) end
174
+
175
+ res = Open.traverse queue, :into => [] do |v|
176
+ v
177
+ end
178
+
179
+ assert_equal array.sort, res
180
+ end
181
+
182
+ def test_traverse_into_path
183
+ size = 100
184
+ array = (1..size).to_a.collect{|n| n.to_s}
185
+ TmpFile.with_file do |tmpfile|
186
+ Path.setup(tmpfile)
187
+ io = TSV.traverse array, :into => tmpfile do |e|
188
+ e
189
+ end
190
+ io.join
191
+ assert_equal size, Open.read(tmpfile).split("\n").length
192
+ end
193
+ end
194
+
195
+ def test_simple_index_key_field
196
+ text=<<-EOF
197
+ #: :sep=' '
198
+ #Y X
199
+ y x
200
+ yy xx
201
+ EOF
202
+
203
+ TmpFile.with_file(text) do |tmp|
204
+ k, f = Open.traverse TSV.open(tmp), key_field: "Y", fields: ["X"] do end
205
+ assert_equal "Y", k
206
+ k, f = Open.traverse tmp, key_field: "Y", fields: ["X"] do end
207
+ assert_equal "Y", k
208
+ end
209
+ end
166
210
  end
167
211
 
@@ -115,6 +115,21 @@ k4 a|A b|B
115
115
  assert_equal 3, tsv.keys.length
116
116
  end
117
117
 
118
+ def test_parse_tsv_grep
119
+ content =<<-EOF
120
+ #: :sep=" "#:type=:double
121
+ #Key ValueA ValueB
122
+ k a|A b|B
123
+ k1 a|A b|B
124
+ k2 a|A b|B
125
+ k3 a|A b|B
126
+ k4 a|A b|B
127
+ EOF
128
+ content = StringIO.new content
129
+
130
+ tsv = TSV.parse(content, :tsv_grep => ["k3","k4"])
131
+ assert_equal %w(k3 k4), tsv.keys.sort
132
+ end
118
133
 
119
134
  def test_parse_fields
120
135
  content =<<-EOF
@@ -247,4 +262,59 @@ k 1 2
247
262
  assert_equal [1.0, 2.0], data["k"]
248
263
  end
249
264
  end
265
+
266
+ def test_merge
267
+ content =<<-EOF
268
+ #: :type=:double
269
+ #PMID:Sentence number:TF:TG Transcription Factor (Associated Gene Name) Target Gene (Associated Gene Name) Sign Negation PMID
270
+ 24265317:3:NR1H3:FASN NR1H3 FASN 24265317
271
+ 17522048:0:NR1H3:FASN NR1H3 FASN + 17522048
272
+ 19903962:0:NR1H3:FASN NR1H3 FASN 19903962
273
+ 19903962:7:NR1H3:FASN NR1H3 FASN 19903962
274
+ 22183856:4:NR1H3:FASN NR1H3 FASN 22183856
275
+ 22641099:4:NR1H3:FASN NR1H3 FASN + 22641099
276
+ 23499676:8:NR1H3:FASN NR1H3 FASN + 23499676
277
+ 11790787:5:NR1H3:FASN NR1H3 FASN 11790787
278
+ 11790787:7:NR1H3:FASN NR1H3 FASN + 11790787
279
+ 11790787:9:NR1H3:FASN NR1H3 FASN + 11790787
280
+ 11790787:11:NR1H3:FASN NR1H3 FASN 11790787
281
+ 17522048:1:NR1H3:FASN NR1H3 FASN + 17522048
282
+ 17522048:3:NR1H3:FASN NR1H3 FASN 17522048
283
+ 22160584:1:NR1H3:FASN NR1H3 FASN 22160584
284
+ 22160584:5:NR1H3:FASN NR1H3 FASN + 22160584
285
+ 22160584:8:NR1H3:FASN NR1H3 FASN + 22160584
286
+ EOF
287
+
288
+ TmpFile.with_file(content) do |filename|
289
+ tsv = TSV.open(filename, key_field: "Transcription Factor (Associated Gene Name)", fields: ["Target Gene (Associated Gene Name)", "Sign", "PMID"], merge: true, one2one: true, type: :double)
290
+ assert_equal 16, tsv["NR1H3"]["Sign"].length
291
+ end
292
+ end
293
+
294
+ def test_load_stream
295
+ content =<<-EOF
296
+ #Key ValueA ValueB
297
+ k a|A b|B
298
+ k1 a|A b|B
299
+ k2 a|A b|B
300
+ k3 a|A b|B
301
+ k4 a|A b|B
302
+ EOF
303
+ content = StringIO.new content.gsub(' ', "\t")
304
+
305
+ TmpFile.with_file do |tmp_logfile|
306
+ old_logfile = Log.logfile
307
+ Log.logfile(tmp_logfile)
308
+ TmpFile.with_file do |persistence|
309
+ data = ScoutCabinet.open persistence, true
310
+ tsv = Log.with_severity(0) do
311
+ TSV.parse(content, data: data)
312
+ end
313
+ assert_equal %w(b B), tsv["k"]["ValueB"]
314
+ assert_equal %w(a A), tsv["k4"]["ValueA"]
315
+ end
316
+ Log.logfile(old_logfile)
317
+ assert Open.read(tmp_logfile).include?("directly into")
318
+ end
319
+ end
250
320
  end
@@ -196,5 +196,27 @@ row2\tAAA
196
196
  assert_equal ["AA", "AAA"], tsv["row2"][0]
197
197
  end
198
198
 
199
+ def test_concat_streams
200
+
201
+ text1=<<-EOF
202
+ #Key\tValueA
203
+ row1\tA
204
+ row2\tAA
205
+ EOF
206
+
207
+ text2=<<-EOF
208
+ #Key\tValueA
209
+ row3\tAAA
210
+ row2\tBB
211
+ EOF
212
+
213
+ s1 = StringIO.new text1
214
+ s2 = StringIO.new text2
215
+ tsv = TSV.open TSV.concat_streams([s1,s2]), :merge => true
216
+ assert_equal ["A"], tsv["row1"][0]
217
+ assert_equal ["AA","BB"], tsv["row2"][0]
218
+ assert_equal ["AAA"], tsv["row3"][0]
219
+ end
220
+
199
221
  end
200
222
 
@@ -3,6 +3,32 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
3
3
 
4
4
  require 'scout/tsv'
5
5
  class TestTSVTransformer < Test::Unit::TestCase
6
+ def test_filename
7
+ content =<<-EOF
8
+ #: :sep=" "
9
+ #ID ValueA ValueB
10
+ row1 A1|A11 B1|B11
11
+ row2 A2|A22 B2|B22
12
+ EOF
13
+
14
+ TmpFile.with_file(content) do |file|
15
+ parser = TSV::Parser.new file
16
+ dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Values"], type: :flat, :filename => parser.options[:filename]
17
+ dumper.init
18
+
19
+ trans = TSV::Transformer.new parser, dumper
20
+ dumper = trans.traverse do |k,values|
21
+ [k, values.flatten]
22
+ end
23
+
24
+ tsv = trans.tsv
25
+ assert_include tsv.filename, File.dirname(file)
26
+
27
+ assert_equal %w(A1 A11 B1 B11), tsv['row1']
28
+ end
29
+
30
+ end
31
+
6
32
  def test_traverse
7
33
  content =<<-EOF
8
34
  #: :sep=" "
@@ -40,12 +66,11 @@ row2 A2|A22 B2|B22
40
66
  dumper.init
41
67
 
42
68
  trans = TSV::Transformer.new parser, dumper
69
+ trans["row3"] = %w(A3 A33)
43
70
  dumper = trans.each do |k,values|
44
71
  values.replace values.flatten
45
72
  end
46
73
 
47
- trans["row3"] = %w(A3 A33)
48
-
49
74
  tsv = trans.tsv
50
75
  assert_equal %w(A1 A11 B1 B11), tsv['row1']
51
76
  assert_equal %w(A3 A33), tsv['row3']
@@ -67,7 +92,6 @@ row2 A2|A22 B2|B22
67
92
  trans.type = :flat
68
93
  trans.sep = "\t"
69
94
 
70
-
71
95
  trans["row3"] = %w(A3 A33)
72
96
 
73
97
  trans.each do |k,values|