scout-gear 7.3.0 → 8.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +20 -9
  3. data/VERSION +1 -1
  4. data/bin/scout +6 -3
  5. data/lib/rbbt-scout.rb +1 -0
  6. data/lib/scout/cmd.rb +1 -1
  7. data/lib/scout/concurrent_stream.rb +26 -23
  8. data/lib/scout/config.rb +1 -1
  9. data/lib/scout/log/color.rb +4 -1
  10. data/lib/scout/log/progress/report.rb +1 -1
  11. data/lib/scout/log/progress/util.rb +58 -54
  12. data/lib/scout/log/progress.rb +1 -1
  13. data/lib/scout/log/trap.rb +107 -0
  14. data/lib/scout/log.rb +56 -21
  15. data/lib/scout/meta_extension.rb +13 -6
  16. data/lib/scout/misc/digest.rb +1 -1
  17. data/lib/scout/misc/format.rb +12 -0
  18. data/lib/scout/misc/insist.rb +1 -1
  19. data/lib/scout/misc/monitor.rb +11 -0
  20. data/lib/scout/misc/system.rb +10 -0
  21. data/lib/scout/named_array.rb +65 -3
  22. data/lib/scout/open/lock/lockfile.rb +587 -0
  23. data/lib/scout/open/lock.rb +28 -2
  24. data/lib/scout/open/remote.rb +4 -0
  25. data/lib/scout/open/stream.rb +90 -15
  26. data/lib/scout/open/util.rb +13 -3
  27. data/lib/scout/path/find.rb +9 -1
  28. data/lib/scout/path/util.rb +35 -0
  29. data/lib/scout/persist/serialize.rb +18 -5
  30. data/lib/scout/persist.rb +28 -12
  31. data/lib/scout/resource/path.rb +53 -0
  32. data/lib/scout/resource/produce.rb +0 -8
  33. data/lib/scout/resource/util.rb +2 -1
  34. data/lib/scout/tmpfile.rb +7 -8
  35. data/lib/scout/tsv/attach.rb +177 -0
  36. data/lib/scout/tsv/change_id.rb +40 -0
  37. data/lib/scout/tsv/dumper.rb +72 -46
  38. data/lib/scout/tsv/index.rb +69 -13
  39. data/lib/scout/tsv/open.rb +138 -84
  40. data/lib/scout/tsv/parser.rb +135 -80
  41. data/lib/scout/tsv/path.rb +1 -2
  42. data/lib/scout/tsv/persist/adapter.rb +15 -45
  43. data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
  44. data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
  45. data/lib/scout/tsv/persist.rb +4 -0
  46. data/lib/scout/tsv/transformer.rb +141 -0
  47. data/lib/scout/tsv/traverse.rb +96 -92
  48. data/lib/scout/tsv/util/filter.rb +9 -0
  49. data/lib/scout/tsv/util/reorder.rb +81 -0
  50. data/lib/scout/tsv/util/select.rb +78 -33
  51. data/lib/scout/tsv/util/unzip.rb +86 -0
  52. data/lib/scout/tsv/util.rb +60 -11
  53. data/lib/scout/tsv.rb +26 -3
  54. data/lib/scout/work_queue/socket.rb +6 -1
  55. data/lib/scout/work_queue/worker.rb +5 -2
  56. data/lib/scout/work_queue.rb +15 -8
  57. data/lib/scout/workflow/definition.rb +21 -2
  58. data/lib/scout/workflow/step/dependencies.rb +24 -4
  59. data/lib/scout/workflow/step/info.rb +36 -5
  60. data/lib/scout/workflow/step/provenance.rb +8 -7
  61. data/lib/scout/workflow/step/status.rb +45 -0
  62. data/lib/scout/workflow/step.rb +100 -34
  63. data/lib/scout/workflow/task/inputs.rb +14 -20
  64. data/lib/scout/workflow/task.rb +81 -46
  65. data/lib/scout/workflow/usage.rb +8 -6
  66. data/scout-gear.gemspec +24 -20
  67. data/scout_commands/workflow/task +34 -7
  68. data/test/scout/open/test_stream.rb +60 -58
  69. data/test/scout/path/test_find.rb +10 -1
  70. data/test/scout/resource/test_produce.rb +15 -0
  71. data/test/scout/test_meta_extension.rb +25 -0
  72. data/test/scout/test_named_array.rb +18 -0
  73. data/test/scout/test_persist.rb +6 -0
  74. data/test/scout/test_tsv.rb +212 -2
  75. data/test/scout/test_work_queue.rb +21 -19
  76. data/test/scout/tsv/persist/test_adapter.rb +1 -1
  77. data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
  78. data/test/scout/tsv/test_attach.rb +227 -0
  79. data/test/scout/tsv/test_change_id.rb +98 -0
  80. data/test/scout/tsv/test_dumper.rb +1 -1
  81. data/test/scout/tsv/test_index.rb +35 -3
  82. data/test/scout/tsv/test_open.rb +160 -2
  83. data/test/scout/tsv/test_parser.rb +19 -2
  84. data/test/scout/tsv/test_persist.rb +2 -0
  85. data/test/scout/tsv/test_transformer.rb +108 -0
  86. data/test/scout/tsv/test_traverse.rb +88 -3
  87. data/test/scout/tsv/test_util.rb +1 -0
  88. data/test/scout/tsv/util/test_reorder.rb +94 -0
  89. data/test/scout/tsv/util/test_select.rb +25 -11
  90. data/test/scout/tsv/util/test_unzip.rb +112 -0
  91. data/test/scout/work_queue/test_socket.rb +0 -1
  92. data/test/scout/workflow/step/test_status.rb +31 -0
  93. data/test/scout/workflow/task/test_inputs.rb +14 -14
  94. data/test/scout/workflow/test_step.rb +3 -3
  95. data/test/scout/workflow/test_task.rb +168 -32
  96. data/test/scout/workflow/test_usage.rb +33 -6
  97. metadata +20 -6
@@ -22,7 +22,7 @@ row4 a a id3
22
22
  tsv = TmpFile.with_file(content) do |filename|
23
23
  TSV.open(filename, :persist => false)
24
24
  end
25
-
25
+
26
26
  TmpFile.with_file(content2) do |filename|
27
27
  TSV.open(filename, :data => tsv)
28
28
  end
@@ -41,9 +41,14 @@ row2 a a id3
41
41
  EOF
42
42
 
43
43
  tsv = TmpFile.with_file(content) do |filename|
44
+ tsv = TSV.open(filename, :persist => true)
45
+ tsv.close
46
+ Persist::CONNECTIONS.clear
44
47
  TSV.open(filename, :persist => true)
45
48
  end
46
49
 
50
+ assert_equal "Id", tsv.key_field
51
+
47
52
  assert tsv.respond_to?(:persistence_class)
48
53
  assert_equal TokyoCabinet::HDB, tsv.persistence_class
49
54
 
@@ -79,5 +84,210 @@ row2 a a id3
79
84
  assert_include tsv.keys, 'row2'
80
85
  assert_equal %w(A a), tsv["row2"][0]
81
86
  end
82
- end
83
87
 
88
+ def test_headerless_fields
89
+ content =<<-EOF
90
+ row1 a|aa|aaa b Id1|Id2
91
+ row2 A B Id3
92
+ EOF
93
+
94
+ TmpFile.with_file(content) do |filename|
95
+ tsv = TSV.open(filename, :sep => /\s+/, :fields => [1])
96
+ assert_equal ["a", "aa", "aaa"], tsv["row1"][0]
97
+ assert_equal :double, tsv.type
98
+ assert_equal [%w(a aa aaa)], tsv["row1"]
99
+ end
100
+ end
101
+
102
+ def test_tsv_field_selection
103
+ content =<<-EOF
104
+ #Id ValueA ValueB OtherID
105
+ row1 a|aa|aaa b Id1|Id2
106
+ row2 A B Id3
107
+ EOF
108
+
109
+ TmpFile.with_file(content) do |filename|
110
+ tsv = TSV.open(filename, :sep => /\s+/, :type => :single)
111
+ assert_equal ["ValueA"], tsv.fields
112
+ end
113
+ end
114
+
115
+ def test_tsv_single_from_flat
116
+ content =<<-EOF
117
+ #: :type=:flat
118
+ #Id Value
119
+ row1 1 2
120
+ row2 4
121
+ EOF
122
+
123
+ TmpFile.with_file(content) do |filename|
124
+ tsv = TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value", :fields => ["Id"])
125
+ assert_equal "row1", tsv["1"]
126
+ end
127
+ end
128
+
129
+ def test_key_field
130
+ content =<<-EOF
131
+ #: :sep=/\\s+/#:type=:single
132
+ #Id Value
133
+ a 1
134
+ b 2
135
+ EOF
136
+
137
+ TmpFile.with_file(content) do |filename|
138
+ tsv = TSV.open(filename, :key_field => "Value")
139
+ assert_equal %w(Id), tsv.fields
140
+ assert_equal "Value", tsv.key_field
141
+ assert_equal "a", tsv["1"]
142
+ end
143
+ end
144
+
145
+ def test_fix
146
+ content =<<-EOF
147
+ #: :sep=/\\s+/#:type=:single
148
+ #Id Value
149
+ a 1
150
+ b 2
151
+ EOF
152
+
153
+ TmpFile.with_file(content) do |filename|
154
+ tsv = TSV.open(filename, :key_field => "Value", :fix => Proc.new{|l| if l =~ /1/;then "a 3" else l end})
155
+ assert_equal "a", tsv["3"]
156
+ end
157
+ end
158
+
159
+ def test_flat
160
+ content =<<-EOF
161
+ #: :type=:flat
162
+ #Id Value
163
+ row1 a|aa|aaa
164
+ row2 A|AA|AAA
165
+ EOF
166
+
167
+ TmpFile.with_file(content) do |filename|
168
+ assert TSV.open(filename, :sep => /\s+/, :type => :flat).include? "row1"
169
+ assert TSV.open(filename, :sep => /\s+/, :type => :flat)["row1"].include? "a"
170
+ assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id")["row1"].include? "a"
171
+ assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id", :fields => ["Value"])["row1"].include? "a"
172
+ end
173
+ end
174
+
175
+ def test_tsv_flat_double
176
+ content =<<-EOF
177
+ #Id ValueA ValueB OtherID
178
+ row1 a|aa|aaa b Id1|Id2
179
+ row2 A B Id3
180
+ EOF
181
+
182
+ TmpFile.with_file(content) do |filename|
183
+ tsv = TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "ValueA", :fields => ["OtherID"], :merge => true)
184
+ assert tsv["aaa"].include? "Id1"
185
+ assert tsv["aaa"].include? "Id2"
186
+ end
187
+ end
188
+
189
+ def test_flat2single
190
+ content =<<-EOF
191
+ #: :type=:flat
192
+ #Id Value
193
+ row1 a aa aaa
194
+ row2 A AA AAA
195
+ EOF
196
+
197
+ TmpFile.with_file(content) do |filename|
198
+ tsv = TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value")
199
+ assert tsv.include? "aaa"
200
+ end
201
+ end
202
+
203
+ def test_flat_key
204
+ content =<<-EOF
205
+ #Id ValueA
206
+ row1 a aa aaa
207
+ row2 b bbb bbbb bb aa
208
+ EOF
209
+
210
+ TmpFile.with_file(content) do |filename|
211
+ tsv = TSV.open(filename, :sep => /\s+/, :merge => true, :type => :flat, :key_field => "ValueA")
212
+ assert_equal ["row1"], tsv["a"]
213
+ assert_equal ["row1", "row2"], tsv["aa"]
214
+ end
215
+ end
216
+
217
+ def test_unnamed_key
218
+ content =<<-EOF
219
+ row1 a|aa|aaa b Id1|Id2
220
+ row2 A B Id3
221
+ EOF
222
+
223
+ TmpFile.with_file(content) do |filename|
224
+ tsv = TSV.open(filename, :sep => /\s+/, :key_field => 1)
225
+ assert tsv.keys.include? "a"
226
+ end
227
+ end
228
+
229
+ def test_grep
230
+ content =<<-EOF
231
+ #: :sep=/\\s+/#:type=:single
232
+ #Id Value
233
+ a 1
234
+ b 2
235
+ c 3
236
+ EOF
237
+
238
+ TmpFile.with_file(content) do |filename|
239
+ tsv = TSV.open(filename, :key_field => "Value", :grep => "#\\|2")
240
+ assert_includes tsv, "2"
241
+ refute_includes tsv, "3"
242
+ end
243
+ end
244
+
245
+ def TODO_test_tsv_grep
246
+ content =<<-EOF
247
+ #: :sep=/\\s+/#:type=:single
248
+ #Id Value
249
+ a 1
250
+ b 2
251
+ b 3
252
+ d 22
253
+ EOF
254
+
255
+ TmpFile.with_file(content) do |filename|
256
+ tsv = TSV.open(filename, :key_field => "Value", :tsv_grep => "2")
257
+ assert_includes tsv, "2"
258
+ refute_includes tsv, "3"
259
+ assert(tsv.include?("2"))
260
+ assert(! tsv.include?("3"))
261
+ end
262
+ end
263
+
264
+ def test_flat_with_field_header
265
+ content =<<-EOF
266
+ #: :type=:flat
267
+ #Id ValueA
268
+ row1 a aa aaa
269
+ row2 b bbb bbbb bb
270
+ EOF
271
+
272
+ TmpFile.with_file(content) do |filename|
273
+ tsv = TSV.open(filename, :sep => /\s+/, :merge => false)
274
+ assert_equal ["a", "aa", "aaa"], tsv["row1"]
275
+ end
276
+ end
277
+
278
+ def test_alt_args
279
+ content =<<-EOF
280
+ #Id ValueA
281
+ row1 a aa aaa
282
+ row2 b bbb bbbb bb
283
+ EOF
284
+
285
+ TmpFile.with_file(content) do |filename|
286
+ tsv = TSV.open(filename, type: :flat, :sep => /\s+/, :merge => false)
287
+ assert_equal ["a", "aa", "aaa"], tsv["row1"]
288
+ end
289
+
290
+ tsv = TSV.str_setup("ID~ValueA,ValueB#:type=:flat", {})
291
+ assert_equal "ID", tsv.key_field
292
+ end
293
+ end
@@ -93,7 +93,6 @@ class TestWorkQueue < Test::Unit::TestCase
93
93
  num = 100
94
94
  reps = 10_000
95
95
 
96
- sss 0
97
96
  q = WorkQueue.new num do |obj|
98
97
  raise ScoutException if rand < 0.1
99
98
  [Process.pid.to_s, obj.to_s] * " "
@@ -104,18 +103,19 @@ class TestWorkQueue < Test::Unit::TestCase
104
103
  res << out
105
104
  end
106
105
 
107
- pid = Process.fork do
108
- reps.times do |i|
109
- q.write i
106
+ Log.with_severity 7 do
107
+ pid = Process.fork do
108
+ reps.times do |i|
109
+ q.write i
110
+ end
110
111
  end
111
- end
112
112
 
113
- Process.waitpid pid
114
- q.close
113
+ Process.waitpid pid
115
114
 
116
- assert_raise ScoutException do
117
- q.join
118
- t.join
115
+ assert_raise ScoutException do
116
+ q.join
117
+ t.join
118
+ end
119
119
  end
120
120
  end
121
121
 
@@ -129,20 +129,22 @@ class TestWorkQueue < Test::Unit::TestCase
129
129
 
130
130
  res = []
131
131
  q.process do |out|
132
- raise ScoutException if rand < 0.01
132
+ raise ScoutException
133
133
  res << out
134
134
  end
135
135
 
136
- pid = Process.fork do
137
- reps.times do |i|
138
- q.write i
136
+ Log.with_severity 7 do
137
+ pid = Process.fork do
138
+ reps.times do |i|
139
+ q.write i
140
+ end
141
+ q.close
139
142
  end
140
- q.close
141
- end
142
143
 
143
- assert_raise ScoutException do
144
- q.join
145
- t.join
144
+ assert_raise ScoutException do
145
+ q.join
146
+ t.join
147
+ end
146
148
  end
147
149
  end
148
150
  end
@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
3
3
 
4
4
  require 'scout/tsv'
5
5
  class TestTSVAdapter < Test::Unit::TestCase
6
- def test_get_set
6
+ def _test_get_set
7
7
  tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
8
8
  tsv.type = :list
9
9
  tsv.extend TSVAdapter
@@ -21,6 +21,11 @@ row2 a a id3
21
21
 
22
22
  assert_equal %w(a aa aaa), tsv["row1"][0]
23
23
 
24
+ assert TSVAdapter === tsv
25
+ assert TSV === tsv
26
+ assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
27
+ assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
28
+
24
29
  tsv_loaded = assert_nothing_raised do
25
30
  TmpFile.with_file(content) do |filename|
26
31
  Persist.persist(__method__, :HDB) do
@@ -29,6 +34,7 @@ row2 a a id3
29
34
  end
30
35
  end
31
36
 
37
+
32
38
  assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
33
39
  end
34
40
 
@@ -54,7 +60,7 @@ row2 a a id3
54
60
  end
55
61
  end
56
62
 
57
- def test_speed
63
+ def __test_speed
58
64
  tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
59
65
 
60
66
  size = 100_000
@@ -88,5 +94,27 @@ row2 a a id3
88
94
  assert_equal tc["key-#{i}"], tsv["key-#{i}"]
89
95
  end
90
96
  end
97
+
98
+ def test_float_array
99
+ content =<<-EOF
100
+ #Id ValueA ValueB OtherID
101
+ row1 0.2 0.3 0
102
+ row2 0.1 4.5 0
103
+ EOF
104
+
105
+ TmpFile.with_file(content) do |filename|
106
+ tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :list, :cast => :to_f)
107
+ tsv.save_extension_attr_hash
108
+ assert_equal [0.2, 0.3, 0], tsv["row1"]
109
+ assert_equal TSVAdapter::FloatArraySerializer, tsv.serializer
110
+ Open.cp tsv.persistence_path, tmpdir.persistence.foo
111
+ tsv2 = ScoutCabinet.open(tmpdir.persistence.foo, false)
112
+ tsv2.extend TSVAdapter
113
+ assert_equal [0.2, 0.3, 0], tsv2["row1"]
114
+ assert_equal TSVAdapter::FloatArraySerializer, tsv2.serializer
115
+ end
116
+
117
+ end
118
+
91
119
  end
92
120
 
@@ -0,0 +1,227 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+
6
+ class TestTSVAttach < Test::Unit::TestCase
7
+ def test_attach_simple
8
+ content1 =<<-EOF
9
+ #: :sep=" "
10
+ #ID ValueA ValueB
11
+ row1 a|aa|aaa b
12
+ row2 A B
13
+ EOF
14
+
15
+ content2 =<<-EOF
16
+ #: :sep=" "
17
+ #ID ValueB OtherID
18
+ row1 b Id1|Id2
19
+ row3 B Id3
20
+ EOF
21
+
22
+ TmpFile.with_file(content1) do |filename1|
23
+ TmpFile.with_file(content2) do |filename2|
24
+ tsv = TSV.open(filename1)
25
+ other = TSV.open(filename2)
26
+ tsv.attach other, :complete => true
27
+ assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
28
+ assert_equal %w(Id3), tsv["row3"]["OtherID"]
29
+ assert_equal %w(B), tsv["row3"]["ValueB"]
30
+ end
31
+ end
32
+ end
33
+
34
+ def test_attach_by_key
35
+ content1 =<<-EOF
36
+ #: :sep=" "
37
+ #ID ValueA ValueB
38
+ row1 A1|A11 B1|B11
39
+ row2 A2|A22 B2|B22
40
+ EOF
41
+
42
+ content2 =<<-EOF
43
+ #: :sep=" "
44
+ #ID ValueB OtherID
45
+ row1 B1|B11 Id1|Id11
46
+ row2.2 B2|B22|B222 Id2.2|Id22.2|Id222.2
47
+ row3 B3 Id3
48
+ EOF
49
+
50
+ TmpFile.with_file(content1) do |filename1|
51
+ TmpFile.with_file(content2) do |filename2|
52
+ tsv = TSV.open(filename1)
53
+ other = TSV.open(filename2)
54
+ tsv.attach other, complete: true, match_key: "ValueB"
55
+ assert_equal %w(A1 A11), tsv["row1"]["ValueA"]
56
+ assert_equal %w(B1 B11), tsv["row1"]["ValueB"]
57
+ assert_equal %w(Id1 Id11), tsv["row1"]["OtherID"]
58
+ assert_equal %w(Id2.2 Id22.2), tsv["row2"]["OtherID"]
59
+ end
60
+ end
61
+ end
62
+
63
+ def test_attach_by_reorder
64
+ content1 =<<-EOF
65
+ #: :sep=" "
66
+ #ID ValueA ValueB
67
+ row1 A1|A11 B1|B11
68
+ row2 A2|A22 B2|B22
69
+ EOF
70
+
71
+ content2 =<<-EOF
72
+ #: :sep=" "
73
+ #ValueB ID OtherID
74
+ B1 row1|row1.1 Id1|Id11
75
+ B2 row2 Id2.2|Id22.2|Id222.2
76
+ B3 row3 Id3
77
+ EOF
78
+
79
+ TmpFile.with_file(content1) do |filename1|
80
+ TmpFile.with_file(content2) do |filename2|
81
+ tsv = TSV.open(filename1)
82
+ other = TSV.open(filename2)
83
+ tsv.attach other, match_key: "ID", one2one: false
84
+ assert_equal %w(A1 A11), tsv["row1"]["ValueA"]
85
+ assert_equal %w(B1 B11), tsv["row1"]["ValueB"]
86
+ assert_equal %w(Id1 Id11), tsv["row1"]["OtherID"]
87
+ assert_equal %w(Id2.2 Id22.2 Id222.2), tsv["row2"]["OtherID"]
88
+ end
89
+ end
90
+ end
91
+
92
+
93
+ def test_attach_same_key
94
+ content1 =<<-EOF
95
+ #ID ValueA ValueB
96
+ row1 a|aa|aaa b
97
+ row2 A B
98
+ EOF
99
+
100
+ content2 =<<-EOF
101
+ #ID ValueB OtherID
102
+ row1 b Id1|Id2
103
+ row3 B Id3
104
+ EOF
105
+
106
+ tsv1 = tsv2 = nil
107
+ TmpFile.with_file(content1) do |filename|
108
+ tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
109
+ end
110
+
111
+ TmpFile.with_file(content2) do |filename|
112
+ tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
113
+ end
114
+
115
+ tsv1.attach tsv2, fields: "OtherID"
116
+
117
+ assert_equal %w(ValueA ValueB OtherID), tsv1.fields
118
+ assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
119
+
120
+ TmpFile.with_file(content1) do |filename|
121
+ tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
122
+ end
123
+
124
+ tsv1.attach tsv2
125
+
126
+ assert_equal %w(ValueA ValueB OtherID), tsv1.fields
127
+
128
+ tsv1 = tsv2 = nil
129
+ TmpFile.with_file(content1) do |filename|
130
+ tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
131
+ end
132
+
133
+ TmpFile.with_file(content2) do |filename|
134
+ tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
135
+ end
136
+
137
+ tsv1.attach tsv2, fields: "OtherID"
138
+
139
+ assert_equal %w(ValueA ValueB OtherID), tsv1.fields
140
+ assert_equal "Id1", tsv1["row1"]["OtherID"]
141
+ end
142
+
143
+ def test_attach_source_field
144
+ content1 =<<-EOF
145
+ #Id ValueA ValueB
146
+ row1 a|aa|aaa b
147
+ row2 A B
148
+ EOF
149
+
150
+ content2 =<<-EOF
151
+ #ValueB OtherID
152
+ b Id1|Id2
153
+ B Id3
154
+ EOF
155
+
156
+ tsv1 = tsv2 = nil
157
+ TmpFile.with_file(content1) do |filename|
158
+ tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
159
+ end
160
+
161
+ TmpFile.with_file(content2) do |filename|
162
+ tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
163
+ end
164
+
165
+ tsv1.attach tsv2, bar: true
166
+
167
+ assert_equal %w(ValueA ValueB OtherID), tsv1.fields
168
+ assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
169
+
170
+ TmpFile.with_file(content1) do |filename|
171
+ tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
172
+ end
173
+
174
+ tsv1.attach tsv2
175
+
176
+ assert_equal %w(ValueA ValueB OtherID), tsv1.fields
177
+ assert_equal "Id1", tsv1["row1"]["OtherID"]
178
+ end
179
+
180
+ def test_attach_transformer
181
+ content1 =<<-EOF
182
+ #: :sep=" "
183
+ #ID ValueA ValueB
184
+ row1 a|aa|aaa b
185
+ row2 A B
186
+ EOF
187
+
188
+ content2 =<<-EOF
189
+ #: :sep=" "
190
+ #ID ValueB OtherID
191
+ row1 b Id1|Id2
192
+ row3 B Id3
193
+ EOF
194
+
195
+ TmpFile.with_file(content1) do |filename1|
196
+ TmpFile.with_file(content2) do |filename2|
197
+ out = TSV.attach filename1, filename2, target: :stream, bar: false
198
+ tsv = out.tsv
199
+ assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
200
+ end
201
+ end
202
+ end
203
+
204
+ def test_attach_flexible_names
205
+ content1 =<<-EOF
206
+ #: :sep=" "
207
+ #ID ValueA ValueB
208
+ row1 a|aa|aaa b
209
+ row2 A B
210
+ EOF
211
+
212
+ content2 =<<-EOF
213
+ #: :sep=" "
214
+ #Identifiers(ID) OtherID
215
+ row1 Id1|Id2
216
+ row3 Id3
217
+ EOF
218
+
219
+ TmpFile.with_file(content1) do |filename1|
220
+ TmpFile.with_file(content2) do |filename2|
221
+ out = TSV.attach filename1, filename2, target: :stream, bar: false
222
+ tsv = out.tsv
223
+ assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
224
+ end
225
+ end
226
+ end
227
+ end
@@ -0,0 +1,98 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+ class TestChangeID < Test::Unit::TestCase
6
+ def test_simple_reorder
7
+ content1 =<<-EOF
8
+ #: :sep=" "
9
+ #ID ValueA ValueB
10
+ row1 A1|A11 B1|B11
11
+ row2 A2|A22 B2|B22
12
+ EOF
13
+
14
+ tsv = TSV.open StringIO.new(content1)
15
+
16
+ res = tsv.change_key "ValueA", keep: true
17
+ assert_equal ["row1"], res["A1"]["ID"]
18
+ assert_equal ["row1"], res["A11"]["ID"]
19
+ assert_equal ["row2"], res["A2"]["ID"]
20
+
21
+ res = tsv.change_key "ValueA", keep: false, one2one: true
22
+ assert_equal ["B1"], res["A1"]["ValueB"]
23
+ assert_equal ["B11"], res["A11"]["ValueB"]
24
+ end
25
+
26
+ def test_simple_reorder_file
27
+ content1 =<<-EOF
28
+ #: :sep=" "
29
+ #ID ValueA ValueB
30
+ row1 A1|A11 B1|B11
31
+ row2 A2|A22 B2|B22
32
+ EOF
33
+
34
+ TmpFile.with_file(content1) do |file1|
35
+ res = TSV.change_key file1, "ValueA", keep: true
36
+ assert_equal ["row1"], res["A1"]["ID"]
37
+ assert_equal ["row1"], res["A11"]["ID"]
38
+ assert_equal ["row2"], res["A2"]["ID"]
39
+ assert_equal ["B1","B11"], res["A1"]["ValueB"]
40
+
41
+ res = TSV.change_key file1, "ValueA", one2one: true, keep: true
42
+ assert_equal ["row1"], res["A1"]["ID"]
43
+ assert_equal ["row1"], res["A11"]["ID"]
44
+ assert_equal ["row2"], res["A2"]["ID"]
45
+ assert_equal ["B1"], res["A1"]["ValueB"]
46
+ end
47
+ end
48
+
49
+ def test_change_key_identifiers
50
+ content1 =<<-EOF
51
+ #: :sep=" "
52
+ #ID ValueA ValueB
53
+ row1 A1|A11 B1|B11
54
+ row2 A2|A22 B2|B22
55
+ EOF
56
+
57
+ identifiers_content =<<-EOF
58
+ #: :sep=" "
59
+ #ID ValueC ValueD
60
+ row1 C1|C11 D1|D11
61
+ row2 C2|C22 D2|D22
62
+ EOF
63
+
64
+
65
+ tsv = TSV.open StringIO.new(content1)
66
+ identifiers = TSV.open StringIO.new(identifiers_content)
67
+
68
+ res = tsv.change_key "ValueC", identifiers: identifiers, keep: true
69
+ assert_equal ["row1"], res["C1"]["ID"]
70
+ assert_equal ["row1"], res["C11"]["ID"]
71
+ assert_equal ["row2"], res["C2"]["ID"]
72
+ end
73
+
74
+ def test_change_id_identifiers
75
+ content1 =<<-EOF
76
+ #: :sep=" "
77
+ #ID ValueA ValueB
78
+ row1 A1|A11 B1|B11
79
+ row2 A2|A22 B2|B22
80
+ EOF
81
+
82
+ identifiers_content =<<-EOF
83
+ #: :sep=" "
84
+ #ID ValueC ValueD
85
+ row1 C1|C11 D1|D11
86
+ row2 C2|C22 D2|D22
87
+ EOF
88
+
89
+
90
+ tsv = TSV.open StringIO.new(content1)
91
+ identifiers = TSV.open StringIO.new(identifiers_content)
92
+
93
+ res = tsv.change_id "ValueA", "ValueC", identifiers: identifiers
94
+ assert_equal ["C1","C11"], res["row1"]["ValueC"]
95
+ assert_equal ["C2","C22"], res["row2"]["ValueC"]
96
+ end
97
+ end
98
+
@@ -37,7 +37,7 @@ a\t1|11\t2|22
37
37
  end
38
38
 
39
39
  assert_raise ScoutException do
40
- dumper.stream.read
40
+ TSV.open(dumper.stream, bar: true)
41
41
  end
42
42
  end
43
43
  end