scout-gear 7.3.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +20 -9
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +26 -23
- data/lib/scout/config.rb +1 -1
- data/lib/scout/log/color.rb +4 -1
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +58 -54
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +11 -0
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +90 -15
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +28 -12
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +72 -46
- data/lib/scout/tsv/index.rb +69 -13
- data/lib/scout/tsv/open.rb +138 -84
- data/lib/scout/tsv/parser.rb +135 -80
- data/lib/scout/tsv/path.rb +1 -2
- data/lib/scout/tsv/persist/adapter.rb +15 -45
- data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
- data/lib/scout/tsv/persist.rb +4 -0
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +96 -92
- data/lib/scout/tsv/util/filter.rb +9 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +78 -33
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +60 -11
- data/lib/scout/tsv.rb +26 -3
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +21 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +36 -5
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +100 -34
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +81 -46
- data/lib/scout/workflow/usage.rb +8 -6
- data/scout-gear.gemspec +24 -20
- data/scout_commands/workflow/task +34 -7
- data/test/scout/open/test_stream.rb +60 -58
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +6 -0
- data/test/scout/test_tsv.rb +212 -2
- data/test/scout/test_work_queue.rb +21 -19
- data/test/scout/tsv/persist/test_adapter.rb +1 -1
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +35 -3
- data/test/scout/tsv/test_open.rb +160 -2
- data/test/scout/tsv/test_parser.rb +19 -2
- data/test/scout/tsv/test_persist.rb +2 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +88 -3
- data/test/scout/tsv/test_util.rb +1 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +25 -11
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +3 -3
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- metadata +20 -6
data/test/scout/test_tsv.rb
CHANGED
@@ -22,7 +22,7 @@ row4 a a id3
|
|
22
22
|
tsv = TmpFile.with_file(content) do |filename|
|
23
23
|
TSV.open(filename, :persist => false)
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
TmpFile.with_file(content2) do |filename|
|
27
27
|
TSV.open(filename, :data => tsv)
|
28
28
|
end
|
@@ -41,9 +41,14 @@ row2 a a id3
|
|
41
41
|
EOF
|
42
42
|
|
43
43
|
tsv = TmpFile.with_file(content) do |filename|
|
44
|
+
tsv = TSV.open(filename, :persist => true)
|
45
|
+
tsv.close
|
46
|
+
Persist::CONNECTIONS.clear
|
44
47
|
TSV.open(filename, :persist => true)
|
45
48
|
end
|
46
49
|
|
50
|
+
assert_equal "Id", tsv.key_field
|
51
|
+
|
47
52
|
assert tsv.respond_to?(:persistence_class)
|
48
53
|
assert_equal TokyoCabinet::HDB, tsv.persistence_class
|
49
54
|
|
@@ -79,5 +84,210 @@ row2 a a id3
|
|
79
84
|
assert_include tsv.keys, 'row2'
|
80
85
|
assert_equal %w(A a), tsv["row2"][0]
|
81
86
|
end
|
82
|
-
end
|
83
87
|
|
88
|
+
def test_headerless_fields
|
89
|
+
content =<<-EOF
|
90
|
+
row1 a|aa|aaa b Id1|Id2
|
91
|
+
row2 A B Id3
|
92
|
+
EOF
|
93
|
+
|
94
|
+
TmpFile.with_file(content) do |filename|
|
95
|
+
tsv = TSV.open(filename, :sep => /\s+/, :fields => [1])
|
96
|
+
assert_equal ["a", "aa", "aaa"], tsv["row1"][0]
|
97
|
+
assert_equal :double, tsv.type
|
98
|
+
assert_equal [%w(a aa aaa)], tsv["row1"]
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
def test_tsv_field_selection
|
103
|
+
content =<<-EOF
|
104
|
+
#Id ValueA ValueB OtherID
|
105
|
+
row1 a|aa|aaa b Id1|Id2
|
106
|
+
row2 A B Id3
|
107
|
+
EOF
|
108
|
+
|
109
|
+
TmpFile.with_file(content) do |filename|
|
110
|
+
tsv = TSV.open(filename, :sep => /\s+/, :type => :single)
|
111
|
+
assert_equal ["ValueA"], tsv.fields
|
112
|
+
end
|
113
|
+
end
|
114
|
+
|
115
|
+
def test_tsv_single_from_flat
|
116
|
+
content =<<-EOF
|
117
|
+
#: :type=:flat
|
118
|
+
#Id Value
|
119
|
+
row1 1 2
|
120
|
+
row2 4
|
121
|
+
EOF
|
122
|
+
|
123
|
+
TmpFile.with_file(content) do |filename|
|
124
|
+
tsv = TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value", :fields => ["Id"])
|
125
|
+
assert_equal "row1", tsv["1"]
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def test_key_field
|
130
|
+
content =<<-EOF
|
131
|
+
#: :sep=/\\s+/#:type=:single
|
132
|
+
#Id Value
|
133
|
+
a 1
|
134
|
+
b 2
|
135
|
+
EOF
|
136
|
+
|
137
|
+
TmpFile.with_file(content) do |filename|
|
138
|
+
tsv = TSV.open(filename, :key_field => "Value")
|
139
|
+
assert_equal %w(Id), tsv.fields
|
140
|
+
assert_equal "Value", tsv.key_field
|
141
|
+
assert_equal "a", tsv["1"]
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def test_fix
|
146
|
+
content =<<-EOF
|
147
|
+
#: :sep=/\\s+/#:type=:single
|
148
|
+
#Id Value
|
149
|
+
a 1
|
150
|
+
b 2
|
151
|
+
EOF
|
152
|
+
|
153
|
+
TmpFile.with_file(content) do |filename|
|
154
|
+
tsv = TSV.open(filename, :key_field => "Value", :fix => Proc.new{|l| if l =~ /1/;then "a 3" else l end})
|
155
|
+
assert_equal "a", tsv["3"]
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def test_flat
|
160
|
+
content =<<-EOF
|
161
|
+
#: :type=:flat
|
162
|
+
#Id Value
|
163
|
+
row1 a|aa|aaa
|
164
|
+
row2 A|AA|AAA
|
165
|
+
EOF
|
166
|
+
|
167
|
+
TmpFile.with_file(content) do |filename|
|
168
|
+
assert TSV.open(filename, :sep => /\s+/, :type => :flat).include? "row1"
|
169
|
+
assert TSV.open(filename, :sep => /\s+/, :type => :flat)["row1"].include? "a"
|
170
|
+
assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id")["row1"].include? "a"
|
171
|
+
assert TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "Id", :fields => ["Value"])["row1"].include? "a"
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
def test_tsv_flat_double
|
176
|
+
content =<<-EOF
|
177
|
+
#Id ValueA ValueB OtherID
|
178
|
+
row1 a|aa|aaa b Id1|Id2
|
179
|
+
row2 A B Id3
|
180
|
+
EOF
|
181
|
+
|
182
|
+
TmpFile.with_file(content) do |filename|
|
183
|
+
tsv = TSV.open(filename, :sep => /\s+/, :type => :flat, :key_field => "ValueA", :fields => ["OtherID"], :merge => true)
|
184
|
+
assert tsv["aaa"].include? "Id1"
|
185
|
+
assert tsv["aaa"].include? "Id2"
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
def test_flat2single
|
190
|
+
content =<<-EOF
|
191
|
+
#: :type=:flat
|
192
|
+
#Id Value
|
193
|
+
row1 a aa aaa
|
194
|
+
row2 A AA AAA
|
195
|
+
EOF
|
196
|
+
|
197
|
+
TmpFile.with_file(content) do |filename|
|
198
|
+
tsv = TSV.open(filename, :sep => /\s+/, :type => :single, :key_field => "Value")
|
199
|
+
assert tsv.include? "aaa"
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
def test_flat_key
|
204
|
+
content =<<-EOF
|
205
|
+
#Id ValueA
|
206
|
+
row1 a aa aaa
|
207
|
+
row2 b bbb bbbb bb aa
|
208
|
+
EOF
|
209
|
+
|
210
|
+
TmpFile.with_file(content) do |filename|
|
211
|
+
tsv = TSV.open(filename, :sep => /\s+/, :merge => true, :type => :flat, :key_field => "ValueA")
|
212
|
+
assert_equal ["row1"], tsv["a"]
|
213
|
+
assert_equal ["row1", "row2"], tsv["aa"]
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def test_unnamed_key
|
218
|
+
content =<<-EOF
|
219
|
+
row1 a|aa|aaa b Id1|Id2
|
220
|
+
row2 A B Id3
|
221
|
+
EOF
|
222
|
+
|
223
|
+
TmpFile.with_file(content) do |filename|
|
224
|
+
tsv = TSV.open(filename, :sep => /\s+/, :key_field => 1)
|
225
|
+
assert tsv.keys.include? "a"
|
226
|
+
end
|
227
|
+
end
|
228
|
+
|
229
|
+
def test_grep
|
230
|
+
content =<<-EOF
|
231
|
+
#: :sep=/\\s+/#:type=:single
|
232
|
+
#Id Value
|
233
|
+
a 1
|
234
|
+
b 2
|
235
|
+
c 3
|
236
|
+
EOF
|
237
|
+
|
238
|
+
TmpFile.with_file(content) do |filename|
|
239
|
+
tsv = TSV.open(filename, :key_field => "Value", :grep => "#\\|2")
|
240
|
+
assert_includes tsv, "2"
|
241
|
+
refute_includes tsv, "3"
|
242
|
+
end
|
243
|
+
end
|
244
|
+
|
245
|
+
def TODO_test_tsv_grep
|
246
|
+
content =<<-EOF
|
247
|
+
#: :sep=/\\s+/#:type=:single
|
248
|
+
#Id Value
|
249
|
+
a 1
|
250
|
+
b 2
|
251
|
+
b 3
|
252
|
+
d 22
|
253
|
+
EOF
|
254
|
+
|
255
|
+
TmpFile.with_file(content) do |filename|
|
256
|
+
tsv = TSV.open(filename, :key_field => "Value", :tsv_grep => "2")
|
257
|
+
assert_includes tsv, "2"
|
258
|
+
refute_includes tsv, "3"
|
259
|
+
assert(tsv.include?("2"))
|
260
|
+
assert(! tsv.include?("3"))
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
264
|
+
def test_flat_with_field_header
|
265
|
+
content =<<-EOF
|
266
|
+
#: :type=:flat
|
267
|
+
#Id ValueA
|
268
|
+
row1 a aa aaa
|
269
|
+
row2 b bbb bbbb bb
|
270
|
+
EOF
|
271
|
+
|
272
|
+
TmpFile.with_file(content) do |filename|
|
273
|
+
tsv = TSV.open(filename, :sep => /\s+/, :merge => false)
|
274
|
+
assert_equal ["a", "aa", "aaa"], tsv["row1"]
|
275
|
+
end
|
276
|
+
end
|
277
|
+
|
278
|
+
def test_alt_args
|
279
|
+
content =<<-EOF
|
280
|
+
#Id ValueA
|
281
|
+
row1 a aa aaa
|
282
|
+
row2 b bbb bbbb bb
|
283
|
+
EOF
|
284
|
+
|
285
|
+
TmpFile.with_file(content) do |filename|
|
286
|
+
tsv = TSV.open(filename, type: :flat, :sep => /\s+/, :merge => false)
|
287
|
+
assert_equal ["a", "aa", "aaa"], tsv["row1"]
|
288
|
+
end
|
289
|
+
|
290
|
+
tsv = TSV.str_setup("ID~ValueA,ValueB#:type=:flat", {})
|
291
|
+
assert_equal "ID", tsv.key_field
|
292
|
+
end
|
293
|
+
end
|
@@ -93,7 +93,6 @@ class TestWorkQueue < Test::Unit::TestCase
|
|
93
93
|
num = 100
|
94
94
|
reps = 10_000
|
95
95
|
|
96
|
-
sss 0
|
97
96
|
q = WorkQueue.new num do |obj|
|
98
97
|
raise ScoutException if rand < 0.1
|
99
98
|
[Process.pid.to_s, obj.to_s] * " "
|
@@ -104,18 +103,19 @@ class TestWorkQueue < Test::Unit::TestCase
|
|
104
103
|
res << out
|
105
104
|
end
|
106
105
|
|
107
|
-
|
108
|
-
|
109
|
-
|
106
|
+
Log.with_severity 7 do
|
107
|
+
pid = Process.fork do
|
108
|
+
reps.times do |i|
|
109
|
+
q.write i
|
110
|
+
end
|
110
111
|
end
|
111
|
-
end
|
112
112
|
|
113
|
-
|
114
|
-
q.close
|
113
|
+
Process.waitpid pid
|
115
114
|
|
116
|
-
|
117
|
-
|
118
|
-
|
115
|
+
assert_raise ScoutException do
|
116
|
+
q.join
|
117
|
+
t.join
|
118
|
+
end
|
119
119
|
end
|
120
120
|
end
|
121
121
|
|
@@ -129,20 +129,22 @@ class TestWorkQueue < Test::Unit::TestCase
|
|
129
129
|
|
130
130
|
res = []
|
131
131
|
q.process do |out|
|
132
|
-
raise ScoutException
|
132
|
+
raise ScoutException
|
133
133
|
res << out
|
134
134
|
end
|
135
135
|
|
136
|
-
|
137
|
-
|
138
|
-
|
136
|
+
Log.with_severity 7 do
|
137
|
+
pid = Process.fork do
|
138
|
+
reps.times do |i|
|
139
|
+
q.write i
|
140
|
+
end
|
141
|
+
q.close
|
139
142
|
end
|
140
|
-
q.close
|
141
|
-
end
|
142
143
|
|
143
|
-
|
144
|
-
|
145
|
-
|
144
|
+
assert_raise ScoutException do
|
145
|
+
q.join
|
146
|
+
t.join
|
147
|
+
end
|
146
148
|
end
|
147
149
|
end
|
148
150
|
end
|
@@ -3,7 +3,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
|
|
3
3
|
|
4
4
|
require 'scout/tsv'
|
5
5
|
class TestTSVAdapter < Test::Unit::TestCase
|
6
|
-
def
|
6
|
+
def _test_get_set
|
7
7
|
tsv = TSV.setup({}, :type => :list, :key_field => "Key", :fields => %w(one two three))
|
8
8
|
tsv.type = :list
|
9
9
|
tsv.extend TSVAdapter
|
@@ -21,6 +21,11 @@ row2 a a id3
|
|
21
21
|
|
22
22
|
assert_equal %w(a aa aaa), tsv["row1"][0]
|
23
23
|
|
24
|
+
assert TSVAdapter === tsv
|
25
|
+
assert TSV === tsv
|
26
|
+
assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
|
27
|
+
assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
|
28
|
+
|
24
29
|
tsv_loaded = assert_nothing_raised do
|
25
30
|
TmpFile.with_file(content) do |filename|
|
26
31
|
Persist.persist(__method__, :HDB) do
|
@@ -29,6 +34,7 @@ row2 a a id3
|
|
29
34
|
end
|
30
35
|
end
|
31
36
|
|
37
|
+
|
32
38
|
assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
|
33
39
|
end
|
34
40
|
|
@@ -54,7 +60,7 @@ row2 a a id3
|
|
54
60
|
end
|
55
61
|
end
|
56
62
|
|
57
|
-
def
|
63
|
+
def __test_speed
|
58
64
|
tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
|
59
65
|
|
60
66
|
size = 100_000
|
@@ -88,5 +94,27 @@ row2 a a id3
|
|
88
94
|
assert_equal tc["key-#{i}"], tsv["key-#{i}"]
|
89
95
|
end
|
90
96
|
end
|
97
|
+
|
98
|
+
def test_float_array
|
99
|
+
content =<<-EOF
|
100
|
+
#Id ValueA ValueB OtherID
|
101
|
+
row1 0.2 0.3 0
|
102
|
+
row2 0.1 4.5 0
|
103
|
+
EOF
|
104
|
+
|
105
|
+
TmpFile.with_file(content) do |filename|
|
106
|
+
tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :list, :cast => :to_f)
|
107
|
+
tsv.save_extension_attr_hash
|
108
|
+
assert_equal [0.2, 0.3, 0], tsv["row1"]
|
109
|
+
assert_equal TSVAdapter::FloatArraySerializer, tsv.serializer
|
110
|
+
Open.cp tsv.persistence_path, tmpdir.persistence.foo
|
111
|
+
tsv2 = ScoutCabinet.open(tmpdir.persistence.foo, false)
|
112
|
+
tsv2.extend TSVAdapter
|
113
|
+
assert_equal [0.2, 0.3, 0], tsv2["row1"]
|
114
|
+
assert_equal TSVAdapter::FloatArraySerializer, tsv2.serializer
|
115
|
+
end
|
116
|
+
|
117
|
+
end
|
118
|
+
|
91
119
|
end
|
92
120
|
|
@@ -0,0 +1,227 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/tsv'
|
5
|
+
|
6
|
+
class TestTSVAttach < Test::Unit::TestCase
|
7
|
+
def test_attach_simple
|
8
|
+
content1 =<<-EOF
|
9
|
+
#: :sep=" "
|
10
|
+
#ID ValueA ValueB
|
11
|
+
row1 a|aa|aaa b
|
12
|
+
row2 A B
|
13
|
+
EOF
|
14
|
+
|
15
|
+
content2 =<<-EOF
|
16
|
+
#: :sep=" "
|
17
|
+
#ID ValueB OtherID
|
18
|
+
row1 b Id1|Id2
|
19
|
+
row3 B Id3
|
20
|
+
EOF
|
21
|
+
|
22
|
+
TmpFile.with_file(content1) do |filename1|
|
23
|
+
TmpFile.with_file(content2) do |filename2|
|
24
|
+
tsv = TSV.open(filename1)
|
25
|
+
other = TSV.open(filename2)
|
26
|
+
tsv.attach other, :complete => true
|
27
|
+
assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
|
28
|
+
assert_equal %w(Id3), tsv["row3"]["OtherID"]
|
29
|
+
assert_equal %w(B), tsv["row3"]["ValueB"]
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_attach_by_key
|
35
|
+
content1 =<<-EOF
|
36
|
+
#: :sep=" "
|
37
|
+
#ID ValueA ValueB
|
38
|
+
row1 A1|A11 B1|B11
|
39
|
+
row2 A2|A22 B2|B22
|
40
|
+
EOF
|
41
|
+
|
42
|
+
content2 =<<-EOF
|
43
|
+
#: :sep=" "
|
44
|
+
#ID ValueB OtherID
|
45
|
+
row1 B1|B11 Id1|Id11
|
46
|
+
row2.2 B2|B22|B222 Id2.2|Id22.2|Id222.2
|
47
|
+
row3 B3 Id3
|
48
|
+
EOF
|
49
|
+
|
50
|
+
TmpFile.with_file(content1) do |filename1|
|
51
|
+
TmpFile.with_file(content2) do |filename2|
|
52
|
+
tsv = TSV.open(filename1)
|
53
|
+
other = TSV.open(filename2)
|
54
|
+
tsv.attach other, complete: true, match_key: "ValueB"
|
55
|
+
assert_equal %w(A1 A11), tsv["row1"]["ValueA"]
|
56
|
+
assert_equal %w(B1 B11), tsv["row1"]["ValueB"]
|
57
|
+
assert_equal %w(Id1 Id11), tsv["row1"]["OtherID"]
|
58
|
+
assert_equal %w(Id2.2 Id22.2), tsv["row2"]["OtherID"]
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
def test_attach_by_reorder
|
64
|
+
content1 =<<-EOF
|
65
|
+
#: :sep=" "
|
66
|
+
#ID ValueA ValueB
|
67
|
+
row1 A1|A11 B1|B11
|
68
|
+
row2 A2|A22 B2|B22
|
69
|
+
EOF
|
70
|
+
|
71
|
+
content2 =<<-EOF
|
72
|
+
#: :sep=" "
|
73
|
+
#ValueB ID OtherID
|
74
|
+
B1 row1|row1.1 Id1|Id11
|
75
|
+
B2 row2 Id2.2|Id22.2|Id222.2
|
76
|
+
B3 row3 Id3
|
77
|
+
EOF
|
78
|
+
|
79
|
+
TmpFile.with_file(content1) do |filename1|
|
80
|
+
TmpFile.with_file(content2) do |filename2|
|
81
|
+
tsv = TSV.open(filename1)
|
82
|
+
other = TSV.open(filename2)
|
83
|
+
tsv.attach other, match_key: "ID", one2one: false
|
84
|
+
assert_equal %w(A1 A11), tsv["row1"]["ValueA"]
|
85
|
+
assert_equal %w(B1 B11), tsv["row1"]["ValueB"]
|
86
|
+
assert_equal %w(Id1 Id11), tsv["row1"]["OtherID"]
|
87
|
+
assert_equal %w(Id2.2 Id22.2 Id222.2), tsv["row2"]["OtherID"]
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
|
93
|
+
def test_attach_same_key
|
94
|
+
content1 =<<-EOF
|
95
|
+
#ID ValueA ValueB
|
96
|
+
row1 a|aa|aaa b
|
97
|
+
row2 A B
|
98
|
+
EOF
|
99
|
+
|
100
|
+
content2 =<<-EOF
|
101
|
+
#ID ValueB OtherID
|
102
|
+
row1 b Id1|Id2
|
103
|
+
row3 B Id3
|
104
|
+
EOF
|
105
|
+
|
106
|
+
tsv1 = tsv2 = nil
|
107
|
+
TmpFile.with_file(content1) do |filename|
|
108
|
+
tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
|
109
|
+
end
|
110
|
+
|
111
|
+
TmpFile.with_file(content2) do |filename|
|
112
|
+
tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
|
113
|
+
end
|
114
|
+
|
115
|
+
tsv1.attach tsv2, fields: "OtherID"
|
116
|
+
|
117
|
+
assert_equal %w(ValueA ValueB OtherID), tsv1.fields
|
118
|
+
assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
|
119
|
+
|
120
|
+
TmpFile.with_file(content1) do |filename|
|
121
|
+
tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
|
122
|
+
end
|
123
|
+
|
124
|
+
tsv1.attach tsv2
|
125
|
+
|
126
|
+
assert_equal %w(ValueA ValueB OtherID), tsv1.fields
|
127
|
+
|
128
|
+
tsv1 = tsv2 = nil
|
129
|
+
TmpFile.with_file(content1) do |filename|
|
130
|
+
tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
|
131
|
+
end
|
132
|
+
|
133
|
+
TmpFile.with_file(content2) do |filename|
|
134
|
+
tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
|
135
|
+
end
|
136
|
+
|
137
|
+
tsv1.attach tsv2, fields: "OtherID"
|
138
|
+
|
139
|
+
assert_equal %w(ValueA ValueB OtherID), tsv1.fields
|
140
|
+
assert_equal "Id1", tsv1["row1"]["OtherID"]
|
141
|
+
end
|
142
|
+
|
143
|
+
def test_attach_source_field
|
144
|
+
content1 =<<-EOF
|
145
|
+
#Id ValueA ValueB
|
146
|
+
row1 a|aa|aaa b
|
147
|
+
row2 A B
|
148
|
+
EOF
|
149
|
+
|
150
|
+
content2 =<<-EOF
|
151
|
+
#ValueB OtherID
|
152
|
+
b Id1|Id2
|
153
|
+
B Id3
|
154
|
+
EOF
|
155
|
+
|
156
|
+
tsv1 = tsv2 = nil
|
157
|
+
TmpFile.with_file(content1) do |filename|
|
158
|
+
tsv1 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
|
159
|
+
end
|
160
|
+
|
161
|
+
TmpFile.with_file(content2) do |filename|
|
162
|
+
tsv2 = TSV.open(File.open(filename), type: :double, :sep => /\s+/)
|
163
|
+
end
|
164
|
+
|
165
|
+
tsv1.attach tsv2, bar: true
|
166
|
+
|
167
|
+
assert_equal %w(ValueA ValueB OtherID), tsv1.fields
|
168
|
+
assert_equal %w(Id1 Id2), tsv1["row1"]["OtherID"]
|
169
|
+
|
170
|
+
TmpFile.with_file(content1) do |filename|
|
171
|
+
tsv1 = TSV.open(File.open(filename), type: :list, :sep => /\s+/)
|
172
|
+
end
|
173
|
+
|
174
|
+
tsv1.attach tsv2
|
175
|
+
|
176
|
+
assert_equal %w(ValueA ValueB OtherID), tsv1.fields
|
177
|
+
assert_equal "Id1", tsv1["row1"]["OtherID"]
|
178
|
+
end
|
179
|
+
|
180
|
+
def test_attach_transformer
|
181
|
+
content1 =<<-EOF
|
182
|
+
#: :sep=" "
|
183
|
+
#ID ValueA ValueB
|
184
|
+
row1 a|aa|aaa b
|
185
|
+
row2 A B
|
186
|
+
EOF
|
187
|
+
|
188
|
+
content2 =<<-EOF
|
189
|
+
#: :sep=" "
|
190
|
+
#ID ValueB OtherID
|
191
|
+
row1 b Id1|Id2
|
192
|
+
row3 B Id3
|
193
|
+
EOF
|
194
|
+
|
195
|
+
TmpFile.with_file(content1) do |filename1|
|
196
|
+
TmpFile.with_file(content2) do |filename2|
|
197
|
+
out = TSV.attach filename1, filename2, target: :stream, bar: false
|
198
|
+
tsv = out.tsv
|
199
|
+
assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
|
200
|
+
end
|
201
|
+
end
|
202
|
+
end
|
203
|
+
|
204
|
+
def test_attach_flexible_names
|
205
|
+
content1 =<<-EOF
|
206
|
+
#: :sep=" "
|
207
|
+
#ID ValueA ValueB
|
208
|
+
row1 a|aa|aaa b
|
209
|
+
row2 A B
|
210
|
+
EOF
|
211
|
+
|
212
|
+
content2 =<<-EOF
|
213
|
+
#: :sep=" "
|
214
|
+
#Identifiers(ID) OtherID
|
215
|
+
row1 Id1|Id2
|
216
|
+
row3 Id3
|
217
|
+
EOF
|
218
|
+
|
219
|
+
TmpFile.with_file(content1) do |filename1|
|
220
|
+
TmpFile.with_file(content2) do |filename2|
|
221
|
+
out = TSV.attach filename1, filename2, target: :stream, bar: false
|
222
|
+
tsv = out.tsv
|
223
|
+
assert_equal %w(Id1 Id2), tsv["row1"]["OtherID"]
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
end
|
@@ -0,0 +1,98 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/tsv'
|
5
|
+
class TestChangeID < Test::Unit::TestCase
|
6
|
+
def test_simple_reorder
|
7
|
+
content1 =<<-EOF
|
8
|
+
#: :sep=" "
|
9
|
+
#ID ValueA ValueB
|
10
|
+
row1 A1|A11 B1|B11
|
11
|
+
row2 A2|A22 B2|B22
|
12
|
+
EOF
|
13
|
+
|
14
|
+
tsv = TSV.open StringIO.new(content1)
|
15
|
+
|
16
|
+
res = tsv.change_key "ValueA", keep: true
|
17
|
+
assert_equal ["row1"], res["A1"]["ID"]
|
18
|
+
assert_equal ["row1"], res["A11"]["ID"]
|
19
|
+
assert_equal ["row2"], res["A2"]["ID"]
|
20
|
+
|
21
|
+
res = tsv.change_key "ValueA", keep: false, one2one: true
|
22
|
+
assert_equal ["B1"], res["A1"]["ValueB"]
|
23
|
+
assert_equal ["B11"], res["A11"]["ValueB"]
|
24
|
+
end
|
25
|
+
|
26
|
+
def test_simple_reorder_file
|
27
|
+
content1 =<<-EOF
|
28
|
+
#: :sep=" "
|
29
|
+
#ID ValueA ValueB
|
30
|
+
row1 A1|A11 B1|B11
|
31
|
+
row2 A2|A22 B2|B22
|
32
|
+
EOF
|
33
|
+
|
34
|
+
TmpFile.with_file(content1) do |file1|
|
35
|
+
res = TSV.change_key file1, "ValueA", keep: true
|
36
|
+
assert_equal ["row1"], res["A1"]["ID"]
|
37
|
+
assert_equal ["row1"], res["A11"]["ID"]
|
38
|
+
assert_equal ["row2"], res["A2"]["ID"]
|
39
|
+
assert_equal ["B1","B11"], res["A1"]["ValueB"]
|
40
|
+
|
41
|
+
res = TSV.change_key file1, "ValueA", one2one: true, keep: true
|
42
|
+
assert_equal ["row1"], res["A1"]["ID"]
|
43
|
+
assert_equal ["row1"], res["A11"]["ID"]
|
44
|
+
assert_equal ["row2"], res["A2"]["ID"]
|
45
|
+
assert_equal ["B1"], res["A1"]["ValueB"]
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_change_key_identifiers
|
50
|
+
content1 =<<-EOF
|
51
|
+
#: :sep=" "
|
52
|
+
#ID ValueA ValueB
|
53
|
+
row1 A1|A11 B1|B11
|
54
|
+
row2 A2|A22 B2|B22
|
55
|
+
EOF
|
56
|
+
|
57
|
+
identifiers_content =<<-EOF
|
58
|
+
#: :sep=" "
|
59
|
+
#ID ValueC ValueD
|
60
|
+
row1 C1|C11 D1|D11
|
61
|
+
row2 C2|C22 D2|D22
|
62
|
+
EOF
|
63
|
+
|
64
|
+
|
65
|
+
tsv = TSV.open StringIO.new(content1)
|
66
|
+
identifiers = TSV.open StringIO.new(identifiers_content)
|
67
|
+
|
68
|
+
res = tsv.change_key "ValueC", identifiers: identifiers, keep: true
|
69
|
+
assert_equal ["row1"], res["C1"]["ID"]
|
70
|
+
assert_equal ["row1"], res["C11"]["ID"]
|
71
|
+
assert_equal ["row2"], res["C2"]["ID"]
|
72
|
+
end
|
73
|
+
|
74
|
+
def test_change_id_identifiers
|
75
|
+
content1 =<<-EOF
|
76
|
+
#: :sep=" "
|
77
|
+
#ID ValueA ValueB
|
78
|
+
row1 A1|A11 B1|B11
|
79
|
+
row2 A2|A22 B2|B22
|
80
|
+
EOF
|
81
|
+
|
82
|
+
identifiers_content =<<-EOF
|
83
|
+
#: :sep=" "
|
84
|
+
#ID ValueC ValueD
|
85
|
+
row1 C1|C11 D1|D11
|
86
|
+
row2 C2|C22 D2|D22
|
87
|
+
EOF
|
88
|
+
|
89
|
+
|
90
|
+
tsv = TSV.open StringIO.new(content1)
|
91
|
+
identifiers = TSV.open StringIO.new(identifiers_content)
|
92
|
+
|
93
|
+
res = tsv.change_id "ValueA", "ValueC", identifiers: identifiers
|
94
|
+
assert_equal ["C1","C11"], res["row1"]["ValueC"]
|
95
|
+
assert_equal ["C2","C22"], res["row2"]["ValueC"]
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|