scout-gear 7.3.0 → 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +20 -9
- data/VERSION +1 -1
- data/bin/scout +6 -3
- data/lib/rbbt-scout.rb +1 -0
- data/lib/scout/cmd.rb +1 -1
- data/lib/scout/concurrent_stream.rb +26 -23
- data/lib/scout/config.rb +1 -1
- data/lib/scout/log/color.rb +4 -1
- data/lib/scout/log/progress/report.rb +1 -1
- data/lib/scout/log/progress/util.rb +58 -54
- data/lib/scout/log/progress.rb +1 -1
- data/lib/scout/log/trap.rb +107 -0
- data/lib/scout/log.rb +56 -21
- data/lib/scout/meta_extension.rb +13 -6
- data/lib/scout/misc/digest.rb +1 -1
- data/lib/scout/misc/format.rb +12 -0
- data/lib/scout/misc/insist.rb +1 -1
- data/lib/scout/misc/monitor.rb +11 -0
- data/lib/scout/misc/system.rb +10 -0
- data/lib/scout/named_array.rb +65 -3
- data/lib/scout/open/lock/lockfile.rb +587 -0
- data/lib/scout/open/lock.rb +28 -2
- data/lib/scout/open/remote.rb +4 -0
- data/lib/scout/open/stream.rb +90 -15
- data/lib/scout/open/util.rb +13 -3
- data/lib/scout/path/find.rb +9 -1
- data/lib/scout/path/util.rb +35 -0
- data/lib/scout/persist/serialize.rb +18 -5
- data/lib/scout/persist.rb +28 -12
- data/lib/scout/resource/path.rb +53 -0
- data/lib/scout/resource/produce.rb +0 -8
- data/lib/scout/resource/util.rb +2 -1
- data/lib/scout/tmpfile.rb +7 -8
- data/lib/scout/tsv/attach.rb +177 -0
- data/lib/scout/tsv/change_id.rb +40 -0
- data/lib/scout/tsv/dumper.rb +72 -46
- data/lib/scout/tsv/index.rb +69 -13
- data/lib/scout/tsv/open.rb +138 -84
- data/lib/scout/tsv/parser.rb +135 -80
- data/lib/scout/tsv/path.rb +1 -2
- data/lib/scout/tsv/persist/adapter.rb +15 -45
- data/lib/scout/tsv/persist/fix_width_table.rb +3 -0
- data/lib/scout/tsv/persist/tokyocabinet.rb +4 -1
- data/lib/scout/tsv/persist.rb +4 -0
- data/lib/scout/tsv/transformer.rb +141 -0
- data/lib/scout/tsv/traverse.rb +96 -92
- data/lib/scout/tsv/util/filter.rb +9 -0
- data/lib/scout/tsv/util/reorder.rb +81 -0
- data/lib/scout/tsv/util/select.rb +78 -33
- data/lib/scout/tsv/util/unzip.rb +86 -0
- data/lib/scout/tsv/util.rb +60 -11
- data/lib/scout/tsv.rb +26 -3
- data/lib/scout/work_queue/socket.rb +6 -1
- data/lib/scout/work_queue/worker.rb +5 -2
- data/lib/scout/work_queue.rb +15 -8
- data/lib/scout/workflow/definition.rb +21 -2
- data/lib/scout/workflow/step/dependencies.rb +24 -4
- data/lib/scout/workflow/step/info.rb +36 -5
- data/lib/scout/workflow/step/provenance.rb +8 -7
- data/lib/scout/workflow/step/status.rb +45 -0
- data/lib/scout/workflow/step.rb +100 -34
- data/lib/scout/workflow/task/inputs.rb +14 -20
- data/lib/scout/workflow/task.rb +81 -46
- data/lib/scout/workflow/usage.rb +8 -6
- data/scout-gear.gemspec +24 -20
- data/scout_commands/workflow/task +34 -7
- data/test/scout/open/test_stream.rb +60 -58
- data/test/scout/path/test_find.rb +10 -1
- data/test/scout/resource/test_produce.rb +15 -0
- data/test/scout/test_meta_extension.rb +25 -0
- data/test/scout/test_named_array.rb +18 -0
- data/test/scout/test_persist.rb +6 -0
- data/test/scout/test_tsv.rb +212 -2
- data/test/scout/test_work_queue.rb +21 -19
- data/test/scout/tsv/persist/test_adapter.rb +1 -1
- data/test/scout/tsv/persist/test_tokyocabinet.rb +29 -1
- data/test/scout/tsv/test_attach.rb +227 -0
- data/test/scout/tsv/test_change_id.rb +98 -0
- data/test/scout/tsv/test_dumper.rb +1 -1
- data/test/scout/tsv/test_index.rb +35 -3
- data/test/scout/tsv/test_open.rb +160 -2
- data/test/scout/tsv/test_parser.rb +19 -2
- data/test/scout/tsv/test_persist.rb +2 -0
- data/test/scout/tsv/test_transformer.rb +108 -0
- data/test/scout/tsv/test_traverse.rb +88 -3
- data/test/scout/tsv/test_util.rb +1 -0
- data/test/scout/tsv/util/test_reorder.rb +94 -0
- data/test/scout/tsv/util/test_select.rb +25 -11
- data/test/scout/tsv/util/test_unzip.rb +112 -0
- data/test/scout/work_queue/test_socket.rb +0 -1
- data/test/scout/workflow/step/test_status.rb +31 -0
- data/test/scout/workflow/task/test_inputs.rb +14 -14
- data/test/scout/workflow/test_step.rb +3 -3
- data/test/scout/workflow/test_task.rb +168 -32
- data/test/scout/workflow/test_usage.rb +33 -6
- metadata +20 -6
@@ -48,19 +48,30 @@ row2 a b id3
|
|
48
48
|
#: :sep=/\s+/#:type=:double#:merge=:concat
|
49
49
|
#Id ValueA ValueB OtherID
|
50
50
|
row1 a|aa|aaa b Id1|Id2
|
51
|
-
row2 A B Id3|a
|
51
|
+
row2 A|b B Id3|a
|
52
52
|
row2 a b id3
|
53
53
|
EOF
|
54
54
|
|
55
55
|
TmpFile.with_file(content) do |filename|
|
56
56
|
tsv = TSV.open(filename)
|
57
|
+
index = TSV.index(tsv, :target => "ValueB")
|
58
|
+
assert_equal 'b', index["a"]
|
59
|
+
assert_equal 'B', index["B"]
|
60
|
+
assert_equal 'b', index["b"]
|
61
|
+
|
62
|
+
index = tsv.index(:target => "ValueB")
|
63
|
+
assert_equal 'b', index["a"]
|
64
|
+
assert_equal 'B', index["B"]
|
65
|
+
assert_equal 'b', index["b"]
|
66
|
+
|
67
|
+
|
57
68
|
index = TSV.index(tsv, :target => "ValueB", :fields => "OtherID")
|
58
69
|
assert_equal 'B', index["a"]
|
59
70
|
assert_nil index["B"]
|
60
71
|
|
61
72
|
index = tsv.index(:target => "ValueB", :fields => "OtherID")
|
62
73
|
assert_equal 'B', index["a"]
|
63
|
-
assert_nil
|
74
|
+
assert_nil index["B"]
|
64
75
|
end
|
65
76
|
end
|
66
77
|
|
@@ -74,7 +85,7 @@ row2 A B Id3|a
|
|
74
85
|
row2 a b id3
|
75
86
|
EOF
|
76
87
|
tsv = TmpFile.with_file(content) do |filename|
|
77
|
-
index = TSV.index(filename, :target => "ValueB", :persist => true)
|
88
|
+
index = TSV.index(filename, :target => "ValueB", :persist => true, bar: true)
|
78
89
|
assert_equal 'b', index["row1"]
|
79
90
|
assert_equal 'b', index["a"]
|
80
91
|
assert_equal 'b', index["aaa"]
|
@@ -107,6 +118,27 @@ g: ____
|
|
107
118
|
end
|
108
119
|
end
|
109
120
|
|
121
|
+
def test_pos_index
|
122
|
+
data =<<-EOF
|
123
|
+
# 012345678901234567890
|
124
|
+
#ID:Range
|
125
|
+
a: ______
|
126
|
+
b: ______
|
127
|
+
c: _______
|
128
|
+
d: ____
|
129
|
+
e: ______
|
130
|
+
f: ___
|
131
|
+
g: ____
|
132
|
+
EOF
|
133
|
+
TmpFile.with_file(data) do |datafile|
|
134
|
+
tsv = load_segment_data(datafile)
|
135
|
+
f = tsv.pos_index("Start", :persist => true)
|
136
|
+
|
137
|
+
assert_equal %w(), f[0].sort
|
138
|
+
assert_equal %w(a c d e), f[(2..4)].sort
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
110
142
|
def test_range_index_persistent
|
111
143
|
data =<<-EOF
|
112
144
|
# 012345678901234567890
|
data/test/scout/tsv/test_open.rb
CHANGED
@@ -1,9 +1,167 @@
|
|
1
1
|
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
2
|
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
3
|
|
4
|
+
require 'scout/tsv'
|
4
5
|
class TestOpenTraverse < Test::Unit::TestCase
|
5
|
-
def
|
6
|
-
|
6
|
+
def test_array
|
7
|
+
num_lines = 100
|
8
|
+
lines = num_lines.times.collect{|i| "line-#{i}" }
|
9
|
+
|
10
|
+
r = TSV.traverse lines, :into => [] do |l|
|
11
|
+
l + "-" + Process.pid.to_s
|
12
|
+
end
|
13
|
+
|
14
|
+
assert_equal num_lines, r.length
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_array_cpus
|
18
|
+
num_lines = 1000
|
19
|
+
lines = num_lines.times.collect{|i| "line-#{i}" }
|
20
|
+
|
21
|
+
r = TSV.traverse lines, :into => [], :cpus => 2 do |l|
|
22
|
+
l + "-" + Process.pid.to_s
|
23
|
+
end
|
24
|
+
|
25
|
+
assert_equal num_lines, r.length
|
26
|
+
assert_equal 2, r.collect{|l| l.split("-").last}.uniq.length
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_tsv_cpus
|
30
|
+
num_lines = 10000
|
31
|
+
lines = num_lines.times.collect{|i| "line-#{i}" }
|
32
|
+
|
33
|
+
tsv = TSV.setup({}, key_field: "Line", :fields => %w(Prefix Number), :type => :list)
|
34
|
+
lines.each do |line|
|
35
|
+
tsv[line] = ["LINE", line.split("-").last]
|
36
|
+
end
|
37
|
+
|
38
|
+
r = TSV.traverse tsv, :into => [], :cpus => 2, :bar => {desc: "Process", severity: 0} do |l,v|
|
39
|
+
pre, num = v
|
40
|
+
pre + "-" + num.to_s + "-" + Process.pid.to_s
|
41
|
+
end
|
42
|
+
|
43
|
+
assert_equal num_lines, r.length
|
44
|
+
assert_equal 2, r.collect{|l| l.split("-").last}.uniq.length
|
45
|
+
assert_equal "LINE", r.collect{|l| l.split("-").first}.first
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_into_stream
|
49
|
+
num_lines = 100
|
50
|
+
lines = num_lines.times.collect{|i| "line-#{i}" }
|
51
|
+
|
52
|
+
r = TSV.traverse lines, :into => :stream do |l|
|
53
|
+
l + "-" + Process.pid.to_s
|
54
|
+
end
|
55
|
+
|
56
|
+
assert_equal num_lines, r.read.split("\n").length
|
57
|
+
end
|
58
|
+
|
59
|
+
def test_into_stream_error
|
60
|
+
num_lines = 100
|
61
|
+
lines = num_lines.times.collect{|i| "line-#{i}" }
|
62
|
+
|
63
|
+
assert_raise ScoutException do
|
64
|
+
Log.with_severity 7 do
|
65
|
+
i = 0
|
66
|
+
r = TSV.traverse lines, :into => :stream, cpus: 3 do |l|
|
67
|
+
raise ScoutException if i > 10
|
68
|
+
i += 1
|
69
|
+
l + "-" + Process.pid.to_s
|
70
|
+
end
|
71
|
+
|
72
|
+
r.read
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_into_dumper_error
|
78
|
+
num_lines = 100
|
79
|
+
lines = num_lines.times.collect{|i| "line-#{i}" }
|
80
|
+
|
81
|
+
assert_raise ScoutException do
|
82
|
+
i = 0
|
83
|
+
Log.with_severity 7 do
|
84
|
+
dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Value"], :type => :single
|
85
|
+
dumper.init
|
86
|
+
dumper = TSV.traverse lines, :into => dumper, :cpus => 3 do |l|
|
87
|
+
raise ScoutException if i > 10
|
88
|
+
i += 1
|
89
|
+
value = l + "-" + Process.pid.to_s
|
90
|
+
|
91
|
+
[i.to_s, value]
|
92
|
+
end
|
93
|
+
ppp dumper.stream.read
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def test_traverse_line
|
99
|
+
text=<<-EOF
|
100
|
+
#: :sep=" "
|
101
|
+
#Row LabelA LabelB LabelC
|
102
|
+
row1 A B C
|
103
|
+
row1 a b c
|
104
|
+
row2 AA BB CC
|
105
|
+
row2 aa bb cc
|
106
|
+
EOF
|
107
|
+
|
108
|
+
TmpFile.with_file(text) do |file|
|
109
|
+
lines = Open.traverse file, :type => :line, :into => [] do |line|
|
110
|
+
line
|
111
|
+
end
|
112
|
+
assert_include lines, "row2 AA BB CC"
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def test_collapse_stream
|
117
|
+
text=<<-EOF
|
118
|
+
#: :sep=" "
|
119
|
+
#Row LabelA LabelB LabelC
|
120
|
+
row1 A B C
|
121
|
+
row1 a b c
|
122
|
+
row2 AA BB CC
|
123
|
+
row2 aa bb cc
|
124
|
+
EOF
|
125
|
+
|
126
|
+
s = StringIO.new text
|
127
|
+
collapsed = TSV.collapse_stream(s)
|
128
|
+
tsv = TSV.open collapsed
|
129
|
+
assert_equal ["A", "a"], tsv["row1"][0]
|
130
|
+
assert_equal ["BB", "bb"], tsv["row2"][1]
|
131
|
+
end
|
132
|
+
|
133
|
+
def test_cpus_error_dumper
|
134
|
+
num_lines = 100
|
135
|
+
lines = num_lines.times.collect{|i| "line-#{i}" }
|
136
|
+
|
137
|
+
dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Field"], type: :single
|
138
|
+
dumper.init
|
139
|
+
assert_raise ScoutException do
|
140
|
+
Log.with_severity 0 do
|
141
|
+
i = 0
|
142
|
+
TSV.traverse lines, :into => dumper, cpus: 3 do |l|
|
143
|
+
raise ScoutException if i > 10
|
144
|
+
i += 1
|
145
|
+
[Process.pid.to_s, l + "-" + Process.pid.to_s]
|
146
|
+
end
|
147
|
+
|
148
|
+
end
|
149
|
+
ppp dumper.stream.read
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
def test_step_travese_cpus
|
154
|
+
|
155
|
+
size = 1000
|
156
|
+
step = Step.new tmpdir.step[__method__] do
|
157
|
+
lines = size.times.collect{|i| "line-#{i}" }
|
158
|
+
Open.traverse lines, :type => :array, :into => :stream, :cpus => 3 do |line|
|
159
|
+
line.reverse
|
160
|
+
end
|
161
|
+
end
|
162
|
+
step.type = :array
|
163
|
+
|
164
|
+
assert_equal size, step.run.length
|
7
165
|
end
|
8
166
|
end
|
9
167
|
|
@@ -172,6 +172,25 @@ k a|A b|B
|
|
172
172
|
assert_equal [["k", [%w(b B)]]], values
|
173
173
|
end
|
174
174
|
|
175
|
+
def test_parser_traverse_all
|
176
|
+
content =<<-EOF
|
177
|
+
Key ValueA ValueB
|
178
|
+
k a|A b|B
|
179
|
+
EOF
|
180
|
+
content = StringIO.new content
|
181
|
+
|
182
|
+
parser = TSV::Parser.new content, sep: " ", header_hash: ''
|
183
|
+
|
184
|
+
assert_equal "Key", parser.key_field
|
185
|
+
|
186
|
+
values = []
|
187
|
+
parser.traverse key_field: "ValueA", fields: :all, type: :double do |k,v|
|
188
|
+
values << v
|
189
|
+
end
|
190
|
+
|
191
|
+
assert_include values.flatten, 'a'
|
192
|
+
end
|
193
|
+
|
175
194
|
|
176
195
|
def test_parse_persist_serializer
|
177
196
|
content =<<-EOF
|
@@ -192,7 +211,5 @@ k 1 2
|
|
192
211
|
TSV.parse content, sep: " ", header_hash: '', data: data, cast: :to_i, type: :list, serializer: :float_array
|
193
212
|
assert_equal [1.0, 2.0], data["k"]
|
194
213
|
end
|
195
|
-
|
196
214
|
end
|
197
|
-
|
198
215
|
end
|
@@ -0,0 +1,108 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/tsv'
|
5
|
+
class TestTSVTransformer < Test::Unit::TestCase
|
6
|
+
def test_traverse
|
7
|
+
content =<<-EOF
|
8
|
+
#: :sep=" "
|
9
|
+
#ID ValueA ValueB
|
10
|
+
row1 A1|A11 B1|B11
|
11
|
+
row2 A2|A22 B2|B22
|
12
|
+
EOF
|
13
|
+
|
14
|
+
TmpFile.with_file(content) do |file|
|
15
|
+
parser = TSV::Parser.new file
|
16
|
+
dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Values"], type: :flat
|
17
|
+
dumper.init
|
18
|
+
|
19
|
+
trans = TSV::Transformer.new parser, dumper
|
20
|
+
dumper = trans.traverse do |k,values|
|
21
|
+
[k, values.flatten]
|
22
|
+
end
|
23
|
+
|
24
|
+
tsv = trans.tsv
|
25
|
+
assert_equal %w(A1 A11 B1 B11), tsv['row1']
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_each
|
30
|
+
content =<<-EOF
|
31
|
+
#: :sep=" "
|
32
|
+
#ID ValueA ValueB
|
33
|
+
row1 A1|A11 B1|B11
|
34
|
+
row2 A2|A22 B2|B22
|
35
|
+
EOF
|
36
|
+
|
37
|
+
TmpFile.with_file(content) do |file|
|
38
|
+
parser = TSV::Parser.new file
|
39
|
+
dumper = TSV::Dumper.new :key_field => "Key", :fields => ["Values"], type: :flat
|
40
|
+
dumper.init
|
41
|
+
|
42
|
+
trans = TSV::Transformer.new parser, dumper
|
43
|
+
dumper = trans.each do |k,values|
|
44
|
+
values.replace values.flatten
|
45
|
+
end
|
46
|
+
|
47
|
+
trans["row3"] = %w(A3 A33)
|
48
|
+
|
49
|
+
tsv = trans.tsv
|
50
|
+
assert_equal %w(A1 A11 B1 B11), tsv['row1']
|
51
|
+
assert_equal %w(A3 A33), tsv['row3']
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def test_no_dumper_no_parser
|
56
|
+
content =<<-EOF
|
57
|
+
#: :sep=" "
|
58
|
+
#ID ValueA ValueB
|
59
|
+
row1 A1|A11 B1|B11
|
60
|
+
row2 A2|A22 B2|B22
|
61
|
+
EOF
|
62
|
+
|
63
|
+
TmpFile.with_file(content) do |file|
|
64
|
+
trans = TSV::Transformer.new file
|
65
|
+
trans.key_field = "Key"
|
66
|
+
trans.fields = ["Values"]
|
67
|
+
trans.type = :flat
|
68
|
+
trans.sep = "\t"
|
69
|
+
|
70
|
+
|
71
|
+
trans["row3"] = %w(A3 A33)
|
72
|
+
|
73
|
+
trans.each do |k,values|
|
74
|
+
values.replace values.flatten
|
75
|
+
end
|
76
|
+
|
77
|
+
tsv = trans.tsv
|
78
|
+
assert_equal %w(A1 A11 B1 B11), tsv['row1']
|
79
|
+
assert_equal %w(A3 A33), tsv['row3']
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def test_to_list
|
84
|
+
content =<<-EOF
|
85
|
+
#: :sep=" "
|
86
|
+
#ID ValueA ValueB
|
87
|
+
row1 A1|A11 B1|B11
|
88
|
+
row2 A2|A22 B2|B22
|
89
|
+
EOF
|
90
|
+
|
91
|
+
tsv = TSV.open(content)
|
92
|
+
assert_equal "A1", tsv.to_list["row1"]["ValueA"]
|
93
|
+
assert_equal "B2", tsv.to_list["row2"]["ValueB"]
|
94
|
+
end
|
95
|
+
|
96
|
+
def test_to_single
|
97
|
+
content =<<-EOF
|
98
|
+
#: :sep=" "
|
99
|
+
#ID ValueA ValueB
|
100
|
+
row1 A1|A11 B1|B11
|
101
|
+
row2 A2|A22 B2|B22
|
102
|
+
EOF
|
103
|
+
|
104
|
+
tsv = TSV.open(content)
|
105
|
+
assert_equal "A1", tsv.to_single["row1"]
|
106
|
+
assert_equal "A2", tsv.to_single["row2"]
|
107
|
+
end
|
108
|
+
end
|
@@ -13,7 +13,6 @@ row2 A B Id3
|
|
13
13
|
row2 AA BB Id33
|
14
14
|
EOF
|
15
15
|
|
16
|
-
sss 0
|
17
16
|
tsv = TmpFile.with_file(content) do |filename|
|
18
17
|
TSV.open(filename, :persist => true)
|
19
18
|
end
|
@@ -31,18 +30,26 @@ row2 AA BB Id33
|
|
31
30
|
assert_equal ["Id3", "B"], res["row2"]
|
32
31
|
|
33
32
|
res = {}
|
34
|
-
tsv.traverse "OtherID", %w(Id ValueB), one2one:
|
33
|
+
tsv.traverse "OtherID", %w(Id ValueB), one2one: :strict do |k,v|
|
35
34
|
res[k] = v
|
36
35
|
end
|
37
36
|
assert_equal [[nil], %w(BB)], res["Id33"]
|
38
37
|
|
39
38
|
res = {}
|
40
|
-
tsv.traverse "OtherID", %w(Id ValueB), one2one:
|
39
|
+
tsv.traverse "OtherID", %w(Id ValueB), one2one: :strict, type: :list do |k,v|
|
41
40
|
res[k] = v
|
42
41
|
end
|
43
42
|
assert_equal ["row2", "B"], res["Id3"]
|
44
43
|
assert_equal [nil, "BB"], res["Id33"]
|
45
44
|
|
45
|
+
res = {}
|
46
|
+
tsv.traverse "OtherID", %w(Id ValueB), one2one: true, type: :list do |k,v|
|
47
|
+
res[k] = v
|
48
|
+
end
|
49
|
+
assert_equal ["row2", "B"], res["Id3"]
|
50
|
+
assert_equal ["row2", "BB"], res["Id33"]
|
51
|
+
|
52
|
+
|
46
53
|
tsv.traverse "OtherID", %w(Id ValueB), one2one: false, type: :list do |k,v|
|
47
54
|
res[k] = v
|
48
55
|
end
|
@@ -57,6 +64,36 @@ row2 AA BB Id33
|
|
57
64
|
assert_equal %w(Id ValueA ValueB), field_names
|
58
65
|
end
|
59
66
|
|
67
|
+
def test_tsv_traverse_all
|
68
|
+
content =<<-'EOF'
|
69
|
+
#: :sep=/\s+/#:type=:double
|
70
|
+
#Id ValueA ValueB OtherID
|
71
|
+
row1 a|aa|aaa b Id1|Id2
|
72
|
+
row2 A B Id3
|
73
|
+
row2 AA BB Id33
|
74
|
+
EOF
|
75
|
+
|
76
|
+
tsv = TmpFile.with_file(content) do |filename|
|
77
|
+
TSV.open(filename, :persist => true)
|
78
|
+
end
|
79
|
+
|
80
|
+
all_values = []
|
81
|
+
tsv.traverse "ValueA", :all do |k,v|
|
82
|
+
all_values.concat(v)
|
83
|
+
end
|
84
|
+
assert_include all_values.flatten, "row1"
|
85
|
+
assert_include all_values.flatten, "a"
|
86
|
+
assert_include all_values.flatten, "aaa"
|
87
|
+
|
88
|
+
all_values = []
|
89
|
+
tsv.traverse "Id", :all do |k,v|
|
90
|
+
all_values.concat(v)
|
91
|
+
end
|
92
|
+
assert_include all_values.flatten, "row1"
|
93
|
+
assert_include all_values.flatten, "a"
|
94
|
+
assert_include all_values.flatten, "aaa"
|
95
|
+
end
|
96
|
+
|
60
97
|
def test_tsv_traverse_list
|
61
98
|
content =<<-'EOF'
|
62
99
|
#: :sep=/\s+/#:type=:list
|
@@ -112,5 +149,53 @@ row2 A
|
|
112
149
|
end
|
113
150
|
assert_equal [["row1"]], res["a"]
|
114
151
|
end
|
152
|
+
|
153
|
+
def test_traverse_reorder_one2one
|
154
|
+
content =<<-'EOF'
|
155
|
+
#: :sep=" "
|
156
|
+
#ID ValueA ValueB
|
157
|
+
row1 A1|A11 B1|B11
|
158
|
+
row2 A2|A22 B2|B22
|
159
|
+
EOF
|
160
|
+
|
161
|
+
tsv = TmpFile.with_file(content) do |filename|
|
162
|
+
TSV.open(filename, :persist => true)
|
163
|
+
end
|
164
|
+
|
165
|
+
res = {}
|
166
|
+
tsv.traverse "ValueA", one2one: true do |k,v|
|
167
|
+
res[k] = v
|
168
|
+
end
|
169
|
+
|
170
|
+
assert_include res["A2"][0], "row2"
|
171
|
+
|
172
|
+
end
|
173
|
+
|
174
|
+
def test_traverse_select
|
175
|
+
content =<<-'EOF'
|
176
|
+
#: :sep=" "
|
177
|
+
#ID ValueA ValueB
|
178
|
+
row1 A1|A11 B1|B11
|
179
|
+
row2 A2|A22 B2|B22
|
180
|
+
EOF
|
181
|
+
|
182
|
+
TmpFile.with_file(content) do |filename|
|
183
|
+
tsv = TSV.open(filename, :persist => true, select: "B1")
|
184
|
+
|
185
|
+
assert_equal %w(row1), tsv.keys
|
186
|
+
|
187
|
+
tsv = TSV.open(filename, :persist => true, select: "B2")
|
188
|
+
|
189
|
+
assert_equal %w(row2), tsv.keys
|
190
|
+
|
191
|
+
tsv = TSV.open(filename, :persist => true, select: "B1")
|
192
|
+
|
193
|
+
assert_equal %w(row1), tsv.keys
|
194
|
+
|
195
|
+
tsv = TSV.open(filename, :persist => true, select: "B2")
|
196
|
+
|
197
|
+
assert_equal %w(row2), tsv.keys
|
198
|
+
end
|
199
|
+
end
|
115
200
|
end
|
116
201
|
|
data/test/scout/tsv/test_util.rb
CHANGED
@@ -0,0 +1,94 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
require 'scout/tsv'
|
5
|
+
|
6
|
+
class TestReorder < Test::Unit::TestCase
|
7
|
+
def test_reorder
|
8
|
+
content =<<-'EOF'
|
9
|
+
#: :sep=/\s+/#:type=:double
|
10
|
+
#Id ValueA ValueB OtherID
|
11
|
+
row1 a1|a2 b1|b2 Id1|Id2
|
12
|
+
row2 A1|A3 B1|B3 Id1|Id3
|
13
|
+
EOF
|
14
|
+
|
15
|
+
tsv = TmpFile.with_file(content) do |filename|
|
16
|
+
TSV.open(filename)
|
17
|
+
end
|
18
|
+
|
19
|
+
r = tsv.reorder "OtherID", %w(ValueB Id)
|
20
|
+
|
21
|
+
assert_equal %w(row1 row2), r["Id1"]["Id"]
|
22
|
+
assert_equal %w(row2), r["Id3"]["Id"]
|
23
|
+
end
|
24
|
+
|
25
|
+
def test_reorder_list
|
26
|
+
content =<<-'EOF'
|
27
|
+
#: :sep=/\s+/#:type=:list
|
28
|
+
#Id ValueA ValueB OtherID
|
29
|
+
row1 a1 b1 Id1
|
30
|
+
row2 A1 B1 Id1
|
31
|
+
EOF
|
32
|
+
|
33
|
+
tsv = TmpFile.with_file(content) do |filename|
|
34
|
+
TSV.open(filename)
|
35
|
+
end
|
36
|
+
|
37
|
+
r = tsv.reorder "ValueB"
|
38
|
+
|
39
|
+
assert_equal "row1", r["b1"]["Id"]
|
40
|
+
assert_equal "row2", r["B1"]["Id"]
|
41
|
+
end
|
42
|
+
|
43
|
+
def test_reorder_single
|
44
|
+
content =<<-'EOF'
|
45
|
+
#: :sep=/\s+/#:type=:single
|
46
|
+
#Id ValueA
|
47
|
+
row1 a1
|
48
|
+
row2 A1
|
49
|
+
EOF
|
50
|
+
|
51
|
+
tsv = TmpFile.with_file(content) do |filename|
|
52
|
+
TSV.open(filename)
|
53
|
+
end
|
54
|
+
|
55
|
+
r = tsv.reorder "ValueA"
|
56
|
+
|
57
|
+
assert_equal "row1", r["a1"]
|
58
|
+
assert_equal "row2", r["A1"]
|
59
|
+
end
|
60
|
+
|
61
|
+
def test_transpose
|
62
|
+
content =<<-EOF
|
63
|
+
#: :type=:list
|
64
|
+
#Row vA vB vID
|
65
|
+
row1 a b Id1
|
66
|
+
row2 A B Id3
|
67
|
+
row3 a C Id4
|
68
|
+
EOF
|
69
|
+
|
70
|
+
TmpFile.with_file(content) do |filename|
|
71
|
+
tsv = TSV.open(filename, :sep => /\s+/)
|
72
|
+
|
73
|
+
assert_equal %w(vA vB vID), tsv.transpose("Values").keys
|
74
|
+
assert_equal %w(Id1 Id3 Id4), tsv.transpose("Values")["vID"]
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def test_column
|
79
|
+
content =<<-EOF
|
80
|
+
#Id ValueA ValueB ValueC
|
81
|
+
rowA A|AA B|BB C|CC
|
82
|
+
rowa a|aa b|BB C|CC
|
83
|
+
EOF
|
84
|
+
|
85
|
+
TmpFile.with_file(content) do |filename|
|
86
|
+
tsv = TSV.open(File.open(filename), :sep => /\s+/, :type => :double)
|
87
|
+
tsv = tsv.column("ValueA", cast: :downcase)
|
88
|
+
assert_equal %w(a aa), tsv["rowA"]
|
89
|
+
assert_equal %w(a aa), tsv["rowa"]
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
end
|
94
|
+
|
@@ -23,22 +23,36 @@ row2 AA BB Id33
|
|
23
23
|
assert_equal ['row2'], s.keys
|
24
24
|
end
|
25
25
|
|
26
|
-
def
|
26
|
+
def test_select_values
|
27
27
|
content =<<-'EOF'
|
28
|
-
#: :sep
|
29
|
-
#
|
30
|
-
row1
|
31
|
-
row2
|
28
|
+
#: :sep=" "
|
29
|
+
#ID ValueA ValueB
|
30
|
+
row1 A1|A11 B1|B11
|
31
|
+
row2 A2|A22 B2|B22
|
32
32
|
EOF
|
33
33
|
|
34
|
-
tsv =
|
35
|
-
|
36
|
-
|
34
|
+
tsv = TSV.open(content)
|
35
|
+
|
36
|
+
assert TSV.select("row1", tsv["row1"], "A1")
|
37
|
+
refute TSV.select("row2", tsv["row2"], "A1")
|
38
|
+
assert TSV.select("row2", tsv["row2"], "A2")
|
37
39
|
|
38
|
-
|
40
|
+
assert TSV.select("row1", tsv["row1"], "B1")
|
41
|
+
refute TSV.select("row2", tsv["row2"], "B1")
|
42
|
+
|
43
|
+
refute TSV.select("row1", tsv["row1"], "B1", invert: true)
|
44
|
+
assert TSV.select("row2", tsv["row2"], "B1", invert: true)
|
45
|
+
|
46
|
+
assert TSV.select("row1", tsv["row1"], {"ValueB" => "B1"})
|
47
|
+
refute TSV.select("row2", tsv["row2"], {"ValueB" => "B1"})
|
48
|
+
|
49
|
+
tsv.with_unnamed do
|
50
|
+
assert TSV.select("row1", tsv["row1"], {"ValueB" => "B1"}, fields: tsv.fields)
|
51
|
+
refute TSV.select("row2", tsv["row2"], {"ValueB" => "B1"}, fields: tsv.fields)
|
52
|
+
end
|
39
53
|
|
40
|
-
|
41
|
-
|
54
|
+
assert TSV.select("row1", tsv["row1"], "B1", field: "ValueB")
|
55
|
+
refute TSV.select("row2", tsv["row2"], "B1", field: "ValueB")
|
42
56
|
end
|
43
57
|
end
|
44
58
|
|