scout-gear 10.4.0 → 10.7.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (143) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +100 -656
  3. data/Rakefile +1 -0
  4. data/VERSION +1 -1
  5. data/bin/scout +1 -3
  6. data/lib/scout/association/fields.rb +170 -0
  7. data/lib/scout/association/index.rb +229 -0
  8. data/lib/scout/association/item.rb +227 -0
  9. data/lib/scout/association/util.rb +7 -0
  10. data/lib/scout/association.rb +100 -0
  11. data/lib/scout/entity/format.rb +62 -0
  12. data/lib/scout/entity/identifiers.rb +111 -0
  13. data/lib/scout/entity/object.rb +20 -0
  14. data/lib/scout/entity/property.rb +165 -0
  15. data/lib/scout/entity.rb +41 -0
  16. data/lib/scout/offsite/step.rb +2 -2
  17. data/lib/scout/{tsv/persist → persist/engine}/fix_width_table.rb +25 -33
  18. data/lib/scout/persist/engine/packed_index.rb +100 -0
  19. data/lib/scout/persist/engine/sharder.rb +219 -0
  20. data/lib/scout/{tsv/persist → persist/engine}/tkrzw.rb +0 -17
  21. data/lib/scout/{tsv/persist → persist/engine}/tokyocabinet.rb +55 -31
  22. data/lib/scout/persist/engine.rb +4 -0
  23. data/lib/scout/{tsv/persist/adapter.rb → persist/tsv/adapter/base.rb} +80 -51
  24. data/lib/scout/persist/tsv/adapter/fix_width_table.rb +106 -0
  25. data/lib/scout/persist/tsv/adapter/packed_index.rb +95 -0
  26. data/lib/scout/persist/tsv/adapter/sharder.rb +54 -0
  27. data/lib/scout/persist/tsv/adapter/tkrzw.rb +18 -0
  28. data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +65 -0
  29. data/lib/scout/persist/tsv/adapter.rb +6 -0
  30. data/lib/scout/{tsv/persist → persist/tsv}/serialize.rb +5 -0
  31. data/lib/scout/persist/tsv.rb +107 -0
  32. data/lib/scout/tsv/annotation/repo.rb +87 -0
  33. data/lib/scout/tsv/annotation.rb +169 -0
  34. data/lib/scout/tsv/attach.rb +97 -21
  35. data/lib/scout/tsv/change_id/translate.rb +148 -0
  36. data/lib/scout/tsv/change_id.rb +3 -0
  37. data/lib/scout/tsv/csv.rb +85 -0
  38. data/lib/scout/tsv/dumper.rb +113 -25
  39. data/lib/scout/tsv/index.rb +88 -36
  40. data/lib/scout/tsv/open.rb +21 -8
  41. data/lib/scout/tsv/parser.rb +153 -90
  42. data/lib/scout/tsv/path.rb +7 -2
  43. data/lib/scout/tsv/stream.rb +48 -6
  44. data/lib/scout/tsv/transformer.rb +5 -3
  45. data/lib/scout/tsv/traverse.rb +28 -19
  46. data/lib/scout/tsv/util/process.rb +7 -0
  47. data/lib/scout/tsv/util/reorder.rb +25 -15
  48. data/lib/scout/tsv/util/select.rb +9 -1
  49. data/lib/scout/tsv/util/sort.rb +90 -2
  50. data/lib/scout/tsv/util/unzip.rb +56 -0
  51. data/lib/scout/tsv/util.rb +52 -5
  52. data/lib/scout/tsv.rb +42 -27
  53. data/lib/scout/work_queue/socket.rb +8 -0
  54. data/lib/scout/work_queue/worker.rb +22 -5
  55. data/lib/scout/work_queue.rb +41 -24
  56. data/lib/scout/workflow/definition.rb +15 -12
  57. data/lib/scout/workflow/deployment/orchestrator.rb +21 -3
  58. data/lib/scout/workflow/deployment/trace.rb +205 -0
  59. data/lib/scout/workflow/deployment.rb +1 -0
  60. data/lib/scout/workflow/documentation.rb +1 -1
  61. data/lib/scout/workflow/step/archive.rb +42 -0
  62. data/lib/scout/workflow/step/children.rb +51 -0
  63. data/lib/scout/workflow/step/config.rb +1 -1
  64. data/lib/scout/workflow/step/dependencies.rb +25 -8
  65. data/lib/scout/workflow/step/file.rb +19 -0
  66. data/lib/scout/workflow/step/info.rb +37 -9
  67. data/lib/scout/workflow/step/progress.rb +11 -2
  68. data/lib/scout/workflow/step/status.rb +9 -1
  69. data/lib/scout/workflow/step.rb +80 -25
  70. data/lib/scout/workflow/task/dependencies.rb +5 -2
  71. data/lib/scout/workflow/task/inputs.rb +91 -41
  72. data/lib/scout/workflow/task.rb +54 -57
  73. data/lib/scout/workflow/usage.rb +1 -1
  74. data/lib/scout/workflow/util.rb +4 -0
  75. data/lib/scout/workflow.rb +110 -13
  76. data/lib/scout-gear.rb +2 -0
  77. data/lib/scout.rb +0 -1
  78. data/scout-gear.gemspec +78 -23
  79. data/scout_commands/rbbt +2 -0
  80. data/test/data/person/brothers +4 -0
  81. data/test/data/person/identifiers +10 -0
  82. data/test/data/person/marriages +3 -0
  83. data/test/data/person/parents +6 -0
  84. data/test/scout/association/test_fields.rb +105 -0
  85. data/test/scout/association/test_index.rb +70 -0
  86. data/test/scout/association/test_item.rb +21 -0
  87. data/test/scout/entity/test_format.rb +19 -0
  88. data/test/scout/entity/test_identifiers.rb +58 -0
  89. data/test/scout/entity/test_object.rb +0 -0
  90. data/test/scout/entity/test_property.rb +345 -0
  91. data/test/scout/{tsv/persist → persist/engine}/test_fix_width_table.rb +0 -1
  92. data/test/scout/persist/engine/test_packed_index.rb +99 -0
  93. data/test/scout/persist/engine/test_sharder.rb +31 -0
  94. data/test/scout/persist/engine/test_tkrzw.rb +0 -0
  95. data/test/scout/persist/engine/test_tokyocabinet.rb +17 -0
  96. data/test/scout/persist/test_tsv.rb +146 -0
  97. data/test/scout/{tsv/persist/test_adapter.rb → persist/tsv/adapter/test_base.rb} +3 -4
  98. data/test/scout/persist/tsv/adapter/test_fix_width_table.rb +46 -0
  99. data/test/scout/persist/tsv/adapter/test_packed_index.rb +37 -0
  100. data/test/scout/persist/tsv/adapter/test_serialize.rb +0 -0
  101. data/test/scout/persist/tsv/adapter/test_sharder.rb +290 -0
  102. data/test/scout/{tsv/persist → persist/tsv/adapter}/test_tkrzw.rb +3 -6
  103. data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +282 -0
  104. data/test/scout/persist/tsv/test_serialize.rb +12 -0
  105. data/test/scout/test_association.rb +51 -0
  106. data/test/scout/test_entity.rb +40 -0
  107. data/test/scout/test_tsv.rb +33 -4
  108. data/test/scout/test_work_queue.rb +5 -2
  109. data/test/scout/test_workflow.rb +31 -14
  110. data/test/scout/tsv/annotation/test_repo.rb +150 -0
  111. data/test/scout/tsv/change_id/test_translate.rb +178 -0
  112. data/test/scout/tsv/test_annotation.rb +52 -0
  113. data/test/scout/tsv/test_attach.rb +255 -1
  114. data/test/scout/tsv/test_change_id.rb +25 -0
  115. data/test/scout/tsv/test_csv.rb +50 -0
  116. data/test/scout/tsv/test_dumper.rb +38 -0
  117. data/test/scout/tsv/test_index.rb +82 -0
  118. data/test/scout/tsv/test_open.rb +44 -0
  119. data/test/scout/tsv/test_parser.rb +70 -0
  120. data/test/scout/tsv/test_stream.rb +22 -0
  121. data/test/scout/tsv/test_transformer.rb +27 -3
  122. data/test/scout/tsv/test_traverse.rb +78 -0
  123. data/test/scout/tsv/util/test_process.rb +16 -0
  124. data/test/scout/tsv/util/test_reorder.rb +67 -0
  125. data/test/scout/tsv/util/test_sort.rb +28 -1
  126. data/test/scout/tsv/util/test_unzip.rb +32 -0
  127. data/test/scout/work_queue/test_socket.rb +4 -1
  128. data/test/scout/workflow/deployment/test_orchestrator.rb +17 -26
  129. data/test/scout/workflow/deployment/test_trace.rb +25 -0
  130. data/test/scout/workflow/step/test_archive.rb +28 -0
  131. data/test/scout/workflow/step/test_children.rb +25 -0
  132. data/test/scout/workflow/step/test_info.rb +16 -0
  133. data/test/scout/workflow/task/test_dependencies.rb +16 -16
  134. data/test/scout/workflow/task/test_inputs.rb +45 -1
  135. data/test/scout/workflow/test_definition.rb +52 -0
  136. data/test/scout/workflow/test_step.rb +57 -0
  137. data/test/scout/workflow/test_task.rb +26 -1
  138. data/test/scout/workflow/test_usage.rb +4 -4
  139. data/test/test_helper.rb +23 -1
  140. metadata +69 -14
  141. data/lib/scout/tsv/persist.rb +0 -27
  142. data/test/scout/tsv/persist/test_tokyocabinet.rb +0 -120
  143. data/test/scout/tsv/test_persist.rb +0 -45
@@ -0,0 +1,290 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/persist/tsv'
5
+ class TestSharder < Test::Unit::TestCase
6
+ def test_open_sharder
7
+ TmpFile.with_file do |dir|
8
+ shard_function = Proc.new do |key|
9
+ key[-1]
10
+ end
11
+
12
+ size = 10
13
+ sharder = Persist.open_sharder(dir, true, :HDB, :path => dir, :shard_function => shard_function, :persist => true, :serializer => :float_array)
14
+ size.times do |v|
15
+ sharder[v.to_s] = [v, v*2]
16
+ end
17
+ assert_equal dir, sharder.persistence_path
18
+ assert_equal size, sharder.size
19
+
20
+ assert_equal [2,4], sharder["2"]
21
+ count = 0
22
+ sharder.each do |k,v|
23
+ count += 1
24
+ end
25
+ assert_equal count, size
26
+
27
+ TSV.setup(sharder, :key_field => "Letter", :fields => ["Value", "Value times 2"], :type => :list)
28
+
29
+ assert_equal "Letter", sharder.key_field
30
+ assert_equal [2,4], sharder["2"]
31
+ assert_equal 2, sharder["2"]["Value"]
32
+
33
+ sharder = Sharder.new dir, false do |key|
34
+ key[-1]
35
+ end
36
+
37
+ sharder.extend ShardAdapter
38
+ sharder.load_annotation_hash
39
+ assert_equal size, sharder.keys.length
40
+ assert_equal [2,4], sharder["2"]
41
+
42
+ sharder = Persist.open_sharder dir do |key|
43
+ key[-1]
44
+ end
45
+
46
+ assert_equal "Letter", sharder.key_field
47
+ assert_equal [2,4], sharder["2"]
48
+ assert_equal 2, sharder["2"]["Value"]
49
+ end
50
+ end
51
+
52
+ def test_shard_tsv
53
+ content =<<-EOF
54
+ #Id,ValueA,ValueB
55
+ id1,a1,b1
56
+ id2,a2,b2
57
+ id3,a3,b3
58
+ id11,a11,b11
59
+ EOF
60
+
61
+ TmpFile.with_file(content.gsub(',',"\t")) do |tsv_file|
62
+ sharder = Persist.tsv(tsv_file, persist_options: { shard_function: proc{|k| k[-1] } }) do |data|
63
+ TSV.open(tsv_file, data: data, type: :list)
64
+ end
65
+ assert_equal 'a1', sharder["id1"]["ValueA"]
66
+ end
67
+ end
68
+
69
+ def test_shard_tsv_BDB
70
+ content =<<-EOF
71
+ #Id,ValueA,ValueB
72
+ id1,a1,b1
73
+ id2,a2,b2
74
+ id3,a3,b3
75
+ id11,a11,b11
76
+ EOF
77
+
78
+ TmpFile.with_file(content.gsub(',',"\t")) do |tsv_file|
79
+ sharder = Persist.tsv(tsv_file, engine: :BDB, persist_options: { shard_function: proc{|k| k[-1] } }) do |data|
80
+ TSV.open(tsv_file, data: data, type: :list)
81
+ end
82
+ assert_equal 'a1', sharder["id1"]["ValueA"]
83
+ assert_equal %w(id1 id11), sharder.prefix('id1')
84
+ end
85
+ end
86
+
87
+ def test_shard_fwt
88
+ TmpFile.with_file do |dir|
89
+ shard_function = Proc.new do |key|
90
+ key[0..(key.index(":")-1)]
91
+ end
92
+
93
+ pos_function = Proc.new do |key|
94
+ key.split(":").last.to_i
95
+ end
96
+
97
+ size = 10
98
+ sharder = Persist.tsv(dir,
99
+ persist_options: {
100
+ :engine => 'fwt',
101
+ :path => dir,
102
+ :serializer => :float,
103
+ :range => false, :value_size => 64,
104
+ :shard_function => shard_function,
105
+ :pos_function => pos_function
106
+ }
107
+ ) do |db|
108
+ size.times do |v|
109
+ v = v + 1
110
+ chr = "chr" << (v % 5).to_s
111
+ key = chr + ":" << v.to_s
112
+ value = v*2
113
+ db[key] = value
114
+ end
115
+ end
116
+
117
+ sharder.read
118
+
119
+ assert_equal dir, sharder.persistence_path
120
+ assert_equal size, sharder.size
121
+
122
+ assert_equal 4.0, sharder["chr2:2"]
123
+
124
+ count = 0
125
+ sharder.through do |k,v|
126
+ count += 1
127
+ end
128
+ assert_equal count, size
129
+
130
+ sharder = Persist.open_sharder(dir, false, 'fwt', {:range => false, :value_size => 64, :pos_function => pos_function}, &shard_function)
131
+
132
+ assert_equal 4.0, sharder["chr2:2"]
133
+
134
+ assert_equal size, sharder.size
135
+ end
136
+ end
137
+
138
+ def test_shard_pki
139
+ TmpFile.with_file do |dir|
140
+ shard_function = Proc.new do |key|
141
+ key[0..(key.index(":")-1)]
142
+ end
143
+
144
+ pos_function = Proc.new do |key|
145
+ key.split(":").last.to_i
146
+ end
147
+
148
+ size = 10
149
+ chrs = (1..10).to_a
150
+ sharder = Persist.tsv(dir,
151
+ engine: 'pki',
152
+ :persist_options => {
153
+ :pattern => %w(f f),
154
+ :range => false,
155
+ :value_size => 64,
156
+ :file => dir,
157
+ :shard_function => shard_function,
158
+ :pos_function => pos_function
159
+ }) do |db|
160
+ chrs.each do |c|
161
+ size.times do |v|
162
+ v = v
163
+ chr = "chr" << c.to_s
164
+ key = chr + ":" << v.to_s
165
+ db[key] = [v, v*2]
166
+ end
167
+ end
168
+ end
169
+ sharder.read
170
+
171
+ assert_equal dir, sharder.persistence_path
172
+
173
+ db = sharder.database("chr2:2")
174
+ db.read
175
+ assert_equal [2.0, 4.0], sharder["chr2:2"]
176
+
177
+ assert_equal size*chrs.length, sharder.size
178
+
179
+
180
+ count = 0
181
+ sharder.through do |k,v|
182
+ count += 1
183
+ end
184
+ assert_equal count, size*chrs.length
185
+
186
+ sharder = Persist.open_sharder(
187
+ dir, false, 'pki',
188
+ {:pattern => %w(f f), :file => dir, :range => false, :value_size => 64, :pos_function => pos_function}, &shard_function
189
+ )
190
+
191
+ db = sharder.database("chr2:2")
192
+ assert_equal [2.0, 4.0], sharder["chr2:2"]
193
+
194
+ assert_equal size*chrs.length, sharder.size
195
+ end
196
+ end
197
+
198
+ def test_shard_pki_skip
199
+ TmpFile.with_file do |dir|
200
+ shard_function = Proc.new do |key|
201
+ key[0..(key.index(":")-1)]
202
+ end
203
+
204
+ pos_function = Proc.new do |key|
205
+ key.split(":").last.to_i
206
+ end
207
+
208
+ size = 10
209
+ chrs = (1..10).to_a
210
+ sharder = Persist.tsv(dir, persist_options: {:pattern => %w(f), :range => false, :value_size => 64, :engine => 'pki', :file => dir, :shard_function => shard_function, :pos_function => pos_function}) do |db|
211
+ chrs.each do |c|
212
+ size.times do |v|
213
+ v = v + 1
214
+ chr = "chr" << c.to_s
215
+ key = chr + ":" << (v*2).to_s
216
+ db[key] = [v*2]
217
+ end
218
+ end
219
+ end
220
+ sharder.read
221
+
222
+ assert_equal dir, sharder.persistence_path
223
+
224
+ assert_equal [2.0], sharder["chr2:2"]
225
+ assert_equal [4.0], sharder["chr2:4"]
226
+
227
+ count = 0
228
+ sharder.through do |k,v|
229
+ count += 1 unless v.nil?
230
+ end
231
+ assert_equal count, size*chrs.length
232
+
233
+ sharder = Persist.open_sharder(dir, false, 'pki', {:range => false, :value_size => 64, :pos_function => pos_function}, &shard_function)
234
+
235
+ assert_equal [2.0], sharder["chr2:2"]
236
+
237
+ end
238
+ end
239
+
240
+ def test_shard_fwt_persist_tsv
241
+ TmpFile.with_file do |dir|
242
+ shard_function = Proc.new do |key|
243
+ key[0..(key.index(":")-1)]
244
+ end
245
+
246
+ pos_function = Proc.new do |key|
247
+ key.split(":").last.to_i
248
+ end
249
+
250
+ size = 10
251
+ sharder = Persist.persist_tsv("ShardTSV_FWT", nil, {}, {
252
+ :engine => 'fwt',
253
+ :path => dir,
254
+ :serializer => :float,
255
+ :range => false, :value_size => 64,
256
+ :shard_function => shard_function,
257
+ :pos_function => pos_function
258
+ }
259
+ ) do |db|
260
+ size.times do |v|
261
+ v = v + 1
262
+ chr = "chr" << (v % 5).to_s
263
+ key = chr + ":" << v.to_s
264
+ value = v*2
265
+ db[key] = value
266
+ end
267
+ end
268
+
269
+ sharder.read
270
+
271
+ assert_equal size, sharder.size
272
+
273
+ assert_equal 4.0, sharder["chr2:2"]
274
+
275
+ count = 0
276
+ sharder.through do |k,v|
277
+ count += 1
278
+ end
279
+ assert_equal count, size
280
+
281
+ sharder = Persist.open_sharder(sharder.persistence_path, false, 'fwt', {:range => false, :value_size => 64, :pos_function => pos_function}, &shard_function)
282
+
283
+ assert_equal 4.0, sharder["chr2:2"]
284
+
285
+ assert_equal size, sharder.size
286
+ end
287
+ end
288
+
289
+ end
290
+
@@ -5,9 +5,6 @@ require 'scout/tsv'
5
5
  begin
6
6
  require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
7
7
  class TestScoutTKRZW < Test::Unit::TestCase
8
- def setup
9
- end
10
-
11
8
  def test_open
12
9
  TmpFile.with_file nil do |tmp|
13
10
  db = ScoutTKRZW.open(tmp, true)
@@ -47,8 +44,8 @@ row2 a a id3
47
44
 
48
45
  assert TSVAdapter === tsv
49
46
  assert TSV === tsv
50
- assert_include tsv.instance_variable_get(:@extension_attrs), :key_field
51
- assert_include tsv.instance_variable_get(:@extension_attrs), :serializer
47
+ assert_include tsv.instance_variable_get(:@annotations), :key_field
48
+ assert_include tsv.instance_variable_get(:@annotations), :serializer
52
49
 
53
50
  tsv.close
54
51
  tsv_loaded = assert_nothing_raised do
@@ -96,7 +93,7 @@ row2 a a id3
96
93
  end
97
94
  end
98
95
 
99
- def test_benchmark2
96
+ def __test_benchmark2
100
97
  TmpFile.with_file nil do |tmp|
101
98
 
102
99
  db = ScoutTKRZW.open(tmp, true)
@@ -0,0 +1,282 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+
6
+ class TestTSVTokyo < Test::Unit::TestCase
7
+ def test_tokyo
8
+ content =<<-'EOF'
9
+ #: :sep=/\s+/#:type=:double#:merge=:concat
10
+ #Id ValueA ValueB OtherID
11
+ row1 a|aa|aaa b Id1|Id2
12
+ row2 A B Id3
13
+ row2 a a id3
14
+ EOF
15
+
16
+ tsv = TmpFile.with_file(content) do |filename|
17
+ Persist.persist(__method__, :HDB) do
18
+ TSV.open(filename)
19
+ end
20
+ end
21
+
22
+ assert_equal %w(a aa aaa), tsv["row1"][0]
23
+
24
+ assert TSVAdapter === tsv
25
+ assert TSV === tsv
26
+ assert_include tsv.instance_variable_get(:@annotations), :key_field
27
+ assert_include tsv.instance_variable_get(:@annotations), :serializer
28
+
29
+ tsv_loaded = assert_nothing_raised do
30
+ TmpFile.with_file(content) do |filename|
31
+ Persist.persist(__method__, :HDB) do
32
+ raise
33
+ end
34
+ end
35
+ end
36
+
37
+
38
+ assert_equal %w(a aa aaa), tsv_loaded["row1"][0]
39
+ end
40
+
41
+ def test_custom_load
42
+ tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
43
+
44
+ size = 100_000
45
+ (0..size).each do |i|
46
+ k = "key-#{i}"
47
+ values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
48
+ values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
49
+
50
+ tsv[k] = [values1, values2]
51
+ end
52
+
53
+ tc = Persist.persist(__method__, :HDB) do |file|
54
+ tsv
55
+ end
56
+
57
+ 100.times do
58
+ i = rand(size).floor
59
+ assert_equal tc["key-#{i}"], tsv["key-#{i}"]
60
+ end
61
+ end
62
+
63
+ def __test_benchmark
64
+ tsv = TSV.setup({}, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
65
+
66
+ size = 100_000
67
+ (0..size).each do |i|
68
+ k = "key-#{i}"
69
+ values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
70
+ values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
71
+
72
+ tsv[k] = [values1, values2]
73
+ end
74
+
75
+ tc = Persist.persist(__method__, :HDB) do |file|
76
+ data = ScoutCabinet.open(file, true, "HDB")
77
+ TSV.setup(data, :type => :double, :key_field => "Key", :fields => %w(Field1 Field2))
78
+ data.extend TSVAdapter
79
+ Log::ProgressBar.with_bar size do |b|
80
+ (0..size).each do |i|
81
+ b.tick
82
+ k = "key-#{i}"
83
+ values1 = 3.times.collect{|j| "value-#{i}-1-#{j}" }
84
+ values2 = 5.times.collect{|j| "value-#{i}-2-#{j}" }
85
+
86
+ data[k] = [values1, values2]
87
+ end
88
+ end
89
+ data
90
+ end
91
+
92
+ 100.times do
93
+ i = rand(size).floor
94
+ assert_equal tc["key-#{i}"], tsv["key-#{i}"]
95
+ end
96
+ end
97
+
98
+ def test_float_array
99
+ content =<<-EOF
100
+ #Id ValueA ValueB OtherID
101
+ row1 0.2 0.3 0
102
+ row2 0.1 4.5 0
103
+ EOF
104
+
105
+ TmpFile.with_file(content) do |filename|
106
+ tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :list, :cast => :to_f, :persist_update => true)
107
+ assert_equal [0.2, 0.3, 0], tsv["row1"]
108
+ assert_equal TSVAdapter::FloatArraySerializer, tsv.serializer
109
+ Open.cp tsv.persistence_path, tmpdir.persistence.foo
110
+ tsv2 = ScoutCabinet.open(tmpdir.persistence.foo, false)
111
+ tsv2.extend TSVAdapter
112
+ assert_equal [0.2, 0.3, 0], tsv2["row1"]
113
+ assert_equal TSVAdapter::FloatArraySerializer, tsv2.serializer
114
+ end
115
+
116
+ end
117
+
118
+ def test_float_double
119
+ content =<<-EOF
120
+ #Id ValueA ValueB OtherID
121
+ row1 0.2 0.3 0
122
+ row2 0.1 4.5 0
123
+ EOF
124
+
125
+ TmpFile.with_file(content) do |filename|
126
+ tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :double, :cast => :to_f)
127
+ assert_equal Marshal, tsv.serializer
128
+ assert_equal [[0.2], [0.3], [0.0]], tsv["row1"]
129
+ tsv.close
130
+
131
+ Persist::CONNECTIONS.clear
132
+
133
+ tsv = TSV.open(filename, :sep => /\s+/, :persist => true, :type => :double, :cast => :to_f)
134
+ assert_equal Marshal, tsv.serializer
135
+ assert_equal [[0.2], [0.3], [0.0]], tsv["row1"]
136
+ end
137
+ end
138
+
139
+ def test_importsv_list
140
+ content =<<-EOF
141
+ #Id ValueA ValueB OtherID
142
+ row1 0.2 0.3 0
143
+ row2 0.1 4.5 0
144
+ EOF
145
+
146
+ TmpFile.with_path(content.gsub(/ +/,"\t")) do |filename|
147
+ TmpFile.with_file do |persistence_path|
148
+ parser = TSV::Parser.new filename, type: :list
149
+ database = ScoutCabinet.open persistence_path, true, :HDB
150
+ parser.with_stream do |stream|
151
+ ScoutCabinet.importtsv(database, stream)
152
+ end
153
+ database.write_and_read do
154
+ TSV.setup(database, **parser.options)
155
+ database.extend TSVAdapter
156
+ end
157
+
158
+ assert_equal '0.2', database["row1"]["ValueA"]
159
+ end
160
+ end
161
+ end
162
+
163
+ def test_importsv_double
164
+ content =<<-EOF
165
+ #Id ValueA ValueB OtherID
166
+ row1 a|aa b|bb c|cc
167
+ row2 A|AA B|BB C|CC
168
+ EOF
169
+
170
+ TmpFile.with_path(content.gsub(/ +/,"\t")) do |filename|
171
+ TmpFile.with_file do |persistence_path|
172
+ parser = TSV::Parser.new filename, type: :double
173
+ database = ScoutCabinet.open persistence_path, true, :HDB
174
+ parser.with_stream do |stream|
175
+ ScoutCabinet.importtsv(database, stream)
176
+ end
177
+ database.write_and_read do
178
+ TSV.setup(database, **parser.options)
179
+ database.extend TSVAdapter
180
+ end
181
+
182
+ assert_equal %w(A AA), database["row2"]["ValueA"]
183
+ end
184
+ end
185
+ end
186
+
187
+ def test_importsv_large
188
+ content =<<-EOF.gsub(/ +/, "\t")
189
+ #Id ValueA ValueB
190
+ EOF
191
+
192
+ 10_000.times do |i|
193
+ content += "row#{i}\ta#{i}\tb#{i}\n"
194
+ end
195
+
196
+ TmpFile.with_path(content) do |filename|
197
+ TmpFile.with_file do |persistence_path|
198
+ parser = TSV::Parser.new filename, type: :list
199
+ database = ScoutCabinet.open persistence_path, true, :HDB
200
+ parser.with_stream do |stream|
201
+ ScoutCabinet.importtsv(database, stream)
202
+ end
203
+ database.write_and_read do
204
+ TSV.setup(database, **parser.options)
205
+ database.extend TSVAdapter
206
+ end
207
+
208
+ assert_equal "a1000", database["row1000"]["ValueA"]
209
+ end
210
+ end
211
+ end
212
+
213
+ def test_importsv_double_BDB
214
+ content =<<-EOF
215
+ #Id ValueA ValueB OtherID
216
+ row1 a|aa b|bb c|cc
217
+ row2 A|AA B|BB C|CC
218
+ EOF
219
+
220
+ TmpFile.with_path(content.gsub(/ +/,"\t")) do |filename|
221
+ TmpFile.with_file do |persistence_path|
222
+ parser = TSV::Parser.new filename, type: :double
223
+ database = ScoutCabinet.open persistence_path, true, :BDB
224
+ parser.with_stream do |stream|
225
+ ScoutCabinet.importtsv(database, stream)
226
+ end
227
+ database.write_and_read do
228
+ TSV.setup(database, **parser.options)
229
+ database.extend TSVAdapter
230
+ end
231
+
232
+ assert_equal %w(A AA), database["row2"]["ValueA"]
233
+ assert_equal %w(row1 row2), database.prefix("row")
234
+ end
235
+ end
236
+ end
237
+
238
+ def test_importsv_from_file
239
+ content =<<-EOF
240
+ #Id ValueA ValueB OtherID
241
+ row1 a|aa b|bb c|cc
242
+ row2 A|AA B|BB C|CC
243
+ EOF
244
+
245
+ TmpFile.with_path(content.gsub(/ +/,"\t")) do |filename|
246
+ TmpFile.with_file do |persistence_path|
247
+ parser = TSV::Parser.new filename, type: :double
248
+ database = ScoutCabinet.open persistence_path, true, :HDB
249
+ parser.with_stream do |stream|
250
+ database.load_stream stream
251
+ end
252
+ database.write_and_read do
253
+ TSV.setup(database, **parser.options)
254
+ database.extend TSVAdapter
255
+ end
256
+
257
+ assert_equal %w(A AA), database["row2"]["ValueA"]
258
+ end
259
+ end
260
+ end
261
+
262
+ def test_open_tokyocabinet
263
+ content =<<-EOF
264
+ #Id ValueA ValueB OtherID
265
+ row1 a|aa b|bb c|cc
266
+ row2 A|AA B|BB C|CC
267
+ EOF
268
+
269
+ TmpFile.with_path(content.gsub(/ +/,"\t")) do |tsv|
270
+ TmpFile.with_path do |filename|
271
+ tk = Persist.open_tokyocabinet(filename, true)
272
+ TSV.setup(tk, key_field: "Id", fields: %w(ValueA ValueB OtherID), type: :double)
273
+ tk.save_annotation_hash
274
+
275
+ tk = Persist.open_tokyocabinet(filename, true)
276
+ assert_equal "Id", tk.key_field
277
+ end
278
+ end
279
+ end
280
+
281
+ end
282
+
@@ -0,0 +1,12 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ class TestClass < Test::Unit::TestCase
5
+ def test_serializer_module
6
+ m = TSVAdapter.serializer_module :marshal
7
+ v = [1, :a]
8
+ d = m.dump(v)
9
+ assert_equal v, m.load(d)
10
+ end
11
+ end
12
+
@@ -0,0 +1,51 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ require 'scout/tsv'
5
+ class TestAssociation < Test::Unit::TestCase
6
+
7
+ def test_marriages_simple
8
+ database = Association.database(datadir_test.person.marriages, :source => "Wife", :target => "Husband")
9
+ assert_equal "001", database["002"]["Husband"]
10
+ assert_equal "2021", database["002"]["Date"]
11
+ end
12
+
13
+ def test_marriages_open
14
+ database = Association.database(datadir_test.person.marriages, :source => "Wife (ID)=>Alias", :target => "Husband (ID)=>Name")
15
+ assert_equal "Miguel", database["Clei"]["Husband"]
16
+ assert_equal "2021", database["Clei"]["Date"]
17
+ assert_include database.key_field, "Alias"
18
+ end
19
+
20
+ def test_marriages_partial_field
21
+ database = Association.database(datadir_test.person.marriages, :source => "Wife=>Alias", :target => "Husband=>Name")
22
+ assert_equal "Miguel", database["Clei"]["Husband"]
23
+ assert_equal "2021", database["Clei"]["Date"]
24
+ assert_include database.key_field, "Alias"
25
+ assert_include database.fields.first, "Name"
26
+ end
27
+
28
+ def test_marriages_open_from_tsv
29
+ database = Association.database(datadir_test.person.marriages.tsv, :source => "Wife (ID)=>Alias", :target => "Husband (ID)=>Name")
30
+ assert_equal "Miguel", database["Clei"]["Husband"]
31
+ assert_equal "2021", database["Clei"]["Date"]
32
+ assert_include database.key_field, "Alias"
33
+ end
34
+
35
+ def test_brothers_id
36
+ database = Association.database(datadir_test.person.brothers, :source => "Older=~Older (Alias)=>Name", :target => "Younger=~Younger (Alias)=>ID")
37
+ assert_equal '001', database["Isabel"]["Younger"]
38
+ end
39
+
40
+ def test_brothers_rename
41
+ database = Association.database(datadir_test.person.brothers, :source => "Older=~Older (Alias)=>Name")
42
+ assert_equal "Older (Name)", database.key_field
43
+ end
44
+
45
+ def test_parents_flat
46
+ tsv = datadir_test.person.parents.tsv type: :flat, fields: ["Parent"]
47
+ database = Association.database(tsv)
48
+ assert_equal database["Miki"], %w(Juan Mariluz)
49
+ end
50
+ end
51
+
@@ -0,0 +1,40 @@
1
+ require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
2
+ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
3
+
4
+ class TestEntity < Test::Unit::TestCase
5
+
6
+ setup do
7
+ Entity.entity_property_cache = tmpdir.property_cache
8
+ end
9
+
10
+ module Person
11
+ extend Entity
12
+
13
+ annotation :language
14
+
15
+ property :salutation do
16
+ case language
17
+ when 'es'
18
+ "Hola #{self}"
19
+ else
20
+ "Hi #{self}"
21
+ end
22
+ end
23
+ end
24
+
25
+ module EmptyEntity
26
+ extend Entity
27
+ end
28
+
29
+ def test_person
30
+ person = Person.setup("Miguel", 'es')
31
+ assert_equal "Hola Miguel", person.salutation
32
+
33
+ person.language = 'en'
34
+ assert_equal "Hi Miguel", person.salutation
35
+ end
36
+
37
+ def test_empty
38
+ refute EmptyEntity.setup("foo").nil?
39
+ end
40
+ end