red-arrow 5.0.0 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +23 -0
  3. data/ext/arrow/converters.cpp +5 -0
  4. data/ext/arrow/converters.hpp +126 -0
  5. data/ext/arrow/extconf.rb +13 -0
  6. data/ext/arrow/raw-records.cpp +1 -0
  7. data/ext/arrow/values.cpp +1 -0
  8. data/lib/arrow/aggregate-node-options.rb +35 -0
  9. data/lib/arrow/aggregation.rb +46 -0
  10. data/lib/arrow/array-builder.rb +5 -0
  11. data/lib/arrow/binary-dictionary-array-builder.rb +27 -0
  12. data/lib/arrow/column-containable.rb +100 -1
  13. data/lib/arrow/datum.rb +2 -0
  14. data/lib/arrow/expression.rb +48 -0
  15. data/lib/arrow/file-system.rb +34 -0
  16. data/lib/arrow/group.rb +116 -124
  17. data/lib/arrow/loader.rb +13 -0
  18. data/lib/arrow/map-array-builder.rb +109 -0
  19. data/lib/arrow/map-array.rb +26 -0
  20. data/lib/arrow/map-data-type.rb +89 -0
  21. data/lib/arrow/path-extension.rb +1 -1
  22. data/lib/arrow/record-batch-reader.rb +41 -0
  23. data/lib/arrow/record-batch.rb +0 -2
  24. data/lib/arrow/slicer.rb +44 -143
  25. data/lib/arrow/source-node-options.rb +32 -0
  26. data/lib/arrow/string-dictionary-array-builder.rb +27 -0
  27. data/lib/arrow/symbol-values-appendable.rb +34 -0
  28. data/lib/arrow/table-concatenate-options.rb +36 -0
  29. data/lib/arrow/table-formatter.rb +141 -17
  30. data/lib/arrow/table-list-formatter.rb +5 -3
  31. data/lib/arrow/table-loader.rb +41 -3
  32. data/lib/arrow/table-saver.rb +29 -3
  33. data/lib/arrow/table-table-formatter.rb +7 -31
  34. data/lib/arrow/table.rb +32 -38
  35. data/lib/arrow/version.rb +1 -1
  36. data/red-arrow.gemspec +1 -1
  37. data/test/raw-records/test-dense-union-array.rb +14 -0
  38. data/test/raw-records/test-list-array.rb +19 -0
  39. data/test/raw-records/test-map-array.rb +441 -0
  40. data/test/raw-records/test-sparse-union-array.rb +14 -0
  41. data/test/raw-records/test-struct-array.rb +15 -0
  42. data/test/test-array-builder.rb +7 -0
  43. data/test/test-binary-dictionary-array-builder.rb +103 -0
  44. data/test/test-csv-loader.rb +8 -8
  45. data/test/test-expression.rb +40 -0
  46. data/test/test-group.rb +75 -51
  47. data/test/test-map-array-builder.rb +110 -0
  48. data/test/test-map-array.rb +33 -0
  49. data/test/test-map-data-type.rb +36 -0
  50. data/test/test-record-batch-reader.rb +46 -0
  51. data/test/test-record-batch.rb +42 -0
  52. data/test/test-slicer.rb +166 -167
  53. data/test/test-string-dictionary-array-builder.rb +103 -0
  54. data/test/test-table.rb +190 -53
  55. data/test/values/test-dense-union-array.rb +14 -0
  56. data/test/values/test-list-array.rb +17 -0
  57. data/test/values/test-map-array.rb +433 -0
  58. data/test/values/test-sparse-union-array.rb +14 -0
  59. data/test/values/test-struct-array.rb +15 -0
  60. metadata +107 -76
data/test/test-slicer.rb CHANGED
@@ -45,15 +45,15 @@ class SlicerTest < Test::Unit::TestCase
45
45
  slicer.visible
46
46
  end
47
47
  assert_equal(<<-TABLE, sliced_table.to_s)
48
- count visible
49
- 0
50
- 1 1 true
51
- 2
52
- 3 8 true
53
- 4 16 true
54
- 5
55
- 6
56
- 7 256 true
48
+ count visible
49
+ 0 (null) (null)
50
+ 1 1 true
51
+ 2 (null) (null)
52
+ 3 8 true
53
+ 4 16 true
54
+ 5 (null) (null)
55
+ 6 (null) (null)
56
+ 7 256 true
57
57
  TABLE
58
58
  end
59
59
 
@@ -62,16 +62,16 @@ class SlicerTest < Test::Unit::TestCase
62
62
  slicer.count
63
63
  end
64
64
  assert_equal(<<-TABLE, sliced_table.to_s)
65
- count visible
66
- 0 1 true
67
- 1 2 false
68
- 2 4
69
- 3 8 true
70
- 4 16 true
71
- 5 32 false
72
- 6 64
73
- 7
74
- 8 256 true
65
+ count visible
66
+ 0 1 true
67
+ 1 2 false
68
+ 2 4 (null)
69
+ 3 8 true
70
+ 4 16 true
71
+ 5 32 false
72
+ 6 64 (null)
73
+ 7 (null) (null)
74
+ 8 256 true
75
75
  TABLE
76
76
  end
77
77
  end
@@ -82,13 +82,13 @@ class SlicerTest < Test::Unit::TestCase
82
82
  !slicer.visible
83
83
  end
84
84
  assert_equal(<<-TABLE, sliced_table.to_s)
85
- count visible
86
- 0
87
- 1 2 false
88
- 2
89
- 3 32 false
90
- 4
91
- 5
85
+ count visible
86
+ 0 (null) (null)
87
+ 1 2 false
88
+ 2 (null) (null)
89
+ 3 32 false
90
+ 4 (null) (null)
91
+ 5 (null) (null)
92
92
  TABLE
93
93
  end
94
94
 
@@ -97,9 +97,9 @@ class SlicerTest < Test::Unit::TestCase
97
97
  !slicer.count
98
98
  end
99
99
  assert_equal(<<-TABLE, sliced_table.to_s)
100
- count visible
101
- 0 0
102
- 1
100
+ count visible
101
+ 0 0 (null)
102
+ 1 (null) (null)
103
103
  TABLE
104
104
  end
105
105
  end
@@ -109,11 +109,11 @@ class SlicerTest < Test::Unit::TestCase
109
109
  slicer.visible.null?
110
110
  end
111
111
  assert_equal(<<-TABLE, sliced_table.to_s)
112
- count visible
113
- 0 0
114
- 1 4
115
- 2 64
116
- 3
112
+ count visible
113
+ 0 0 (null)
114
+ 1 4 (null)
115
+ 2 64 (null)
116
+ 3 (null) (null)
117
117
  TABLE
118
118
  end
119
119
 
@@ -138,11 +138,11 @@ class SlicerTest < Test::Unit::TestCase
138
138
  slicer.visible == nil
139
139
  end
140
140
  assert_equal(<<-TABLE, sliced_table.to_s)
141
- count visible
142
- 0 0
143
- 1 4
144
- 2 64
145
- 3
141
+ count visible
142
+ 0 0 (null)
143
+ 1 4 (null)
144
+ 2 64 (null)
145
+ 3 (null) (null)
146
146
  TABLE
147
147
  end
148
148
 
@@ -151,15 +151,15 @@ class SlicerTest < Test::Unit::TestCase
151
151
  slicer.visible == true
152
152
  end
153
153
  assert_equal(<<-TABLE, sliced_table.to_s)
154
- count visible
155
- 0
156
- 1 1 true
157
- 2
158
- 3 8 true
159
- 4 16 true
160
- 5
161
- 6
162
- 7 256 true
154
+ count visible
155
+ 0 (null) (null)
156
+ 1 1 true
157
+ 2 (null) (null)
158
+ 3 8 true
159
+ 4 16 true
160
+ 5 (null) (null)
161
+ 6 (null) (null)
162
+ 7 256 true
163
163
  TABLE
164
164
  end
165
165
  end
@@ -185,13 +185,13 @@ class SlicerTest < Test::Unit::TestCase
185
185
  !(slicer.visible == true)
186
186
  end
187
187
  assert_equal(<<-TABLE, sliced_table.to_s)
188
- count visible
189
- 0
190
- 1 2 false
191
- 2
192
- 3 32 false
193
- 4
194
- 5
188
+ count visible
189
+ 0 (null) (null)
190
+ 1 2 false
191
+ 2 (null) (null)
192
+ 3 32 false
193
+ 4 (null) (null)
194
+ 5 (null) (null)
195
195
  TABLE
196
196
  end
197
197
  end
@@ -217,13 +217,13 @@ class SlicerTest < Test::Unit::TestCase
217
217
  slicer.visible != true
218
218
  end
219
219
  assert_equal(<<-TABLE, sliced_table.to_s)
220
- count visible
221
- 0
222
- 1 2 false
223
- 2
224
- 3 32 false
225
- 4
226
- 5
220
+ count visible
221
+ 0 (null) (null)
222
+ 1 2 false
223
+ 2 (null) (null)
224
+ 3 32 false
225
+ 4 (null) (null)
226
+ 5 (null) (null)
227
227
  TABLE
228
228
  end
229
229
  end
@@ -233,13 +233,13 @@ class SlicerTest < Test::Unit::TestCase
233
233
  slicer.count < 16
234
234
  end
235
235
  assert_equal(<<-TABLE, sliced_table.to_s)
236
- count visible
237
- 0 0
238
- 1 1 true
239
- 2 2 false
240
- 3 4
241
- 4 8 true
242
- 5
236
+ count visible
237
+ 0 0 (null)
238
+ 1 1 true
239
+ 2 2 false
240
+ 3 4 (null)
241
+ 4 8 true
242
+ 5 (null) (null)
243
243
  TABLE
244
244
  end
245
245
 
@@ -248,12 +248,12 @@ class SlicerTest < Test::Unit::TestCase
248
248
  !(slicer.count < 16)
249
249
  end
250
250
  assert_equal(<<-TABLE, sliced_table.to_s)
251
- count visible
252
- 0 16 true
253
- 1 32 false
254
- 2 64
255
- 3
256
- 4 256 true
251
+ count visible
252
+ 0 16 true
253
+ 1 32 false
254
+ 2 64 (null)
255
+ 3 (null) (null)
256
+ 4 256 true
257
257
  TABLE
258
258
  end
259
259
 
@@ -262,14 +262,14 @@ class SlicerTest < Test::Unit::TestCase
262
262
  slicer.count <= 16
263
263
  end
264
264
  assert_equal(<<-TABLE, sliced_table.to_s)
265
- count visible
266
- 0 0
267
- 1 1 true
268
- 2 2 false
269
- 3 4
270
- 4 8 true
271
- 5 16 true
272
- 6
265
+ count visible
266
+ 0 0 (null)
267
+ 1 1 true
268
+ 2 2 false
269
+ 3 4 (null)
270
+ 4 8 true
271
+ 5 16 true
272
+ 6 (null) (null)
273
273
  TABLE
274
274
  end
275
275
 
@@ -278,11 +278,11 @@ class SlicerTest < Test::Unit::TestCase
278
278
  !(slicer.count <= 16)
279
279
  end
280
280
  assert_equal(<<-TABLE, sliced_table.to_s)
281
- count visible
282
- 0 32 false
283
- 1 64
284
- 2
285
- 3 256 true
281
+ count visible
282
+ 0 32 false
283
+ 1 64 (null)
284
+ 2 (null) (null)
285
+ 3 256 true
286
286
  TABLE
287
287
  end
288
288
 
@@ -291,11 +291,11 @@ class SlicerTest < Test::Unit::TestCase
291
291
  slicer.count > 16
292
292
  end
293
293
  assert_equal(<<-TABLE, sliced_table.to_s)
294
- count visible
295
- 0 32 false
296
- 1 64
297
- 2
298
- 3 256 true
294
+ count visible
295
+ 0 32 false
296
+ 1 64 (null)
297
+ 2 (null) (null)
298
+ 3 256 true
299
299
  TABLE
300
300
  end
301
301
 
@@ -304,14 +304,14 @@ class SlicerTest < Test::Unit::TestCase
304
304
  !(slicer.count > 16)
305
305
  end
306
306
  assert_equal(<<-TABLE, sliced_table.to_s)
307
- count visible
308
- 0 0
309
- 1 1 true
310
- 2 2 false
311
- 3 4
312
- 4 8 true
313
- 5 16 true
314
- 6
307
+ count visible
308
+ 0 0 (null)
309
+ 1 1 true
310
+ 2 2 false
311
+ 3 4 (null)
312
+ 4 8 true
313
+ 5 16 true
314
+ 6 (null) (null)
315
315
  TABLE
316
316
  end
317
317
 
@@ -320,12 +320,12 @@ class SlicerTest < Test::Unit::TestCase
320
320
  slicer.count >= 16
321
321
  end
322
322
  assert_equal(<<-TABLE, sliced_table.to_s)
323
- count visible
324
- 0 16 true
325
- 1 32 false
326
- 2 64
327
- 3
328
- 4 256 true
323
+ count visible
324
+ 0 16 true
325
+ 1 32 false
326
+ 2 64 (null)
327
+ 3 (null) (null)
328
+ 4 256 true
329
329
  TABLE
330
330
  end
331
331
 
@@ -334,13 +334,13 @@ class SlicerTest < Test::Unit::TestCase
334
334
  !(slicer.count >= 16)
335
335
  end
336
336
  assert_equal(<<-TABLE, sliced_table.to_s)
337
- count visible
338
- 0 0
339
- 1 1 true
340
- 2 2 false
341
- 3 4
342
- 4 8 true
343
- 5
337
+ count visible
338
+ 0 0 (null)
339
+ 1 1 true
340
+ 2 2 false
341
+ 3 4 (null)
342
+ 4 8 true
343
+ 5 (null) (null)
344
344
  TABLE
345
345
  end
346
346
 
@@ -351,10 +351,9 @@ class SlicerTest < Test::Unit::TestCase
351
351
  assert_equal(<<-TABLE, sliced_table.to_s)
352
352
  count visible
353
353
  0 1 true
354
- 1 4
354
+ 1 4 (null)
355
355
  2 16 true
356
- 3 64
357
- 4
356
+ 3 64 (null)
358
357
  TABLE
359
358
  end
360
359
 
@@ -363,13 +362,13 @@ class SlicerTest < Test::Unit::TestCase
363
362
  !slicer.count.in?([1, 4, 16, 64])
364
363
  end
365
364
  assert_equal(<<-TABLE, sliced_table.to_s)
366
- count visible
367
- 0 0
368
- 1 2 false
369
- 2 8 true
370
- 3 32 false
371
- 4
372
- 5 256 true
365
+ count visible
366
+ 0 0 (null)
367
+ 1 2 false
368
+ 2 8 true
369
+ 3 32 false
370
+ 4 (null) (null)
371
+ 5 256 true
373
372
  TABLE
374
373
  end
375
374
 
@@ -378,13 +377,13 @@ class SlicerTest < Test::Unit::TestCase
378
377
  slicer.visible & (slicer.count >= 16)
379
378
  end
380
379
  assert_equal(<<-TABLE, sliced_table.to_s)
381
- count visible
382
- 0
383
- 1
384
- 2 16 true
385
- 3
386
- 4
387
- 5 256 true
380
+ count visible
381
+ 0 (null) (null)
382
+ 1 (null) (null)
383
+ 2 16 true
384
+ 3 (null) (null)
385
+ 4 (null) (null)
386
+ 5 256 true
388
387
  TABLE
389
388
  end
390
389
 
@@ -393,16 +392,16 @@ class SlicerTest < Test::Unit::TestCase
393
392
  slicer.visible | (slicer.count >= 16)
394
393
  end
395
394
  assert_equal(<<-TABLE, sliced_table.to_s)
396
- count visible
397
- 0
398
- 1 1 true
399
- 2
400
- 3 8 true
401
- 4 16 true
402
- 5 32 false
403
- 6
404
- 7
405
- 8 256 true
395
+ count visible
396
+ 0 (null) (null)
397
+ 1 1 true
398
+ 2 (null) (null)
399
+ 3 8 true
400
+ 4 16 true
401
+ 5 32 false
402
+ 6 (null) (null)
403
+ 7 (null) (null)
404
+ 8 256 true
406
405
  TABLE
407
406
  end
408
407
 
@@ -411,14 +410,14 @@ class SlicerTest < Test::Unit::TestCase
411
410
  slicer.visible ^ (slicer.count >= 16)
412
411
  end
413
412
  assert_equal(<<-TABLE, sliced_table.to_s)
414
- count visible
415
- 0
416
- 1 1 true
417
- 2
418
- 3 8 true
419
- 4 32 false
420
- 5
421
- 6
413
+ count visible
414
+ 0 (null) (null)
415
+ 1 1 true
416
+ 2 (null) (null)
417
+ 3 8 true
418
+ 4 32 false
419
+ 5 (null) (null)
420
+ 6 (null) (null)
422
421
  TABLE
423
422
  end
424
423
 
@@ -429,15 +428,15 @@ class SlicerTest < Test::Unit::TestCase
429
428
  end
430
429
  end
431
430
  assert_equal(<<-TABLE, sliced_table.to_s)
432
- count visible
433
- 0 0
434
- 1 1 true
435
- 2 4
436
- 3 8 true
437
- 4 16 true
438
- 5 64
439
- 6
440
- 7 256 true
431
+ count visible
432
+ 0 0 (null)
433
+ 1 1 true
434
+ 2 4 (null)
435
+ 3 8 true
436
+ 4 16 true
437
+ 5 64 (null)
438
+ 6 (null) (null)
439
+ 7 256 true
441
440
  TABLE
442
441
  end
443
442
 
@@ -474,15 +473,15 @@ class SlicerTest < Test::Unit::TestCase
474
473
  end
475
474
  end
476
475
  assert_equal(<<-TABLE, sliced_table.to_s)
477
- count visible
478
- 0 0
479
- 1 1 true
480
- 2 4
481
- 3 8 true
482
- 4 16 true
483
- 5 64
484
- 6
485
- 7 256 true
476
+ count visible
477
+ 0 0 (null)
478
+ 1 1 true
479
+ 2 4 (null)
480
+ 3 8 true
481
+ 4 16 true
482
+ 5 64 (null)
483
+ 6 (null) (null)
484
+ 7 256 true
486
485
  TABLE
487
486
  end
488
487
  end
data/test/test-string-dictionary-array-builder.rb ADDED
@@ -0,0 +1,103 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ class StringDictionaryArrayBuilderTest < Test::Unit::TestCase
19
+ def setup
20
+ @builder = Arrow::StringDictionaryArrayBuilder.new
21
+ end
22
+
23
+ sub_test_case("#append_values") do
24
+ test("[nil]") do
25
+ @builder.append_values([nil])
26
+ array = @builder.finish
27
+ assert_equal([
28
+ [],
29
+ [nil],
30
+ ],
31
+ [
32
+ array.dictionary.to_a,
33
+ array.indices.to_a,
34
+ ])
35
+ end
36
+
37
+ test("[String]") do
38
+ @builder.append_values(["hello"])
39
+ array = @builder.finish
40
+ assert_equal([
41
+ ["hello"],
42
+ [0],
43
+ ],
44
+ [
45
+ array.dictionary.to_a,
46
+ array.indices.to_a,
47
+ ])
48
+ end
49
+
50
+ test("[Symbol]") do
51
+ @builder.append_values([:hello])
52
+ array = @builder.finish
53
+ assert_equal([
54
+ ["hello"],
55
+ [0],
56
+ ],
57
+ [
58
+ array.dictionary.to_a,
59
+ array.indices.to_a,
60
+ ])
61
+ end
62
+
63
+ test("[nil, String, Symbol]") do
64
+ @builder.append_values([
65
+ nil,
66
+ "Hello",
67
+ :world,
68
+ "world",
69
+ ])
70
+ array = @builder.finish
71
+ assert_equal([
72
+ ["Hello", "world"],
73
+ [nil, 0, 1, 1],
74
+ ],
75
+ [
76
+ array.dictionary.to_a,
77
+ array.indices.to_a,
78
+ ])
79
+ end
80
+
81
+ test("is_valids") do
82
+ @builder.append_values([
83
+ "Hello",
84
+ :world,
85
+ :goodbye,
86
+ ],
87
+ [
88
+ true,
89
+ false,
90
+ true,
91
+ ])
92
+ array = @builder.finish
93
+ assert_equal([
94
+ ["Hello", "goodbye"],
95
+ [0, nil, 1],
96
+ ],
97
+ [
98
+ array.dictionary.to_a,
99
+ array.indices.to_a,
100
+ ])
101
+ end
102
+ end
103
+ end