red-arrow 4.0.1 → 7.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +10 -0
  3. data/README.md +23 -0
  4. data/ext/arrow/arrow.cpp +3 -0
  5. data/ext/arrow/converters.cpp +5 -0
  6. data/ext/arrow/converters.hpp +126 -0
  7. data/ext/arrow/extconf.rb +13 -0
  8. data/ext/arrow/memory-view.cpp +311 -0
  9. data/ext/arrow/memory-view.hpp +26 -0
  10. data/ext/arrow/raw-records.cpp +1 -0
  11. data/ext/arrow/values.cpp +1 -0
  12. data/lib/arrow/aggregate-node-options.rb +35 -0
  13. data/lib/arrow/aggregation.rb +46 -0
  14. data/lib/arrow/array-builder.rb +5 -0
  15. data/lib/arrow/array.rb +12 -0
  16. data/lib/arrow/binary-dictionary-array-builder.rb +27 -0
  17. data/lib/arrow/buffer.rb +10 -6
  18. data/lib/arrow/column-containable.rb +100 -1
  19. data/lib/arrow/constructor-arguments-gc-guardable.rb +25 -0
  20. data/lib/arrow/datum.rb +102 -0
  21. data/lib/arrow/equal-options.rb +38 -0
  22. data/lib/arrow/expression.rb +48 -0
  23. data/lib/arrow/file-system.rb +34 -0
  24. data/lib/arrow/function.rb +52 -0
  25. data/lib/arrow/group.rb +116 -124
  26. data/lib/arrow/loader.rb +58 -0
  27. data/lib/arrow/map-array-builder.rb +109 -0
  28. data/lib/arrow/map-array.rb +26 -0
  29. data/lib/arrow/map-data-type.rb +89 -0
  30. data/lib/arrow/path-extension.rb +1 -1
  31. data/lib/arrow/record-batch-reader.rb +41 -0
  32. data/lib/arrow/record-batch.rb +0 -2
  33. data/lib/arrow/s3-global-options.rb +38 -0
  34. data/lib/arrow/scalar.rb +32 -0
  35. data/lib/arrow/slicer.rb +44 -143
  36. data/lib/arrow/sort-key.rb +61 -55
  37. data/lib/arrow/sort-options.rb +8 -8
  38. data/lib/arrow/source-node-options.rb +32 -0
  39. data/lib/arrow/string-dictionary-array-builder.rb +27 -0
  40. data/lib/arrow/symbol-values-appendable.rb +34 -0
  41. data/lib/arrow/table-concatenate-options.rb +36 -0
  42. data/lib/arrow/table-formatter.rb +141 -17
  43. data/lib/arrow/table-list-formatter.rb +5 -3
  44. data/lib/arrow/table-loader.rb +119 -44
  45. data/lib/arrow/table-saver.rb +36 -5
  46. data/lib/arrow/table-table-formatter.rb +7 -31
  47. data/lib/arrow/table.rb +112 -40
  48. data/lib/arrow/version.rb +1 -1
  49. data/red-arrow.gemspec +1 -9
  50. data/test/helper.rb +3 -0
  51. data/test/raw-records/test-dense-union-array.rb +14 -0
  52. data/test/raw-records/test-list-array.rb +19 -0
  53. data/test/raw-records/test-map-array.rb +441 -0
  54. data/test/raw-records/test-sparse-union-array.rb +14 -0
  55. data/test/raw-records/test-struct-array.rb +15 -0
  56. data/test/test-array-builder.rb +7 -0
  57. data/test/test-array.rb +34 -0
  58. data/test/test-binary-dictionary-array-builder.rb +103 -0
  59. data/test/test-boolean-scalar.rb +26 -0
  60. data/test/test-csv-loader.rb +8 -8
  61. data/test/test-expression.rb +40 -0
  62. data/test/test-float-scalar.rb +46 -0
  63. data/test/test-function.rb +210 -0
  64. data/test/test-group.rb +75 -51
  65. data/test/test-map-array-builder.rb +110 -0
  66. data/test/test-map-array.rb +33 -0
  67. data/test/test-map-data-type.rb +36 -0
  68. data/test/test-memory-view.rb +434 -0
  69. data/test/test-record-batch-reader.rb +46 -0
  70. data/test/test-record-batch.rb +42 -0
  71. data/test/test-slicer.rb +166 -167
  72. data/test/test-string-dictionary-array-builder.rb +103 -0
  73. data/test/test-table.rb +376 -56
  74. data/test/values/test-dense-union-array.rb +14 -0
  75. data/test/values/test-list-array.rb +17 -0
  76. data/test/values/test-map-array.rb +433 -0
  77. data/test/values/test-sparse-union-array.rb +14 -0
  78. data/test/values/test-struct-array.rb +15 -0
  79. metadata +117 -168
data/test/test-slicer.rb CHANGED
@@ -45,15 +45,15 @@ class SlicerTest < Test::Unit::TestCase
45
45
  slicer.visible
46
46
  end
47
47
  assert_equal(<<-TABLE, sliced_table.to_s)
48
- count visible
49
- 0
50
- 1 1 true
51
- 2
52
- 3 8 true
53
- 4 16 true
54
- 5
55
- 6
56
- 7 256 true
48
+ count visible
49
+ 0 (null) (null)
50
+ 1 1 true
51
+ 2 (null) (null)
52
+ 3 8 true
53
+ 4 16 true
54
+ 5 (null) (null)
55
+ 6 (null) (null)
56
+ 7 256 true
57
57
  TABLE
58
58
  end
59
59
 
@@ -62,16 +62,16 @@ class SlicerTest < Test::Unit::TestCase
62
62
  slicer.count
63
63
  end
64
64
  assert_equal(<<-TABLE, sliced_table.to_s)
65
- count visible
66
- 0 1 true
67
- 1 2 false
68
- 2 4
69
- 3 8 true
70
- 4 16 true
71
- 5 32 false
72
- 6 64
73
- 7
74
- 8 256 true
65
+ count visible
66
+ 0 1 true
67
+ 1 2 false
68
+ 2 4 (null)
69
+ 3 8 true
70
+ 4 16 true
71
+ 5 32 false
72
+ 6 64 (null)
73
+ 7 (null) (null)
74
+ 8 256 true
75
75
  TABLE
76
76
  end
77
77
  end
@@ -82,13 +82,13 @@ class SlicerTest < Test::Unit::TestCase
82
82
  !slicer.visible
83
83
  end
84
84
  assert_equal(<<-TABLE, sliced_table.to_s)
85
- count visible
86
- 0
87
- 1 2 false
88
- 2
89
- 3 32 false
90
- 4
91
- 5
85
+ count visible
86
+ 0 (null) (null)
87
+ 1 2 false
88
+ 2 (null) (null)
89
+ 3 32 false
90
+ 4 (null) (null)
91
+ 5 (null) (null)
92
92
  TABLE
93
93
  end
94
94
 
@@ -97,9 +97,9 @@ class SlicerTest < Test::Unit::TestCase
97
97
  !slicer.count
98
98
  end
99
99
  assert_equal(<<-TABLE, sliced_table.to_s)
100
- count visible
101
- 0 0
102
- 1
100
+ count visible
101
+ 0 0 (null)
102
+ 1 (null) (null)
103
103
  TABLE
104
104
  end
105
105
  end
@@ -109,11 +109,11 @@ class SlicerTest < Test::Unit::TestCase
109
109
  slicer.visible.null?
110
110
  end
111
111
  assert_equal(<<-TABLE, sliced_table.to_s)
112
- count visible
113
- 0 0
114
- 1 4
115
- 2 64
116
- 3
112
+ count visible
113
+ 0 0 (null)
114
+ 1 4 (null)
115
+ 2 64 (null)
116
+ 3 (null) (null)
117
117
  TABLE
118
118
  end
119
119
 
@@ -138,11 +138,11 @@ class SlicerTest < Test::Unit::TestCase
138
138
  slicer.visible == nil
139
139
  end
140
140
  assert_equal(<<-TABLE, sliced_table.to_s)
141
- count visible
142
- 0 0
143
- 1 4
144
- 2 64
145
- 3
141
+ count visible
142
+ 0 0 (null)
143
+ 1 4 (null)
144
+ 2 64 (null)
145
+ 3 (null) (null)
146
146
  TABLE
147
147
  end
148
148
 
@@ -151,15 +151,15 @@ class SlicerTest < Test::Unit::TestCase
151
151
  slicer.visible == true
152
152
  end
153
153
  assert_equal(<<-TABLE, sliced_table.to_s)
154
- count visible
155
- 0
156
- 1 1 true
157
- 2
158
- 3 8 true
159
- 4 16 true
160
- 5
161
- 6
162
- 7 256 true
154
+ count visible
155
+ 0 (null) (null)
156
+ 1 1 true
157
+ 2 (null) (null)
158
+ 3 8 true
159
+ 4 16 true
160
+ 5 (null) (null)
161
+ 6 (null) (null)
162
+ 7 256 true
163
163
  TABLE
164
164
  end
165
165
  end
@@ -185,13 +185,13 @@ class SlicerTest < Test::Unit::TestCase
185
185
  !(slicer.visible == true)
186
186
  end
187
187
  assert_equal(<<-TABLE, sliced_table.to_s)
188
- count visible
189
- 0
190
- 1 2 false
191
- 2
192
- 3 32 false
193
- 4
194
- 5
188
+ count visible
189
+ 0 (null) (null)
190
+ 1 2 false
191
+ 2 (null) (null)
192
+ 3 32 false
193
+ 4 (null) (null)
194
+ 5 (null) (null)
195
195
  TABLE
196
196
  end
197
197
  end
@@ -217,13 +217,13 @@ class SlicerTest < Test::Unit::TestCase
217
217
  slicer.visible != true
218
218
  end
219
219
  assert_equal(<<-TABLE, sliced_table.to_s)
220
- count visible
221
- 0
222
- 1 2 false
223
- 2
224
- 3 32 false
225
- 4
226
- 5
220
+ count visible
221
+ 0 (null) (null)
222
+ 1 2 false
223
+ 2 (null) (null)
224
+ 3 32 false
225
+ 4 (null) (null)
226
+ 5 (null) (null)
227
227
  TABLE
228
228
  end
229
229
  end
@@ -233,13 +233,13 @@ class SlicerTest < Test::Unit::TestCase
233
233
  slicer.count < 16
234
234
  end
235
235
  assert_equal(<<-TABLE, sliced_table.to_s)
236
- count visible
237
- 0 0
238
- 1 1 true
239
- 2 2 false
240
- 3 4
241
- 4 8 true
242
- 5
236
+ count visible
237
+ 0 0 (null)
238
+ 1 1 true
239
+ 2 2 false
240
+ 3 4 (null)
241
+ 4 8 true
242
+ 5 (null) (null)
243
243
  TABLE
244
244
  end
245
245
 
@@ -248,12 +248,12 @@ class SlicerTest < Test::Unit::TestCase
248
248
  !(slicer.count < 16)
249
249
  end
250
250
  assert_equal(<<-TABLE, sliced_table.to_s)
251
- count visible
252
- 0 16 true
253
- 1 32 false
254
- 2 64
255
- 3
256
- 4 256 true
251
+ count visible
252
+ 0 16 true
253
+ 1 32 false
254
+ 2 64 (null)
255
+ 3 (null) (null)
256
+ 4 256 true
257
257
  TABLE
258
258
  end
259
259
 
@@ -262,14 +262,14 @@ class SlicerTest < Test::Unit::TestCase
262
262
  slicer.count <= 16
263
263
  end
264
264
  assert_equal(<<-TABLE, sliced_table.to_s)
265
- count visible
266
- 0 0
267
- 1 1 true
268
- 2 2 false
269
- 3 4
270
- 4 8 true
271
- 5 16 true
272
- 6
265
+ count visible
266
+ 0 0 (null)
267
+ 1 1 true
268
+ 2 2 false
269
+ 3 4 (null)
270
+ 4 8 true
271
+ 5 16 true
272
+ 6 (null) (null)
273
273
  TABLE
274
274
  end
275
275
 
@@ -278,11 +278,11 @@ class SlicerTest < Test::Unit::TestCase
278
278
  !(slicer.count <= 16)
279
279
  end
280
280
  assert_equal(<<-TABLE, sliced_table.to_s)
281
- count visible
282
- 0 32 false
283
- 1 64
284
- 2
285
- 3 256 true
281
+ count visible
282
+ 0 32 false
283
+ 1 64 (null)
284
+ 2 (null) (null)
285
+ 3 256 true
286
286
  TABLE
287
287
  end
288
288
 
@@ -291,11 +291,11 @@ class SlicerTest < Test::Unit::TestCase
291
291
  slicer.count > 16
292
292
  end
293
293
  assert_equal(<<-TABLE, sliced_table.to_s)
294
- count visible
295
- 0 32 false
296
- 1 64
297
- 2
298
- 3 256 true
294
+ count visible
295
+ 0 32 false
296
+ 1 64 (null)
297
+ 2 (null) (null)
298
+ 3 256 true
299
299
  TABLE
300
300
  end
301
301
 
@@ -304,14 +304,14 @@ class SlicerTest < Test::Unit::TestCase
304
304
  !(slicer.count > 16)
305
305
  end
306
306
  assert_equal(<<-TABLE, sliced_table.to_s)
307
- count visible
308
- 0 0
309
- 1 1 true
310
- 2 2 false
311
- 3 4
312
- 4 8 true
313
- 5 16 true
314
- 6
307
+ count visible
308
+ 0 0 (null)
309
+ 1 1 true
310
+ 2 2 false
311
+ 3 4 (null)
312
+ 4 8 true
313
+ 5 16 true
314
+ 6 (null) (null)
315
315
  TABLE
316
316
  end
317
317
 
@@ -320,12 +320,12 @@ class SlicerTest < Test::Unit::TestCase
320
320
  slicer.count >= 16
321
321
  end
322
322
  assert_equal(<<-TABLE, sliced_table.to_s)
323
- count visible
324
- 0 16 true
325
- 1 32 false
326
- 2 64
327
- 3
328
- 4 256 true
323
+ count visible
324
+ 0 16 true
325
+ 1 32 false
326
+ 2 64 (null)
327
+ 3 (null) (null)
328
+ 4 256 true
329
329
  TABLE
330
330
  end
331
331
 
@@ -334,13 +334,13 @@ class SlicerTest < Test::Unit::TestCase
334
334
  !(slicer.count >= 16)
335
335
  end
336
336
  assert_equal(<<-TABLE, sliced_table.to_s)
337
- count visible
338
- 0 0
339
- 1 1 true
340
- 2 2 false
341
- 3 4
342
- 4 8 true
343
- 5
337
+ count visible
338
+ 0 0 (null)
339
+ 1 1 true
340
+ 2 2 false
341
+ 3 4 (null)
342
+ 4 8 true
343
+ 5 (null) (null)
344
344
  TABLE
345
345
  end
346
346
 
@@ -351,10 +351,9 @@ class SlicerTest < Test::Unit::TestCase
351
351
  assert_equal(<<-TABLE, sliced_table.to_s)
352
352
  count visible
353
353
  0 1 true
354
- 1 4
354
+ 1 4 (null)
355
355
  2 16 true
356
- 3 64
357
- 4
356
+ 3 64 (null)
358
357
  TABLE
359
358
  end
360
359
 
@@ -363,13 +362,13 @@ class SlicerTest < Test::Unit::TestCase
363
362
  !slicer.count.in?([1, 4, 16, 64])
364
363
  end
365
364
  assert_equal(<<-TABLE, sliced_table.to_s)
366
- count visible
367
- 0 0
368
- 1 2 false
369
- 2 8 true
370
- 3 32 false
371
- 4
372
- 5 256 true
365
+ count visible
366
+ 0 0 (null)
367
+ 1 2 false
368
+ 2 8 true
369
+ 3 32 false
370
+ 4 (null) (null)
371
+ 5 256 true
373
372
  TABLE
374
373
  end
375
374
 
@@ -378,13 +377,13 @@ class SlicerTest < Test::Unit::TestCase
378
377
  slicer.visible & (slicer.count >= 16)
379
378
  end
380
379
  assert_equal(<<-TABLE, sliced_table.to_s)
381
- count visible
382
- 0
383
- 1
384
- 2 16 true
385
- 3
386
- 4
387
- 5 256 true
380
+ count visible
381
+ 0 (null) (null)
382
+ 1 (null) (null)
383
+ 2 16 true
384
+ 3 (null) (null)
385
+ 4 (null) (null)
386
+ 5 256 true
388
387
  TABLE
389
388
  end
390
389
 
@@ -393,16 +392,16 @@ class SlicerTest < Test::Unit::TestCase
393
392
  slicer.visible | (slicer.count >= 16)
394
393
  end
395
394
  assert_equal(<<-TABLE, sliced_table.to_s)
396
- count visible
397
- 0
398
- 1 1 true
399
- 2
400
- 3 8 true
401
- 4 16 true
402
- 5 32 false
403
- 6
404
- 7
405
- 8 256 true
395
+ count visible
396
+ 0 (null) (null)
397
+ 1 1 true
398
+ 2 (null) (null)
399
+ 3 8 true
400
+ 4 16 true
401
+ 5 32 false
402
+ 6 (null) (null)
403
+ 7 (null) (null)
404
+ 8 256 true
406
405
  TABLE
407
406
  end
408
407
 
@@ -411,14 +410,14 @@ class SlicerTest < Test::Unit::TestCase
411
410
  slicer.visible ^ (slicer.count >= 16)
412
411
  end
413
412
  assert_equal(<<-TABLE, sliced_table.to_s)
414
- count visible
415
- 0
416
- 1 1 true
417
- 2
418
- 3 8 true
419
- 4 32 false
420
- 5
421
- 6
413
+ count visible
414
+ 0 (null) (null)
415
+ 1 1 true
416
+ 2 (null) (null)
417
+ 3 8 true
418
+ 4 32 false
419
+ 5 (null) (null)
420
+ 6 (null) (null)
422
421
  TABLE
423
422
  end
424
423
 
@@ -429,15 +428,15 @@ class SlicerTest < Test::Unit::TestCase
429
428
  end
430
429
  end
431
430
  assert_equal(<<-TABLE, sliced_table.to_s)
432
- count visible
433
- 0 0
434
- 1 1 true
435
- 2 4
436
- 3 8 true
437
- 4 16 true
438
- 5 64
439
- 6
440
- 7 256 true
431
+ count visible
432
+ 0 0 (null)
433
+ 1 1 true
434
+ 2 4 (null)
435
+ 3 8 true
436
+ 4 16 true
437
+ 5 64 (null)
438
+ 6 (null) (null)
439
+ 7 256 true
441
440
  TABLE
442
441
  end
443
442
 
@@ -474,15 +473,15 @@ class SlicerTest < Test::Unit::TestCase
474
473
  end
475
474
  end
476
475
  assert_equal(<<-TABLE, sliced_table.to_s)
477
- count visible
478
- 0 0
479
- 1 1 true
480
- 2 4
481
- 3 8 true
482
- 4 16 true
483
- 5 64
484
- 6
485
- 7 256 true
476
+ count visible
477
+ 0 0 (null)
478
+ 1 1 true
479
+ 2 4 (null)
480
+ 3 8 true
481
+ 4 16 true
482
+ 5 64 (null)
483
+ 6 (null) (null)
484
+ 7 256 true
486
485
  TABLE
487
486
  end
488
487
  end
data/test/test-string-dictionary-array-builder.rb ADDED
@@ -0,0 +1,103 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ class StringDictionaryArrayBuilderTest < Test::Unit::TestCase
19
+ def setup
20
+ @builder = Arrow::StringDictionaryArrayBuilder.new
21
+ end
22
+
23
+ sub_test_case("#append_values") do
24
+ test("[nil]") do
25
+ @builder.append_values([nil])
26
+ array = @builder.finish
27
+ assert_equal([
28
+ [],
29
+ [nil],
30
+ ],
31
+ [
32
+ array.dictionary.to_a,
33
+ array.indices.to_a,
34
+ ])
35
+ end
36
+
37
+ test("[String]") do
38
+ @builder.append_values(["hello"])
39
+ array = @builder.finish
40
+ assert_equal([
41
+ ["hello"],
42
+ [0],
43
+ ],
44
+ [
45
+ array.dictionary.to_a,
46
+ array.indices.to_a,
47
+ ])
48
+ end
49
+
50
+ test("[Symbol]") do
51
+ @builder.append_values([:hello])
52
+ array = @builder.finish
53
+ assert_equal([
54
+ ["hello"],
55
+ [0],
56
+ ],
57
+ [
58
+ array.dictionary.to_a,
59
+ array.indices.to_a,
60
+ ])
61
+ end
62
+
63
+ test("[nil, String, Symbol]") do
64
+ @builder.append_values([
65
+ nil,
66
+ "Hello",
67
+ :world,
68
+ "world",
69
+ ])
70
+ array = @builder.finish
71
+ assert_equal([
72
+ ["Hello", "world"],
73
+ [nil, 0, 1, 1],
74
+ ],
75
+ [
76
+ array.dictionary.to_a,
77
+ array.indices.to_a,
78
+ ])
79
+ end
80
+
81
+ test("is_valids") do
82
+ @builder.append_values([
83
+ "Hello",
84
+ :world,
85
+ :goodbye,
86
+ ],
87
+ [
88
+ true,
89
+ false,
90
+ true,
91
+ ])
92
+ array = @builder.finish
93
+ assert_equal([
94
+ ["Hello", "goodbye"],
95
+ [0, nil, 1],
96
+ ],
97
+ [
98
+ array.dictionary.to_a,
99
+ array.indices.to_a,
100
+ ])
101
+ end
102
+ end
103
+ end