red-arrow 11.0.0 → 13.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/ext/arrow/arrow.cpp +25 -0
- data/ext/arrow/converters.hpp +12 -27
- data/ext/arrow/extconf.rb +2 -2
- data/lib/arrow/array-computable.rb +13 -0
- data/lib/arrow/chunked-array.rb +5 -1
- data/lib/arrow/data-type.rb +9 -0
- data/lib/arrow/dense-union-array-builder.rb +49 -0
- data/lib/arrow/dense-union-array.rb +26 -0
- data/lib/arrow/expression.rb +6 -2
- data/lib/arrow/function.rb +0 -1
- data/lib/arrow/loader.rb +5 -0
- data/lib/arrow/record-batch-file-reader.rb +2 -0
- data/lib/arrow/record-batch-stream-reader.rb +2 -0
- data/lib/arrow/scalar.rb +67 -0
- data/lib/arrow/slicer.rb +61 -0
- data/lib/arrow/sparse-union-array-builder.rb +56 -0
- data/lib/arrow/sparse-union-array.rb +26 -0
- data/lib/arrow/struct-array-builder.rb +0 -5
- data/lib/arrow/table.rb +130 -10
- data/lib/arrow/union-array-builder.rb +59 -0
- data/lib/arrow/version.rb +1 -1
- data/test/raw-records/test-dense-union-array.rb +90 -45
- data/test/raw-records/test-list-array.rb +28 -10
- data/test/raw-records/test-map-array.rb +39 -10
- data/test/raw-records/test-sparse-union-array.rb +86 -41
- data/test/raw-records/test-struct-array.rb +22 -8
- data/test/test-array.rb +7 -0
- data/test/test-chunked-array.rb +9 -0
- data/test/test-dense-union-array.rb +42 -0
- data/test/test-dense-union-data-type.rb +1 -1
- data/test/test-expression.rb +11 -0
- data/test/test-function.rb +7 -7
- data/test/test-group.rb +58 -58
- data/test/test-record-batch-file-reader.rb +21 -0
- data/test/test-record-batch-stream-reader.rb +129 -0
- data/test/test-scalar.rb +65 -0
- data/test/test-slicer.rb +194 -129
- data/test/test-sparse-union-array.rb +38 -0
- data/test/test-table.rb +207 -38
- data/test/values/test-dense-union-array.rb +88 -45
- data/test/values/test-list-array.rb +26 -10
- data/test/values/test-map-array.rb +33 -10
- data/test/values/test-sparse-union-array.rb +84 -41
- data/test/values/test-struct-array.rb +20 -8
- metadata +20 -7
data/test/test-slicer.rb
CHANGED
@@ -45,15 +45,11 @@ class SlicerTest < Test::Unit::TestCase
|
|
45
45
|
slicer.visible
|
46
46
|
end
|
47
47
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
48
|
-
|
49
|
-
0
|
50
|
-
1
|
51
|
-
2
|
52
|
-
3
|
53
|
-
4 16 true
|
54
|
-
5 (null) (null)
|
55
|
-
6 (null) (null)
|
56
|
-
7 256 true
|
48
|
+
count visible
|
49
|
+
0 1 true
|
50
|
+
1 8 true
|
51
|
+
2 16 true
|
52
|
+
3 256 true
|
57
53
|
TABLE
|
58
54
|
end
|
59
55
|
|
@@ -62,16 +58,15 @@ class SlicerTest < Test::Unit::TestCase
|
|
62
58
|
slicer.count
|
63
59
|
end
|
64
60
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
65
|
-
|
66
|
-
0
|
67
|
-
1
|
68
|
-
2
|
69
|
-
3
|
70
|
-
4
|
71
|
-
5
|
72
|
-
6
|
73
|
-
7
|
74
|
-
8 256 true
|
61
|
+
count visible
|
62
|
+
0 1 true
|
63
|
+
1 2 false
|
64
|
+
2 4 (null)
|
65
|
+
3 8 true
|
66
|
+
4 16 true
|
67
|
+
5 32 false
|
68
|
+
6 64 (null)
|
69
|
+
7 256 true
|
75
70
|
TABLE
|
76
71
|
end
|
77
72
|
end
|
@@ -82,13 +77,9 @@ class SlicerTest < Test::Unit::TestCase
|
|
82
77
|
!slicer.visible
|
83
78
|
end
|
84
79
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
85
|
-
|
86
|
-
0
|
87
|
-
1
|
88
|
-
2 (null) (null)
|
89
|
-
3 32 false
|
90
|
-
4 (null) (null)
|
91
|
-
5 (null) (null)
|
80
|
+
count visible
|
81
|
+
0 2 false
|
82
|
+
1 32 false
|
92
83
|
TABLE
|
93
84
|
end
|
94
85
|
|
@@ -97,9 +88,8 @@ class SlicerTest < Test::Unit::TestCase
|
|
97
88
|
!slicer.count
|
98
89
|
end
|
99
90
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
100
|
-
|
101
|
-
0
|
102
|
-
1 (null) (null)
|
91
|
+
count visible
|
92
|
+
0 0 (null)
|
103
93
|
TABLE
|
104
94
|
end
|
105
95
|
end
|
@@ -151,15 +141,11 @@ class SlicerTest < Test::Unit::TestCase
|
|
151
141
|
slicer.visible == true
|
152
142
|
end
|
153
143
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
154
|
-
|
155
|
-
0
|
156
|
-
1
|
157
|
-
2
|
158
|
-
3
|
159
|
-
4 16 true
|
160
|
-
5 (null) (null)
|
161
|
-
6 (null) (null)
|
162
|
-
7 256 true
|
144
|
+
count visible
|
145
|
+
0 1 true
|
146
|
+
1 8 true
|
147
|
+
2 16 true
|
148
|
+
3 256 true
|
163
149
|
TABLE
|
164
150
|
end
|
165
151
|
end
|
@@ -185,13 +171,9 @@ class SlicerTest < Test::Unit::TestCase
|
|
185
171
|
!(slicer.visible == true)
|
186
172
|
end
|
187
173
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
188
|
-
|
189
|
-
0
|
190
|
-
1
|
191
|
-
2 (null) (null)
|
192
|
-
3 32 false
|
193
|
-
4 (null) (null)
|
194
|
-
5 (null) (null)
|
174
|
+
count visible
|
175
|
+
0 2 false
|
176
|
+
1 32 false
|
195
177
|
TABLE
|
196
178
|
end
|
197
179
|
end
|
@@ -217,13 +199,9 @@ class SlicerTest < Test::Unit::TestCase
|
|
217
199
|
slicer.visible != true
|
218
200
|
end
|
219
201
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
220
|
-
|
221
|
-
0
|
222
|
-
1
|
223
|
-
2 (null) (null)
|
224
|
-
3 32 false
|
225
|
-
4 (null) (null)
|
226
|
-
5 (null) (null)
|
202
|
+
count visible
|
203
|
+
0 2 false
|
204
|
+
1 32 false
|
227
205
|
TABLE
|
228
206
|
end
|
229
207
|
end
|
@@ -233,13 +211,12 @@ class SlicerTest < Test::Unit::TestCase
|
|
233
211
|
slicer.count < 16
|
234
212
|
end
|
235
213
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
236
|
-
|
237
|
-
0
|
238
|
-
1
|
239
|
-
2
|
240
|
-
3
|
241
|
-
4
|
242
|
-
5 (null) (null)
|
214
|
+
count visible
|
215
|
+
0 0 (null)
|
216
|
+
1 1 true
|
217
|
+
2 2 false
|
218
|
+
3 4 (null)
|
219
|
+
4 8 true
|
243
220
|
TABLE
|
244
221
|
end
|
245
222
|
|
@@ -248,12 +225,11 @@ class SlicerTest < Test::Unit::TestCase
|
|
248
225
|
!(slicer.count < 16)
|
249
226
|
end
|
250
227
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
251
|
-
|
252
|
-
0
|
253
|
-
1
|
254
|
-
2
|
255
|
-
3
|
256
|
-
4 256 true
|
228
|
+
count visible
|
229
|
+
0 16 true
|
230
|
+
1 32 false
|
231
|
+
2 64 (null)
|
232
|
+
3 256 true
|
257
233
|
TABLE
|
258
234
|
end
|
259
235
|
|
@@ -262,14 +238,13 @@ class SlicerTest < Test::Unit::TestCase
|
|
262
238
|
slicer.count <= 16
|
263
239
|
end
|
264
240
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
265
|
-
|
266
|
-
0
|
267
|
-
1
|
268
|
-
2
|
269
|
-
3
|
270
|
-
4
|
271
|
-
5
|
272
|
-
6 (null) (null)
|
241
|
+
count visible
|
242
|
+
0 0 (null)
|
243
|
+
1 1 true
|
244
|
+
2 2 false
|
245
|
+
3 4 (null)
|
246
|
+
4 8 true
|
247
|
+
5 16 true
|
273
248
|
TABLE
|
274
249
|
end
|
275
250
|
|
@@ -278,11 +253,10 @@ class SlicerTest < Test::Unit::TestCase
|
|
278
253
|
!(slicer.count <= 16)
|
279
254
|
end
|
280
255
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
281
|
-
|
282
|
-
0
|
283
|
-
1
|
284
|
-
2
|
285
|
-
3 256 true
|
256
|
+
count visible
|
257
|
+
0 32 false
|
258
|
+
1 64 (null)
|
259
|
+
2 256 true
|
286
260
|
TABLE
|
287
261
|
end
|
288
262
|
|
@@ -291,11 +265,10 @@ class SlicerTest < Test::Unit::TestCase
|
|
291
265
|
slicer.count > 16
|
292
266
|
end
|
293
267
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
294
|
-
|
295
|
-
0
|
296
|
-
1
|
297
|
-
2
|
298
|
-
3 256 true
|
268
|
+
count visible
|
269
|
+
0 32 false
|
270
|
+
1 64 (null)
|
271
|
+
2 256 true
|
299
272
|
TABLE
|
300
273
|
end
|
301
274
|
|
@@ -304,14 +277,13 @@ class SlicerTest < Test::Unit::TestCase
|
|
304
277
|
!(slicer.count > 16)
|
305
278
|
end
|
306
279
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
307
|
-
|
308
|
-
0
|
309
|
-
1
|
310
|
-
2
|
311
|
-
3
|
312
|
-
4
|
313
|
-
5
|
314
|
-
6 (null) (null)
|
280
|
+
count visible
|
281
|
+
0 0 (null)
|
282
|
+
1 1 true
|
283
|
+
2 2 false
|
284
|
+
3 4 (null)
|
285
|
+
4 8 true
|
286
|
+
5 16 true
|
315
287
|
TABLE
|
316
288
|
end
|
317
289
|
|
@@ -320,12 +292,11 @@ class SlicerTest < Test::Unit::TestCase
|
|
320
292
|
slicer.count >= 16
|
321
293
|
end
|
322
294
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
323
|
-
|
324
|
-
0
|
325
|
-
1
|
326
|
-
2
|
327
|
-
3
|
328
|
-
4 256 true
|
295
|
+
count visible
|
296
|
+
0 16 true
|
297
|
+
1 32 false
|
298
|
+
2 64 (null)
|
299
|
+
3 256 true
|
329
300
|
TABLE
|
330
301
|
end
|
331
302
|
|
@@ -334,13 +305,12 @@ class SlicerTest < Test::Unit::TestCase
|
|
334
305
|
!(slicer.count >= 16)
|
335
306
|
end
|
336
307
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
337
|
-
|
338
|
-
0
|
339
|
-
1
|
340
|
-
2
|
341
|
-
3
|
342
|
-
4
|
343
|
-
5 (null) (null)
|
308
|
+
count visible
|
309
|
+
0 0 (null)
|
310
|
+
1 1 true
|
311
|
+
2 2 false
|
312
|
+
3 4 (null)
|
313
|
+
4 8 true
|
344
314
|
TABLE
|
345
315
|
end
|
346
316
|
|
@@ -377,13 +347,9 @@ class SlicerTest < Test::Unit::TestCase
|
|
377
347
|
slicer.visible & (slicer.count >= 16)
|
378
348
|
end
|
379
349
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
380
|
-
|
381
|
-
0
|
382
|
-
1
|
383
|
-
2 16 true
|
384
|
-
3 (null) (null)
|
385
|
-
4 (null) (null)
|
386
|
-
5 256 true
|
350
|
+
count visible
|
351
|
+
0 16 true
|
352
|
+
1 256 true
|
387
353
|
TABLE
|
388
354
|
end
|
389
355
|
|
@@ -392,16 +358,12 @@ class SlicerTest < Test::Unit::TestCase
|
|
392
358
|
slicer.visible | (slicer.count >= 16)
|
393
359
|
end
|
394
360
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
395
|
-
|
396
|
-
0
|
397
|
-
1
|
398
|
-
2
|
399
|
-
3
|
400
|
-
4
|
401
|
-
5 32 false
|
402
|
-
6 (null) (null)
|
403
|
-
7 (null) (null)
|
404
|
-
8 256 true
|
361
|
+
count visible
|
362
|
+
0 1 true
|
363
|
+
1 8 true
|
364
|
+
2 16 true
|
365
|
+
3 32 false
|
366
|
+
4 256 true
|
405
367
|
TABLE
|
406
368
|
end
|
407
369
|
|
@@ -410,14 +372,10 @@ class SlicerTest < Test::Unit::TestCase
|
|
410
372
|
slicer.visible ^ (slicer.count >= 16)
|
411
373
|
end
|
412
374
|
assert_equal(<<-TABLE, sliced_table.to_s)
|
413
|
-
|
414
|
-
0
|
415
|
-
1
|
416
|
-
2
|
417
|
-
3 8 true
|
418
|
-
4 32 false
|
419
|
-
5 (null) (null)
|
420
|
-
6 (null) (null)
|
375
|
+
count visible
|
376
|
+
0 1 true
|
377
|
+
1 8 true
|
378
|
+
2 32 false
|
421
379
|
TABLE
|
422
380
|
end
|
423
381
|
|
@@ -484,4 +442,111 @@ class SlicerTest < Test::Unit::TestCase
|
|
484
442
|
7 256 true
|
485
443
|
TABLE
|
486
444
|
end
|
445
|
+
|
446
|
+
sub_test_case "MatchSubstringOptions family" do
|
447
|
+
def setup
|
448
|
+
@table = Arrow::Table.new(
|
449
|
+
string: ["array", "Arrow", "carrot", nil, "window"]
|
450
|
+
)
|
451
|
+
end
|
452
|
+
|
453
|
+
test("end_with?") do
|
454
|
+
sliced_table = @table.slice do |slicer|
|
455
|
+
slicer.string.end_with?("ow")
|
456
|
+
end
|
457
|
+
assert_equal(<<~TABLE, sliced_table.to_s)
|
458
|
+
string
|
459
|
+
0 Arrow
|
460
|
+
1 window
|
461
|
+
TABLE
|
462
|
+
end
|
463
|
+
|
464
|
+
test("match_like?") do
|
465
|
+
sliced_table = @table.slice do |slicer|
|
466
|
+
slicer.string.match_like?("_rr%")
|
467
|
+
end
|
468
|
+
assert_equal(<<~TABLE, sliced_table.to_s)
|
469
|
+
string
|
470
|
+
0 array
|
471
|
+
1 Arrow
|
472
|
+
TABLE
|
473
|
+
end
|
474
|
+
|
475
|
+
test("match_substring?") do
|
476
|
+
sliced_table = @table.slice do |slicer|
|
477
|
+
slicer.string.match_substring?("arr")
|
478
|
+
end
|
479
|
+
assert_equal(<<~TABLE, sliced_table.to_s)
|
480
|
+
string
|
481
|
+
0 array
|
482
|
+
1 carrot
|
483
|
+
TABLE
|
484
|
+
end
|
485
|
+
|
486
|
+
test("match_substring?(ignore_case:)") do
|
487
|
+
sliced_table = @table.slice do |slicer|
|
488
|
+
slicer.string.match_substring?("arr", ignore_case: true)
|
489
|
+
end
|
490
|
+
assert_equal(<<~TABLE, sliced_table.to_s)
|
491
|
+
string
|
492
|
+
0 array
|
493
|
+
1 Arrow
|
494
|
+
2 carrot
|
495
|
+
TABLE
|
496
|
+
end
|
497
|
+
|
498
|
+
test("!match_substring?") do
|
499
|
+
sliced_table = @table.slice do |slicer|
|
500
|
+
!slicer.string.match_substring?("arr")
|
501
|
+
end
|
502
|
+
assert_equal(<<~TABLE, sliced_table.to_s)
|
503
|
+
string
|
504
|
+
0 Arrow
|
505
|
+
1 window
|
506
|
+
TABLE
|
507
|
+
end
|
508
|
+
|
509
|
+
test("match_substring?(Regexp)") do
|
510
|
+
sliced_table = @table.slice do |slicer|
|
511
|
+
slicer.string.match_substring?(/[dr]ow/)
|
512
|
+
end
|
513
|
+
assert_equal(<<~TABLE, sliced_table.to_s)
|
514
|
+
string
|
515
|
+
0 Arrow
|
516
|
+
1 window
|
517
|
+
TABLE
|
518
|
+
end
|
519
|
+
|
520
|
+
test("match_substring?(/String/i)") do
|
521
|
+
sliced_table = @table.slice do |slicer|
|
522
|
+
slicer.string.match_substring?(/arr/i)
|
523
|
+
end
|
524
|
+
assert_equal(<<~TABLE, sliced_table.to_s)
|
525
|
+
string
|
526
|
+
0 array
|
527
|
+
1 Arrow
|
528
|
+
2 carrot
|
529
|
+
TABLE
|
530
|
+
end
|
531
|
+
|
532
|
+
test("match_substring? - invalid") do
|
533
|
+
message =
|
534
|
+
'pattern must be either String or Regexp: ["arr"]'
|
535
|
+
assert_raise(ArgumentError.new(message)) do
|
536
|
+
@table.slice do |slicer|
|
537
|
+
slicer.string.match_substring?(["arr"])
|
538
|
+
end
|
539
|
+
end
|
540
|
+
end
|
541
|
+
|
542
|
+
test("start_with?") do
|
543
|
+
sliced_table = @table.slice do |slicer|
|
544
|
+
slicer.string.start_with?("ca")
|
545
|
+
end
|
546
|
+
assert_equal(<<~TABLE, sliced_table.to_s)
|
547
|
+
string
|
548
|
+
0 carrot
|
549
|
+
TABLE
|
550
|
+
end
|
551
|
+
end
|
487
552
|
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
class SparseUnionArrayTest < Test::Unit::TestCase
|
19
|
+
def setup
|
20
|
+
data_type_fields = [
|
21
|
+
Arrow::Field.new("number", :int16),
|
22
|
+
Arrow::Field.new("text", :string),
|
23
|
+
]
|
24
|
+
type_codes = [11, 13]
|
25
|
+
@data_type = Arrow::SparseUnionDataType.new(data_type_fields, type_codes)
|
26
|
+
type_ids = Arrow::Int8Array.new([11, 13, 11, 13, 11])
|
27
|
+
fields = [
|
28
|
+
Arrow::Int16Array.new([1, nil, nil, nil, 5]),
|
29
|
+
Arrow::StringArray.new([nil, "b", nil, "d", nil]),
|
30
|
+
]
|
31
|
+
@array = Arrow::SparseUnionArray.new(@data_type, type_ids, fields)
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_get_value
|
35
|
+
assert_equal([1, "b", nil, "d", 5],
|
36
|
+
@array.length.times.collect {|i| @array[i]})
|
37
|
+
end
|
38
|
+
end
|