red-arrow 16.1.0 → 18.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (21) hide show
  1. checksums.yaml +4 -4
  2. data/ext/arrow/extconf.rb +8 -1
  3. data/lib/arrow/field-containable.rb +1 -1
  4. data/lib/arrow/loader.rb +12 -0
  5. data/lib/arrow/stream-decoder.rb +29 -0
  6. data/lib/arrow/stream-listener.rb +47 -0
  7. data/lib/arrow/table-formatter.rb +33 -7
  8. data/lib/arrow/table-list-formatter.rb +3 -3
  9. data/lib/arrow/table-table-formatter.rb +7 -0
  10. data/lib/arrow/version.rb +1 -1
  11. data/red-arrow.gemspec +1 -1
  12. data/test/test-csv-loader.rb +40 -32
  13. data/test/test-decimal128-array.rb +6 -0
  14. data/test/test-decimal256-array.rb +6 -0
  15. data/test/test-group.rb +13 -0
  16. data/test/test-schema.rb +1 -1
  17. data/test/test-slicer.rb +191 -154
  18. data/test/test-stream-listener.rb +60 -0
  19. data/test/test-struct-data-type.rb +1 -1
  20. data/test/test-table.rb +305 -230
  21. metadata +13 -9
data/test/test-slicer.rb CHANGED
@@ -45,11 +45,12 @@ class SlicerTest < Test::Unit::TestCase
45
45
  slicer.visible
46
46
  end
47
47
  assert_equal(<<-TABLE, sliced_table.to_s)
48
- count visible
49
- 0 1 true
50
- 1 8 true
51
- 2 16 true
52
- 3 256 true
48
+ count visible
49
+ (uint32) (bool)
50
+ 0 1 true
51
+ 1 8 true
52
+ 2 16 true
53
+ 3 256 true
53
54
  TABLE
54
55
  end
55
56
 
@@ -58,15 +59,16 @@ class SlicerTest < Test::Unit::TestCase
58
59
  slicer.count
59
60
  end
60
61
  assert_equal(<<-TABLE, sliced_table.to_s)
61
- count visible
62
- 0 1 true
63
- 1 2 false
64
- 2 4 (null)
65
- 3 8 true
66
- 4 16 true
67
- 5 32 false
68
- 6 64 (null)
69
- 7 256 true
62
+ count visible
63
+ (uint32) (bool)
64
+ 0 1 true
65
+ 1 2 false
66
+ 2 4 (null)
67
+ 3 8 true
68
+ 4 16 true
69
+ 5 32 false
70
+ 6 64 (null)
71
+ 7 256 true
70
72
  TABLE
71
73
  end
72
74
  end
@@ -77,9 +79,10 @@ class SlicerTest < Test::Unit::TestCase
77
79
  !slicer.visible
78
80
  end
79
81
  assert_equal(<<-TABLE, sliced_table.to_s)
80
- count visible
81
- 0 2 false
82
- 1 32 false
82
+ count visible
83
+ (uint32) (bool)
84
+ 0 2 false
85
+ 1 32 false
83
86
  TABLE
84
87
  end
85
88
 
@@ -88,8 +91,9 @@ class SlicerTest < Test::Unit::TestCase
88
91
  !slicer.count
89
92
  end
90
93
  assert_equal(<<-TABLE, sliced_table.to_s)
91
- count visible
92
- 0 0 (null)
94
+ count visible
95
+ (uint32) (bool)
96
+ 0 0 (null)
93
97
  TABLE
94
98
  end
95
99
  end
@@ -99,11 +103,12 @@ class SlicerTest < Test::Unit::TestCase
99
103
  slicer.visible.null?
100
104
  end
101
105
  assert_equal(<<-TABLE, sliced_table.to_s)
102
- count visible
103
- 0 0 (null)
104
- 1 4 (null)
105
- 2 64 (null)
106
- 3 (null) (null)
106
+ count visible
107
+ (uint32) (bool)
108
+ 0 0 (null)
109
+ 1 4 (null)
110
+ 2 64 (null)
111
+ 3 (null) (null)
107
112
  TABLE
108
113
  end
109
114
 
@@ -112,13 +117,14 @@ class SlicerTest < Test::Unit::TestCase
112
117
  slicer.visible.valid?
113
118
  end
114
119
  assert_equal(<<-TABLE, sliced_table.to_s)
115
- count visible
116
- 0 1 true
117
- 1 2 false
118
- 2 8 true
119
- 3 16 true
120
- 4 32 false
121
- 5 256 true
120
+ count visible
121
+ (uint32) (bool)
122
+ 0 1 true
123
+ 1 2 false
124
+ 2 8 true
125
+ 3 16 true
126
+ 4 32 false
127
+ 5 256 true
122
128
  TABLE
123
129
  end
124
130
 
@@ -128,11 +134,12 @@ class SlicerTest < Test::Unit::TestCase
128
134
  slicer.visible == nil
129
135
  end
130
136
  assert_equal(<<-TABLE, sliced_table.to_s)
131
- count visible
132
- 0 0 (null)
133
- 1 4 (null)
134
- 2 64 (null)
135
- 3 (null) (null)
137
+ count visible
138
+ (uint32) (bool)
139
+ 0 0 (null)
140
+ 1 4 (null)
141
+ 2 64 (null)
142
+ 3 (null) (null)
136
143
  TABLE
137
144
  end
138
145
 
@@ -141,11 +148,12 @@ class SlicerTest < Test::Unit::TestCase
141
148
  slicer.visible == true
142
149
  end
143
150
  assert_equal(<<-TABLE, sliced_table.to_s)
144
- count visible
145
- 0 1 true
146
- 1 8 true
147
- 2 16 true
148
- 3 256 true
151
+ count visible
152
+ (uint32) (bool)
153
+ 0 1 true
154
+ 1 8 true
155
+ 2 16 true
156
+ 3 256 true
149
157
  TABLE
150
158
  end
151
159
  end
@@ -156,13 +164,14 @@ class SlicerTest < Test::Unit::TestCase
156
164
  !(slicer.visible == nil)
157
165
  end
158
166
  assert_equal(<<-TABLE, sliced_table.to_s)
159
- count visible
160
- 0 1 true
161
- 1 2 false
162
- 2 8 true
163
- 3 16 true
164
- 4 32 false
165
- 5 256 true
167
+ count visible
168
+ (uint32) (bool)
169
+ 0 1 true
170
+ 1 2 false
171
+ 2 8 true
172
+ 3 16 true
173
+ 4 32 false
174
+ 5 256 true
166
175
  TABLE
167
176
  end
168
177
 
@@ -171,9 +180,10 @@ class SlicerTest < Test::Unit::TestCase
171
180
  !(slicer.visible == true)
172
181
  end
173
182
  assert_equal(<<-TABLE, sliced_table.to_s)
174
- count visible
175
- 0 2 false
176
- 1 32 false
183
+ count visible
184
+ (uint32) (bool)
185
+ 0 2 false
186
+ 1 32 false
177
187
  TABLE
178
188
  end
179
189
  end
@@ -184,13 +194,14 @@ class SlicerTest < Test::Unit::TestCase
184
194
  slicer.visible != nil
185
195
  end
186
196
  assert_equal(<<-TABLE, sliced_table.to_s)
187
- count visible
188
- 0 1 true
189
- 1 2 false
190
- 2 8 true
191
- 3 16 true
192
- 4 32 false
193
- 5 256 true
197
+ count visible
198
+ (uint32) (bool)
199
+ 0 1 true
200
+ 1 2 false
201
+ 2 8 true
202
+ 3 16 true
203
+ 4 32 false
204
+ 5 256 true
194
205
  TABLE
195
206
  end
196
207
 
@@ -199,9 +210,10 @@ class SlicerTest < Test::Unit::TestCase
199
210
  slicer.visible != true
200
211
  end
201
212
  assert_equal(<<-TABLE, sliced_table.to_s)
202
- count visible
203
- 0 2 false
204
- 1 32 false
213
+ count visible
214
+ (uint32) (bool)
215
+ 0 2 false
216
+ 1 32 false
205
217
  TABLE
206
218
  end
207
219
  end
@@ -211,12 +223,13 @@ class SlicerTest < Test::Unit::TestCase
211
223
  slicer.count < 16
212
224
  end
213
225
  assert_equal(<<-TABLE, sliced_table.to_s)
214
- count visible
215
- 0 0 (null)
216
- 1 1 true
217
- 2 2 false
218
- 3 4 (null)
219
- 4 8 true
226
+ count visible
227
+ (uint32) (bool)
228
+ 0 0 (null)
229
+ 1 1 true
230
+ 2 2 false
231
+ 3 4 (null)
232
+ 4 8 true
220
233
  TABLE
221
234
  end
222
235
 
@@ -225,11 +238,12 @@ class SlicerTest < Test::Unit::TestCase
225
238
  !(slicer.count < 16)
226
239
  end
227
240
  assert_equal(<<-TABLE, sliced_table.to_s)
228
- count visible
229
- 0 16 true
230
- 1 32 false
231
- 2 64 (null)
232
- 3 256 true
241
+ count visible
242
+ (uint32) (bool)
243
+ 0 16 true
244
+ 1 32 false
245
+ 2 64 (null)
246
+ 3 256 true
233
247
  TABLE
234
248
  end
235
249
 
@@ -238,13 +252,14 @@ class SlicerTest < Test::Unit::TestCase
238
252
  slicer.count <= 16
239
253
  end
240
254
  assert_equal(<<-TABLE, sliced_table.to_s)
241
- count visible
242
- 0 0 (null)
243
- 1 1 true
244
- 2 2 false
245
- 3 4 (null)
246
- 4 8 true
247
- 5 16 true
255
+ count visible
256
+ (uint32) (bool)
257
+ 0 0 (null)
258
+ 1 1 true
259
+ 2 2 false
260
+ 3 4 (null)
261
+ 4 8 true
262
+ 5 16 true
248
263
  TABLE
249
264
  end
250
265
 
@@ -253,10 +268,11 @@ class SlicerTest < Test::Unit::TestCase
253
268
  !(slicer.count <= 16)
254
269
  end
255
270
  assert_equal(<<-TABLE, sliced_table.to_s)
256
- count visible
257
- 0 32 false
258
- 1 64 (null)
259
- 2 256 true
271
+ count visible
272
+ (uint32) (bool)
273
+ 0 32 false
274
+ 1 64 (null)
275
+ 2 256 true
260
276
  TABLE
261
277
  end
262
278
 
@@ -265,10 +281,11 @@ class SlicerTest < Test::Unit::TestCase
265
281
  slicer.count > 16
266
282
  end
267
283
  assert_equal(<<-TABLE, sliced_table.to_s)
268
- count visible
269
- 0 32 false
270
- 1 64 (null)
271
- 2 256 true
284
+ count visible
285
+ (uint32) (bool)
286
+ 0 32 false
287
+ 1 64 (null)
288
+ 2 256 true
272
289
  TABLE
273
290
  end
274
291
 
@@ -277,13 +294,14 @@ class SlicerTest < Test::Unit::TestCase
277
294
  !(slicer.count > 16)
278
295
  end
279
296
  assert_equal(<<-TABLE, sliced_table.to_s)
280
- count visible
281
- 0 0 (null)
282
- 1 1 true
283
- 2 2 false
284
- 3 4 (null)
285
- 4 8 true
286
- 5 16 true
297
+ count visible
298
+ (uint32) (bool)
299
+ 0 0 (null)
300
+ 1 1 true
301
+ 2 2 false
302
+ 3 4 (null)
303
+ 4 8 true
304
+ 5 16 true
287
305
  TABLE
288
306
  end
289
307
 
@@ -292,11 +310,12 @@ class SlicerTest < Test::Unit::TestCase
292
310
  slicer.count >= 16
293
311
  end
294
312
  assert_equal(<<-TABLE, sliced_table.to_s)
295
- count visible
296
- 0 16 true
297
- 1 32 false
298
- 2 64 (null)
299
- 3 256 true
313
+ count visible
314
+ (uint32) (bool)
315
+ 0 16 true
316
+ 1 32 false
317
+ 2 64 (null)
318
+ 3 256 true
300
319
  TABLE
301
320
  end
302
321
 
@@ -305,12 +324,13 @@ class SlicerTest < Test::Unit::TestCase
305
324
  !(slicer.count >= 16)
306
325
  end
307
326
  assert_equal(<<-TABLE, sliced_table.to_s)
308
- count visible
309
- 0 0 (null)
310
- 1 1 true
311
- 2 2 false
312
- 3 4 (null)
313
- 4 8 true
327
+ count visible
328
+ (uint32) (bool)
329
+ 0 0 (null)
330
+ 1 1 true
331
+ 2 2 false
332
+ 3 4 (null)
333
+ 4 8 true
314
334
  TABLE
315
335
  end
316
336
 
@@ -319,11 +339,12 @@ class SlicerTest < Test::Unit::TestCase
319
339
  slicer.count.in?([1, 4, 16, 64])
320
340
  end
321
341
  assert_equal(<<-TABLE, sliced_table.to_s)
322
- count visible
323
- 0 1 true
324
- 1 4 (null)
325
- 2 16 true
326
- 3 64 (null)
342
+ count visible
343
+ (uint32) (bool)
344
+ 0 1 true
345
+ 1 4 (null)
346
+ 2 16 true
347
+ 3 64 (null)
327
348
  TABLE
328
349
  end
329
350
 
@@ -332,13 +353,14 @@ class SlicerTest < Test::Unit::TestCase
332
353
  !slicer.count.in?([1, 4, 16, 64])
333
354
  end
334
355
  assert_equal(<<-TABLE, sliced_table.to_s)
335
- count visible
336
- 0 0 (null)
337
- 1 2 false
338
- 2 8 true
339
- 3 32 false
340
- 4 (null) (null)
341
- 5 256 true
356
+ count visible
357
+ (uint32) (bool)
358
+ 0 0 (null)
359
+ 1 2 false
360
+ 2 8 true
361
+ 3 32 false
362
+ 4 (null) (null)
363
+ 5 256 true
342
364
  TABLE
343
365
  end
344
366
 
@@ -347,9 +369,10 @@ class SlicerTest < Test::Unit::TestCase
347
369
  slicer.visible & (slicer.count >= 16)
348
370
  end
349
371
  assert_equal(<<-TABLE, sliced_table.to_s)
350
- count visible
351
- 0 16 true
352
- 1 256 true
372
+ count visible
373
+ (uint32) (bool)
374
+ 0 16 true
375
+ 1 256 true
353
376
  TABLE
354
377
  end
355
378
 
@@ -358,12 +381,13 @@ class SlicerTest < Test::Unit::TestCase
358
381
  slicer.visible | (slicer.count >= 16)
359
382
  end
360
383
  assert_equal(<<-TABLE, sliced_table.to_s)
361
- count visible
362
- 0 1 true
363
- 1 8 true
364
- 2 16 true
365
- 3 32 false
366
- 4 256 true
384
+ count visible
385
+ (uint32) (bool)
386
+ 0 1 true
387
+ 1 8 true
388
+ 2 16 true
389
+ 3 32 false
390
+ 4 256 true
367
391
  TABLE
368
392
  end
369
393
 
@@ -372,10 +396,11 @@ class SlicerTest < Test::Unit::TestCase
372
396
  slicer.visible ^ (slicer.count >= 16)
373
397
  end
374
398
  assert_equal(<<-TABLE, sliced_table.to_s)
375
- count visible
376
- 0 1 true
377
- 1 8 true
378
- 2 32 false
399
+ count visible
400
+ (uint32) (bool)
401
+ 0 1 true
402
+ 1 8 true
403
+ 2 32 false
379
404
  TABLE
380
405
  end
381
406
 
@@ -386,15 +411,16 @@ class SlicerTest < Test::Unit::TestCase
386
411
  end
387
412
  end
388
413
  assert_equal(<<-TABLE, sliced_table.to_s)
389
- count visible
390
- 0 0 (null)
391
- 1 1 true
392
- 2 4 (null)
393
- 3 8 true
394
- 4 16 true
395
- 5 64 (null)
396
- 6 (null) (null)
397
- 7 256 true
414
+ count visible
415
+ (uint32) (bool)
416
+ 0 0 (null)
417
+ 1 1 true
418
+ 2 4 (null)
419
+ 3 8 true
420
+ 4 16 true
421
+ 5 64 (null)
422
+ 6 (null) (null)
423
+ 7 256 true
398
424
  TABLE
399
425
  end
400
426
 
@@ -405,9 +431,10 @@ class SlicerTest < Test::Unit::TestCase
405
431
  end
406
432
  end
407
433
  assert_equal(<<-TABLE, sliced_table.to_s)
408
- count visible
409
- 0 2 false
410
- 1 32 false
434
+ count visible
435
+ (uint32) (bool)
436
+ 0 2 false
437
+ 1 32 false
411
438
  TABLE
412
439
  end
413
440
 
@@ -418,9 +445,10 @@ class SlicerTest < Test::Unit::TestCase
418
445
  end
419
446
  end
420
447
  assert_equal(<<-TABLE, sliced_table.to_s)
421
- count visible
422
- 0 2 false
423
- 1 32 false
448
+ count visible
449
+ (uint32) (bool)
450
+ 0 2 false
451
+ 1 32 false
424
452
  TABLE
425
453
  end
426
454
 
@@ -431,15 +459,16 @@ class SlicerTest < Test::Unit::TestCase
431
459
  end
432
460
  end
433
461
  assert_equal(<<-TABLE, sliced_table.to_s)
434
- count visible
435
- 0 0 (null)
436
- 1 1 true
437
- 2 4 (null)
438
- 3 8 true
439
- 4 16 true
440
- 5 64 (null)
441
- 6 (null) (null)
442
- 7 256 true
462
+ count visible
463
+ (uint32) (bool)
464
+ 0 0 (null)
465
+ 1 1 true
466
+ 2 4 (null)
467
+ 3 8 true
468
+ 4 16 true
469
+ 5 64 (null)
470
+ 6 (null) (null)
471
+ 7 256 true
443
472
  TABLE
444
473
  end
445
474
 
@@ -456,6 +485,7 @@ class SlicerTest < Test::Unit::TestCase
456
485
  end
457
486
  assert_equal(<<~TABLE, sliced_table.to_s)
458
487
  string
488
+ (utf8)
459
489
  0 Arrow
460
490
  1 window
461
491
  TABLE
@@ -467,6 +497,7 @@ class SlicerTest < Test::Unit::TestCase
467
497
  end
468
498
  assert_equal(<<~TABLE, sliced_table.to_s)
469
499
  string
500
+ (utf8)
470
501
  0 array
471
502
  1 Arrow
472
503
  TABLE
@@ -478,6 +509,7 @@ class SlicerTest < Test::Unit::TestCase
478
509
  end
479
510
  assert_equal(<<~TABLE, sliced_table.to_s)
480
511
  string
512
+ (utf8)
481
513
  0 array
482
514
  1 carrot
483
515
  TABLE
@@ -489,6 +521,7 @@ class SlicerTest < Test::Unit::TestCase
489
521
  end
490
522
  assert_equal(<<~TABLE, sliced_table.to_s)
491
523
  string
524
+ (utf8)
492
525
  0 array
493
526
  1 Arrow
494
527
  2 carrot
@@ -501,6 +534,7 @@ class SlicerTest < Test::Unit::TestCase
501
534
  end
502
535
  assert_equal(<<~TABLE, sliced_table.to_s)
503
536
  string
537
+ (utf8)
504
538
  0 Arrow
505
539
  1 window
506
540
  TABLE
@@ -512,6 +546,7 @@ class SlicerTest < Test::Unit::TestCase
512
546
  end
513
547
  assert_equal(<<~TABLE, sliced_table.to_s)
514
548
  string
549
+ (utf8)
515
550
  0 Arrow
516
551
  1 window
517
552
  TABLE
@@ -523,6 +558,7 @@ class SlicerTest < Test::Unit::TestCase
523
558
  end
524
559
  assert_equal(<<~TABLE, sliced_table.to_s)
525
560
  string
561
+ (utf8)
526
562
  0 array
527
563
  1 Arrow
528
564
  2 carrot
@@ -545,6 +581,7 @@ class SlicerTest < Test::Unit::TestCase
545
581
  end
546
582
  assert_equal(<<~TABLE, sliced_table.to_s)
547
583
  string
584
+ (utf8)
548
585
  0 carrot
549
586
  TABLE
550
587
  end
data/test/test-stream-listener.rb ADDED
@@ -0,0 +1,60 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ class TestStreamListener < Test::Unit::TestCase
19
+ class Listener < Arrow::StreamListener
20
+ attr_reader :events
21
+ def initialize
22
+ super
23
+ @events = []
24
+ end
25
+
26
+ def on_eos
27
+ @events << [:eos]
28
+ end
29
+
30
+ def on_record_batch_decoded(record_batch, metadata)
31
+ @events << [:record_batch_decoded, record_batch, metadata]
32
+ end
33
+
34
+ def on_schema_decoded(schema, filtered_schema)
35
+ @events << [:schema_decoded, schema, filtered_schema]
36
+ end
37
+ end
38
+
39
+ def setup
40
+ @record_batch = Arrow::RecordBatch.new(enabled: [true, false, nil, true])
41
+ @schema = @record_batch.schema
42
+
43
+ @buffer = Arrow::ResizableBuffer.new(0)
44
+ table = Arrow::Table.new(@schema, [@record_batch])
45
+ table.save(@buffer, format: :stream)
46
+
47
+ @listener = Listener.new
48
+ @decoder = Arrow::StreamDecoder.new(@listener)
49
+ end
50
+
51
+ def test_consume
52
+ @decoder.consume(@buffer)
53
+ assert_equal([
54
+ [:schema_decoded, @schema, @schema],
55
+ [:record_batch_decoded, @record_batch, nil],
56
+ [:eos],
57
+ ],
58
+ @listener.events)
59
+ end
60
+ end
data/test/test-struct-data-type.rb CHANGED
@@ -101,7 +101,7 @@ class StructDataTypeTest < Test::Unit::TestCase
101
101
 
102
102
  test("[invalid]") do
103
103
  invalid = []
104
- message = "field name or index must be String, Symbol or Integer"
104
+ message = +"field name or index must be String, Symbol or Integer"
105
105
  message << ": <#{invalid.inspect}>"
106
106
  assert_raise(ArgumentError.new(message)) do
107
107
  @data_type[invalid]