red-arrow 16.1.0 → 18.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (21) hide show
  1. checksums.yaml +4 -4
  2. data/ext/arrow/extconf.rb +8 -1
  3. data/lib/arrow/field-containable.rb +1 -1
  4. data/lib/arrow/loader.rb +12 -0
  5. data/lib/arrow/stream-decoder.rb +29 -0
  6. data/lib/arrow/stream-listener.rb +47 -0
  7. data/lib/arrow/table-formatter.rb +33 -7
  8. data/lib/arrow/table-list-formatter.rb +3 -3
  9. data/lib/arrow/table-table-formatter.rb +7 -0
  10. data/lib/arrow/version.rb +1 -1
  11. data/red-arrow.gemspec +1 -1
  12. data/test/test-csv-loader.rb +40 -32
  13. data/test/test-decimal128-array.rb +6 -0
  14. data/test/test-decimal256-array.rb +6 -0
  15. data/test/test-group.rb +13 -0
  16. data/test/test-schema.rb +1 -1
  17. data/test/test-slicer.rb +191 -154
  18. data/test/test-stream-listener.rb +60 -0
  19. data/test/test-struct-data-type.rb +1 -1
  20. data/test/test-table.rb +305 -230
  21. metadata +13 -9
data/test/test-slicer.rb CHANGED
@@ -45,11 +45,12 @@ class SlicerTest < Test::Unit::TestCase
45
45
  slicer.visible
46
46
  end
47
47
  assert_equal(<<-TABLE, sliced_table.to_s)
48
- count visible
49
- 0 1 true
50
- 1 8 true
51
- 2 16 true
52
- 3 256 true
48
+ count visible
49
+ (uint32) (bool)
50
+ 0 1 true
51
+ 1 8 true
52
+ 2 16 true
53
+ 3 256 true
53
54
  TABLE
54
55
  end
55
56
 
@@ -58,15 +59,16 @@ class SlicerTest < Test::Unit::TestCase
58
59
  slicer.count
59
60
  end
60
61
  assert_equal(<<-TABLE, sliced_table.to_s)
61
- count visible
62
- 0 1 true
63
- 1 2 false
64
- 2 4 (null)
65
- 3 8 true
66
- 4 16 true
67
- 5 32 false
68
- 6 64 (null)
69
- 7 256 true
62
+ count visible
63
+ (uint32) (bool)
64
+ 0 1 true
65
+ 1 2 false
66
+ 2 4 (null)
67
+ 3 8 true
68
+ 4 16 true
69
+ 5 32 false
70
+ 6 64 (null)
71
+ 7 256 true
70
72
  TABLE
71
73
  end
72
74
  end
@@ -77,9 +79,10 @@ class SlicerTest < Test::Unit::TestCase
77
79
  !slicer.visible
78
80
  end
79
81
  assert_equal(<<-TABLE, sliced_table.to_s)
80
- count visible
81
- 0 2 false
82
- 1 32 false
82
+ count visible
83
+ (uint32) (bool)
84
+ 0 2 false
85
+ 1 32 false
83
86
  TABLE
84
87
  end
85
88
 
@@ -88,8 +91,9 @@ class SlicerTest < Test::Unit::TestCase
88
91
  !slicer.count
89
92
  end
90
93
  assert_equal(<<-TABLE, sliced_table.to_s)
91
- count visible
92
- 0 0 (null)
94
+ count visible
95
+ (uint32) (bool)
96
+ 0 0 (null)
93
97
  TABLE
94
98
  end
95
99
  end
@@ -99,11 +103,12 @@ class SlicerTest < Test::Unit::TestCase
99
103
  slicer.visible.null?
100
104
  end
101
105
  assert_equal(<<-TABLE, sliced_table.to_s)
102
- count visible
103
- 0 0 (null)
104
- 1 4 (null)
105
- 2 64 (null)
106
- 3 (null) (null)
106
+ count visible
107
+ (uint32) (bool)
108
+ 0 0 (null)
109
+ 1 4 (null)
110
+ 2 64 (null)
111
+ 3 (null) (null)
107
112
  TABLE
108
113
  end
109
114
 
@@ -112,13 +117,14 @@ class SlicerTest < Test::Unit::TestCase
112
117
  slicer.visible.valid?
113
118
  end
114
119
  assert_equal(<<-TABLE, sliced_table.to_s)
115
- count visible
116
- 0 1 true
117
- 1 2 false
118
- 2 8 true
119
- 3 16 true
120
- 4 32 false
121
- 5 256 true
120
+ count visible
121
+ (uint32) (bool)
122
+ 0 1 true
123
+ 1 2 false
124
+ 2 8 true
125
+ 3 16 true
126
+ 4 32 false
127
+ 5 256 true
122
128
  TABLE
123
129
  end
124
130
 
@@ -128,11 +134,12 @@ class SlicerTest < Test::Unit::TestCase
128
134
  slicer.visible == nil
129
135
  end
130
136
  assert_equal(<<-TABLE, sliced_table.to_s)
131
- count visible
132
- 0 0 (null)
133
- 1 4 (null)
134
- 2 64 (null)
135
- 3 (null) (null)
137
+ count visible
138
+ (uint32) (bool)
139
+ 0 0 (null)
140
+ 1 4 (null)
141
+ 2 64 (null)
142
+ 3 (null) (null)
136
143
  TABLE
137
144
  end
138
145
 
@@ -141,11 +148,12 @@ class SlicerTest < Test::Unit::TestCase
141
148
  slicer.visible == true
142
149
  end
143
150
  assert_equal(<<-TABLE, sliced_table.to_s)
144
- count visible
145
- 0 1 true
146
- 1 8 true
147
- 2 16 true
148
- 3 256 true
151
+ count visible
152
+ (uint32) (bool)
153
+ 0 1 true
154
+ 1 8 true
155
+ 2 16 true
156
+ 3 256 true
149
157
  TABLE
150
158
  end
151
159
  end
@@ -156,13 +164,14 @@ class SlicerTest < Test::Unit::TestCase
156
164
  !(slicer.visible == nil)
157
165
  end
158
166
  assert_equal(<<-TABLE, sliced_table.to_s)
159
- count visible
160
- 0 1 true
161
- 1 2 false
162
- 2 8 true
163
- 3 16 true
164
- 4 32 false
165
- 5 256 true
167
+ count visible
168
+ (uint32) (bool)
169
+ 0 1 true
170
+ 1 2 false
171
+ 2 8 true
172
+ 3 16 true
173
+ 4 32 false
174
+ 5 256 true
166
175
  TABLE
167
176
  end
168
177
 
@@ -171,9 +180,10 @@ class SlicerTest < Test::Unit::TestCase
171
180
  !(slicer.visible == true)
172
181
  end
173
182
  assert_equal(<<-TABLE, sliced_table.to_s)
174
- count visible
175
- 0 2 false
176
- 1 32 false
183
+ count visible
184
+ (uint32) (bool)
185
+ 0 2 false
186
+ 1 32 false
177
187
  TABLE
178
188
  end
179
189
  end
@@ -184,13 +194,14 @@ class SlicerTest < Test::Unit::TestCase
184
194
  slicer.visible != nil
185
195
  end
186
196
  assert_equal(<<-TABLE, sliced_table.to_s)
187
- count visible
188
- 0 1 true
189
- 1 2 false
190
- 2 8 true
191
- 3 16 true
192
- 4 32 false
193
- 5 256 true
197
+ count visible
198
+ (uint32) (bool)
199
+ 0 1 true
200
+ 1 2 false
201
+ 2 8 true
202
+ 3 16 true
203
+ 4 32 false
204
+ 5 256 true
194
205
  TABLE
195
206
  end
196
207
 
@@ -199,9 +210,10 @@ class SlicerTest < Test::Unit::TestCase
199
210
  slicer.visible != true
200
211
  end
201
212
  assert_equal(<<-TABLE, sliced_table.to_s)
202
- count visible
203
- 0 2 false
204
- 1 32 false
213
+ count visible
214
+ (uint32) (bool)
215
+ 0 2 false
216
+ 1 32 false
205
217
  TABLE
206
218
  end
207
219
  end
@@ -211,12 +223,13 @@ class SlicerTest < Test::Unit::TestCase
211
223
  slicer.count < 16
212
224
  end
213
225
  assert_equal(<<-TABLE, sliced_table.to_s)
214
- count visible
215
- 0 0 (null)
216
- 1 1 true
217
- 2 2 false
218
- 3 4 (null)
219
- 4 8 true
226
+ count visible
227
+ (uint32) (bool)
228
+ 0 0 (null)
229
+ 1 1 true
230
+ 2 2 false
231
+ 3 4 (null)
232
+ 4 8 true
220
233
  TABLE
221
234
  end
222
235
 
@@ -225,11 +238,12 @@ class SlicerTest < Test::Unit::TestCase
225
238
  !(slicer.count < 16)
226
239
  end
227
240
  assert_equal(<<-TABLE, sliced_table.to_s)
228
- count visible
229
- 0 16 true
230
- 1 32 false
231
- 2 64 (null)
232
- 3 256 true
241
+ count visible
242
+ (uint32) (bool)
243
+ 0 16 true
244
+ 1 32 false
245
+ 2 64 (null)
246
+ 3 256 true
233
247
  TABLE
234
248
  end
235
249
 
@@ -238,13 +252,14 @@ class SlicerTest < Test::Unit::TestCase
238
252
  slicer.count <= 16
239
253
  end
240
254
  assert_equal(<<-TABLE, sliced_table.to_s)
241
- count visible
242
- 0 0 (null)
243
- 1 1 true
244
- 2 2 false
245
- 3 4 (null)
246
- 4 8 true
247
- 5 16 true
255
+ count visible
256
+ (uint32) (bool)
257
+ 0 0 (null)
258
+ 1 1 true
259
+ 2 2 false
260
+ 3 4 (null)
261
+ 4 8 true
262
+ 5 16 true
248
263
  TABLE
249
264
  end
250
265
 
@@ -253,10 +268,11 @@ class SlicerTest < Test::Unit::TestCase
253
268
  !(slicer.count <= 16)
254
269
  end
255
270
  assert_equal(<<-TABLE, sliced_table.to_s)
256
- count visible
257
- 0 32 false
258
- 1 64 (null)
259
- 2 256 true
271
+ count visible
272
+ (uint32) (bool)
273
+ 0 32 false
274
+ 1 64 (null)
275
+ 2 256 true
260
276
  TABLE
261
277
  end
262
278
 
@@ -265,10 +281,11 @@ class SlicerTest < Test::Unit::TestCase
265
281
  slicer.count > 16
266
282
  end
267
283
  assert_equal(<<-TABLE, sliced_table.to_s)
268
- count visible
269
- 0 32 false
270
- 1 64 (null)
271
- 2 256 true
284
+ count visible
285
+ (uint32) (bool)
286
+ 0 32 false
287
+ 1 64 (null)
288
+ 2 256 true
272
289
  TABLE
273
290
  end
274
291
 
@@ -277,13 +294,14 @@ class SlicerTest < Test::Unit::TestCase
277
294
  !(slicer.count > 16)
278
295
  end
279
296
  assert_equal(<<-TABLE, sliced_table.to_s)
280
- count visible
281
- 0 0 (null)
282
- 1 1 true
283
- 2 2 false
284
- 3 4 (null)
285
- 4 8 true
286
- 5 16 true
297
+ count visible
298
+ (uint32) (bool)
299
+ 0 0 (null)
300
+ 1 1 true
301
+ 2 2 false
302
+ 3 4 (null)
303
+ 4 8 true
304
+ 5 16 true
287
305
  TABLE
288
306
  end
289
307
 
@@ -292,11 +310,12 @@ class SlicerTest < Test::Unit::TestCase
292
310
  slicer.count >= 16
293
311
  end
294
312
  assert_equal(<<-TABLE, sliced_table.to_s)
295
- count visible
296
- 0 16 true
297
- 1 32 false
298
- 2 64 (null)
299
- 3 256 true
313
+ count visible
314
+ (uint32) (bool)
315
+ 0 16 true
316
+ 1 32 false
317
+ 2 64 (null)
318
+ 3 256 true
300
319
  TABLE
301
320
  end
302
321
 
@@ -305,12 +324,13 @@ class SlicerTest < Test::Unit::TestCase
305
324
  !(slicer.count >= 16)
306
325
  end
307
326
  assert_equal(<<-TABLE, sliced_table.to_s)
308
- count visible
309
- 0 0 (null)
310
- 1 1 true
311
- 2 2 false
312
- 3 4 (null)
313
- 4 8 true
327
+ count visible
328
+ (uint32) (bool)
329
+ 0 0 (null)
330
+ 1 1 true
331
+ 2 2 false
332
+ 3 4 (null)
333
+ 4 8 true
314
334
  TABLE
315
335
  end
316
336
 
@@ -319,11 +339,12 @@ class SlicerTest < Test::Unit::TestCase
319
339
  slicer.count.in?([1, 4, 16, 64])
320
340
  end
321
341
  assert_equal(<<-TABLE, sliced_table.to_s)
322
- count visible
323
- 0 1 true
324
- 1 4 (null)
325
- 2 16 true
326
- 3 64 (null)
342
+ count visible
343
+ (uint32) (bool)
344
+ 0 1 true
345
+ 1 4 (null)
346
+ 2 16 true
347
+ 3 64 (null)
327
348
  TABLE
328
349
  end
329
350
 
@@ -332,13 +353,14 @@ class SlicerTest < Test::Unit::TestCase
332
353
  !slicer.count.in?([1, 4, 16, 64])
333
354
  end
334
355
  assert_equal(<<-TABLE, sliced_table.to_s)
335
- count visible
336
- 0 0 (null)
337
- 1 2 false
338
- 2 8 true
339
- 3 32 false
340
- 4 (null) (null)
341
- 5 256 true
356
+ count visible
357
+ (uint32) (bool)
358
+ 0 0 (null)
359
+ 1 2 false
360
+ 2 8 true
361
+ 3 32 false
362
+ 4 (null) (null)
363
+ 5 256 true
342
364
  TABLE
343
365
  end
344
366
 
@@ -347,9 +369,10 @@ class SlicerTest < Test::Unit::TestCase
347
369
  slicer.visible & (slicer.count >= 16)
348
370
  end
349
371
  assert_equal(<<-TABLE, sliced_table.to_s)
350
- count visible
351
- 0 16 true
352
- 1 256 true
372
+ count visible
373
+ (uint32) (bool)
374
+ 0 16 true
375
+ 1 256 true
353
376
  TABLE
354
377
  end
355
378
 
@@ -358,12 +381,13 @@ class SlicerTest < Test::Unit::TestCase
358
381
  slicer.visible | (slicer.count >= 16)
359
382
  end
360
383
  assert_equal(<<-TABLE, sliced_table.to_s)
361
- count visible
362
- 0 1 true
363
- 1 8 true
364
- 2 16 true
365
- 3 32 false
366
- 4 256 true
384
+ count visible
385
+ (uint32) (bool)
386
+ 0 1 true
387
+ 1 8 true
388
+ 2 16 true
389
+ 3 32 false
390
+ 4 256 true
367
391
  TABLE
368
392
  end
369
393
 
@@ -372,10 +396,11 @@ class SlicerTest < Test::Unit::TestCase
372
396
  slicer.visible ^ (slicer.count >= 16)
373
397
  end
374
398
  assert_equal(<<-TABLE, sliced_table.to_s)
375
- count visible
376
- 0 1 true
377
- 1 8 true
378
- 2 32 false
399
+ count visible
400
+ (uint32) (bool)
401
+ 0 1 true
402
+ 1 8 true
403
+ 2 32 false
379
404
  TABLE
380
405
  end
381
406
 
@@ -386,15 +411,16 @@ class SlicerTest < Test::Unit::TestCase
386
411
  end
387
412
  end
388
413
  assert_equal(<<-TABLE, sliced_table.to_s)
389
- count visible
390
- 0 0 (null)
391
- 1 1 true
392
- 2 4 (null)
393
- 3 8 true
394
- 4 16 true
395
- 5 64 (null)
396
- 6 (null) (null)
397
- 7 256 true
414
+ count visible
415
+ (uint32) (bool)
416
+ 0 0 (null)
417
+ 1 1 true
418
+ 2 4 (null)
419
+ 3 8 true
420
+ 4 16 true
421
+ 5 64 (null)
422
+ 6 (null) (null)
423
+ 7 256 true
398
424
  TABLE
399
425
  end
400
426
 
@@ -405,9 +431,10 @@ class SlicerTest < Test::Unit::TestCase
405
431
  end
406
432
  end
407
433
  assert_equal(<<-TABLE, sliced_table.to_s)
408
- count visible
409
- 0 2 false
410
- 1 32 false
434
+ count visible
435
+ (uint32) (bool)
436
+ 0 2 false
437
+ 1 32 false
411
438
  TABLE
412
439
  end
413
440
 
@@ -418,9 +445,10 @@ class SlicerTest < Test::Unit::TestCase
418
445
  end
419
446
  end
420
447
  assert_equal(<<-TABLE, sliced_table.to_s)
421
- count visible
422
- 0 2 false
423
- 1 32 false
448
+ count visible
449
+ (uint32) (bool)
450
+ 0 2 false
451
+ 1 32 false
424
452
  TABLE
425
453
  end
426
454
 
@@ -431,15 +459,16 @@ class SlicerTest < Test::Unit::TestCase
431
459
  end
432
460
  end
433
461
  assert_equal(<<-TABLE, sliced_table.to_s)
434
- count visible
435
- 0 0 (null)
436
- 1 1 true
437
- 2 4 (null)
438
- 3 8 true
439
- 4 16 true
440
- 5 64 (null)
441
- 6 (null) (null)
442
- 7 256 true
462
+ count visible
463
+ (uint32) (bool)
464
+ 0 0 (null)
465
+ 1 1 true
466
+ 2 4 (null)
467
+ 3 8 true
468
+ 4 16 true
469
+ 5 64 (null)
470
+ 6 (null) (null)
471
+ 7 256 true
443
472
  TABLE
444
473
  end
445
474
 
@@ -456,6 +485,7 @@ class SlicerTest < Test::Unit::TestCase
456
485
  end
457
486
  assert_equal(<<~TABLE, sliced_table.to_s)
458
487
  string
488
+ (utf8)
459
489
  0 Arrow
460
490
  1 window
461
491
  TABLE
@@ -467,6 +497,7 @@ class SlicerTest < Test::Unit::TestCase
467
497
  end
468
498
  assert_equal(<<~TABLE, sliced_table.to_s)
469
499
  string
500
+ (utf8)
470
501
  0 array
471
502
  1 Arrow
472
503
  TABLE
@@ -478,6 +509,7 @@ class SlicerTest < Test::Unit::TestCase
478
509
  end
479
510
  assert_equal(<<~TABLE, sliced_table.to_s)
480
511
  string
512
+ (utf8)
481
513
  0 array
482
514
  1 carrot
483
515
  TABLE
@@ -489,6 +521,7 @@ class SlicerTest < Test::Unit::TestCase
489
521
  end
490
522
  assert_equal(<<~TABLE, sliced_table.to_s)
491
523
  string
524
+ (utf8)
492
525
  0 array
493
526
  1 Arrow
494
527
  2 carrot
@@ -501,6 +534,7 @@ class SlicerTest < Test::Unit::TestCase
501
534
  end
502
535
  assert_equal(<<~TABLE, sliced_table.to_s)
503
536
  string
537
+ (utf8)
504
538
  0 Arrow
505
539
  1 window
506
540
  TABLE
@@ -512,6 +546,7 @@ class SlicerTest < Test::Unit::TestCase
512
546
  end
513
547
  assert_equal(<<~TABLE, sliced_table.to_s)
514
548
  string
549
+ (utf8)
515
550
  0 Arrow
516
551
  1 window
517
552
  TABLE
@@ -523,6 +558,7 @@ class SlicerTest < Test::Unit::TestCase
523
558
  end
524
559
  assert_equal(<<~TABLE, sliced_table.to_s)
525
560
  string
561
+ (utf8)
526
562
  0 array
527
563
  1 Arrow
528
564
  2 carrot
@@ -545,6 +581,7 @@ class SlicerTest < Test::Unit::TestCase
545
581
  end
546
582
  assert_equal(<<~TABLE, sliced_table.to_s)
547
583
  string
584
+ (utf8)
548
585
  0 carrot
549
586
  TABLE
550
587
  end
data/test/test-stream-listener.rb ADDED
@@ -0,0 +1,60 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ class TestStreamListener < Test::Unit::TestCase
19
+ class Listener < Arrow::StreamListener
20
+ attr_reader :events
21
+ def initialize
22
+ super
23
+ @events = []
24
+ end
25
+
26
+ def on_eos
27
+ @events << [:eos]
28
+ end
29
+
30
+ def on_record_batch_decoded(record_batch, metadata)
31
+ @events << [:record_batch_decoded, record_batch, metadata]
32
+ end
33
+
34
+ def on_schema_decoded(schema, filtered_schema)
35
+ @events << [:schema_decoded, schema, filtered_schema]
36
+ end
37
+ end
38
+
39
+ def setup
40
+ @record_batch = Arrow::RecordBatch.new(enabled: [true, false, nil, true])
41
+ @schema = @record_batch.schema
42
+
43
+ @buffer = Arrow::ResizableBuffer.new(0)
44
+ table = Arrow::Table.new(@schema, [@record_batch])
45
+ table.save(@buffer, format: :stream)
46
+
47
+ @listener = Listener.new
48
+ @decoder = Arrow::StreamDecoder.new(@listener)
49
+ end
50
+
51
+ def test_consume
52
+ @decoder.consume(@buffer)
53
+ assert_equal([
54
+ [:schema_decoded, @schema, @schema],
55
+ [:record_batch_decoded, @record_batch, nil],
56
+ [:eos],
57
+ ],
58
+ @listener.events)
59
+ end
60
+ end
data/test/test-struct-data-type.rb CHANGED
@@ -101,7 +101,7 @@ class StructDataTypeTest < Test::Unit::TestCase
101
101
 
102
102
  test("[invalid]") do
103
103
  invalid = []
104
- message = "field name or index must be String, Symbol or Integer"
104
+ message = +"field name or index must be String, Symbol or Integer"
105
105
  message << ": <#{invalid.inspect}>"
106
106
  assert_raise(ArgumentError.new(message)) do
107
107
  @data_type[invalid]