red-arrow 10.0.0 → 16.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/ext/arrow/arrow.cpp +31 -0
- data/ext/arrow/converters.hpp +45 -41
- data/ext/arrow/extconf.rb +16 -4
- data/ext/arrow/raw-records.cpp +155 -2
- data/ext/arrow/red-arrow.hpp +2 -0
- data/ext/arrow/values.cpp +1 -2
- data/lib/arrow/array-computable.rb +13 -0
- data/lib/arrow/array.rb +6 -1
- data/lib/arrow/chunked-array.rb +35 -1
- data/lib/arrow/column-containable.rb +9 -0
- data/lib/arrow/column.rb +1 -0
- data/lib/arrow/data-type.rb +9 -0
- data/lib/arrow/dense-union-array-builder.rb +49 -0
- data/lib/arrow/dense-union-array.rb +26 -0
- data/lib/arrow/expression.rb +6 -2
- data/lib/arrow/function.rb +0 -1
- data/lib/arrow/half-float-array-builder.rb +32 -0
- data/lib/arrow/half-float-array.rb +24 -0
- data/lib/arrow/half-float.rb +118 -0
- data/lib/arrow/input-referable.rb +29 -0
- data/lib/arrow/loader.rb +11 -0
- data/lib/arrow/raw-table-converter.rb +7 -5
- data/lib/arrow/record-batch-file-reader.rb +2 -0
- data/lib/arrow/record-batch-stream-reader.rb +2 -0
- data/lib/arrow/record-batch.rb +6 -2
- data/lib/arrow/scalar.rb +67 -0
- data/lib/arrow/slicer.rb +61 -0
- data/lib/arrow/sort-key.rb +3 -3
- data/lib/arrow/sparse-union-array-builder.rb +56 -0
- data/lib/arrow/sparse-union-array.rb +26 -0
- data/lib/arrow/struct-array-builder.rb +0 -5
- data/lib/arrow/table-loader.rb +11 -5
- data/lib/arrow/table-saver.rb +1 -0
- data/lib/arrow/table.rb +180 -33
- data/lib/arrow/tensor.rb +4 -0
- data/lib/arrow/timestamp-parser.rb +33 -0
- data/lib/arrow/union-array-builder.rb +59 -0
- data/lib/arrow/version.rb +1 -1
- data/red-arrow.gemspec +2 -1
- data/test/each-raw-record/test-basic-arrays.rb +411 -0
- data/test/each-raw-record/test-dense-union-array.rb +566 -0
- data/test/each-raw-record/test-dictionary-array.rb +341 -0
- data/test/each-raw-record/test-list-array.rb +628 -0
- data/test/each-raw-record/test-map-array.rb +507 -0
- data/test/each-raw-record/test-multiple-columns.rb +72 -0
- data/test/each-raw-record/test-sparse-union-array.rb +528 -0
- data/test/each-raw-record/test-struct-array.rb +529 -0
- data/test/each-raw-record/test-table.rb +47 -0
- data/test/helper/omittable.rb +13 -0
- data/test/helper.rb +1 -0
- data/test/raw-records/test-basic-arrays.rb +11 -1
- data/test/raw-records/test-dense-union-array.rb +90 -45
- data/test/raw-records/test-list-array.rb +28 -10
- data/test/raw-records/test-map-array.rb +39 -10
- data/test/raw-records/test-sparse-union-array.rb +86 -41
- data/test/raw-records/test-struct-array.rb +22 -8
- data/test/test-array.rb +7 -0
- data/test/test-chunked-array.rb +9 -0
- data/test/test-csv-loader.rb +39 -0
- data/test/test-data-type.rb +2 -1
- data/test/test-dense-union-array.rb +42 -0
- data/test/test-dense-union-data-type.rb +1 -1
- data/test/test-expression.rb +11 -0
- data/test/test-function.rb +7 -7
- data/test/test-group.rb +58 -58
- data/test/test-half-float-array.rb +43 -0
- data/test/test-half-float.rb +130 -0
- data/test/test-ractor.rb +34 -0
- data/test/test-record-batch-file-reader.rb +21 -0
- data/test/test-record-batch-stream-reader.rb +129 -0
- data/test/test-scalar.rb +65 -0
- data/test/test-slicer.rb +194 -129
- data/test/test-sparse-union-array.rb +38 -0
- data/test/test-table.rb +356 -40
- data/test/values/test-basic-arrays.rb +10 -0
- data/test/values/test-dense-union-array.rb +88 -45
- data/test/values/test-list-array.rb +26 -10
- data/test/values/test-map-array.rb +33 -10
- data/test/values/test-sparse-union-array.rb +84 -41
- data/test/values/test-struct-array.rb +20 -8
- metadata +62 -9
|
@@ -76,12 +76,25 @@ module RawRecordsDenseUnionArrayTests
|
|
|
76
76
|
[union_array])
|
|
77
77
|
end
|
|
78
78
|
|
|
79
|
+
def remove_field_names(records)
|
|
80
|
+
records.collect do |record|
|
|
81
|
+
record.collect do |column|
|
|
82
|
+
if column.nil?
|
|
83
|
+
column
|
|
84
|
+
else
|
|
85
|
+
column.values[0]
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
79
91
|
def test_null
|
|
80
92
|
records = [
|
|
81
93
|
[{"0" => nil}],
|
|
82
94
|
]
|
|
83
95
|
target = build(:null, records)
|
|
84
|
-
assert_equal(records,
|
|
96
|
+
assert_equal(remove_field_names(records),
|
|
97
|
+
target.raw_records)
|
|
85
98
|
end
|
|
86
99
|
|
|
87
100
|
def test_boolean
|
|
@@ -90,7 +103,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
90
103
|
[{"1" => nil}],
|
|
91
104
|
]
|
|
92
105
|
target = build(:boolean, records)
|
|
93
|
-
assert_equal(records,
|
|
106
|
+
assert_equal(remove_field_names(records),
|
|
107
|
+
target.raw_records)
|
|
94
108
|
end
|
|
95
109
|
|
|
96
110
|
def test_int8
|
|
@@ -99,7 +113,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
99
113
|
[{"1" => nil}],
|
|
100
114
|
]
|
|
101
115
|
target = build(:int8, records)
|
|
102
|
-
assert_equal(records,
|
|
116
|
+
assert_equal(remove_field_names(records),
|
|
117
|
+
target.raw_records)
|
|
103
118
|
end
|
|
104
119
|
|
|
105
120
|
def test_uint8
|
|
@@ -108,7 +123,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
108
123
|
[{"1" => nil}],
|
|
109
124
|
]
|
|
110
125
|
target = build(:uint8, records)
|
|
111
|
-
assert_equal(records,
|
|
126
|
+
assert_equal(remove_field_names(records),
|
|
127
|
+
target.raw_records)
|
|
112
128
|
end
|
|
113
129
|
|
|
114
130
|
def test_int16
|
|
@@ -117,7 +133,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
117
133
|
[{"1" => nil}],
|
|
118
134
|
]
|
|
119
135
|
target = build(:int16, records)
|
|
120
|
-
assert_equal(records,
|
|
136
|
+
assert_equal(remove_field_names(records),
|
|
137
|
+
target.raw_records)
|
|
121
138
|
end
|
|
122
139
|
|
|
123
140
|
def test_uint16
|
|
@@ -126,7 +143,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
126
143
|
[{"1" => nil}],
|
|
127
144
|
]
|
|
128
145
|
target = build(:uint16, records)
|
|
129
|
-
assert_equal(records,
|
|
146
|
+
assert_equal(remove_field_names(records),
|
|
147
|
+
target.raw_records)
|
|
130
148
|
end
|
|
131
149
|
|
|
132
150
|
def test_int32
|
|
@@ -135,7 +153,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
135
153
|
[{"1" => nil}],
|
|
136
154
|
]
|
|
137
155
|
target = build(:int32, records)
|
|
138
|
-
assert_equal(records,
|
|
156
|
+
assert_equal(remove_field_names(records),
|
|
157
|
+
target.raw_records)
|
|
139
158
|
end
|
|
140
159
|
|
|
141
160
|
def test_uint32
|
|
@@ -144,7 +163,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
144
163
|
[{"1" => nil}],
|
|
145
164
|
]
|
|
146
165
|
target = build(:uint32, records)
|
|
147
|
-
assert_equal(records,
|
|
166
|
+
assert_equal(remove_field_names(records),
|
|
167
|
+
target.raw_records)
|
|
148
168
|
end
|
|
149
169
|
|
|
150
170
|
def test_int64
|
|
@@ -153,7 +173,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
153
173
|
[{"1" => nil}],
|
|
154
174
|
]
|
|
155
175
|
target = build(:int64, records)
|
|
156
|
-
assert_equal(records,
|
|
176
|
+
assert_equal(remove_field_names(records),
|
|
177
|
+
target.raw_records)
|
|
157
178
|
end
|
|
158
179
|
|
|
159
180
|
def test_uint64
|
|
@@ -162,7 +183,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
162
183
|
[{"1" => nil}],
|
|
163
184
|
]
|
|
164
185
|
target = build(:uint64, records)
|
|
165
|
-
assert_equal(records,
|
|
186
|
+
assert_equal(remove_field_names(records),
|
|
187
|
+
target.raw_records)
|
|
166
188
|
end
|
|
167
189
|
|
|
168
190
|
def test_float
|
|
@@ -171,7 +193,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
171
193
|
[{"1" => nil}],
|
|
172
194
|
]
|
|
173
195
|
target = build(:float, records)
|
|
174
|
-
assert_equal(records,
|
|
196
|
+
assert_equal(remove_field_names(records),
|
|
197
|
+
target.raw_records)
|
|
175
198
|
end
|
|
176
199
|
|
|
177
200
|
def test_double
|
|
@@ -180,7 +203,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
180
203
|
[{"1" => nil}],
|
|
181
204
|
]
|
|
182
205
|
target = build(:double, records)
|
|
183
|
-
assert_equal(records,
|
|
206
|
+
assert_equal(remove_field_names(records),
|
|
207
|
+
target.raw_records)
|
|
184
208
|
end
|
|
185
209
|
|
|
186
210
|
def test_binary
|
|
@@ -189,7 +213,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
189
213
|
[{"1" => nil}],
|
|
190
214
|
]
|
|
191
215
|
target = build(:binary, records)
|
|
192
|
-
assert_equal(records,
|
|
216
|
+
assert_equal(remove_field_names(records),
|
|
217
|
+
target.raw_records)
|
|
193
218
|
end
|
|
194
219
|
|
|
195
220
|
def test_string
|
|
@@ -198,7 +223,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
198
223
|
[{"1" => nil}],
|
|
199
224
|
]
|
|
200
225
|
target = build(:string, records)
|
|
201
|
-
assert_equal(records,
|
|
226
|
+
assert_equal(remove_field_names(records),
|
|
227
|
+
target.raw_records)
|
|
202
228
|
end
|
|
203
229
|
|
|
204
230
|
def test_date32
|
|
@@ -207,7 +233,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
207
233
|
[{"1" => nil}],
|
|
208
234
|
]
|
|
209
235
|
target = build(:date32, records)
|
|
210
|
-
assert_equal(records,
|
|
236
|
+
assert_equal(remove_field_names(records),
|
|
237
|
+
target.raw_records)
|
|
211
238
|
end
|
|
212
239
|
|
|
213
240
|
def test_date64
|
|
@@ -216,7 +243,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
216
243
|
[{"1" => nil}],
|
|
217
244
|
]
|
|
218
245
|
target = build(:date64, records)
|
|
219
|
-
assert_equal(records,
|
|
246
|
+
assert_equal(remove_field_names(records),
|
|
247
|
+
target.raw_records)
|
|
220
248
|
end
|
|
221
249
|
|
|
222
250
|
def test_timestamp_second
|
|
@@ -229,7 +257,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
229
257
|
unit: :second,
|
|
230
258
|
},
|
|
231
259
|
records)
|
|
232
|
-
assert_equal(records,
|
|
260
|
+
assert_equal(remove_field_names(records),
|
|
261
|
+
target.raw_records)
|
|
233
262
|
end
|
|
234
263
|
|
|
235
264
|
def test_timestamp_milli
|
|
@@ -242,7 +271,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
242
271
|
unit: :milli,
|
|
243
272
|
},
|
|
244
273
|
records)
|
|
245
|
-
assert_equal(records,
|
|
274
|
+
assert_equal(remove_field_names(records),
|
|
275
|
+
target.raw_records)
|
|
246
276
|
end
|
|
247
277
|
|
|
248
278
|
def test_timestamp_micro
|
|
@@ -255,7 +285,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
255
285
|
unit: :micro,
|
|
256
286
|
},
|
|
257
287
|
records)
|
|
258
|
-
assert_equal(records,
|
|
288
|
+
assert_equal(remove_field_names(records),
|
|
289
|
+
target.raw_records)
|
|
259
290
|
end
|
|
260
291
|
|
|
261
292
|
def test_timestamp_nano
|
|
@@ -268,7 +299,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
268
299
|
unit: :nano,
|
|
269
300
|
},
|
|
270
301
|
records)
|
|
271
|
-
assert_equal(records,
|
|
302
|
+
assert_equal(remove_field_names(records),
|
|
303
|
+
target.raw_records)
|
|
272
304
|
end
|
|
273
305
|
|
|
274
306
|
def test_time32_second
|
|
@@ -283,7 +315,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
283
315
|
unit: :second,
|
|
284
316
|
},
|
|
285
317
|
records)
|
|
286
|
-
assert_equal(records,
|
|
318
|
+
assert_equal(remove_field_names(records),
|
|
319
|
+
target.raw_records)
|
|
287
320
|
end
|
|
288
321
|
|
|
289
322
|
def test_time32_milli
|
|
@@ -298,7 +331,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
298
331
|
unit: :milli,
|
|
299
332
|
},
|
|
300
333
|
records)
|
|
301
|
-
assert_equal(records,
|
|
334
|
+
assert_equal(remove_field_names(records),
|
|
335
|
+
target.raw_records)
|
|
302
336
|
end
|
|
303
337
|
|
|
304
338
|
def test_time64_micro
|
|
@@ -313,7 +347,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
313
347
|
unit: :micro,
|
|
314
348
|
},
|
|
315
349
|
records)
|
|
316
|
-
assert_equal(records,
|
|
350
|
+
assert_equal(remove_field_names(records),
|
|
351
|
+
target.raw_records)
|
|
317
352
|
end
|
|
318
353
|
|
|
319
354
|
def test_time64_nano
|
|
@@ -328,7 +363,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
328
363
|
unit: :nano,
|
|
329
364
|
},
|
|
330
365
|
records)
|
|
331
|
-
assert_equal(records,
|
|
366
|
+
assert_equal(remove_field_names(records),
|
|
367
|
+
target.raw_records)
|
|
332
368
|
end
|
|
333
369
|
|
|
334
370
|
def test_decimal128
|
|
@@ -342,7 +378,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
342
378
|
scale: 2,
|
|
343
379
|
},
|
|
344
380
|
records)
|
|
345
|
-
assert_equal(records,
|
|
381
|
+
assert_equal(remove_field_names(records),
|
|
382
|
+
target.raw_records)
|
|
346
383
|
end
|
|
347
384
|
|
|
348
385
|
def test_decimal256
|
|
@@ -356,7 +393,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
356
393
|
scale: 2,
|
|
357
394
|
},
|
|
358
395
|
records)
|
|
359
|
-
assert_equal(records,
|
|
396
|
+
assert_equal(remove_field_names(records),
|
|
397
|
+
target.raw_records)
|
|
360
398
|
end
|
|
361
399
|
|
|
362
400
|
def test_month_interval
|
|
@@ -365,7 +403,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
365
403
|
[{"1" => nil}],
|
|
366
404
|
]
|
|
367
405
|
target = build(:month_interval, records)
|
|
368
|
-
assert_equal(records,
|
|
406
|
+
assert_equal(remove_field_names(records),
|
|
407
|
+
target.raw_records)
|
|
369
408
|
end
|
|
370
409
|
|
|
371
410
|
def test_day_time_interval
|
|
@@ -374,7 +413,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
374
413
|
[{"1" => nil}],
|
|
375
414
|
]
|
|
376
415
|
target = build(:day_time_interval, records)
|
|
377
|
-
assert_equal(records,
|
|
416
|
+
assert_equal(remove_field_names(records),
|
|
417
|
+
target.raw_records)
|
|
378
418
|
end
|
|
379
419
|
|
|
380
420
|
def test_month_day_nano_interval
|
|
@@ -383,7 +423,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
383
423
|
[{"1" => nil}],
|
|
384
424
|
]
|
|
385
425
|
target = build(:month_day_nano_interval, records)
|
|
386
|
-
assert_equal(records,
|
|
426
|
+
assert_equal(remove_field_names(records),
|
|
427
|
+
target.raw_records)
|
|
387
428
|
end
|
|
388
429
|
|
|
389
430
|
def test_list
|
|
@@ -399,7 +440,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
399
440
|
},
|
|
400
441
|
},
|
|
401
442
|
records)
|
|
402
|
-
assert_equal(records,
|
|
443
|
+
assert_equal(remove_field_names(records),
|
|
444
|
+
target.raw_records)
|
|
403
445
|
end
|
|
404
446
|
|
|
405
447
|
def test_struct
|
|
@@ -418,7 +460,8 @@ module RawRecordsDenseUnionArrayTests
|
|
|
418
460
|
],
|
|
419
461
|
},
|
|
420
462
|
records)
|
|
421
|
-
assert_equal(records,
|
|
463
|
+
assert_equal(remove_field_names(records),
|
|
464
|
+
target.raw_records)
|
|
422
465
|
end
|
|
423
466
|
|
|
424
467
|
def test_map
|
|
@@ -432,14 +475,15 @@ module RawRecordsDenseUnionArrayTests
|
|
|
432
475
|
item: :boolean,
|
|
433
476
|
},
|
|
434
477
|
records)
|
|
435
|
-
assert_equal(records,
|
|
478
|
+
assert_equal(remove_field_names(records),
|
|
479
|
+
target.raw_records)
|
|
436
480
|
end
|
|
437
481
|
|
|
438
482
|
def test_sparse_union
|
|
439
|
-
omit("Need to add support for SparseUnionArrayBuilder")
|
|
440
483
|
records = [
|
|
441
484
|
[{"0" => {"field1" => true}}],
|
|
442
485
|
[{"1" => nil}],
|
|
486
|
+
[{"0" => {"field2" => 29}}],
|
|
443
487
|
[{"0" => {"field2" => nil}}],
|
|
444
488
|
]
|
|
445
489
|
target = build({
|
|
@@ -457,14 +501,15 @@ module RawRecordsDenseUnionArrayTests
|
|
|
457
501
|
type_codes: [0, 1],
|
|
458
502
|
},
|
|
459
503
|
records)
|
|
460
|
-
assert_equal(records,
|
|
504
|
+
assert_equal(remove_field_names(remove_field_names(records)),
|
|
505
|
+
target.raw_records)
|
|
461
506
|
end
|
|
462
507
|
|
|
463
508
|
def test_dense_union
|
|
464
|
-
omit("Need to add support for DenseUnionArrayBuilder")
|
|
465
509
|
records = [
|
|
466
510
|
[{"0" => {"field1" => true}}],
|
|
467
511
|
[{"1" => nil}],
|
|
512
|
+
[{"0" => {"field2" => 29}}],
|
|
468
513
|
[{"0" => {"field2" => nil}}],
|
|
469
514
|
]
|
|
470
515
|
target = build({
|
|
@@ -482,25 +527,25 @@ module RawRecordsDenseUnionArrayTests
|
|
|
482
527
|
type_codes: [0, 1],
|
|
483
528
|
},
|
|
484
529
|
records)
|
|
485
|
-
assert_equal(records,
|
|
530
|
+
assert_equal(remove_field_names(remove_field_names(records)),
|
|
531
|
+
target.raw_records)
|
|
486
532
|
end
|
|
487
533
|
|
|
488
534
|
def test_dictionary
|
|
489
|
-
omit("Need to add support for DictionaryArrayBuilder")
|
|
490
535
|
records = [
|
|
491
536
|
[{"0" => "Ruby"}],
|
|
492
537
|
[{"1" => nil}],
|
|
493
538
|
[{"0" => "GLib"}],
|
|
494
539
|
]
|
|
495
|
-
dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
|
|
496
540
|
target = build({
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
assert_equal(records,
|
|
541
|
+
type: :dictionary,
|
|
542
|
+
index_data_type: :int8,
|
|
543
|
+
value_data_type: :string,
|
|
544
|
+
ordered: false,
|
|
545
|
+
},
|
|
546
|
+
records)
|
|
547
|
+
assert_equal(remove_field_names(records),
|
|
548
|
+
target.raw_records)
|
|
504
549
|
end
|
|
505
550
|
end
|
|
506
551
|
|
|
@@ -509,13 +509,31 @@ module RawRecordsListArrayTests
|
|
|
509
509
|
assert_equal(records, target.raw_records)
|
|
510
510
|
end
|
|
511
511
|
|
|
512
|
-
def
|
|
513
|
-
|
|
512
|
+
def remove_union_field_names(records)
|
|
513
|
+
records.collect do |record|
|
|
514
|
+
record.collect do |column|
|
|
515
|
+
if column.nil?
|
|
516
|
+
column
|
|
517
|
+
else
|
|
518
|
+
column.collect do |value|
|
|
519
|
+
if value.nil?
|
|
520
|
+
value
|
|
521
|
+
else
|
|
522
|
+
value.values[0]
|
|
523
|
+
end
|
|
524
|
+
end
|
|
525
|
+
end
|
|
526
|
+
end
|
|
527
|
+
end
|
|
528
|
+
end
|
|
529
|
+
|
|
530
|
+
def test_sparse_union
|
|
514
531
|
records = [
|
|
515
532
|
[
|
|
516
533
|
[
|
|
517
534
|
{"field1" => true},
|
|
518
535
|
nil,
|
|
536
|
+
{"field2" => 29},
|
|
519
537
|
{"field2" => nil},
|
|
520
538
|
],
|
|
521
539
|
],
|
|
@@ -536,16 +554,17 @@ module RawRecordsListArrayTests
|
|
|
536
554
|
type_codes: [0, 1],
|
|
537
555
|
},
|
|
538
556
|
records)
|
|
539
|
-
assert_equal(records,
|
|
557
|
+
assert_equal(remove_union_field_names(records),
|
|
558
|
+
target.raw_records)
|
|
540
559
|
end
|
|
541
560
|
|
|
542
|
-
def
|
|
543
|
-
omit("Need to add support for DenseUnionArrayBuilder")
|
|
561
|
+
def test_dense_union
|
|
544
562
|
records = [
|
|
545
563
|
[
|
|
546
564
|
[
|
|
547
565
|
{"field1" => true},
|
|
548
566
|
nil,
|
|
567
|
+
{"field2" => 29},
|
|
549
568
|
{"field2" => nil},
|
|
550
569
|
],
|
|
551
570
|
],
|
|
@@ -566,11 +585,11 @@ module RawRecordsListArrayTests
|
|
|
566
585
|
type_codes: [0, 1],
|
|
567
586
|
},
|
|
568
587
|
records)
|
|
569
|
-
assert_equal(records,
|
|
588
|
+
assert_equal(remove_union_field_names(records),
|
|
589
|
+
target.raw_records)
|
|
570
590
|
end
|
|
571
591
|
|
|
572
592
|
def test_dictionary
|
|
573
|
-
omit("Need to add support for DictionaryArrayBuilder")
|
|
574
593
|
records = [
|
|
575
594
|
[
|
|
576
595
|
[
|
|
@@ -581,12 +600,11 @@ module RawRecordsListArrayTests
|
|
|
581
600
|
],
|
|
582
601
|
[nil],
|
|
583
602
|
]
|
|
584
|
-
dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
|
|
585
603
|
target = build({
|
|
586
604
|
type: :dictionary,
|
|
587
605
|
index_data_type: :int8,
|
|
588
|
-
|
|
589
|
-
ordered:
|
|
606
|
+
value_data_type: :string,
|
|
607
|
+
ordered: false,
|
|
590
608
|
},
|
|
591
609
|
records)
|
|
592
610
|
assert_equal(records, target.raw_records)
|
|
@@ -395,10 +395,33 @@ module RawRecordsMapArrayTests
|
|
|
395
395
|
assert_equal(records, target.raw_records)
|
|
396
396
|
end
|
|
397
397
|
|
|
398
|
+
def remove_union_field_names(records)
|
|
399
|
+
records.collect do |record|
|
|
400
|
+
record.collect do |column|
|
|
401
|
+
if column.nil?
|
|
402
|
+
column
|
|
403
|
+
else
|
|
404
|
+
value = {}
|
|
405
|
+
column.each do |k, v|
|
|
406
|
+
v = v.values[0] unless v.nil?
|
|
407
|
+
value[k] = v
|
|
408
|
+
end
|
|
409
|
+
value
|
|
410
|
+
end
|
|
411
|
+
end
|
|
412
|
+
end
|
|
413
|
+
end
|
|
414
|
+
|
|
398
415
|
def test_sparse_union
|
|
399
|
-
omit("Need to add support for SparseUnionArrayBuilder")
|
|
400
416
|
records = [
|
|
401
|
-
[
|
|
417
|
+
[
|
|
418
|
+
{
|
|
419
|
+
"key1" => {"field1" => true},
|
|
420
|
+
"key2" => nil,
|
|
421
|
+
"key3" => {"field2" => 29},
|
|
422
|
+
"key4" => {"field2" => nil},
|
|
423
|
+
},
|
|
424
|
+
],
|
|
402
425
|
[nil],
|
|
403
426
|
]
|
|
404
427
|
target = build({
|
|
@@ -416,13 +439,20 @@ module RawRecordsMapArrayTests
|
|
|
416
439
|
type_codes: [0, 1],
|
|
417
440
|
},
|
|
418
441
|
records)
|
|
419
|
-
assert_equal(records,
|
|
442
|
+
assert_equal(remove_union_field_names(records),
|
|
443
|
+
target.raw_records)
|
|
420
444
|
end
|
|
421
445
|
|
|
422
446
|
def test_dense_union
|
|
423
|
-
omit("Need to add support for DenseUnionArrayBuilder")
|
|
424
447
|
records = [
|
|
425
|
-
[
|
|
448
|
+
[
|
|
449
|
+
{
|
|
450
|
+
"key1" => {"field1" => true},
|
|
451
|
+
"key2" => nil,
|
|
452
|
+
"key3" => {"field2" => 29},
|
|
453
|
+
"key4" => {"field2" => nil},
|
|
454
|
+
},
|
|
455
|
+
],
|
|
426
456
|
[nil],
|
|
427
457
|
]
|
|
428
458
|
target = build({
|
|
@@ -440,21 +470,20 @@ module RawRecordsMapArrayTests
|
|
|
440
470
|
type_codes: [0, 1],
|
|
441
471
|
},
|
|
442
472
|
records)
|
|
443
|
-
assert_equal(records,
|
|
473
|
+
assert_equal(remove_union_field_names(records),
|
|
474
|
+
target.raw_records)
|
|
444
475
|
end
|
|
445
476
|
|
|
446
477
|
def test_dictionary
|
|
447
|
-
omit("Need to add support for DictionaryArrayBuilder")
|
|
448
478
|
records = [
|
|
449
479
|
[{"key1" => "Ruby", "key2" => nil, "key3" => "GLib"}],
|
|
450
480
|
[nil],
|
|
451
481
|
]
|
|
452
|
-
dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
|
|
453
482
|
target = build({
|
|
454
483
|
type: :dictionary,
|
|
455
484
|
index_data_type: :int8,
|
|
456
|
-
|
|
457
|
-
ordered:
|
|
485
|
+
value_data_type: :string,
|
|
486
|
+
ordered: false,
|
|
458
487
|
},
|
|
459
488
|
records)
|
|
460
489
|
assert_equal(records, target.raw_records)
|