red-arrow 11.0.0 → 12.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -3
  3. data/ext/arrow/converters.hpp +12 -27
  4. data/lib/arrow/array-computable.rb +13 -0
  5. data/lib/arrow/data-type.rb +9 -0
  6. data/lib/arrow/dense-union-array-builder.rb +49 -0
  7. data/lib/arrow/dense-union-array.rb +26 -0
  8. data/lib/arrow/loader.rb +5 -0
  9. data/lib/arrow/record-batch-file-reader.rb +2 -0
  10. data/lib/arrow/record-batch-stream-reader.rb +2 -0
  11. data/lib/arrow/scalar.rb +67 -0
  12. data/lib/arrow/slicer.rb +61 -0
  13. data/lib/arrow/sparse-union-array-builder.rb +56 -0
  14. data/lib/arrow/sparse-union-array.rb +26 -0
  15. data/lib/arrow/struct-array-builder.rb +0 -5
  16. data/lib/arrow/table.rb +130 -10
  17. data/lib/arrow/union-array-builder.rb +59 -0
  18. data/lib/arrow/version.rb +1 -1
  19. data/test/raw-records/test-dense-union-array.rb +90 -45
  20. data/test/raw-records/test-list-array.rb +28 -10
  21. data/test/raw-records/test-map-array.rb +39 -10
  22. data/test/raw-records/test-sparse-union-array.rb +86 -41
  23. data/test/raw-records/test-struct-array.rb +22 -8
  24. data/test/test-array.rb +7 -0
  25. data/test/test-chunked-array.rb +9 -0
  26. data/test/test-dense-union-array.rb +42 -0
  27. data/test/test-dense-union-data-type.rb +1 -1
  28. data/test/test-function.rb +7 -7
  29. data/test/test-group.rb +58 -58
  30. data/test/test-record-batch-file-reader.rb +21 -0
  31. data/test/test-record-batch-stream-reader.rb +129 -0
  32. data/test/test-scalar.rb +65 -0
  33. data/test/test-slicer.rb +194 -129
  34. data/test/test-sparse-union-array.rb +38 -0
  35. data/test/test-table.rb +200 -38
  36. data/test/values/test-dense-union-array.rb +88 -45
  37. data/test/values/test-list-array.rb +26 -10
  38. data/test/values/test-map-array.rb +33 -10
  39. data/test/values/test-sparse-union-array.rb +84 -41
  40. data/test/values/test-struct-array.rb +20 -8
  41. metadata +20 -7
@@ -55,12 +55,23 @@ module ValuesSparseUnionArrayTests
55
55
  arrays)
56
56
  end
57
57
 
58
+ def remove_field_names(values)
59
+ values.collect do |value|
60
+ if value.nil?
61
+ value
62
+ else
63
+ value.values[0]
64
+ end
65
+ end
66
+ end
67
+
58
68
  def test_null
59
69
  values = [
60
70
  {"0" => nil},
61
71
  ]
62
72
  target = build(:null, values)
63
- assert_equal(values, target.values)
73
+ assert_equal(remove_field_names(values),
74
+ target.values)
64
75
  end
65
76
 
66
77
  def test_boolean
@@ -69,7 +80,8 @@ module ValuesSparseUnionArrayTests
69
80
  {"1" => nil},
70
81
  ]
71
82
  target = build(:boolean, values)
72
- assert_equal(values, target.values)
83
+ assert_equal(remove_field_names(values),
84
+ target.values)
73
85
  end
74
86
 
75
87
  def test_int8
@@ -78,7 +90,8 @@ module ValuesSparseUnionArrayTests
78
90
  {"1" => nil},
79
91
  ]
80
92
  target = build(:int8, values)
81
- assert_equal(values, target.values)
93
+ assert_equal(remove_field_names(values),
94
+ target.values)
82
95
  end
83
96
 
84
97
  def test_uint8
@@ -87,7 +100,8 @@ module ValuesSparseUnionArrayTests
87
100
  {"1" => nil},
88
101
  ]
89
102
  target = build(:uint8, values)
90
- assert_equal(values, target.values)
103
+ assert_equal(remove_field_names(values),
104
+ target.values)
91
105
  end
92
106
 
93
107
  def test_int16
@@ -96,7 +110,8 @@ module ValuesSparseUnionArrayTests
96
110
  {"1" => nil},
97
111
  ]
98
112
  target = build(:int16, values)
99
- assert_equal(values, target.values)
113
+ assert_equal(remove_field_names(values),
114
+ target.values)
100
115
  end
101
116
 
102
117
  def test_uint16
@@ -105,7 +120,8 @@ module ValuesSparseUnionArrayTests
105
120
  {"1" => nil},
106
121
  ]
107
122
  target = build(:uint16, values)
108
- assert_equal(values, target.values)
123
+ assert_equal(remove_field_names(values),
124
+ target.values)
109
125
  end
110
126
 
111
127
  def test_int32
@@ -114,7 +130,8 @@ module ValuesSparseUnionArrayTests
114
130
  {"1" => nil},
115
131
  ]
116
132
  target = build(:int32, values)
117
- assert_equal(values, target.values)
133
+ assert_equal(remove_field_names(values),
134
+ target.values)
118
135
  end
119
136
 
120
137
  def test_uint32
@@ -123,7 +140,8 @@ module ValuesSparseUnionArrayTests
123
140
  {"1" => nil},
124
141
  ]
125
142
  target = build(:uint32, values)
126
- assert_equal(values, target.values)
143
+ assert_equal(remove_field_names(values),
144
+ target.values)
127
145
  end
128
146
 
129
147
  def test_int64
@@ -132,7 +150,8 @@ module ValuesSparseUnionArrayTests
132
150
  {"1" => nil},
133
151
  ]
134
152
  target = build(:int64, values)
135
- assert_equal(values, target.values)
153
+ assert_equal(remove_field_names(values),
154
+ target.values)
136
155
  end
137
156
 
138
157
  def test_uint64
@@ -141,7 +160,8 @@ module ValuesSparseUnionArrayTests
141
160
  {"1" => nil},
142
161
  ]
143
162
  target = build(:uint64, values)
144
- assert_equal(values, target.values)
163
+ assert_equal(remove_field_names(values),
164
+ target.values)
145
165
  end
146
166
 
147
167
  def test_float
@@ -150,7 +170,8 @@ module ValuesSparseUnionArrayTests
150
170
  {"1" => nil},
151
171
  ]
152
172
  target = build(:float, values)
153
- assert_equal(values, target.values)
173
+ assert_equal(remove_field_names(values),
174
+ target.values)
154
175
  end
155
176
 
156
177
  def test_double
@@ -159,7 +180,8 @@ module ValuesSparseUnionArrayTests
159
180
  {"1" => nil},
160
181
  ]
161
182
  target = build(:double, values)
162
- assert_equal(values, target.values)
183
+ assert_equal(remove_field_names(values),
184
+ target.values)
163
185
  end
164
186
 
165
187
  def test_binary
@@ -168,7 +190,8 @@ module ValuesSparseUnionArrayTests
168
190
  {"1" => nil},
169
191
  ]
170
192
  target = build(:binary, values)
171
- assert_equal(values, target.values)
193
+ assert_equal(remove_field_names(values),
194
+ target.values)
172
195
  end
173
196
 
174
197
  def test_string
@@ -177,7 +200,8 @@ module ValuesSparseUnionArrayTests
177
200
  {"1" => nil},
178
201
  ]
179
202
  target = build(:string, values)
180
- assert_equal(values, target.values)
203
+ assert_equal(remove_field_names(values),
204
+ target.values)
181
205
  end
182
206
 
183
207
  def test_date32
@@ -186,7 +210,8 @@ module ValuesSparseUnionArrayTests
186
210
  {"1" => nil},
187
211
  ]
188
212
  target = build(:date32, values)
189
- assert_equal(values, target.values)
213
+ assert_equal(remove_field_names(values),
214
+ target.values)
190
215
  end
191
216
 
192
217
  def test_date64
@@ -195,7 +220,8 @@ module ValuesSparseUnionArrayTests
195
220
  {"1" => nil},
196
221
  ]
197
222
  target = build(:date64, values)
198
- assert_equal(values, target.values)
223
+ assert_equal(remove_field_names(values),
224
+ target.values)
199
225
  end
200
226
 
201
227
  def test_timestamp_second
@@ -208,7 +234,8 @@ module ValuesSparseUnionArrayTests
208
234
  unit: :second,
209
235
  },
210
236
  values)
211
- assert_equal(values, target.values)
237
+ assert_equal(remove_field_names(values),
238
+ target.values)
212
239
  end
213
240
 
214
241
  def test_timestamp_milli
@@ -221,7 +248,8 @@ module ValuesSparseUnionArrayTests
221
248
  unit: :milli,
222
249
  },
223
250
  values)
224
- assert_equal(values, target.values)
251
+ assert_equal(remove_field_names(values),
252
+ target.values)
225
253
  end
226
254
 
227
255
  def test_timestamp_micro
@@ -234,7 +262,8 @@ module ValuesSparseUnionArrayTests
234
262
  unit: :micro,
235
263
  },
236
264
  values)
237
- assert_equal(values, target.values)
265
+ assert_equal(remove_field_names(values),
266
+ target.values)
238
267
  end
239
268
 
240
269
  def test_timestamp_nano
@@ -247,7 +276,8 @@ module ValuesSparseUnionArrayTests
247
276
  unit: :nano,
248
277
  },
249
278
  values)
250
- assert_equal(values, target.values)
279
+ assert_equal(remove_field_names(values),
280
+ target.values)
251
281
  end
252
282
 
253
283
  def test_time32_second
@@ -262,7 +292,8 @@ module ValuesSparseUnionArrayTests
262
292
  unit: :second,
263
293
  },
264
294
  values)
265
- assert_equal(values, target.values)
295
+ assert_equal(remove_field_names(values),
296
+ target.values)
266
297
  end
267
298
 
268
299
  def test_time32_milli
@@ -277,7 +308,8 @@ module ValuesSparseUnionArrayTests
277
308
  unit: :milli,
278
309
  },
279
310
  values)
280
- assert_equal(values, target.values)
311
+ assert_equal(remove_field_names(values),
312
+ target.values)
281
313
  end
282
314
 
283
315
  def test_time64_micro
@@ -292,7 +324,8 @@ module ValuesSparseUnionArrayTests
292
324
  unit: :micro,
293
325
  },
294
326
  values)
295
- assert_equal(values, target.values)
327
+ assert_equal(remove_field_names(values),
328
+ target.values)
296
329
  end
297
330
 
298
331
  def test_time64_nano
@@ -307,7 +340,8 @@ module ValuesSparseUnionArrayTests
307
340
  unit: :nano,
308
341
  },
309
342
  values)
310
- assert_equal(values, target.values)
343
+ assert_equal(remove_field_names(values),
344
+ target.values)
311
345
  end
312
346
 
313
347
  def test_decimal128
@@ -321,7 +355,8 @@ module ValuesSparseUnionArrayTests
321
355
  scale: 2,
322
356
  },
323
357
  values)
324
- assert_equal(values, target.values)
358
+ assert_equal(remove_field_names(values),
359
+ target.values)
325
360
  end
326
361
 
327
362
  def test_month_interval
@@ -330,7 +365,8 @@ module ValuesSparseUnionArrayTests
330
365
  {"1" => nil},
331
366
  ]
332
367
  target = build(:month_interval, values)
333
- assert_equal(values, target.values)
368
+ assert_equal(remove_field_names(values),
369
+ target.values)
334
370
  end
335
371
 
336
372
  def test_day_time_interval
@@ -339,7 +375,8 @@ module ValuesSparseUnionArrayTests
339
375
  {"1" => nil},
340
376
  ]
341
377
  target = build(:day_time_interval, values)
342
- assert_equal(values, target.values)
378
+ assert_equal(remove_field_names(values),
379
+ target.values)
343
380
  end
344
381
 
345
382
  def test_month_day_nano_interval
@@ -348,7 +385,8 @@ module ValuesSparseUnionArrayTests
348
385
  {"1" => nil},
349
386
  ]
350
387
  target = build(:month_day_nano_interval, values)
351
- assert_equal(values, target.values)
388
+ assert_equal(remove_field_names(values),
389
+ target.values)
352
390
  end
353
391
 
354
392
  def test_decimal256
@@ -362,7 +400,8 @@ module ValuesSparseUnionArrayTests
362
400
  scale: 2,
363
401
  },
364
402
  values)
365
- assert_equal(values, target.values)
403
+ assert_equal(remove_field_names(values),
404
+ target.values)
366
405
  end
367
406
 
368
407
  def test_list
@@ -378,7 +417,8 @@ module ValuesSparseUnionArrayTests
378
417
  },
379
418
  },
380
419
  values)
381
- assert_equal(values, target.values)
420
+ assert_equal(remove_field_names(values),
421
+ target.values)
382
422
  end
383
423
 
384
424
  def test_struct
@@ -397,7 +437,8 @@ module ValuesSparseUnionArrayTests
397
437
  ],
398
438
  },
399
439
  values)
400
- assert_equal(values, target.values)
440
+ assert_equal(remove_field_names(values),
441
+ target.values)
401
442
  end
402
443
 
403
444
  def test_map
@@ -411,14 +452,15 @@ module ValuesSparseUnionArrayTests
411
452
  item: :boolean,
412
453
  },
413
454
  values)
414
- assert_equal(values, target.values)
455
+ assert_equal(remove_field_names(values),
456
+ target.values)
415
457
  end
416
458
 
417
459
  def test_sparse_union
418
- omit("Need to add support for SparseUnionArrayBuilder")
419
460
  values = [
420
461
  {"0" => {"field1" => true}},
421
462
  {"1" => nil},
463
+ {"0" => {"field2" => 29}},
422
464
  {"0" => {"field2" => nil}},
423
465
  ]
424
466
  target = build({
@@ -436,14 +478,15 @@ module ValuesSparseUnionArrayTests
436
478
  type_codes: [0, 1],
437
479
  },
438
480
  values)
439
- assert_equal(values, target.values)
481
+ assert_equal(remove_field_names(remove_field_names(values)),
482
+ target.values)
440
483
  end
441
484
 
442
485
  def test_dense_union
443
- omit("Need to add support for DenseUnionArrayBuilder")
444
486
  values = [
445
487
  {"0" => {"field1" => true}},
446
488
  {"1" => nil},
489
+ {"0" => {"field2" => 29}},
447
490
  {"0" => {"field2" => nil}},
448
491
  ]
449
492
  target = build({
@@ -461,25 +504,25 @@ module ValuesSparseUnionArrayTests
461
504
  type_codes: [0, 1],
462
505
  },
463
506
  values)
464
- assert_equal(values, target.values)
507
+ assert_equal(remove_field_names(remove_field_names(values)),
508
+ target.values)
465
509
  end
466
510
 
467
511
  def test_dictionary
468
- omit("Need to add support for DictionaryArrayBuilder")
469
512
  values = [
470
513
  {"0" => "Ruby"},
471
514
  {"1" => nil},
472
515
  {"0" => "GLib"},
473
516
  ]
474
- dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
475
517
  target = build({
476
518
  type: :dictionary,
477
519
  index_data_type: :int8,
478
- dictionary: dictionary,
479
- ordered: true,
520
+ value_data_type: :string,
521
+ ordered: false,
480
522
  },
481
523
  values)
482
- assert_equal(values, target.values)
524
+ assert_equal(remove_field_names(values),
525
+ target.values)
483
526
  end
484
527
  end
485
528
 
@@ -423,12 +423,24 @@ module ValuesStructArrayTests
423
423
  assert_equal(values, target.values)
424
424
  end
425
425
 
426
+ def remove_union_field_names(values)
427
+ values.collect do |value|
428
+ if value.nil?
429
+ value
430
+ else
431
+ v = value["field"]
432
+ v = v.values[0] unless v.nil?
433
+ {"field" => v}
434
+ end
435
+ end
436
+ end
437
+
426
438
  def test_sparse_union
427
- omit("Need to add support for SparseUnionArrayBuilder")
428
439
  values = [
429
440
  {"field" => {"field1" => true}},
430
441
  nil,
431
442
  {"field" => nil},
443
+ {"field" => {"field2" => 29}},
432
444
  {"field" => {"field2" => nil}},
433
445
  ]
434
446
  target = build({
@@ -446,15 +458,16 @@ module ValuesStructArrayTests
446
458
  type_codes: [0, 1],
447
459
  },
448
460
  values)
449
- assert_equal(values, target.values)
461
+ assert_equal(remove_union_field_names(values),
462
+ target.values)
450
463
  end
451
464
 
452
465
  def test_dense_union
453
- omit("Need to add support for DenseUnionArrayBuilder")
454
466
  values = [
455
467
  {"field" => {"field1" => true}},
456
468
  nil,
457
469
  {"field" => nil},
470
+ {"field" => {"field2" => 29}},
458
471
  {"field" => {"field2" => nil}},
459
472
  ]
460
473
  target = build({
@@ -472,23 +485,22 @@ module ValuesStructArrayTests
472
485
  type_codes: [0, 1],
473
486
  },
474
487
  values)
475
- assert_equal(values, target.values)
488
+ assert_equal(remove_union_field_names(values),
489
+ target.values)
476
490
  end
477
491
 
478
492
  def test_dictionary
479
- omit("Need to add support for DictionaryArrayBuilder")
480
493
  values = [
481
494
  {"field" => "Ruby"},
482
495
  nil,
483
496
  {"field" => nil},
484
497
  {"field" => "GLib"},
485
498
  ]
486
- dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
487
499
  target = build({
488
500
  type: :dictionary,
489
501
  index_data_type: :int8,
490
- dictionary: dictionary,
491
- ordered: true,
502
+ value_data_type: :string,
503
+ ordered: false,
492
504
  },
493
505
  values)
494
506
  assert_equal(values, target.values)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: red-arrow
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.0.0
4
+ version: 12.0.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Apache Arrow Developers
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-02-24 00:00:00.000000000 Z
11
+ date: 2023-06-15 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bigdecimal
@@ -137,6 +137,8 @@ files:
137
137
  - lib/arrow/decimal256-array.rb
138
138
  - lib/arrow/decimal256-data-type.rb
139
139
  - lib/arrow/decimal256.rb
140
+ - lib/arrow/dense-union-array-builder.rb
141
+ - lib/arrow/dense-union-array.rb
140
142
  - lib/arrow/dense-union-data-type.rb
141
143
  - lib/arrow/dictionary-array.rb
142
144
  - lib/arrow/dictionary-data-type.rb
@@ -184,6 +186,8 @@ files:
184
186
  - lib/arrow/sort-key.rb
185
187
  - lib/arrow/sort-options.rb
186
188
  - lib/arrow/source-node-options.rb
189
+ - lib/arrow/sparse-union-array-builder.rb
190
+ - lib/arrow/sparse-union-array.rb
187
191
  - lib/arrow/sparse-union-data-type.rb
188
192
  - lib/arrow/string-array-builder.rb
189
193
  - lib/arrow/string-dictionary-array-builder.rb
@@ -210,6 +214,7 @@ files:
210
214
  - lib/arrow/timestamp-array-builder.rb
211
215
  - lib/arrow/timestamp-array.rb
212
216
  - lib/arrow/timestamp-data-type.rb
217
+ - lib/arrow/union-array-builder.rb
213
218
  - lib/arrow/version.rb
214
219
  - lib/arrow/writable.rb
215
220
  - red-arrow.gemspec
@@ -255,6 +260,7 @@ files:
255
260
  - test/test-decimal256-array.rb
256
261
  - test/test-decimal256-data-type.rb
257
262
  - test/test-decimal256.rb
263
+ - test/test-dense-union-array.rb
258
264
  - test/test-dense-union-data-type.rb
259
265
  - test/test-dictionary-array.rb
260
266
  - test/test-dictionary-data-type.rb
@@ -282,13 +288,16 @@ files:
282
288
  - test/test-record-batch-file-reader.rb
283
289
  - test/test-record-batch-iterator.rb
284
290
  - test/test-record-batch-reader.rb
291
+ - test/test-record-batch-stream-reader.rb
285
292
  - test/test-record-batch.rb
286
293
  - test/test-rolling-window.rb
294
+ - test/test-scalar.rb
287
295
  - test/test-schema.rb
288
296
  - test/test-slicer.rb
289
297
  - test/test-sort-indices.rb
290
298
  - test/test-sort-key.rb
291
299
  - test/test-sort-options.rb
300
+ - test/test-sparse-union-array.rb
292
301
  - test/test-sparse-union-data-type.rb
293
302
  - test/test-string-dictionary-array-builder.rb
294
303
  - test/test-struct-array-builder.rb
@@ -314,8 +323,8 @@ homepage: https://arrow.apache.org/
314
323
  licenses:
315
324
  - Apache-2.0
316
325
  metadata:
317
- msys2_mingw_dependencies: arrow>=11.0.0
318
- post_install_message:
326
+ msys2_mingw_dependencies: arrow>=12.0.1
327
+ post_install_message:
319
328
  rdoc_options: []
320
329
  require_paths:
321
330
  - lib
@@ -330,8 +339,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
330
339
  - !ruby/object:Gem::Version
331
340
  version: '0'
332
341
  requirements: []
333
- rubygems_version: 3.5.0.dev
334
- signing_key:
342
+ rubygems_version: 3.3.5
343
+ signing_key:
335
344
  specification_version: 4
336
345
  summary: Red Arrow is the Ruby bindings of Apache Arrow
337
346
  test_files:
@@ -377,6 +386,7 @@ test_files:
377
386
  - test/test-decimal256-array.rb
378
387
  - test/test-decimal256-data-type.rb
379
388
  - test/test-decimal256.rb
389
+ - test/test-dense-union-array.rb
380
390
  - test/test-dense-union-data-type.rb
381
391
  - test/test-dictionary-array.rb
382
392
  - test/test-dictionary-data-type.rb
@@ -404,13 +414,16 @@ test_files:
404
414
  - test/test-record-batch-file-reader.rb
405
415
  - test/test-record-batch-iterator.rb
406
416
  - test/test-record-batch-reader.rb
417
+ - test/test-record-batch-stream-reader.rb
407
418
  - test/test-record-batch.rb
408
419
  - test/test-rolling-window.rb
420
+ - test/test-scalar.rb
409
421
  - test/test-schema.rb
410
422
  - test/test-slicer.rb
411
423
  - test/test-sort-indices.rb
412
424
  - test/test-sort-key.rb
413
425
  - test/test-sort-options.rb
426
+ - test/test-sparse-union-array.rb
414
427
  - test/test-sparse-union-data-type.rb
415
428
  - test/test-string-dictionary-array-builder.rb
416
429
  - test/test-struct-array-builder.rb