red-arrow 11.0.0 → 13.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (47) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -3
  3. data/ext/arrow/arrow.cpp +25 -0
  4. data/ext/arrow/converters.hpp +12 -27
  5. data/ext/arrow/extconf.rb +2 -2
  6. data/lib/arrow/array-computable.rb +13 -0
  7. data/lib/arrow/chunked-array.rb +5 -1
  8. data/lib/arrow/data-type.rb +9 -0
  9. data/lib/arrow/dense-union-array-builder.rb +49 -0
  10. data/lib/arrow/dense-union-array.rb +26 -0
  11. data/lib/arrow/expression.rb +6 -2
  12. data/lib/arrow/function.rb +0 -1
  13. data/lib/arrow/loader.rb +5 -0
  14. data/lib/arrow/record-batch-file-reader.rb +2 -0
  15. data/lib/arrow/record-batch-stream-reader.rb +2 -0
  16. data/lib/arrow/scalar.rb +67 -0
  17. data/lib/arrow/slicer.rb +61 -0
  18. data/lib/arrow/sparse-union-array-builder.rb +56 -0
  19. data/lib/arrow/sparse-union-array.rb +26 -0
  20. data/lib/arrow/struct-array-builder.rb +0 -5
  21. data/lib/arrow/table.rb +130 -10
  22. data/lib/arrow/union-array-builder.rb +59 -0
  23. data/lib/arrow/version.rb +1 -1
  24. data/test/raw-records/test-dense-union-array.rb +90 -45
  25. data/test/raw-records/test-list-array.rb +28 -10
  26. data/test/raw-records/test-map-array.rb +39 -10
  27. data/test/raw-records/test-sparse-union-array.rb +86 -41
  28. data/test/raw-records/test-struct-array.rb +22 -8
  29. data/test/test-array.rb +7 -0
  30. data/test/test-chunked-array.rb +9 -0
  31. data/test/test-dense-union-array.rb +42 -0
  32. data/test/test-dense-union-data-type.rb +1 -1
  33. data/test/test-expression.rb +11 -0
  34. data/test/test-function.rb +7 -7
  35. data/test/test-group.rb +58 -58
  36. data/test/test-record-batch-file-reader.rb +21 -0
  37. data/test/test-record-batch-stream-reader.rb +129 -0
  38. data/test/test-scalar.rb +65 -0
  39. data/test/test-slicer.rb +194 -129
  40. data/test/test-sparse-union-array.rb +38 -0
  41. data/test/test-table.rb +207 -38
  42. data/test/values/test-dense-union-array.rb +88 -45
  43. data/test/values/test-list-array.rb +26 -10
  44. data/test/values/test-map-array.rb +33 -10
  45. data/test/values/test-sparse-union-array.rb +84 -41
  46. data/test/values/test-struct-array.rb +20 -8
  47. metadata +20 -7
@@ -55,12 +55,23 @@ module ValuesSparseUnionArrayTests
55
55
  arrays)
56
56
  end
57
57
 
58
+ def remove_field_names(values)
59
+ values.collect do |value|
60
+ if value.nil?
61
+ value
62
+ else
63
+ value.values[0]
64
+ end
65
+ end
66
+ end
67
+
58
68
  def test_null
59
69
  values = [
60
70
  {"0" => nil},
61
71
  ]
62
72
  target = build(:null, values)
63
- assert_equal(values, target.values)
73
+ assert_equal(remove_field_names(values),
74
+ target.values)
64
75
  end
65
76
 
66
77
  def test_boolean
@@ -69,7 +80,8 @@ module ValuesSparseUnionArrayTests
69
80
  {"1" => nil},
70
81
  ]
71
82
  target = build(:boolean, values)
72
- assert_equal(values, target.values)
83
+ assert_equal(remove_field_names(values),
84
+ target.values)
73
85
  end
74
86
 
75
87
  def test_int8
@@ -78,7 +90,8 @@ module ValuesSparseUnionArrayTests
78
90
  {"1" => nil},
79
91
  ]
80
92
  target = build(:int8, values)
81
- assert_equal(values, target.values)
93
+ assert_equal(remove_field_names(values),
94
+ target.values)
82
95
  end
83
96
 
84
97
  def test_uint8
@@ -87,7 +100,8 @@ module ValuesSparseUnionArrayTests
87
100
  {"1" => nil},
88
101
  ]
89
102
  target = build(:uint8, values)
90
- assert_equal(values, target.values)
103
+ assert_equal(remove_field_names(values),
104
+ target.values)
91
105
  end
92
106
 
93
107
  def test_int16
@@ -96,7 +110,8 @@ module ValuesSparseUnionArrayTests
96
110
  {"1" => nil},
97
111
  ]
98
112
  target = build(:int16, values)
99
- assert_equal(values, target.values)
113
+ assert_equal(remove_field_names(values),
114
+ target.values)
100
115
  end
101
116
 
102
117
  def test_uint16
@@ -105,7 +120,8 @@ module ValuesSparseUnionArrayTests
105
120
  {"1" => nil},
106
121
  ]
107
122
  target = build(:uint16, values)
108
- assert_equal(values, target.values)
123
+ assert_equal(remove_field_names(values),
124
+ target.values)
109
125
  end
110
126
 
111
127
  def test_int32
@@ -114,7 +130,8 @@ module ValuesSparseUnionArrayTests
114
130
  {"1" => nil},
115
131
  ]
116
132
  target = build(:int32, values)
117
- assert_equal(values, target.values)
133
+ assert_equal(remove_field_names(values),
134
+ target.values)
118
135
  end
119
136
 
120
137
  def test_uint32
@@ -123,7 +140,8 @@ module ValuesSparseUnionArrayTests
123
140
  {"1" => nil},
124
141
  ]
125
142
  target = build(:uint32, values)
126
- assert_equal(values, target.values)
143
+ assert_equal(remove_field_names(values),
144
+ target.values)
127
145
  end
128
146
 
129
147
  def test_int64
@@ -132,7 +150,8 @@ module ValuesSparseUnionArrayTests
132
150
  {"1" => nil},
133
151
  ]
134
152
  target = build(:int64, values)
135
- assert_equal(values, target.values)
153
+ assert_equal(remove_field_names(values),
154
+ target.values)
136
155
  end
137
156
 
138
157
  def test_uint64
@@ -141,7 +160,8 @@ module ValuesSparseUnionArrayTests
141
160
  {"1" => nil},
142
161
  ]
143
162
  target = build(:uint64, values)
144
- assert_equal(values, target.values)
163
+ assert_equal(remove_field_names(values),
164
+ target.values)
145
165
  end
146
166
 
147
167
  def test_float
@@ -150,7 +170,8 @@ module ValuesSparseUnionArrayTests
150
170
  {"1" => nil},
151
171
  ]
152
172
  target = build(:float, values)
153
- assert_equal(values, target.values)
173
+ assert_equal(remove_field_names(values),
174
+ target.values)
154
175
  end
155
176
 
156
177
  def test_double
@@ -159,7 +180,8 @@ module ValuesSparseUnionArrayTests
159
180
  {"1" => nil},
160
181
  ]
161
182
  target = build(:double, values)
162
- assert_equal(values, target.values)
183
+ assert_equal(remove_field_names(values),
184
+ target.values)
163
185
  end
164
186
 
165
187
  def test_binary
@@ -168,7 +190,8 @@ module ValuesSparseUnionArrayTests
168
190
  {"1" => nil},
169
191
  ]
170
192
  target = build(:binary, values)
171
- assert_equal(values, target.values)
193
+ assert_equal(remove_field_names(values),
194
+ target.values)
172
195
  end
173
196
 
174
197
  def test_string
@@ -177,7 +200,8 @@ module ValuesSparseUnionArrayTests
177
200
  {"1" => nil},
178
201
  ]
179
202
  target = build(:string, values)
180
- assert_equal(values, target.values)
203
+ assert_equal(remove_field_names(values),
204
+ target.values)
181
205
  end
182
206
 
183
207
  def test_date32
@@ -186,7 +210,8 @@ module ValuesSparseUnionArrayTests
186
210
  {"1" => nil},
187
211
  ]
188
212
  target = build(:date32, values)
189
- assert_equal(values, target.values)
213
+ assert_equal(remove_field_names(values),
214
+ target.values)
190
215
  end
191
216
 
192
217
  def test_date64
@@ -195,7 +220,8 @@ module ValuesSparseUnionArrayTests
195
220
  {"1" => nil},
196
221
  ]
197
222
  target = build(:date64, values)
198
- assert_equal(values, target.values)
223
+ assert_equal(remove_field_names(values),
224
+ target.values)
199
225
  end
200
226
 
201
227
  def test_timestamp_second
@@ -208,7 +234,8 @@ module ValuesSparseUnionArrayTests
208
234
  unit: :second,
209
235
  },
210
236
  values)
211
- assert_equal(values, target.values)
237
+ assert_equal(remove_field_names(values),
238
+ target.values)
212
239
  end
213
240
 
214
241
  def test_timestamp_milli
@@ -221,7 +248,8 @@ module ValuesSparseUnionArrayTests
221
248
  unit: :milli,
222
249
  },
223
250
  values)
224
- assert_equal(values, target.values)
251
+ assert_equal(remove_field_names(values),
252
+ target.values)
225
253
  end
226
254
 
227
255
  def test_timestamp_micro
@@ -234,7 +262,8 @@ module ValuesSparseUnionArrayTests
234
262
  unit: :micro,
235
263
  },
236
264
  values)
237
- assert_equal(values, target.values)
265
+ assert_equal(remove_field_names(values),
266
+ target.values)
238
267
  end
239
268
 
240
269
  def test_timestamp_nano
@@ -247,7 +276,8 @@ module ValuesSparseUnionArrayTests
247
276
  unit: :nano,
248
277
  },
249
278
  values)
250
- assert_equal(values, target.values)
279
+ assert_equal(remove_field_names(values),
280
+ target.values)
251
281
  end
252
282
 
253
283
  def test_time32_second
@@ -262,7 +292,8 @@ module ValuesSparseUnionArrayTests
262
292
  unit: :second,
263
293
  },
264
294
  values)
265
- assert_equal(values, target.values)
295
+ assert_equal(remove_field_names(values),
296
+ target.values)
266
297
  end
267
298
 
268
299
  def test_time32_milli
@@ -277,7 +308,8 @@ module ValuesSparseUnionArrayTests
277
308
  unit: :milli,
278
309
  },
279
310
  values)
280
- assert_equal(values, target.values)
311
+ assert_equal(remove_field_names(values),
312
+ target.values)
281
313
  end
282
314
 
283
315
  def test_time64_micro
@@ -292,7 +324,8 @@ module ValuesSparseUnionArrayTests
292
324
  unit: :micro,
293
325
  },
294
326
  values)
295
- assert_equal(values, target.values)
327
+ assert_equal(remove_field_names(values),
328
+ target.values)
296
329
  end
297
330
 
298
331
  def test_time64_nano
@@ -307,7 +340,8 @@ module ValuesSparseUnionArrayTests
307
340
  unit: :nano,
308
341
  },
309
342
  values)
310
- assert_equal(values, target.values)
343
+ assert_equal(remove_field_names(values),
344
+ target.values)
311
345
  end
312
346
 
313
347
  def test_decimal128
@@ -321,7 +355,8 @@ module ValuesSparseUnionArrayTests
321
355
  scale: 2,
322
356
  },
323
357
  values)
324
- assert_equal(values, target.values)
358
+ assert_equal(remove_field_names(values),
359
+ target.values)
325
360
  end
326
361
 
327
362
  def test_month_interval
@@ -330,7 +365,8 @@ module ValuesSparseUnionArrayTests
330
365
  {"1" => nil},
331
366
  ]
332
367
  target = build(:month_interval, values)
333
- assert_equal(values, target.values)
368
+ assert_equal(remove_field_names(values),
369
+ target.values)
334
370
  end
335
371
 
336
372
  def test_day_time_interval
@@ -339,7 +375,8 @@ module ValuesSparseUnionArrayTests
339
375
  {"1" => nil},
340
376
  ]
341
377
  target = build(:day_time_interval, values)
342
- assert_equal(values, target.values)
378
+ assert_equal(remove_field_names(values),
379
+ target.values)
343
380
  end
344
381
 
345
382
  def test_month_day_nano_interval
@@ -348,7 +385,8 @@ module ValuesSparseUnionArrayTests
348
385
  {"1" => nil},
349
386
  ]
350
387
  target = build(:month_day_nano_interval, values)
351
- assert_equal(values, target.values)
388
+ assert_equal(remove_field_names(values),
389
+ target.values)
352
390
  end
353
391
 
354
392
  def test_decimal256
@@ -362,7 +400,8 @@ module ValuesSparseUnionArrayTests
362
400
  scale: 2,
363
401
  },
364
402
  values)
365
- assert_equal(values, target.values)
403
+ assert_equal(remove_field_names(values),
404
+ target.values)
366
405
  end
367
406
 
368
407
  def test_list
@@ -378,7 +417,8 @@ module ValuesSparseUnionArrayTests
378
417
  },
379
418
  },
380
419
  values)
381
- assert_equal(values, target.values)
420
+ assert_equal(remove_field_names(values),
421
+ target.values)
382
422
  end
383
423
 
384
424
  def test_struct
@@ -397,7 +437,8 @@ module ValuesSparseUnionArrayTests
397
437
  ],
398
438
  },
399
439
  values)
400
- assert_equal(values, target.values)
440
+ assert_equal(remove_field_names(values),
441
+ target.values)
401
442
  end
402
443
 
403
444
  def test_map
@@ -411,14 +452,15 @@ module ValuesSparseUnionArrayTests
411
452
  item: :boolean,
412
453
  },
413
454
  values)
414
- assert_equal(values, target.values)
455
+ assert_equal(remove_field_names(values),
456
+ target.values)
415
457
  end
416
458
 
417
459
  def test_sparse_union
418
- omit("Need to add support for SparseUnionArrayBuilder")
419
460
  values = [
420
461
  {"0" => {"field1" => true}},
421
462
  {"1" => nil},
463
+ {"0" => {"field2" => 29}},
422
464
  {"0" => {"field2" => nil}},
423
465
  ]
424
466
  target = build({
@@ -436,14 +478,15 @@ module ValuesSparseUnionArrayTests
436
478
  type_codes: [0, 1],
437
479
  },
438
480
  values)
439
- assert_equal(values, target.values)
481
+ assert_equal(remove_field_names(remove_field_names(values)),
482
+ target.values)
440
483
  end
441
484
 
442
485
  def test_dense_union
443
- omit("Need to add support for DenseUnionArrayBuilder")
444
486
  values = [
445
487
  {"0" => {"field1" => true}},
446
488
  {"1" => nil},
489
+ {"0" => {"field2" => 29}},
447
490
  {"0" => {"field2" => nil}},
448
491
  ]
449
492
  target = build({
@@ -461,25 +504,25 @@ module ValuesSparseUnionArrayTests
461
504
  type_codes: [0, 1],
462
505
  },
463
506
  values)
464
- assert_equal(values, target.values)
507
+ assert_equal(remove_field_names(remove_field_names(values)),
508
+ target.values)
465
509
  end
466
510
 
467
511
  def test_dictionary
468
- omit("Need to add support for DictionaryArrayBuilder")
469
512
  values = [
470
513
  {"0" => "Ruby"},
471
514
  {"1" => nil},
472
515
  {"0" => "GLib"},
473
516
  ]
474
- dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
475
517
  target = build({
476
518
  type: :dictionary,
477
519
  index_data_type: :int8,
478
- dictionary: dictionary,
479
- ordered: true,
520
+ value_data_type: :string,
521
+ ordered: false,
480
522
  },
481
523
  values)
482
- assert_equal(values, target.values)
524
+ assert_equal(remove_field_names(values),
525
+ target.values)
483
526
  end
484
527
  end
485
528
 
@@ -423,12 +423,24 @@ module ValuesStructArrayTests
423
423
  assert_equal(values, target.values)
424
424
  end
425
425
 
426
+ def remove_union_field_names(values)
427
+ values.collect do |value|
428
+ if value.nil?
429
+ value
430
+ else
431
+ v = value["field"]
432
+ v = v.values[0] unless v.nil?
433
+ {"field" => v}
434
+ end
435
+ end
436
+ end
437
+
426
438
  def test_sparse_union
427
- omit("Need to add support for SparseUnionArrayBuilder")
428
439
  values = [
429
440
  {"field" => {"field1" => true}},
430
441
  nil,
431
442
  {"field" => nil},
443
+ {"field" => {"field2" => 29}},
432
444
  {"field" => {"field2" => nil}},
433
445
  ]
434
446
  target = build({
@@ -446,15 +458,16 @@ module ValuesStructArrayTests
446
458
  type_codes: [0, 1],
447
459
  },
448
460
  values)
449
- assert_equal(values, target.values)
461
+ assert_equal(remove_union_field_names(values),
462
+ target.values)
450
463
  end
451
464
 
452
465
  def test_dense_union
453
- omit("Need to add support for DenseUnionArrayBuilder")
454
466
  values = [
455
467
  {"field" => {"field1" => true}},
456
468
  nil,
457
469
  {"field" => nil},
470
+ {"field" => {"field2" => 29}},
458
471
  {"field" => {"field2" => nil}},
459
472
  ]
460
473
  target = build({
@@ -472,23 +485,22 @@ module ValuesStructArrayTests
472
485
  type_codes: [0, 1],
473
486
  },
474
487
  values)
475
- assert_equal(values, target.values)
488
+ assert_equal(remove_union_field_names(values),
489
+ target.values)
476
490
  end
477
491
 
478
492
  def test_dictionary
479
- omit("Need to add support for DictionaryArrayBuilder")
480
493
  values = [
481
494
  {"field" => "Ruby"},
482
495
  nil,
483
496
  {"field" => nil},
484
497
  {"field" => "GLib"},
485
498
  ]
486
- dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
487
499
  target = build({
488
500
  type: :dictionary,
489
501
  index_data_type: :int8,
490
- dictionary: dictionary,
491
- ordered: true,
502
+ value_data_type: :string,
503
+ ordered: false,
492
504
  },
493
505
  values)
494
506
  assert_equal(values, target.values)
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: red-arrow
3
3
  version: !ruby/object:Gem::Version
4
- version: 11.0.0
4
+ version: 13.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Apache Arrow Developers
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-02-24 00:00:00.000000000 Z
11
+ date: 2023-08-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bigdecimal
@@ -137,6 +137,8 @@ files:
137
137
  - lib/arrow/decimal256-array.rb
138
138
  - lib/arrow/decimal256-data-type.rb
139
139
  - lib/arrow/decimal256.rb
140
+ - lib/arrow/dense-union-array-builder.rb
141
+ - lib/arrow/dense-union-array.rb
140
142
  - lib/arrow/dense-union-data-type.rb
141
143
  - lib/arrow/dictionary-array.rb
142
144
  - lib/arrow/dictionary-data-type.rb
@@ -184,6 +186,8 @@ files:
184
186
  - lib/arrow/sort-key.rb
185
187
  - lib/arrow/sort-options.rb
186
188
  - lib/arrow/source-node-options.rb
189
+ - lib/arrow/sparse-union-array-builder.rb
190
+ - lib/arrow/sparse-union-array.rb
187
191
  - lib/arrow/sparse-union-data-type.rb
188
192
  - lib/arrow/string-array-builder.rb
189
193
  - lib/arrow/string-dictionary-array-builder.rb
@@ -210,6 +214,7 @@ files:
210
214
  - lib/arrow/timestamp-array-builder.rb
211
215
  - lib/arrow/timestamp-array.rb
212
216
  - lib/arrow/timestamp-data-type.rb
217
+ - lib/arrow/union-array-builder.rb
213
218
  - lib/arrow/version.rb
214
219
  - lib/arrow/writable.rb
215
220
  - red-arrow.gemspec
@@ -255,6 +260,7 @@ files:
255
260
  - test/test-decimal256-array.rb
256
261
  - test/test-decimal256-data-type.rb
257
262
  - test/test-decimal256.rb
263
+ - test/test-dense-union-array.rb
258
264
  - test/test-dense-union-data-type.rb
259
265
  - test/test-dictionary-array.rb
260
266
  - test/test-dictionary-data-type.rb
@@ -282,13 +288,16 @@ files:
282
288
  - test/test-record-batch-file-reader.rb
283
289
  - test/test-record-batch-iterator.rb
284
290
  - test/test-record-batch-reader.rb
291
+ - test/test-record-batch-stream-reader.rb
285
292
  - test/test-record-batch.rb
286
293
  - test/test-rolling-window.rb
294
+ - test/test-scalar.rb
287
295
  - test/test-schema.rb
288
296
  - test/test-slicer.rb
289
297
  - test/test-sort-indices.rb
290
298
  - test/test-sort-key.rb
291
299
  - test/test-sort-options.rb
300
+ - test/test-sparse-union-array.rb
292
301
  - test/test-sparse-union-data-type.rb
293
302
  - test/test-string-dictionary-array-builder.rb
294
303
  - test/test-struct-array-builder.rb
@@ -314,8 +323,8 @@ homepage: https://arrow.apache.org/
314
323
  licenses:
315
324
  - Apache-2.0
316
325
  metadata:
317
- msys2_mingw_dependencies: arrow>=11.0.0
318
- post_install_message:
326
+ msys2_mingw_dependencies: arrow>=13.0.0
327
+ post_install_message:
319
328
  rdoc_options: []
320
329
  require_paths:
321
330
  - lib
@@ -330,8 +339,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
330
339
  - !ruby/object:Gem::Version
331
340
  version: '0'
332
341
  requirements: []
333
- rubygems_version: 3.5.0.dev
334
- signing_key:
342
+ rubygems_version: 3.3.5
343
+ signing_key:
335
344
  specification_version: 4
336
345
  summary: Red Arrow is the Ruby bindings of Apache Arrow
337
346
  test_files:
@@ -377,6 +386,7 @@ test_files:
377
386
  - test/test-decimal256-array.rb
378
387
  - test/test-decimal256-data-type.rb
379
388
  - test/test-decimal256.rb
389
+ - test/test-dense-union-array.rb
380
390
  - test/test-dense-union-data-type.rb
381
391
  - test/test-dictionary-array.rb
382
392
  - test/test-dictionary-data-type.rb
@@ -404,13 +414,16 @@ test_files:
404
414
  - test/test-record-batch-file-reader.rb
405
415
  - test/test-record-batch-iterator.rb
406
416
  - test/test-record-batch-reader.rb
417
+ - test/test-record-batch-stream-reader.rb
407
418
  - test/test-record-batch.rb
408
419
  - test/test-rolling-window.rb
420
+ - test/test-scalar.rb
409
421
  - test/test-schema.rb
410
422
  - test/test-slicer.rb
411
423
  - test/test-sort-indices.rb
412
424
  - test/test-sort-key.rb
413
425
  - test/test-sort-options.rb
426
+ - test/test-sparse-union-array.rb
414
427
  - test/test-sparse-union-data-type.rb
415
428
  - test/test-string-dictionary-array-builder.rb
416
429
  - test/test-struct-array-builder.rb