red-arrow 0.15.0 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +28 -16
  3. data/ext/arrow/converters.hpp +63 -33
  4. data/ext/arrow/raw-records.cpp +2 -1
  5. data/ext/arrow/values.cpp +2 -1
  6. data/lib/arrow/array-builder.rb +101 -52
  7. data/lib/arrow/array.rb +28 -10
  8. data/lib/arrow/{binary-array-builder.rb → buffer.rb} +7 -15
  9. data/lib/arrow/chunked-array.rb +2 -0
  10. data/lib/arrow/csv-loader.rb +15 -3
  11. data/lib/arrow/csv-read-options.rb +18 -0
  12. data/lib/arrow/data-type.rb +35 -2
  13. data/lib/arrow/decimal128-array-builder.rb +0 -2
  14. data/lib/arrow/dictionary-array.rb +24 -0
  15. data/lib/arrow/field.rb +1 -1
  16. data/lib/arrow/generic-filterable.rb +43 -0
  17. data/lib/arrow/generic-takeable.rb +38 -0
  18. data/lib/arrow/list-data-type.rb +58 -8
  19. data/lib/arrow/loader.rb +12 -1
  20. data/lib/arrow/null-array-builder.rb +1 -1
  21. data/lib/arrow/null-array.rb +24 -0
  22. data/lib/arrow/raw-table-converter.rb +47 -0
  23. data/lib/arrow/record-batch-iterator.rb +22 -0
  24. data/lib/arrow/record-batch.rb +8 -3
  25. data/lib/arrow/schema.rb +5 -2
  26. data/lib/arrow/struct-array-builder.rb +13 -7
  27. data/lib/arrow/struct-data-type.rb +0 -2
  28. data/lib/arrow/table-loader.rb +29 -6
  29. data/lib/arrow/table-saver.rb +37 -13
  30. data/lib/arrow/table.rb +20 -73
  31. data/lib/arrow/version.rb +1 -1
  32. data/red-arrow.gemspec +4 -2
  33. data/test/helper.rb +1 -0
  34. data/test/helper/omittable.rb +36 -0
  35. data/test/raw-records/test-dense-union-array.rb +1 -34
  36. data/test/raw-records/test-sparse-union-array.rb +1 -33
  37. data/test/run-test.rb +14 -3
  38. data/test/test-array-builder.rb +17 -0
  39. data/test/test-array.rb +104 -0
  40. data/test/test-buffer.rb +11 -0
  41. data/test/test-chunked-array.rb +96 -0
  42. data/test/test-csv-loader.rb +77 -2
  43. data/test/test-data-type.rb +11 -0
  44. data/test/test-dense-union-data-type.rb +2 -2
  45. data/test/test-dictionary-array.rb +41 -0
  46. data/test/test-feather.rb +21 -6
  47. data/test/test-list-data-type.rb +27 -1
  48. data/test/test-null-array.rb +23 -0
  49. data/test/test-record-batch-iterator.rb +37 -0
  50. data/test/test-record-batch.rb +14 -0
  51. data/test/test-schema.rb +16 -0
  52. data/test/test-slicer.rb +74 -30
  53. data/test/test-sparse-union-data-type.rb +2 -2
  54. data/test/test-struct-array-builder.rb +8 -4
  55. data/test/test-table.rb +153 -14
  56. data/test/test-timestamp-array.rb +19 -0
  57. data/test/values/test-dense-union-array.rb +1 -34
  58. data/test/values/test-sparse-union-array.rb +1 -33
  59. metadata +76 -63
@@ -44,9 +44,7 @@ module ValuesSparseUnionArrayTests
44
44
  sub_record_batch.columns[0].data
45
45
  end
46
46
  values.each do |value|
47
- if value.nil?
48
- type_ids << nil
49
- elsif value.key?("0")
47
+ if value.key?("0")
50
48
  type_ids << type_codes[0]
51
49
  elsif value.key?("1")
52
50
  type_ids << type_codes[1]
@@ -60,7 +58,6 @@ module ValuesSparseUnionArrayTests
60
58
  def test_null
61
59
  values = [
62
60
  {"0" => nil},
63
- nil,
64
61
  ]
65
62
  target = build(:null, values)
66
63
  assert_equal(values, target.values)
@@ -69,7 +66,6 @@ module ValuesSparseUnionArrayTests
69
66
  def test_boolean
70
67
  values = [
71
68
  {"0" => true},
72
- nil,
73
69
  {"1" => nil},
74
70
  ]
75
71
  target = build(:boolean, values)
@@ -79,7 +75,6 @@ module ValuesSparseUnionArrayTests
79
75
  def test_int8
80
76
  values = [
81
77
  {"0" => -(2 ** 7)},
82
- nil,
83
78
  {"1" => nil},
84
79
  ]
85
80
  target = build(:int8, values)
@@ -89,7 +84,6 @@ module ValuesSparseUnionArrayTests
89
84
  def test_uint8
90
85
  values = [
91
86
  {"0" => (2 ** 8) - 1},
92
- nil,
93
87
  {"1" => nil},
94
88
  ]
95
89
  target = build(:uint8, values)
@@ -99,7 +93,6 @@ module ValuesSparseUnionArrayTests
99
93
  def test_int16
100
94
  values = [
101
95
  {"0" => -(2 ** 15)},
102
- nil,
103
96
  {"1" => nil},
104
97
  ]
105
98
  target = build(:int16, values)
@@ -109,7 +102,6 @@ module ValuesSparseUnionArrayTests
109
102
  def test_uint16
110
103
  values = [
111
104
  {"0" => (2 ** 16) - 1},
112
- nil,
113
105
  {"1" => nil},
114
106
  ]
115
107
  target = build(:uint16, values)
@@ -119,7 +111,6 @@ module ValuesSparseUnionArrayTests
119
111
  def test_int32
120
112
  values = [
121
113
  {"0" => -(2 ** 31)},
122
- nil,
123
114
  {"1" => nil},
124
115
  ]
125
116
  target = build(:int32, values)
@@ -129,7 +120,6 @@ module ValuesSparseUnionArrayTests
129
120
  def test_uint32
130
121
  values = [
131
122
  {"0" => (2 ** 32) - 1},
132
- nil,
133
123
  {"1" => nil},
134
124
  ]
135
125
  target = build(:uint32, values)
@@ -139,7 +129,6 @@ module ValuesSparseUnionArrayTests
139
129
  def test_int64
140
130
  values = [
141
131
  {"0" => -(2 ** 63)},
142
- nil,
143
132
  {"1" => nil},
144
133
  ]
145
134
  target = build(:int64, values)
@@ -149,7 +138,6 @@ module ValuesSparseUnionArrayTests
149
138
  def test_uint64
150
139
  values = [
151
140
  {"0" => (2 ** 64) - 1},
152
- nil,
153
141
  {"1" => nil},
154
142
  ]
155
143
  target = build(:uint64, values)
@@ -159,7 +147,6 @@ module ValuesSparseUnionArrayTests
159
147
  def test_float
160
148
  values = [
161
149
  {"0" => -1.0},
162
- nil,
163
150
  {"1" => nil},
164
151
  ]
165
152
  target = build(:float, values)
@@ -169,7 +156,6 @@ module ValuesSparseUnionArrayTests
169
156
  def test_double
170
157
  values = [
171
158
  {"0" => -1.0},
172
- nil,
173
159
  {"1" => nil},
174
160
  ]
175
161
  target = build(:double, values)
@@ -179,7 +165,6 @@ module ValuesSparseUnionArrayTests
179
165
  def test_binary
180
166
  values = [
181
167
  {"0" => "\xff".b},
182
- nil,
183
168
  {"1" => nil},
184
169
  ]
185
170
  target = build(:binary, values)
@@ -189,7 +174,6 @@ module ValuesSparseUnionArrayTests
189
174
  def test_string
190
175
  values = [
191
176
  {"0" => "Ruby"},
192
- nil,
193
177
  {"1" => nil},
194
178
  ]
195
179
  target = build(:string, values)
@@ -199,7 +183,6 @@ module ValuesSparseUnionArrayTests
199
183
  def test_date32
200
184
  values = [
201
185
  {"0" => Date.new(1960, 1, 1)},
202
- nil,
203
186
  {"1" => nil},
204
187
  ]
205
188
  target = build(:date32, values)
@@ -209,7 +192,6 @@ module ValuesSparseUnionArrayTests
209
192
  def test_date64
210
193
  values = [
211
194
  {"0" => DateTime.new(1960, 1, 1, 2, 9, 30)},
212
- nil,
213
195
  {"1" => nil},
214
196
  ]
215
197
  target = build(:date64, values)
@@ -219,7 +201,6 @@ module ValuesSparseUnionArrayTests
219
201
  def test_timestamp_second
220
202
  values = [
221
203
  {"0" => Time.parse("1960-01-01T02:09:30Z")},
222
- nil,
223
204
  {"1" => nil},
224
205
  ]
225
206
  target = build({
@@ -233,7 +214,6 @@ module ValuesSparseUnionArrayTests
233
214
  def test_timestamp_milli
234
215
  values = [
235
216
  {"0" => Time.parse("1960-01-01T02:09:30.123Z")},
236
- nil,
237
217
  {"1" => nil},
238
218
  ]
239
219
  target = build({
@@ -247,7 +227,6 @@ module ValuesSparseUnionArrayTests
247
227
  def test_timestamp_micro
248
228
  values = [
249
229
  {"0" => Time.parse("1960-01-01T02:09:30.123456Z")},
250
- nil,
251
230
  {"1" => nil},
252
231
  ]
253
232
  target = build({
@@ -261,7 +240,6 @@ module ValuesSparseUnionArrayTests
261
240
  def test_timestamp_nano
262
241
  values = [
263
242
  {"0" => Time.parse("1960-01-01T02:09:30.123456789Z")},
264
- nil,
265
243
  {"1" => nil},
266
244
  ]
267
245
  target = build({
@@ -277,7 +255,6 @@ module ValuesSparseUnionArrayTests
277
255
  values = [
278
256
  # 00:10:00
279
257
  {"0" => Arrow::Time.new(unit, 60 * 10)},
280
- nil,
281
258
  {"1" => nil},
282
259
  ]
283
260
  target = build({
@@ -293,7 +270,6 @@ module ValuesSparseUnionArrayTests
293
270
  values = [
294
271
  # 00:10:00.123
295
272
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)},
296
- nil,
297
273
  {"1" => nil},
298
274
  ]
299
275
  target = build({
@@ -309,7 +285,6 @@ module ValuesSparseUnionArrayTests
309
285
  values = [
310
286
  # 00:10:00.123456
311
287
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)},
312
- nil,
313
288
  {"1" => nil},
314
289
  ]
315
290
  target = build({
@@ -325,7 +300,6 @@ module ValuesSparseUnionArrayTests
325
300
  values = [
326
301
  # 00:10:00.123456789
327
302
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)},
328
- nil,
329
303
  {"1" => nil},
330
304
  ]
331
305
  target = build({
@@ -339,7 +313,6 @@ module ValuesSparseUnionArrayTests
339
313
  def test_decimal128
340
314
  values = [
341
315
  {"0" => BigDecimal("92.92")},
342
- nil,
343
316
  {"1" => nil},
344
317
  ]
345
318
  target = build({
@@ -354,7 +327,6 @@ module ValuesSparseUnionArrayTests
354
327
  def test_list
355
328
  values = [
356
329
  {"0" => [true, nil, false]},
357
- nil,
358
330
  {"1" => nil},
359
331
  ]
360
332
  target = build({
@@ -371,7 +343,6 @@ module ValuesSparseUnionArrayTests
371
343
  def test_struct
372
344
  values = [
373
345
  {"0" => {"sub_field" => true}},
374
- nil,
375
346
  {"1" => nil},
376
347
  {"0" => {"sub_field" => nil}},
377
348
  ]
@@ -392,7 +363,6 @@ module ValuesSparseUnionArrayTests
392
363
  omit("Need to add support for SparseUnionArrayBuilder")
393
364
  values = [
394
365
  {"0" => {"field1" => true}},
395
- nil,
396
366
  {"1" => nil},
397
367
  {"0" => {"field2" => nil}},
398
368
  ]
@@ -418,7 +388,6 @@ module ValuesSparseUnionArrayTests
418
388
  omit("Need to add support for DenseUnionArrayBuilder")
419
389
  values = [
420
390
  {"0" => {"field1" => true}},
421
- nil,
422
391
  {"1" => nil},
423
392
  {"0" => {"field2" => nil}},
424
393
  ]
@@ -444,7 +413,6 @@ module ValuesSparseUnionArrayTests
444
413
  omit("Need to add support for DictionaryArrayBuilder")
445
414
  values = [
446
415
  {"0" => "Ruby"},
447
- nil,
448
416
  {"1" => nil},
449
417
  {"0" => "GLib"},
450
418
  ]
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: red-arrow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.15.0
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Apache Arrow Developers
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-10-05 00:00:00.000000000 Z
11
+ date: 2020-07-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: extpp
@@ -28,16 +28,16 @@ dependencies:
28
28
  name: gio2
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - '='
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: 3.3.7
33
+ version: 3.3.6
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: 3.3.7
40
+ version: 3.3.6
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: native-package-installer
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -191,8 +191,8 @@ files:
191
191
  - lib/arrow/array-builder.rb
192
192
  - lib/arrow/array.rb
193
193
  - lib/arrow/bigdecimal-extension.rb
194
- - lib/arrow/binary-array-builder.rb
195
194
  - lib/arrow/block-closable.rb
195
+ - lib/arrow/buffer.rb
196
196
  - lib/arrow/chunked-array.rb
197
197
  - lib/arrow/column-containable.rb
198
198
  - lib/arrow/column.rb
@@ -209,18 +209,24 @@ files:
209
209
  - lib/arrow/decimal128-data-type.rb
210
210
  - lib/arrow/decimal128.rb
211
211
  - lib/arrow/dense-union-data-type.rb
212
+ - lib/arrow/dictionary-array.rb
212
213
  - lib/arrow/dictionary-data-type.rb
213
214
  - lib/arrow/field-containable.rb
214
215
  - lib/arrow/field.rb
215
216
  - lib/arrow/file-output-stream.rb
217
+ - lib/arrow/generic-filterable.rb
218
+ - lib/arrow/generic-takeable.rb
216
219
  - lib/arrow/group.rb
217
220
  - lib/arrow/list-array-builder.rb
218
221
  - lib/arrow/list-data-type.rb
219
222
  - lib/arrow/loader.rb
220
223
  - lib/arrow/null-array-builder.rb
224
+ - lib/arrow/null-array.rb
221
225
  - lib/arrow/path-extension.rb
226
+ - lib/arrow/raw-table-converter.rb
222
227
  - lib/arrow/record-batch-builder.rb
223
228
  - lib/arrow/record-batch-file-reader.rb
229
+ - lib/arrow/record-batch-iterator.rb
224
230
  - lib/arrow/record-batch-stream-reader.rb
225
231
  - lib/arrow/record-batch.rb
226
232
  - lib/arrow/record-containable.rb
@@ -263,6 +269,7 @@ files:
263
269
  - test/fixture/without-header.csv
264
270
  - test/helper.rb
265
271
  - test/helper/fixture.rb
272
+ - test/helper/omittable.rb
266
273
  - test/raw-records/test-basic-arrays.rb
267
274
  - test/raw-records/test-dense-union-array.rb
268
275
  - test/raw-records/test-list-array.rb
@@ -286,6 +293,7 @@ files:
286
293
  - test/test-decimal128-data-type.rb
287
294
  - test/test-decimal128.rb
288
295
  - test/test-dense-union-data-type.rb
296
+ - test/test-dictionary-array.rb
289
297
  - test/test-dictionary-data-type.rb
290
298
  - test/test-feather.rb
291
299
  - test/test-field.rb
@@ -294,9 +302,11 @@ files:
294
302
  - test/test-list-array-builder.rb
295
303
  - test/test-list-array.rb
296
304
  - test/test-list-data-type.rb
305
+ - test/test-null-array.rb
297
306
  - test/test-orc.rb
298
307
  - test/test-record-batch-builder.rb
299
308
  - test/test-record-batch-file-reader.rb
309
+ - test/test-record-batch-iterator.rb
300
310
  - test/test-record-batch.rb
301
311
  - test/test-rolling-window.rb
302
312
  - test/test-schema.rb
@@ -323,8 +333,8 @@ homepage: https://arrow.apache.org/
323
333
  licenses:
324
334
  - Apache-2.0
325
335
  metadata:
326
- msys2_mingw_dependencies: arrow
327
- post_install_message:
336
+ msys2_mingw_dependencies: arrow>=1.0.0
337
+ post_install_message:
328
338
  rdoc_options: []
329
339
  require_paths:
330
340
  - lib
@@ -339,76 +349,79 @@ required_rubygems_version: !ruby/object:Gem::Requirement
339
349
  - !ruby/object:Gem::Version
340
350
  version: '0'
341
351
  requirements: []
342
- rubyforge_project:
343
- rubygems_version: 2.7.6.2
344
- signing_key:
352
+ rubygems_version: 3.1.2
353
+ signing_key:
345
354
  specification_version: 4
346
355
  summary: Red Arrow is the Ruby bindings of Apache Arrow
347
356
  test_files:
348
- - test/test-time64-data-type.rb
349
- - test/test-feather.rb
350
- - test/test-decimal128.rb
351
- - test/test-struct-array.rb
352
- - test/test-data-type.rb
353
- - test/test-list-data-type.rb
357
+ - test/test-orc.rb
358
+ - test/test-file-output-stream.rb
354
359
  - test/test-dense-union-data-type.rb
355
- - test/helper.rb
360
+ - test/test-record-batch-builder.rb
356
361
  - test/test-record-batch.rb
362
+ - test/test-list-data-type.rb
363
+ - test/test-decimal128.rb
364
+ - test/test-chunked-array.rb
365
+ - test/test-array.rb
366
+ - test/test-null-array.rb
367
+ - test/test-struct-array-builder.rb
357
368
  - test/test-table.rb
358
- - test/run-test.rb
359
- - test/test-timestamp-array.rb
360
- - test/test-date32-array.rb
361
- - test/test-array-builder.rb
362
- - test/test-date64-array.rb
363
- - test/test-record-batch-file-reader.rb
364
369
  - test/test-decimal128-data-type.rb
365
- - test/test-time.rb
366
- - test/test-timestamp-data-type.rb
367
- - test/test-column.rb
368
- - test/test-field.rb
369
- - test/test-decimal128-array.rb
370
- - test/test-csv-loader.rb
371
- - test/test-bigdecimal.rb
372
- - test/test-list-array.rb
373
- - test/test-time64-array.rb
374
- - test/test-rolling-window.rb
375
- - test/test-dictionary-data-type.rb
370
+ - test/fixture/TestOrcFile.test1.orc
376
371
  - test/fixture/integer-float.csv
377
- - test/fixture/without-header-float.csv
378
372
  - test/fixture/null-with-double-quote.csv
373
+ - test/fixture/with-header.csv
374
+ - test/fixture/without-header.csv
375
+ - test/fixture/without-header-float.csv
379
376
  - test/fixture/with-header-float.csv
380
- - test/fixture/TestOrcFile.test1.orc
381
377
  - test/fixture/null-without-double-quote.csv
382
- - test/fixture/without-header.csv
383
- - test/fixture/with-header.csv
384
378
  - test/fixture/float-integer.csv
385
- - test/test-orc.rb
386
- - test/test-time32-data-type.rb
387
- - test/test-struct-data-type.rb
379
+ - test/run-test.rb
388
380
  - test/test-group.rb
389
- - test/test-buffer.rb
390
- - test/values/test-struct-array.rb
381
+ - test/helper.rb
382
+ - test/test-time64-array.rb
383
+ - test/test-csv-loader.rb
384
+ - test/test-feather.rb
385
+ - test/test-time64-data-type.rb
386
+ - test/values/test-sparse-union-array.rb
391
387
  - test/values/test-basic-arrays.rb
392
- - test/values/test-list-array.rb
393
388
  - test/values/test-dense-union-array.rb
394
- - test/values/test-sparse-union-array.rb
395
- - test/test-slicer.rb
396
- - test/test-list-array-builder.rb
397
- - test/test-sparse-union-data-type.rb
398
- - test/test-record-batch-builder.rb
399
- - test/raw-records/test-struct-array.rb
400
- - test/raw-records/test-table.rb
401
- - test/raw-records/test-basic-arrays.rb
402
- - test/raw-records/test-list-array.rb
403
- - test/raw-records/test-dense-union-array.rb
389
+ - test/values/test-list-array.rb
390
+ - test/values/test-struct-array.rb
391
+ - test/test-struct-data-type.rb
392
+ - test/test-date64-array.rb
404
393
  - test/raw-records/test-sparse-union-array.rb
405
394
  - test/raw-records/test-multiple-columns.rb
406
- - test/test-array.rb
407
- - test/test-file-output-stream.rb
408
- - test/test-tensor.rb
395
+ - test/raw-records/test-basic-arrays.rb
396
+ - test/raw-records/test-table.rb
397
+ - test/raw-records/test-dense-union-array.rb
398
+ - test/raw-records/test-list-array.rb
399
+ - test/raw-records/test-struct-array.rb
400
+ - test/test-list-array.rb
401
+ - test/test-timestamp-data-type.rb
402
+ - test/test-timestamp-array.rb
403
+ - test/test-data-type.rb
404
+ - test/test-array-builder.rb
405
+ - test/test-time32-data-type.rb
406
+ - test/test-record-batch-file-reader.rb
407
+ - test/test-rolling-window.rb
408
+ - test/test-list-array-builder.rb
409
+ - test/test-slicer.rb
409
410
  - test/test-decimal128-array-builder.rb
410
- - test/test-chunked-array.rb
411
411
  - test/test-schema.rb
412
- - test/test-struct-array-builder.rb
413
- - test/helper/fixture.rb
412
+ - test/test-time.rb
413
+ - test/test-column.rb
414
+ - test/test-bigdecimal.rb
415
+ - test/test-sparse-union-data-type.rb
416
+ - test/test-dictionary-array.rb
417
+ - test/test-tensor.rb
414
418
  - test/test-time32-array.rb
419
+ - test/test-buffer.rb
420
+ - test/test-decimal128-array.rb
421
+ - test/helper/omittable.rb
422
+ - test/helper/fixture.rb
423
+ - test/test-dictionary-data-type.rb
424
+ - test/test-date32-array.rb
425
+ - test/test-field.rb
426
+ - test/test-record-batch-iterator.rb
427
+ - test/test-struct-array.rb