red-arrow 0.15.0 → 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +28 -16
  3. data/ext/arrow/converters.hpp +63 -33
  4. data/ext/arrow/raw-records.cpp +2 -1
  5. data/ext/arrow/values.cpp +2 -1
  6. data/lib/arrow/array-builder.rb +101 -52
  7. data/lib/arrow/array.rb +28 -10
  8. data/lib/arrow/{binary-array-builder.rb → buffer.rb} +7 -15
  9. data/lib/arrow/chunked-array.rb +2 -0
  10. data/lib/arrow/csv-loader.rb +15 -3
  11. data/lib/arrow/csv-read-options.rb +18 -0
  12. data/lib/arrow/data-type.rb +35 -2
  13. data/lib/arrow/decimal128-array-builder.rb +0 -2
  14. data/lib/arrow/dictionary-array.rb +24 -0
  15. data/lib/arrow/field.rb +1 -1
  16. data/lib/arrow/generic-filterable.rb +43 -0
  17. data/lib/arrow/generic-takeable.rb +38 -0
  18. data/lib/arrow/list-data-type.rb +58 -8
  19. data/lib/arrow/loader.rb +12 -1
  20. data/lib/arrow/null-array-builder.rb +1 -1
  21. data/lib/arrow/null-array.rb +24 -0
  22. data/lib/arrow/raw-table-converter.rb +47 -0
  23. data/lib/arrow/record-batch-iterator.rb +22 -0
  24. data/lib/arrow/record-batch.rb +8 -3
  25. data/lib/arrow/schema.rb +5 -2
  26. data/lib/arrow/struct-array-builder.rb +13 -7
  27. data/lib/arrow/struct-data-type.rb +0 -2
  28. data/lib/arrow/table-loader.rb +29 -6
  29. data/lib/arrow/table-saver.rb +37 -13
  30. data/lib/arrow/table.rb +20 -73
  31. data/lib/arrow/version.rb +1 -1
  32. data/red-arrow.gemspec +4 -2
  33. data/test/helper.rb +1 -0
  34. data/test/helper/omittable.rb +36 -0
  35. data/test/raw-records/test-dense-union-array.rb +1 -34
  36. data/test/raw-records/test-sparse-union-array.rb +1 -33
  37. data/test/run-test.rb +14 -3
  38. data/test/test-array-builder.rb +17 -0
  39. data/test/test-array.rb +104 -0
  40. data/test/test-buffer.rb +11 -0
  41. data/test/test-chunked-array.rb +96 -0
  42. data/test/test-csv-loader.rb +77 -2
  43. data/test/test-data-type.rb +11 -0
  44. data/test/test-dense-union-data-type.rb +2 -2
  45. data/test/test-dictionary-array.rb +41 -0
  46. data/test/test-feather.rb +21 -6
  47. data/test/test-list-data-type.rb +27 -1
  48. data/test/test-null-array.rb +23 -0
  49. data/test/test-record-batch-iterator.rb +37 -0
  50. data/test/test-record-batch.rb +14 -0
  51. data/test/test-schema.rb +16 -0
  52. data/test/test-slicer.rb +74 -30
  53. data/test/test-sparse-union-data-type.rb +2 -2
  54. data/test/test-struct-array-builder.rb +8 -4
  55. data/test/test-table.rb +153 -14
  56. data/test/test-timestamp-array.rb +19 -0
  57. data/test/values/test-dense-union-array.rb +1 -34
  58. data/test/values/test-sparse-union-array.rb +1 -33
  59. metadata +76 -63
@@ -44,9 +44,7 @@ module ValuesSparseUnionArrayTests
44
44
  sub_record_batch.columns[0].data
45
45
  end
46
46
  values.each do |value|
47
- if value.nil?
48
- type_ids << nil
49
- elsif value.key?("0")
47
+ if value.key?("0")
50
48
  type_ids << type_codes[0]
51
49
  elsif value.key?("1")
52
50
  type_ids << type_codes[1]
@@ -60,7 +58,6 @@ module ValuesSparseUnionArrayTests
60
58
  def test_null
61
59
  values = [
62
60
  {"0" => nil},
63
- nil,
64
61
  ]
65
62
  target = build(:null, values)
66
63
  assert_equal(values, target.values)
@@ -69,7 +66,6 @@ module ValuesSparseUnionArrayTests
69
66
  def test_boolean
70
67
  values = [
71
68
  {"0" => true},
72
- nil,
73
69
  {"1" => nil},
74
70
  ]
75
71
  target = build(:boolean, values)
@@ -79,7 +75,6 @@ module ValuesSparseUnionArrayTests
79
75
  def test_int8
80
76
  values = [
81
77
  {"0" => -(2 ** 7)},
82
- nil,
83
78
  {"1" => nil},
84
79
  ]
85
80
  target = build(:int8, values)
@@ -89,7 +84,6 @@ module ValuesSparseUnionArrayTests
89
84
  def test_uint8
90
85
  values = [
91
86
  {"0" => (2 ** 8) - 1},
92
- nil,
93
87
  {"1" => nil},
94
88
  ]
95
89
  target = build(:uint8, values)
@@ -99,7 +93,6 @@ module ValuesSparseUnionArrayTests
99
93
  def test_int16
100
94
  values = [
101
95
  {"0" => -(2 ** 15)},
102
- nil,
103
96
  {"1" => nil},
104
97
  ]
105
98
  target = build(:int16, values)
@@ -109,7 +102,6 @@ module ValuesSparseUnionArrayTests
109
102
  def test_uint16
110
103
  values = [
111
104
  {"0" => (2 ** 16) - 1},
112
- nil,
113
105
  {"1" => nil},
114
106
  ]
115
107
  target = build(:uint16, values)
@@ -119,7 +111,6 @@ module ValuesSparseUnionArrayTests
119
111
  def test_int32
120
112
  values = [
121
113
  {"0" => -(2 ** 31)},
122
- nil,
123
114
  {"1" => nil},
124
115
  ]
125
116
  target = build(:int32, values)
@@ -129,7 +120,6 @@ module ValuesSparseUnionArrayTests
129
120
  def test_uint32
130
121
  values = [
131
122
  {"0" => (2 ** 32) - 1},
132
- nil,
133
123
  {"1" => nil},
134
124
  ]
135
125
  target = build(:uint32, values)
@@ -139,7 +129,6 @@ module ValuesSparseUnionArrayTests
139
129
  def test_int64
140
130
  values = [
141
131
  {"0" => -(2 ** 63)},
142
- nil,
143
132
  {"1" => nil},
144
133
  ]
145
134
  target = build(:int64, values)
@@ -149,7 +138,6 @@ module ValuesSparseUnionArrayTests
149
138
  def test_uint64
150
139
  values = [
151
140
  {"0" => (2 ** 64) - 1},
152
- nil,
153
141
  {"1" => nil},
154
142
  ]
155
143
  target = build(:uint64, values)
@@ -159,7 +147,6 @@ module ValuesSparseUnionArrayTests
159
147
  def test_float
160
148
  values = [
161
149
  {"0" => -1.0},
162
- nil,
163
150
  {"1" => nil},
164
151
  ]
165
152
  target = build(:float, values)
@@ -169,7 +156,6 @@ module ValuesSparseUnionArrayTests
169
156
  def test_double
170
157
  values = [
171
158
  {"0" => -1.0},
172
- nil,
173
159
  {"1" => nil},
174
160
  ]
175
161
  target = build(:double, values)
@@ -179,7 +165,6 @@ module ValuesSparseUnionArrayTests
179
165
  def test_binary
180
166
  values = [
181
167
  {"0" => "\xff".b},
182
- nil,
183
168
  {"1" => nil},
184
169
  ]
185
170
  target = build(:binary, values)
@@ -189,7 +174,6 @@ module ValuesSparseUnionArrayTests
189
174
  def test_string
190
175
  values = [
191
176
  {"0" => "Ruby"},
192
- nil,
193
177
  {"1" => nil},
194
178
  ]
195
179
  target = build(:string, values)
@@ -199,7 +183,6 @@ module ValuesSparseUnionArrayTests
199
183
  def test_date32
200
184
  values = [
201
185
  {"0" => Date.new(1960, 1, 1)},
202
- nil,
203
186
  {"1" => nil},
204
187
  ]
205
188
  target = build(:date32, values)
@@ -209,7 +192,6 @@ module ValuesSparseUnionArrayTests
209
192
  def test_date64
210
193
  values = [
211
194
  {"0" => DateTime.new(1960, 1, 1, 2, 9, 30)},
212
- nil,
213
195
  {"1" => nil},
214
196
  ]
215
197
  target = build(:date64, values)
@@ -219,7 +201,6 @@ module ValuesSparseUnionArrayTests
219
201
  def test_timestamp_second
220
202
  values = [
221
203
  {"0" => Time.parse("1960-01-01T02:09:30Z")},
222
- nil,
223
204
  {"1" => nil},
224
205
  ]
225
206
  target = build({
@@ -233,7 +214,6 @@ module ValuesSparseUnionArrayTests
233
214
  def test_timestamp_milli
234
215
  values = [
235
216
  {"0" => Time.parse("1960-01-01T02:09:30.123Z")},
236
- nil,
237
217
  {"1" => nil},
238
218
  ]
239
219
  target = build({
@@ -247,7 +227,6 @@ module ValuesSparseUnionArrayTests
247
227
  def test_timestamp_micro
248
228
  values = [
249
229
  {"0" => Time.parse("1960-01-01T02:09:30.123456Z")},
250
- nil,
251
230
  {"1" => nil},
252
231
  ]
253
232
  target = build({
@@ -261,7 +240,6 @@ module ValuesSparseUnionArrayTests
261
240
  def test_timestamp_nano
262
241
  values = [
263
242
  {"0" => Time.parse("1960-01-01T02:09:30.123456789Z")},
264
- nil,
265
243
  {"1" => nil},
266
244
  ]
267
245
  target = build({
@@ -277,7 +255,6 @@ module ValuesSparseUnionArrayTests
277
255
  values = [
278
256
  # 00:10:00
279
257
  {"0" => Arrow::Time.new(unit, 60 * 10)},
280
- nil,
281
258
  {"1" => nil},
282
259
  ]
283
260
  target = build({
@@ -293,7 +270,6 @@ module ValuesSparseUnionArrayTests
293
270
  values = [
294
271
  # 00:10:00.123
295
272
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)},
296
- nil,
297
273
  {"1" => nil},
298
274
  ]
299
275
  target = build({
@@ -309,7 +285,6 @@ module ValuesSparseUnionArrayTests
309
285
  values = [
310
286
  # 00:10:00.123456
311
287
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)},
312
- nil,
313
288
  {"1" => nil},
314
289
  ]
315
290
  target = build({
@@ -325,7 +300,6 @@ module ValuesSparseUnionArrayTests
325
300
  values = [
326
301
  # 00:10:00.123456789
327
302
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)},
328
- nil,
329
303
  {"1" => nil},
330
304
  ]
331
305
  target = build({
@@ -339,7 +313,6 @@ module ValuesSparseUnionArrayTests
339
313
  def test_decimal128
340
314
  values = [
341
315
  {"0" => BigDecimal("92.92")},
342
- nil,
343
316
  {"1" => nil},
344
317
  ]
345
318
  target = build({
@@ -354,7 +327,6 @@ module ValuesSparseUnionArrayTests
354
327
  def test_list
355
328
  values = [
356
329
  {"0" => [true, nil, false]},
357
- nil,
358
330
  {"1" => nil},
359
331
  ]
360
332
  target = build({
@@ -371,7 +343,6 @@ module ValuesSparseUnionArrayTests
371
343
  def test_struct
372
344
  values = [
373
345
  {"0" => {"sub_field" => true}},
374
- nil,
375
346
  {"1" => nil},
376
347
  {"0" => {"sub_field" => nil}},
377
348
  ]
@@ -392,7 +363,6 @@ module ValuesSparseUnionArrayTests
392
363
  omit("Need to add support for SparseUnionArrayBuilder")
393
364
  values = [
394
365
  {"0" => {"field1" => true}},
395
- nil,
396
366
  {"1" => nil},
397
367
  {"0" => {"field2" => nil}},
398
368
  ]
@@ -418,7 +388,6 @@ module ValuesSparseUnionArrayTests
418
388
  omit("Need to add support for DenseUnionArrayBuilder")
419
389
  values = [
420
390
  {"0" => {"field1" => true}},
421
- nil,
422
391
  {"1" => nil},
423
392
  {"0" => {"field2" => nil}},
424
393
  ]
@@ -444,7 +413,6 @@ module ValuesSparseUnionArrayTests
444
413
  omit("Need to add support for DictionaryArrayBuilder")
445
414
  values = [
446
415
  {"0" => "Ruby"},
447
- nil,
448
416
  {"1" => nil},
449
417
  {"0" => "GLib"},
450
418
  ]
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: red-arrow
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.15.0
4
+ version: 1.0.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Apache Arrow Developers
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-10-05 00:00:00.000000000 Z
11
+ date: 2020-07-24 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: extpp
@@ -28,16 +28,16 @@ dependencies:
28
28
  name: gio2
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - '='
31
+ - - ">="
32
32
  - !ruby/object:Gem::Version
33
- version: 3.3.7
33
+ version: 3.3.6
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - '='
38
+ - - ">="
39
39
  - !ruby/object:Gem::Version
40
- version: 3.3.7
40
+ version: 3.3.6
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: native-package-installer
43
43
  requirement: !ruby/object:Gem::Requirement
@@ -191,8 +191,8 @@ files:
191
191
  - lib/arrow/array-builder.rb
192
192
  - lib/arrow/array.rb
193
193
  - lib/arrow/bigdecimal-extension.rb
194
- - lib/arrow/binary-array-builder.rb
195
194
  - lib/arrow/block-closable.rb
195
+ - lib/arrow/buffer.rb
196
196
  - lib/arrow/chunked-array.rb
197
197
  - lib/arrow/column-containable.rb
198
198
  - lib/arrow/column.rb
@@ -209,18 +209,24 @@ files:
209
209
  - lib/arrow/decimal128-data-type.rb
210
210
  - lib/arrow/decimal128.rb
211
211
  - lib/arrow/dense-union-data-type.rb
212
+ - lib/arrow/dictionary-array.rb
212
213
  - lib/arrow/dictionary-data-type.rb
213
214
  - lib/arrow/field-containable.rb
214
215
  - lib/arrow/field.rb
215
216
  - lib/arrow/file-output-stream.rb
217
+ - lib/arrow/generic-filterable.rb
218
+ - lib/arrow/generic-takeable.rb
216
219
  - lib/arrow/group.rb
217
220
  - lib/arrow/list-array-builder.rb
218
221
  - lib/arrow/list-data-type.rb
219
222
  - lib/arrow/loader.rb
220
223
  - lib/arrow/null-array-builder.rb
224
+ - lib/arrow/null-array.rb
221
225
  - lib/arrow/path-extension.rb
226
+ - lib/arrow/raw-table-converter.rb
222
227
  - lib/arrow/record-batch-builder.rb
223
228
  - lib/arrow/record-batch-file-reader.rb
229
+ - lib/arrow/record-batch-iterator.rb
224
230
  - lib/arrow/record-batch-stream-reader.rb
225
231
  - lib/arrow/record-batch.rb
226
232
  - lib/arrow/record-containable.rb
@@ -263,6 +269,7 @@ files:
263
269
  - test/fixture/without-header.csv
264
270
  - test/helper.rb
265
271
  - test/helper/fixture.rb
272
+ - test/helper/omittable.rb
266
273
  - test/raw-records/test-basic-arrays.rb
267
274
  - test/raw-records/test-dense-union-array.rb
268
275
  - test/raw-records/test-list-array.rb
@@ -286,6 +293,7 @@ files:
286
293
  - test/test-decimal128-data-type.rb
287
294
  - test/test-decimal128.rb
288
295
  - test/test-dense-union-data-type.rb
296
+ - test/test-dictionary-array.rb
289
297
  - test/test-dictionary-data-type.rb
290
298
  - test/test-feather.rb
291
299
  - test/test-field.rb
@@ -294,9 +302,11 @@ files:
294
302
  - test/test-list-array-builder.rb
295
303
  - test/test-list-array.rb
296
304
  - test/test-list-data-type.rb
305
+ - test/test-null-array.rb
297
306
  - test/test-orc.rb
298
307
  - test/test-record-batch-builder.rb
299
308
  - test/test-record-batch-file-reader.rb
309
+ - test/test-record-batch-iterator.rb
300
310
  - test/test-record-batch.rb
301
311
  - test/test-rolling-window.rb
302
312
  - test/test-schema.rb
@@ -323,8 +333,8 @@ homepage: https://arrow.apache.org/
323
333
  licenses:
324
334
  - Apache-2.0
325
335
  metadata:
326
- msys2_mingw_dependencies: arrow
327
- post_install_message:
336
+ msys2_mingw_dependencies: arrow>=1.0.0
337
+ post_install_message:
328
338
  rdoc_options: []
329
339
  require_paths:
330
340
  - lib
@@ -339,76 +349,79 @@ required_rubygems_version: !ruby/object:Gem::Requirement
339
349
  - !ruby/object:Gem::Version
340
350
  version: '0'
341
351
  requirements: []
342
- rubyforge_project:
343
- rubygems_version: 2.7.6.2
344
- signing_key:
352
+ rubygems_version: 3.1.2
353
+ signing_key:
345
354
  specification_version: 4
346
355
  summary: Red Arrow is the Ruby bindings of Apache Arrow
347
356
  test_files:
348
- - test/test-time64-data-type.rb
349
- - test/test-feather.rb
350
- - test/test-decimal128.rb
351
- - test/test-struct-array.rb
352
- - test/test-data-type.rb
353
- - test/test-list-data-type.rb
357
+ - test/test-orc.rb
358
+ - test/test-file-output-stream.rb
354
359
  - test/test-dense-union-data-type.rb
355
- - test/helper.rb
360
+ - test/test-record-batch-builder.rb
356
361
  - test/test-record-batch.rb
362
+ - test/test-list-data-type.rb
363
+ - test/test-decimal128.rb
364
+ - test/test-chunked-array.rb
365
+ - test/test-array.rb
366
+ - test/test-null-array.rb
367
+ - test/test-struct-array-builder.rb
357
368
  - test/test-table.rb
358
- - test/run-test.rb
359
- - test/test-timestamp-array.rb
360
- - test/test-date32-array.rb
361
- - test/test-array-builder.rb
362
- - test/test-date64-array.rb
363
- - test/test-record-batch-file-reader.rb
364
369
  - test/test-decimal128-data-type.rb
365
- - test/test-time.rb
366
- - test/test-timestamp-data-type.rb
367
- - test/test-column.rb
368
- - test/test-field.rb
369
- - test/test-decimal128-array.rb
370
- - test/test-csv-loader.rb
371
- - test/test-bigdecimal.rb
372
- - test/test-list-array.rb
373
- - test/test-time64-array.rb
374
- - test/test-rolling-window.rb
375
- - test/test-dictionary-data-type.rb
370
+ - test/fixture/TestOrcFile.test1.orc
376
371
  - test/fixture/integer-float.csv
377
- - test/fixture/without-header-float.csv
378
372
  - test/fixture/null-with-double-quote.csv
373
+ - test/fixture/with-header.csv
374
+ - test/fixture/without-header.csv
375
+ - test/fixture/without-header-float.csv
379
376
  - test/fixture/with-header-float.csv
380
- - test/fixture/TestOrcFile.test1.orc
381
377
  - test/fixture/null-without-double-quote.csv
382
- - test/fixture/without-header.csv
383
- - test/fixture/with-header.csv
384
378
  - test/fixture/float-integer.csv
385
- - test/test-orc.rb
386
- - test/test-time32-data-type.rb
387
- - test/test-struct-data-type.rb
379
+ - test/run-test.rb
388
380
  - test/test-group.rb
389
- - test/test-buffer.rb
390
- - test/values/test-struct-array.rb
381
+ - test/helper.rb
382
+ - test/test-time64-array.rb
383
+ - test/test-csv-loader.rb
384
+ - test/test-feather.rb
385
+ - test/test-time64-data-type.rb
386
+ - test/values/test-sparse-union-array.rb
391
387
  - test/values/test-basic-arrays.rb
392
- - test/values/test-list-array.rb
393
388
  - test/values/test-dense-union-array.rb
394
- - test/values/test-sparse-union-array.rb
395
- - test/test-slicer.rb
396
- - test/test-list-array-builder.rb
397
- - test/test-sparse-union-data-type.rb
398
- - test/test-record-batch-builder.rb
399
- - test/raw-records/test-struct-array.rb
400
- - test/raw-records/test-table.rb
401
- - test/raw-records/test-basic-arrays.rb
402
- - test/raw-records/test-list-array.rb
403
- - test/raw-records/test-dense-union-array.rb
389
+ - test/values/test-list-array.rb
390
+ - test/values/test-struct-array.rb
391
+ - test/test-struct-data-type.rb
392
+ - test/test-date64-array.rb
404
393
  - test/raw-records/test-sparse-union-array.rb
405
394
  - test/raw-records/test-multiple-columns.rb
406
- - test/test-array.rb
407
- - test/test-file-output-stream.rb
408
- - test/test-tensor.rb
395
+ - test/raw-records/test-basic-arrays.rb
396
+ - test/raw-records/test-table.rb
397
+ - test/raw-records/test-dense-union-array.rb
398
+ - test/raw-records/test-list-array.rb
399
+ - test/raw-records/test-struct-array.rb
400
+ - test/test-list-array.rb
401
+ - test/test-timestamp-data-type.rb
402
+ - test/test-timestamp-array.rb
403
+ - test/test-data-type.rb
404
+ - test/test-array-builder.rb
405
+ - test/test-time32-data-type.rb
406
+ - test/test-record-batch-file-reader.rb
407
+ - test/test-rolling-window.rb
408
+ - test/test-list-array-builder.rb
409
+ - test/test-slicer.rb
409
410
  - test/test-decimal128-array-builder.rb
410
- - test/test-chunked-array.rb
411
411
  - test/test-schema.rb
412
- - test/test-struct-array-builder.rb
413
- - test/helper/fixture.rb
412
+ - test/test-time.rb
413
+ - test/test-column.rb
414
+ - test/test-bigdecimal.rb
415
+ - test/test-sparse-union-data-type.rb
416
+ - test/test-dictionary-array.rb
417
+ - test/test-tensor.rb
414
418
  - test/test-time32-array.rb
419
+ - test/test-buffer.rb
420
+ - test/test-decimal128-array.rb
421
+ - test/helper/omittable.rb
422
+ - test/helper/fixture.rb
423
+ - test/test-dictionary-data-type.rb
424
+ - test/test-date32-array.rb
425
+ - test/test-field.rb
426
+ - test/test-record-batch-iterator.rb
427
+ - test/test-struct-array.rb