red-arrow 0.15.1 → 1.0.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (59) hide show
  1. checksums.yaml +4 -4
  2. data/Rakefile +28 -16
  3. data/ext/arrow/converters.hpp +63 -33
  4. data/ext/arrow/raw-records.cpp +2 -1
  5. data/ext/arrow/values.cpp +2 -1
  6. data/lib/arrow/array-builder.rb +101 -52
  7. data/lib/arrow/array.rb +28 -10
  8. data/lib/arrow/{binary-array-builder.rb → buffer.rb} +7 -15
  9. data/lib/arrow/chunked-array.rb +2 -0
  10. data/lib/arrow/csv-loader.rb +5 -0
  11. data/lib/arrow/csv-read-options.rb +18 -0
  12. data/lib/arrow/data-type.rb +35 -2
  13. data/lib/arrow/decimal128-array-builder.rb +0 -2
  14. data/lib/arrow/dictionary-array.rb +24 -0
  15. data/lib/arrow/field.rb +1 -1
  16. data/lib/arrow/generic-filterable.rb +43 -0
  17. data/lib/arrow/generic-takeable.rb +38 -0
  18. data/lib/arrow/list-data-type.rb +58 -8
  19. data/lib/arrow/loader.rb +12 -1
  20. data/lib/arrow/null-array-builder.rb +1 -1
  21. data/lib/arrow/null-array.rb +24 -0
  22. data/lib/arrow/raw-table-converter.rb +47 -0
  23. data/lib/arrow/record-batch-iterator.rb +22 -0
  24. data/lib/arrow/record-batch.rb +8 -3
  25. data/lib/arrow/schema.rb +5 -2
  26. data/lib/arrow/struct-array-builder.rb +13 -7
  27. data/lib/arrow/struct-data-type.rb +0 -2
  28. data/lib/arrow/table-loader.rb +29 -6
  29. data/lib/arrow/table-saver.rb +37 -13
  30. data/lib/arrow/table.rb +20 -73
  31. data/lib/arrow/version.rb +1 -1
  32. data/red-arrow.gemspec +3 -1
  33. data/test/helper.rb +1 -0
  34. data/test/helper/omittable.rb +36 -0
  35. data/test/raw-records/test-dense-union-array.rb +1 -34
  36. data/test/raw-records/test-sparse-union-array.rb +1 -33
  37. data/test/run-test.rb +14 -3
  38. data/test/test-array-builder.rb +17 -0
  39. data/test/test-array.rb +104 -0
  40. data/test/test-buffer.rb +11 -0
  41. data/test/test-chunked-array.rb +96 -0
  42. data/test/test-csv-loader.rb +2 -2
  43. data/test/test-data-type.rb +11 -0
  44. data/test/test-dense-union-data-type.rb +2 -2
  45. data/test/test-dictionary-array.rb +41 -0
  46. data/test/test-feather.rb +21 -6
  47. data/test/test-list-data-type.rb +27 -1
  48. data/test/test-null-array.rb +23 -0
  49. data/test/test-record-batch-iterator.rb +37 -0
  50. data/test/test-record-batch.rb +14 -0
  51. data/test/test-schema.rb +16 -0
  52. data/test/test-slicer.rb +74 -30
  53. data/test/test-sparse-union-data-type.rb +2 -2
  54. data/test/test-struct-array-builder.rb +8 -4
  55. data/test/test-table.rb +153 -14
  56. data/test/test-timestamp-array.rb +19 -0
  57. data/test/values/test-dense-union-array.rb +1 -34
  58. data/test/values/test-sparse-union-array.rb +1 -33
  59. metadata +22 -8
data/test/test-timestamp-array.rb CHANGED
@@ -23,4 +23,23 @@ class TimestampArrayTest < Test::Unit::TestCase
23
23
  time = Time.at(sec, usec)
24
24
  assert_equal(time, array[0])
25
25
  end
26
+
27
+ sub_test_case("#is_in") do
28
+ def setup
29
+ values = [
30
+ Time.parse("2019-11-18T00:09:11"),
31
+ Time.parse("2019-11-18T00:09:12"),
32
+ Time.parse("2019-11-18T00:09:13"),
33
+ ]
34
+ @array = Arrow::TimestampArray.new(:micro, values)
35
+ end
36
+
37
+ test("Arrow: Array") do
38
+ right = [
39
+ Time.parse("2019-11-18T00:09:12"),
40
+ ]
41
+ assert_equal(Arrow::BooleanArray.new([false, true, false]),
42
+ @array.is_in(right))
43
+ end
44
+ end
26
45
  end
data/test/values/test-dense-union-array.rb CHANGED
@@ -48,10 +48,7 @@ module ValuesDenseUnionArrayTests
48
48
  sub_record_batch.columns[0].data
49
49
  end
50
50
  values.each do |value|
51
- if value.nil?
52
- type_ids << nil
53
- offsets << 0
54
- elsif value.key?("0")
51
+ if value.key?("0")
55
52
  type_id = type_codes[0]
56
53
  type_ids << type_id
57
54
  offsets << (type_ids.count(type_id) - 1)
@@ -70,7 +67,6 @@ module ValuesDenseUnionArrayTests
70
67
  def test_null
71
68
  values = [
72
69
  {"0" => nil},
73
- nil,
74
70
  ]
75
71
  target = build(:null, values)
76
72
  assert_equal(values, target.values)
@@ -79,7 +75,6 @@ module ValuesDenseUnionArrayTests
79
75
  def test_boolean
80
76
  values = [
81
77
  {"0" => true},
82
- nil,
83
78
  {"1" => nil},
84
79
  ]
85
80
  target = build(:boolean, values)
@@ -89,7 +84,6 @@ module ValuesDenseUnionArrayTests
89
84
  def test_int8
90
85
  values = [
91
86
  {"0" => -(2 ** 7)},
92
- nil,
93
87
  {"1" => nil},
94
88
  ]
95
89
  target = build(:int8, values)
@@ -99,7 +93,6 @@ module ValuesDenseUnionArrayTests
99
93
  def test_uint8
100
94
  values = [
101
95
  {"0" => (2 ** 8) - 1},
102
- nil,
103
96
  {"1" => nil},
104
97
  ]
105
98
  target = build(:uint8, values)
@@ -109,7 +102,6 @@ module ValuesDenseUnionArrayTests
109
102
  def test_int16
110
103
  values = [
111
104
  {"0" => -(2 ** 15)},
112
- nil,
113
105
  {"1" => nil},
114
106
  ]
115
107
  target = build(:int16, values)
@@ -119,7 +111,6 @@ module ValuesDenseUnionArrayTests
119
111
  def test_uint16
120
112
  values = [
121
113
  {"0" => (2 ** 16) - 1},
122
- nil,
123
114
  {"1" => nil},
124
115
  ]
125
116
  target = build(:uint16, values)
@@ -129,7 +120,6 @@ module ValuesDenseUnionArrayTests
129
120
  def test_int32
130
121
  values = [
131
122
  {"0" => -(2 ** 31)},
132
- nil,
133
123
  {"1" => nil},
134
124
  ]
135
125
  target = build(:int32, values)
@@ -139,7 +129,6 @@ module ValuesDenseUnionArrayTests
139
129
  def test_uint32
140
130
  values = [
141
131
  {"0" => (2 ** 32) - 1},
142
- nil,
143
132
  {"1" => nil},
144
133
  ]
145
134
  target = build(:uint32, values)
@@ -149,7 +138,6 @@ module ValuesDenseUnionArrayTests
149
138
  def test_int64
150
139
  values = [
151
140
  {"0" => -(2 ** 63)},
152
- nil,
153
141
  {"1" => nil},
154
142
  ]
155
143
  target = build(:int64, values)
@@ -159,7 +147,6 @@ module ValuesDenseUnionArrayTests
159
147
  def test_uint64
160
148
  values = [
161
149
  {"0" => (2 ** 64) - 1},
162
- nil,
163
150
  {"1" => nil},
164
151
  ]
165
152
  target = build(:uint64, values)
@@ -169,7 +156,6 @@ module ValuesDenseUnionArrayTests
169
156
  def test_float
170
157
  values = [
171
158
  {"0" => -1.0},
172
- nil,
173
159
  {"1" => nil},
174
160
  ]
175
161
  target = build(:float, values)
@@ -179,7 +165,6 @@ module ValuesDenseUnionArrayTests
179
165
  def test_double
180
166
  values = [
181
167
  {"0" => -1.0},
182
- nil,
183
168
  {"1" => nil},
184
169
  ]
185
170
  target = build(:double, values)
@@ -189,7 +174,6 @@ module ValuesDenseUnionArrayTests
189
174
  def test_binary
190
175
  values = [
191
176
  {"0" => "\xff".b},
192
- nil,
193
177
  {"1" => nil},
194
178
  ]
195
179
  target = build(:binary, values)
@@ -199,7 +183,6 @@ module ValuesDenseUnionArrayTests
199
183
  def test_string
200
184
  values = [
201
185
  {"0" => "Ruby"},
202
- nil,
203
186
  {"1" => nil},
204
187
  ]
205
188
  target = build(:string, values)
@@ -209,7 +192,6 @@ module ValuesDenseUnionArrayTests
209
192
  def test_date32
210
193
  values = [
211
194
  {"0" => Date.new(1960, 1, 1)},
212
- nil,
213
195
  {"1" => nil},
214
196
  ]
215
197
  target = build(:date32, values)
@@ -219,7 +201,6 @@ module ValuesDenseUnionArrayTests
219
201
  def test_date64
220
202
  values = [
221
203
  {"0" => DateTime.new(1960, 1, 1, 2, 9, 30)},
222
- nil,
223
204
  {"1" => nil},
224
205
  ]
225
206
  target = build(:date64, values)
@@ -229,7 +210,6 @@ module ValuesDenseUnionArrayTests
229
210
  def test_timestamp_second
230
211
  values = [
231
212
  {"0" => Time.parse("1960-01-01T02:09:30Z")},
232
- nil,
233
213
  {"1" => nil},
234
214
  ]
235
215
  target = build({
@@ -243,7 +223,6 @@ module ValuesDenseUnionArrayTests
243
223
  def test_timestamp_milli
244
224
  values = [
245
225
  {"0" => Time.parse("1960-01-01T02:09:30.123Z")},
246
- nil,
247
226
  {"1" => nil},
248
227
  ]
249
228
  target = build({
@@ -257,7 +236,6 @@ module ValuesDenseUnionArrayTests
257
236
  def test_timestamp_micro
258
237
  values = [
259
238
  {"0" => Time.parse("1960-01-01T02:09:30.123456Z")},
260
- nil,
261
239
  {"1" => nil},
262
240
  ]
263
241
  target = build({
@@ -271,7 +249,6 @@ module ValuesDenseUnionArrayTests
271
249
  def test_timestamp_nano
272
250
  values = [
273
251
  {"0" => Time.parse("1960-01-01T02:09:30.123456789Z")},
274
- nil,
275
252
  {"1" => nil},
276
253
  ]
277
254
  target = build({
@@ -287,7 +264,6 @@ module ValuesDenseUnionArrayTests
287
264
  values = [
288
265
  # 00:10:00
289
266
  {"0" => Arrow::Time.new(unit, 60 * 10)},
290
- nil,
291
267
  {"1" => nil},
292
268
  ]
293
269
  target = build({
@@ -303,7 +279,6 @@ module ValuesDenseUnionArrayTests
303
279
  values = [
304
280
  # 00:10:00.123
305
281
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)},
306
- nil,
307
282
  {"1" => nil},
308
283
  ]
309
284
  target = build({
@@ -319,7 +294,6 @@ module ValuesDenseUnionArrayTests
319
294
  values = [
320
295
  # 00:10:00.123456
321
296
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)},
322
- nil,
323
297
  {"1" => nil},
324
298
  ]
325
299
  target = build({
@@ -335,7 +309,6 @@ module ValuesDenseUnionArrayTests
335
309
  values = [
336
310
  # 00:10:00.123456789
337
311
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)},
338
- nil,
339
312
  {"1" => nil},
340
313
  ]
341
314
  target = build({
@@ -349,7 +322,6 @@ module ValuesDenseUnionArrayTests
349
322
  def test_decimal128
350
323
  values = [
351
324
  {"0" => BigDecimal("92.92")},
352
- nil,
353
325
  {"1" => nil},
354
326
  ]
355
327
  target = build({
@@ -364,7 +336,6 @@ module ValuesDenseUnionArrayTests
364
336
  def test_list
365
337
  values = [
366
338
  {"0" => [true, nil, false]},
367
- nil,
368
339
  {"1" => nil},
369
340
  ]
370
341
  target = build({
@@ -381,7 +352,6 @@ module ValuesDenseUnionArrayTests
381
352
  def test_struct
382
353
  values = [
383
354
  {"0" => {"sub_field" => true}},
384
- nil,
385
355
  {"1" => nil},
386
356
  {"0" => {"sub_field" => nil}},
387
357
  ]
@@ -402,7 +372,6 @@ module ValuesDenseUnionArrayTests
402
372
  omit("Need to add support for SparseUnionArrayBuilder")
403
373
  values = [
404
374
  {"0" => {"field1" => true}},
405
- nil,
406
375
  {"1" => nil},
407
376
  {"0" => {"field2" => nil}},
408
377
  ]
@@ -428,7 +397,6 @@ module ValuesDenseUnionArrayTests
428
397
  omit("Need to add support for DenseUnionArrayBuilder")
429
398
  values = [
430
399
  {"0" => {"field1" => true}},
431
- nil,
432
400
  {"1" => nil},
433
401
  {"0" => {"field2" => nil}},
434
402
  ]
@@ -454,7 +422,6 @@ module ValuesDenseUnionArrayTests
454
422
  omit("Need to add support for DictionaryArrayBuilder")
455
423
  values = [
456
424
  {"0" => "Ruby"},
457
- nil,
458
425
  {"1" => nil},
459
426
  {"0" => "GLib"},
460
427
  ]
data/test/values/test-sparse-union-array.rb CHANGED
@@ -44,9 +44,7 @@ module ValuesSparseUnionArrayTests
44
44
  sub_record_batch.columns[0].data
45
45
  end
46
46
  values.each do |value|
47
- if value.nil?
48
- type_ids << nil
49
- elsif value.key?("0")
47
+ if value.key?("0")
50
48
  type_ids << type_codes[0]
51
49
  elsif value.key?("1")
52
50
  type_ids << type_codes[1]
@@ -60,7 +58,6 @@ module ValuesSparseUnionArrayTests
60
58
  def test_null
61
59
  values = [
62
60
  {"0" => nil},
63
- nil,
64
61
  ]
65
62
  target = build(:null, values)
66
63
  assert_equal(values, target.values)
@@ -69,7 +66,6 @@ module ValuesSparseUnionArrayTests
69
66
  def test_boolean
70
67
  values = [
71
68
  {"0" => true},
72
- nil,
73
69
  {"1" => nil},
74
70
  ]
75
71
  target = build(:boolean, values)
@@ -79,7 +75,6 @@ module ValuesSparseUnionArrayTests
79
75
  def test_int8
80
76
  values = [
81
77
  {"0" => -(2 ** 7)},
82
- nil,
83
78
  {"1" => nil},
84
79
  ]
85
80
  target = build(:int8, values)
@@ -89,7 +84,6 @@ module ValuesSparseUnionArrayTests
89
84
  def test_uint8
90
85
  values = [
91
86
  {"0" => (2 ** 8) - 1},
92
- nil,
93
87
  {"1" => nil},
94
88
  ]
95
89
  target = build(:uint8, values)
@@ -99,7 +93,6 @@ module ValuesSparseUnionArrayTests
99
93
  def test_int16
100
94
  values = [
101
95
  {"0" => -(2 ** 15)},
102
- nil,
103
96
  {"1" => nil},
104
97
  ]
105
98
  target = build(:int16, values)
@@ -109,7 +102,6 @@ module ValuesSparseUnionArrayTests
109
102
  def test_uint16
110
103
  values = [
111
104
  {"0" => (2 ** 16) - 1},
112
- nil,
113
105
  {"1" => nil},
114
106
  ]
115
107
  target = build(:uint16, values)
@@ -119,7 +111,6 @@ module ValuesSparseUnionArrayTests
119
111
  def test_int32
120
112
  values = [
121
113
  {"0" => -(2 ** 31)},
122
- nil,
123
114
  {"1" => nil},
124
115
  ]
125
116
  target = build(:int32, values)
@@ -129,7 +120,6 @@ module ValuesSparseUnionArrayTests
129
120
  def test_uint32
130
121
  values = [
131
122
  {"0" => (2 ** 32) - 1},
132
- nil,
133
123
  {"1" => nil},
134
124
  ]
135
125
  target = build(:uint32, values)
@@ -139,7 +129,6 @@ module ValuesSparseUnionArrayTests
139
129
  def test_int64
140
130
  values = [
141
131
  {"0" => -(2 ** 63)},
142
- nil,
143
132
  {"1" => nil},
144
133
  ]
145
134
  target = build(:int64, values)
@@ -149,7 +138,6 @@ module ValuesSparseUnionArrayTests
149
138
  def test_uint64
150
139
  values = [
151
140
  {"0" => (2 ** 64) - 1},
152
- nil,
153
141
  {"1" => nil},
154
142
  ]
155
143
  target = build(:uint64, values)
@@ -159,7 +147,6 @@ module ValuesSparseUnionArrayTests
159
147
  def test_float
160
148
  values = [
161
149
  {"0" => -1.0},
162
- nil,
163
150
  {"1" => nil},
164
151
  ]
165
152
  target = build(:float, values)
@@ -169,7 +156,6 @@ module ValuesSparseUnionArrayTests
169
156
  def test_double
170
157
  values = [
171
158
  {"0" => -1.0},
172
- nil,
173
159
  {"1" => nil},
174
160
  ]
175
161
  target = build(:double, values)
@@ -179,7 +165,6 @@ module ValuesSparseUnionArrayTests
179
165
  def test_binary
180
166
  values = [
181
167
  {"0" => "\xff".b},
182
- nil,
183
168
  {"1" => nil},
184
169
  ]
185
170
  target = build(:binary, values)
@@ -189,7 +174,6 @@ module ValuesSparseUnionArrayTests
189
174
  def test_string
190
175
  values = [
191
176
  {"0" => "Ruby"},
192
- nil,
193
177
  {"1" => nil},
194
178
  ]
195
179
  target = build(:string, values)
@@ -199,7 +183,6 @@ module ValuesSparseUnionArrayTests
199
183
  def test_date32
200
184
  values = [
201
185
  {"0" => Date.new(1960, 1, 1)},
202
- nil,
203
186
  {"1" => nil},
204
187
  ]
205
188
  target = build(:date32, values)
@@ -209,7 +192,6 @@ module ValuesSparseUnionArrayTests
209
192
  def test_date64
210
193
  values = [
211
194
  {"0" => DateTime.new(1960, 1, 1, 2, 9, 30)},
212
- nil,
213
195
  {"1" => nil},
214
196
  ]
215
197
  target = build(:date64, values)
@@ -219,7 +201,6 @@ module ValuesSparseUnionArrayTests
219
201
  def test_timestamp_second
220
202
  values = [
221
203
  {"0" => Time.parse("1960-01-01T02:09:30Z")},
222
- nil,
223
204
  {"1" => nil},
224
205
  ]
225
206
  target = build({
@@ -233,7 +214,6 @@ module ValuesSparseUnionArrayTests
233
214
  def test_timestamp_milli
234
215
  values = [
235
216
  {"0" => Time.parse("1960-01-01T02:09:30.123Z")},
236
- nil,
237
217
  {"1" => nil},
238
218
  ]
239
219
  target = build({
@@ -247,7 +227,6 @@ module ValuesSparseUnionArrayTests
247
227
  def test_timestamp_micro
248
228
  values = [
249
229
  {"0" => Time.parse("1960-01-01T02:09:30.123456Z")},
250
- nil,
251
230
  {"1" => nil},
252
231
  ]
253
232
  target = build({
@@ -261,7 +240,6 @@ module ValuesSparseUnionArrayTests
261
240
  def test_timestamp_nano
262
241
  values = [
263
242
  {"0" => Time.parse("1960-01-01T02:09:30.123456789Z")},
264
- nil,
265
243
  {"1" => nil},
266
244
  ]
267
245
  target = build({
@@ -277,7 +255,6 @@ module ValuesSparseUnionArrayTests
277
255
  values = [
278
256
  # 00:10:00
279
257
  {"0" => Arrow::Time.new(unit, 60 * 10)},
280
- nil,
281
258
  {"1" => nil},
282
259
  ]
283
260
  target = build({
@@ -293,7 +270,6 @@ module ValuesSparseUnionArrayTests
293
270
  values = [
294
271
  # 00:10:00.123
295
272
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)},
296
- nil,
297
273
  {"1" => nil},
298
274
  ]
299
275
  target = build({
@@ -309,7 +285,6 @@ module ValuesSparseUnionArrayTests
309
285
  values = [
310
286
  # 00:10:00.123456
311
287
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)},
312
- nil,
313
288
  {"1" => nil},
314
289
  ]
315
290
  target = build({
@@ -325,7 +300,6 @@ module ValuesSparseUnionArrayTests
325
300
  values = [
326
301
  # 00:10:00.123456789
327
302
  {"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)},
328
- nil,
329
303
  {"1" => nil},
330
304
  ]
331
305
  target = build({
@@ -339,7 +313,6 @@ module ValuesSparseUnionArrayTests
339
313
  def test_decimal128
340
314
  values = [
341
315
  {"0" => BigDecimal("92.92")},
342
- nil,
343
316
  {"1" => nil},
344
317
  ]
345
318
  target = build({
@@ -354,7 +327,6 @@ module ValuesSparseUnionArrayTests
354
327
  def test_list
355
328
  values = [
356
329
  {"0" => [true, nil, false]},
357
- nil,
358
330
  {"1" => nil},
359
331
  ]
360
332
  target = build({
@@ -371,7 +343,6 @@ module ValuesSparseUnionArrayTests
371
343
  def test_struct
372
344
  values = [
373
345
  {"0" => {"sub_field" => true}},
374
- nil,
375
346
  {"1" => nil},
376
347
  {"0" => {"sub_field" => nil}},
377
348
  ]
@@ -392,7 +363,6 @@ module ValuesSparseUnionArrayTests
392
363
  omit("Need to add support for SparseUnionArrayBuilder")
393
364
  values = [
394
365
  {"0" => {"field1" => true}},
395
- nil,
396
366
  {"1" => nil},
397
367
  {"0" => {"field2" => nil}},
398
368
  ]
@@ -418,7 +388,6 @@ module ValuesSparseUnionArrayTests
418
388
  omit("Need to add support for DenseUnionArrayBuilder")
419
389
  values = [
420
390
  {"0" => {"field1" => true}},
421
- nil,
422
391
  {"1" => nil},
423
392
  {"0" => {"field2" => nil}},
424
393
  ]
@@ -444,7 +413,6 @@ module ValuesSparseUnionArrayTests
444
413
  omit("Need to add support for DictionaryArrayBuilder")
445
414
  values = [
446
415
  {"0" => "Ruby"},
447
- nil,
448
416
  {"1" => nil},
449
417
  {"0" => "GLib"},
450
418
  ]