red-arrow 18.1.0 → 19.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/lib/arrow/array.rb +7 -4
  3. data/lib/arrow/column.rb +4 -4
  4. data/lib/arrow/jruby/array-builder.rb +114 -0
  5. data/lib/arrow/jruby/array.rb +109 -0
  6. data/{test/helper/fixture.rb → lib/arrow/jruby/chunked-array.rb} +14 -6
  7. data/lib/arrow/jruby/compression-type.rb +26 -0
  8. data/{test/test-boolean-scalar.rb → lib/arrow/jruby/csv-read-options.rb} +12 -6
  9. data/{test/test-map-data-type.rb → lib/arrow/jruby/data-type.rb} +24 -12
  10. data/lib/arrow/jruby/decimal128.rb +28 -0
  11. data/lib/arrow/jruby/decimal256.rb +28 -0
  12. data/{test/fixture/null-with-double-quote.csv → lib/arrow/jruby/error.rb} +7 -4
  13. data/lib/arrow/jruby/file-system.rb +24 -0
  14. data/{test/test-null-array.rb → lib/arrow/jruby/function.rb} +5 -4
  15. data/lib/arrow/jruby/record-batch-iterator.rb +24 -0
  16. data/{test/fixture/integer-float.csv → lib/arrow/jruby/record-batch.rb} +8 -4
  17. data/{test/fixture/float-integer.csv → lib/arrow/jruby/sort-key.rb} +8 -4
  18. data/lib/arrow/jruby/sort-options.rb +24 -0
  19. data/lib/arrow/jruby/stream-listener-raw.rb +25 -0
  20. data/{test/test-rolling-window.rb → lib/arrow/jruby/table.rb} +19 -19
  21. data/lib/arrow/jruby/writable.rb +24 -0
  22. data/lib/arrow/jruby.rb +52 -0
  23. data/lib/arrow/libraries.rb +126 -0
  24. data/lib/arrow/list-array-builder.rb +1 -0
  25. data/lib/arrow/loader.rb +3 -111
  26. data/{test/fixture/null-without-double-quote.csv → lib/arrow/ruby.rb} +6 -4
  27. data/lib/arrow/version.rb +1 -1
  28. data/lib/arrow.rb +2 -7
  29. data/red-arrow.gemspec +15 -6
  30. metadata +26 -229
  31. data/test/each-raw-record/test-basic-arrays.rb +0 -411
  32. data/test/each-raw-record/test-dense-union-array.rb +0 -566
  33. data/test/each-raw-record/test-dictionary-array.rb +0 -341
  34. data/test/each-raw-record/test-list-array.rb +0 -628
  35. data/test/each-raw-record/test-map-array.rb +0 -507
  36. data/test/each-raw-record/test-multiple-columns.rb +0 -72
  37. data/test/each-raw-record/test-sparse-union-array.rb +0 -528
  38. data/test/each-raw-record/test-struct-array.rb +0 -529
  39. data/test/each-raw-record/test-table.rb +0 -47
  40. data/test/fixture/TestOrcFile.test1.orc +0 -0
  41. data/test/fixture/with-header-float.csv +0 -20
  42. data/test/fixture/with-header.csv +0 -20
  43. data/test/fixture/without-header-float.csv +0 -19
  44. data/test/fixture/without-header.csv +0 -19
  45. data/test/helper/omittable.rb +0 -49
  46. data/test/helper.rb +0 -31
  47. data/test/raw-records/test-basic-arrays.rb +0 -405
  48. data/test/raw-records/test-dense-union-array.rb +0 -566
  49. data/test/raw-records/test-dictionary-array.rb +0 -341
  50. data/test/raw-records/test-list-array.rb +0 -628
  51. data/test/raw-records/test-map-array.rb +0 -507
  52. data/test/raw-records/test-multiple-columns.rb +0 -65
  53. data/test/raw-records/test-sparse-union-array.rb +0 -556
  54. data/test/raw-records/test-struct-array.rb +0 -529
  55. data/test/raw-records/test-table.rb +0 -47
  56. data/test/run-test.rb +0 -71
  57. data/test/test-array-builder.rb +0 -198
  58. data/test/test-array.rb +0 -332
  59. data/test/test-bigdecimal.rb +0 -40
  60. data/test/test-binary-dictionary-array-builder.rb +0 -103
  61. data/test/test-buffer.rb +0 -49
  62. data/test/test-chunked-array.rb +0 -198
  63. data/test/test-column.rb +0 -123
  64. data/test/test-csv-loader.rb +0 -297
  65. data/test/test-data-type.rb +0 -84
  66. data/test/test-date32-array.rb +0 -24
  67. data/test/test-date64-array.rb +0 -25
  68. data/test/test-decimal128-array-builder.rb +0 -126
  69. data/test/test-decimal128-array.rb +0 -47
  70. data/test/test-decimal128-data-type.rb +0 -31
  71. data/test/test-decimal128.rb +0 -126
  72. data/test/test-decimal256-array-builder.rb +0 -126
  73. data/test/test-decimal256-array.rb +0 -47
  74. data/test/test-decimal256-data-type.rb +0 -31
  75. data/test/test-decimal256.rb +0 -126
  76. data/test/test-dense-union-array.rb +0 -42
  77. data/test/test-dense-union-data-type.rb +0 -41
  78. data/test/test-dictionary-array.rb +0 -41
  79. data/test/test-dictionary-data-type.rb +0 -40
  80. data/test/test-expression.rb +0 -51
  81. data/test/test-feather.rb +0 -49
  82. data/test/test-field.rb +0 -117
  83. data/test/test-file-output-stream.rb +0 -54
  84. data/test/test-fixed-size-binary-array-builder.rb +0 -92
  85. data/test/test-fixed-size-binary-array.rb +0 -36
  86. data/test/test-float-scalar.rb +0 -46
  87. data/test/test-function.rb +0 -210
  88. data/test/test-group.rb +0 -193
  89. data/test/test-half-float-array.rb +0 -43
  90. data/test/test-half-float.rb +0 -130
  91. data/test/test-list-array-builder.rb +0 -79
  92. data/test/test-list-array.rb +0 -32
  93. data/test/test-list-data-type.rb +0 -69
  94. data/test/test-map-array-builder.rb +0 -110
  95. data/test/test-map-array.rb +0 -33
  96. data/test/test-memory-view.rb +0 -434
  97. data/test/test-orc.rb +0 -173
  98. data/test/test-ractor.rb +0 -34
  99. data/test/test-record-batch-builder.rb +0 -125
  100. data/test/test-record-batch-file-reader.rb +0 -136
  101. data/test/test-record-batch-iterator.rb +0 -37
  102. data/test/test-record-batch-reader.rb +0 -46
  103. data/test/test-record-batch-stream-reader.rb +0 -129
  104. data/test/test-record-batch.rb +0 -182
  105. data/test/test-scalar.rb +0 -65
  106. data/test/test-schema.rb +0 -134
  107. data/test/test-slicer.rb +0 -589
  108. data/test/test-sort-indices.rb +0 -40
  109. data/test/test-sort-key.rb +0 -81
  110. data/test/test-sort-options.rb +0 -58
  111. data/test/test-sparse-union-array.rb +0 -38
  112. data/test/test-sparse-union-data-type.rb +0 -41
  113. data/test/test-stream-listener.rb +0 -60
  114. data/test/test-string-dictionary-array-builder.rb +0 -103
  115. data/test/test-struct-array-builder.rb +0 -184
  116. data/test/test-struct-array.rb +0 -94
  117. data/test/test-struct-data-type.rb +0 -112
  118. data/test/test-table.rb +0 -1530
  119. data/test/test-tensor.rb +0 -297
  120. data/test/test-time.rb +0 -288
  121. data/test/test-time32-array.rb +0 -81
  122. data/test/test-time32-data-type.rb +0 -42
  123. data/test/test-time64-array.rb +0 -81
  124. data/test/test-time64-data-type.rb +0 -42
  125. data/test/test-timestamp-array.rb +0 -45
  126. data/test/test-timestamp-data-type.rb +0 -42
  127. data/test/values/test-basic-arrays.rb +0 -335
  128. data/test/values/test-dense-union-array.rb +0 -552
  129. data/test/values/test-dictionary-array.rb +0 -325
  130. data/test/values/test-list-array.rb +0 -587
  131. data/test/values/test-map-array.rb +0 -489
  132. data/test/values/test-sparse-union-array.rb +0 -543
  133. data/test/values/test-struct-array.rb +0 -524
@@ -1,524 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- module ValuesStructArrayTests
19
- def build_data_type(type)
20
- field_description = {
21
- name: :field,
22
- }
23
- if type.is_a?(Hash)
24
- field_description = field_description.merge(type)
25
- else
26
- field_description[:type] = type
27
- end
28
- Arrow::StructDataType.new([field_description])
29
- end
30
-
31
- def build_array(type, values)
32
- Arrow::StructArray.new(build_data_type(type), values)
33
- end
34
-
35
- def test_null
36
- values = [
37
- {"field" => nil},
38
- nil,
39
- ]
40
- target = build(:null, values)
41
- assert_equal(values, target.values)
42
- end
43
-
44
- def test_boolean
45
- values = [
46
- {"field" => true},
47
- nil,
48
- {"field" => nil},
49
- ]
50
- target = build(:boolean, values)
51
- assert_equal(values, target.values)
52
- end
53
-
54
- def test_int8
55
- values = [
56
- {"field" => -(2 ** 7)},
57
- nil,
58
- {"field" => nil},
59
- ]
60
- target = build(:int8, values)
61
- assert_equal(values, target.values)
62
- end
63
-
64
- def test_uint8
65
- values = [
66
- {"field" => (2 ** 8) - 1},
67
- nil,
68
- {"field" => nil},
69
- ]
70
- target = build(:uint8, values)
71
- assert_equal(values, target.values)
72
- end
73
-
74
- def test_int16
75
- values = [
76
- {"field" => -(2 ** 15)},
77
- nil,
78
- {"field" => nil},
79
- ]
80
- target = build(:int16, values)
81
- assert_equal(values, target.values)
82
- end
83
-
84
- def test_uint16
85
- values = [
86
- {"field" => (2 ** 16) - 1},
87
- nil,
88
- {"field" => nil},
89
- ]
90
- target = build(:uint16, values)
91
- assert_equal(values, target.values)
92
- end
93
-
94
- def test_int32
95
- values = [
96
- {"field" => -(2 ** 31)},
97
- nil,
98
- {"field" => nil},
99
- ]
100
- target = build(:int32, values)
101
- assert_equal(values, target.values)
102
- end
103
-
104
- def test_uint32
105
- values = [
106
- {"field" => (2 ** 32) - 1},
107
- nil,
108
- {"field" => nil},
109
- ]
110
- target = build(:uint32, values)
111
- assert_equal(values, target.values)
112
- end
113
-
114
- def test_int64
115
- values = [
116
- {"field" => -(2 ** 63)},
117
- nil,
118
- {"field" => nil},
119
- ]
120
- target = build(:int64, values)
121
- assert_equal(values, target.values)
122
- end
123
-
124
- def test_uint64
125
- values = [
126
- {"field" => (2 ** 64) - 1},
127
- nil,
128
- {"field" => nil},
129
- ]
130
- target = build(:uint64, values)
131
- assert_equal(values, target.values)
132
- end
133
-
134
- def test_float
135
- values = [
136
- {"field" => -1.0},
137
- nil,
138
- {"field" => nil},
139
- ]
140
- target = build(:float, values)
141
- assert_equal(values, target.values)
142
- end
143
-
144
- def test_double
145
- values = [
146
- {"field" => -1.0},
147
- nil,
148
- {"field" => nil},
149
- ]
150
- target = build(:double, values)
151
- assert_equal(values, target.values)
152
- end
153
-
154
- def test_binary
155
- values = [
156
- {"field" => "\xff".b},
157
- nil,
158
- {"field" => nil},
159
- ]
160
- target = build(:binary, values)
161
- assert_equal(values, target.values)
162
- end
163
-
164
- def test_string
165
- values = [
166
- {"field" => "Ruby"},
167
- nil,
168
- {"field" => nil},
169
- ]
170
- target = build(:string, values)
171
- assert_equal(values, target.values)
172
- end
173
-
174
- def test_date32
175
- values = [
176
- {"field" => Date.new(1960, 1, 1)},
177
- nil,
178
- {"field" => nil},
179
- ]
180
- target = build(:date32, values)
181
- assert_equal(values, target.values)
182
- end
183
-
184
- def test_date64
185
- values = [
186
- {"field" => DateTime.new(1960, 1, 1, 2, 9, 30)},
187
- nil,
188
- {"field" => nil},
189
- ]
190
- target = build(:date64, values)
191
- assert_equal(values, target.values)
192
- end
193
-
194
- def test_timestamp_second
195
- values = [
196
- {"field" => Time.parse("1960-01-01T02:09:30Z")},
197
- nil,
198
- {"field" => nil},
199
- ]
200
- target = build({
201
- type: :timestamp,
202
- unit: :second,
203
- },
204
- values)
205
- assert_equal(values, target.values)
206
- end
207
-
208
- def test_timestamp_milli
209
- values = [
210
- {"field" => Time.parse("1960-01-01T02:09:30.123Z")},
211
- nil,
212
- {"field" => nil},
213
- ]
214
- target = build({
215
- type: :timestamp,
216
- unit: :milli,
217
- },
218
- values)
219
- assert_equal(values, target.values)
220
- end
221
-
222
- def test_timestamp_micro
223
- values = [
224
- {"field" => Time.parse("1960-01-01T02:09:30.123456Z")},
225
- nil,
226
- {"field" => nil},
227
- ]
228
- target = build({
229
- type: :timestamp,
230
- unit: :micro,
231
- },
232
- values)
233
- assert_equal(values, target.values)
234
- end
235
-
236
- def test_timestamp_nano
237
- values = [
238
- {"field" => Time.parse("1960-01-01T02:09:30.123456789Z")},
239
- nil,
240
- {"field" => nil},
241
- ]
242
- target = build({
243
- type: :timestamp,
244
- unit: :nano,
245
- },
246
- values)
247
- assert_equal(values, target.values)
248
- end
249
-
250
- def test_time32_second
251
- unit = Arrow::TimeUnit::SECOND
252
- values = [
253
- # 00:10:00
254
- {"field" => Arrow::Time.new(unit, 60 * 10)},
255
- nil,
256
- {"field" => nil},
257
- ]
258
- target = build({
259
- type: :time32,
260
- unit: :second,
261
- },
262
- values)
263
- assert_equal(values, target.values)
264
- end
265
-
266
- def test_time32_milli
267
- unit = Arrow::TimeUnit::MILLI
268
- values = [
269
- # 00:10:00.123
270
- {"field" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)},
271
- nil,
272
- {"field" => nil},
273
- ]
274
- target = build({
275
- type: :time32,
276
- unit: :milli,
277
- },
278
- values)
279
- assert_equal(values, target.values)
280
- end
281
-
282
- def test_time64_micro
283
- unit = Arrow::TimeUnit::MICRO
284
- values = [
285
- # 00:10:00.123456
286
- {"field" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)},
287
- nil,
288
- {"field" => nil},
289
- ]
290
- target = build({
291
- type: :time64,
292
- unit: :micro,
293
- },
294
- values)
295
- assert_equal(values, target.values)
296
- end
297
-
298
- def test_time64_nano
299
- unit = Arrow::TimeUnit::NANO
300
- values = [
301
- # 00:10:00.123456789
302
- {"field" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)},
303
- nil,
304
- {"field" => nil},
305
- ]
306
- target = build({
307
- type: :time64,
308
- unit: :nano,
309
- },
310
- values)
311
- assert_equal(values, target.values)
312
- end
313
-
314
- def test_decimal128
315
- values = [
316
- {"field" => BigDecimal("92.92")},
317
- nil,
318
- {"field" => nil},
319
- ]
320
- target = build({
321
- type: :decimal128,
322
- precision: 8,
323
- scale: 2,
324
- },
325
- values)
326
- assert_equal(values, target.values)
327
- end
328
-
329
- def test_decimal256
330
- values = [
331
- {"field" => BigDecimal("92.92")},
332
- nil,
333
- {"field" => nil},
334
- ]
335
- target = build({
336
- type: :decimal256,
337
- precision: 38,
338
- scale: 2,
339
- },
340
- values)
341
- assert_equal(values, target.values)
342
- end
343
-
344
- def test_month_interval
345
- values = [
346
- {"field" => 1},
347
- nil,
348
- {"field" => nil},
349
- ]
350
- target = build(:month_interval, values)
351
- assert_equal(values, target.values)
352
- end
353
-
354
- def test_day_time_interval
355
- values = [
356
- {"field" => {day: 1, millisecond: 100}},
357
- nil,
358
- {"field" => nil},
359
- ]
360
- target = build(:day_time_interval, values)
361
- assert_equal(values, target.values)
362
- end
363
-
364
- def test_month_day_nano_interval
365
- values = [
366
- {"field" => {month: 1, day: 1, nanosecond: 100}},
367
- nil,
368
- {"field" => nil},
369
- ]
370
- target = build(:month_day_nano_interval, values)
371
- assert_equal(values, target.values)
372
- end
373
-
374
- def test_list
375
- values = [
376
- {"field" => [true, nil, false]},
377
- nil,
378
- {"field" => nil},
379
- ]
380
- target = build({
381
- type: :list,
382
- field: {
383
- name: :sub_element,
384
- type: :boolean,
385
- },
386
- },
387
- values)
388
- assert_equal(values, target.values)
389
- end
390
-
391
- def test_struct
392
- values = [
393
- {"field" => {"sub_field" => true}},
394
- nil,
395
- {"field" => nil},
396
- {"field" => {"sub_field" => nil}},
397
- ]
398
- target = build({
399
- type: :struct,
400
- fields: [
401
- {
402
- name: :sub_field,
403
- type: :boolean,
404
- },
405
- ],
406
- },
407
- values)
408
- assert_equal(values, target.values)
409
- end
410
-
411
- def test_map
412
- values = [
413
- {"field" => {"key1" => true, "key2" => nil}},
414
- nil,
415
- {"field" => nil},
416
- ]
417
- target = build({
418
- type: :map,
419
- key: :string,
420
- item: :boolean,
421
- },
422
- values)
423
- assert_equal(values, target.values)
424
- end
425
-
426
- def remove_union_field_names(values)
427
- values.collect do |value|
428
- if value.nil?
429
- value
430
- else
431
- v = value["field"]
432
- v = v.values[0] unless v.nil?
433
- {"field" => v}
434
- end
435
- end
436
- end
437
-
438
- def test_sparse_union
439
- values = [
440
- {"field" => {"field1" => true}},
441
- nil,
442
- {"field" => nil},
443
- {"field" => {"field2" => 29}},
444
- {"field" => {"field2" => nil}},
445
- ]
446
- target = build({
447
- type: :sparse_union,
448
- fields: [
449
- {
450
- name: :field1,
451
- type: :boolean,
452
- },
453
- {
454
- name: :field2,
455
- type: :uint8,
456
- },
457
- ],
458
- type_codes: [0, 1],
459
- },
460
- values)
461
- assert_equal(remove_union_field_names(values),
462
- target.values)
463
- end
464
-
465
- def test_dense_union
466
- values = [
467
- {"field" => {"field1" => true}},
468
- nil,
469
- {"field" => nil},
470
- {"field" => {"field2" => 29}},
471
- {"field" => {"field2" => nil}},
472
- ]
473
- target = build({
474
- type: :dense_union,
475
- fields: [
476
- {
477
- name: :field1,
478
- type: :boolean,
479
- },
480
- {
481
- name: :field2,
482
- type: :uint8,
483
- },
484
- ],
485
- type_codes: [0, 1],
486
- },
487
- values)
488
- assert_equal(remove_union_field_names(values),
489
- target.values)
490
- end
491
-
492
- def test_dictionary
493
- values = [
494
- {"field" => "Ruby"},
495
- nil,
496
- {"field" => nil},
497
- {"field" => "GLib"},
498
- ]
499
- target = build({
500
- type: :dictionary,
501
- index_data_type: :int8,
502
- value_data_type: :string,
503
- ordered: false,
504
- },
505
- values)
506
- assert_equal(values, target.values)
507
- end
508
- end
509
-
510
- class ValuesArrayStructArrayTest < Test::Unit::TestCase
511
- include ValuesStructArrayTests
512
-
513
- def build(type, values)
514
- build_array(type, values)
515
- end
516
- end
517
-
518
- class ValuesChunkedArrayStructArrayTest < Test::Unit::TestCase
519
- include ValuesStructArrayTests
520
-
521
- def build(type, values)
522
- Arrow::ChunkedArray.new([build_array(type, values)])
523
- end
524
- end