red-arrow 18.1.0 → 19.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/lib/arrow/array.rb +7 -4
  3. data/lib/arrow/column.rb +4 -4
  4. data/lib/arrow/jruby/array-builder.rb +114 -0
  5. data/lib/arrow/jruby/array.rb +109 -0
  6. data/{test/helper/fixture.rb → lib/arrow/jruby/chunked-array.rb} +14 -6
  7. data/lib/arrow/jruby/compression-type.rb +26 -0
  8. data/{test/test-boolean-scalar.rb → lib/arrow/jruby/csv-read-options.rb} +12 -6
  9. data/{test/test-map-data-type.rb → lib/arrow/jruby/data-type.rb} +24 -12
  10. data/lib/arrow/jruby/decimal128.rb +28 -0
  11. data/lib/arrow/jruby/decimal256.rb +28 -0
  12. data/{test/fixture/null-with-double-quote.csv → lib/arrow/jruby/error.rb} +7 -4
  13. data/lib/arrow/jruby/file-system.rb +24 -0
  14. data/{test/test-null-array.rb → lib/arrow/jruby/function.rb} +5 -4
  15. data/lib/arrow/jruby/record-batch-iterator.rb +24 -0
  16. data/{test/fixture/integer-float.csv → lib/arrow/jruby/record-batch.rb} +8 -4
  17. data/{test/fixture/float-integer.csv → lib/arrow/jruby/sort-key.rb} +8 -4
  18. data/lib/arrow/jruby/sort-options.rb +24 -0
  19. data/lib/arrow/jruby/stream-listener-raw.rb +25 -0
  20. data/{test/test-rolling-window.rb → lib/arrow/jruby/table.rb} +19 -19
  21. data/lib/arrow/jruby/writable.rb +24 -0
  22. data/lib/arrow/jruby.rb +52 -0
  23. data/lib/arrow/libraries.rb +126 -0
  24. data/lib/arrow/list-array-builder.rb +1 -0
  25. data/lib/arrow/loader.rb +3 -111
  26. data/{test/fixture/null-without-double-quote.csv → lib/arrow/ruby.rb} +6 -4
  27. data/lib/arrow/version.rb +1 -1
  28. data/lib/arrow.rb +2 -7
  29. data/red-arrow.gemspec +15 -6
  30. metadata +26 -229
  31. data/test/each-raw-record/test-basic-arrays.rb +0 -411
  32. data/test/each-raw-record/test-dense-union-array.rb +0 -566
  33. data/test/each-raw-record/test-dictionary-array.rb +0 -341
  34. data/test/each-raw-record/test-list-array.rb +0 -628
  35. data/test/each-raw-record/test-map-array.rb +0 -507
  36. data/test/each-raw-record/test-multiple-columns.rb +0 -72
  37. data/test/each-raw-record/test-sparse-union-array.rb +0 -528
  38. data/test/each-raw-record/test-struct-array.rb +0 -529
  39. data/test/each-raw-record/test-table.rb +0 -47
  40. data/test/fixture/TestOrcFile.test1.orc +0 -0
  41. data/test/fixture/with-header-float.csv +0 -20
  42. data/test/fixture/with-header.csv +0 -20
  43. data/test/fixture/without-header-float.csv +0 -19
  44. data/test/fixture/without-header.csv +0 -19
  45. data/test/helper/omittable.rb +0 -49
  46. data/test/helper.rb +0 -31
  47. data/test/raw-records/test-basic-arrays.rb +0 -405
  48. data/test/raw-records/test-dense-union-array.rb +0 -566
  49. data/test/raw-records/test-dictionary-array.rb +0 -341
  50. data/test/raw-records/test-list-array.rb +0 -628
  51. data/test/raw-records/test-map-array.rb +0 -507
  52. data/test/raw-records/test-multiple-columns.rb +0 -65
  53. data/test/raw-records/test-sparse-union-array.rb +0 -556
  54. data/test/raw-records/test-struct-array.rb +0 -529
  55. data/test/raw-records/test-table.rb +0 -47
  56. data/test/run-test.rb +0 -71
  57. data/test/test-array-builder.rb +0 -198
  58. data/test/test-array.rb +0 -332
  59. data/test/test-bigdecimal.rb +0 -40
  60. data/test/test-binary-dictionary-array-builder.rb +0 -103
  61. data/test/test-buffer.rb +0 -49
  62. data/test/test-chunked-array.rb +0 -198
  63. data/test/test-column.rb +0 -123
  64. data/test/test-csv-loader.rb +0 -297
  65. data/test/test-data-type.rb +0 -84
  66. data/test/test-date32-array.rb +0 -24
  67. data/test/test-date64-array.rb +0 -25
  68. data/test/test-decimal128-array-builder.rb +0 -126
  69. data/test/test-decimal128-array.rb +0 -47
  70. data/test/test-decimal128-data-type.rb +0 -31
  71. data/test/test-decimal128.rb +0 -126
  72. data/test/test-decimal256-array-builder.rb +0 -126
  73. data/test/test-decimal256-array.rb +0 -47
  74. data/test/test-decimal256-data-type.rb +0 -31
  75. data/test/test-decimal256.rb +0 -126
  76. data/test/test-dense-union-array.rb +0 -42
  77. data/test/test-dense-union-data-type.rb +0 -41
  78. data/test/test-dictionary-array.rb +0 -41
  79. data/test/test-dictionary-data-type.rb +0 -40
  80. data/test/test-expression.rb +0 -51
  81. data/test/test-feather.rb +0 -49
  82. data/test/test-field.rb +0 -117
  83. data/test/test-file-output-stream.rb +0 -54
  84. data/test/test-fixed-size-binary-array-builder.rb +0 -92
  85. data/test/test-fixed-size-binary-array.rb +0 -36
  86. data/test/test-float-scalar.rb +0 -46
  87. data/test/test-function.rb +0 -210
  88. data/test/test-group.rb +0 -193
  89. data/test/test-half-float-array.rb +0 -43
  90. data/test/test-half-float.rb +0 -130
  91. data/test/test-list-array-builder.rb +0 -79
  92. data/test/test-list-array.rb +0 -32
  93. data/test/test-list-data-type.rb +0 -69
  94. data/test/test-map-array-builder.rb +0 -110
  95. data/test/test-map-array.rb +0 -33
  96. data/test/test-memory-view.rb +0 -434
  97. data/test/test-orc.rb +0 -173
  98. data/test/test-ractor.rb +0 -34
  99. data/test/test-record-batch-builder.rb +0 -125
  100. data/test/test-record-batch-file-reader.rb +0 -136
  101. data/test/test-record-batch-iterator.rb +0 -37
  102. data/test/test-record-batch-reader.rb +0 -46
  103. data/test/test-record-batch-stream-reader.rb +0 -129
  104. data/test/test-record-batch.rb +0 -182
  105. data/test/test-scalar.rb +0 -65
  106. data/test/test-schema.rb +0 -134
  107. data/test/test-slicer.rb +0 -589
  108. data/test/test-sort-indices.rb +0 -40
  109. data/test/test-sort-key.rb +0 -81
  110. data/test/test-sort-options.rb +0 -58
  111. data/test/test-sparse-union-array.rb +0 -38
  112. data/test/test-sparse-union-data-type.rb +0 -41
  113. data/test/test-stream-listener.rb +0 -60
  114. data/test/test-string-dictionary-array-builder.rb +0 -103
  115. data/test/test-struct-array-builder.rb +0 -184
  116. data/test/test-struct-array.rb +0 -94
  117. data/test/test-struct-data-type.rb +0 -112
  118. data/test/test-table.rb +0 -1530
  119. data/test/test-tensor.rb +0 -297
  120. data/test/test-time.rb +0 -288
  121. data/test/test-time32-array.rb +0 -81
  122. data/test/test-time32-data-type.rb +0 -42
  123. data/test/test-time64-array.rb +0 -81
  124. data/test/test-time64-data-type.rb +0 -42
  125. data/test/test-timestamp-array.rb +0 -45
  126. data/test/test-timestamp-data-type.rb +0 -42
  127. data/test/values/test-basic-arrays.rb +0 -335
  128. data/test/values/test-dense-union-array.rb +0 -552
  129. data/test/values/test-dictionary-array.rb +0 -325
  130. data/test/values/test-list-array.rb +0 -587
  131. data/test/values/test-map-array.rb +0 -489
  132. data/test/values/test-sparse-union-array.rb +0 -543
  133. data/test/values/test-struct-array.rb +0 -524
@@ -1,489 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- module ValuesMapArrayTests
19
- def build_data_type(item_type)
20
- Arrow::MapDataType.new(
21
- key: :string,
22
- item: item_type
23
- )
24
- end
25
-
26
- def build_array(item_type, values)
27
- Arrow::MapArray.new(build_data_type(item_type), values)
28
- end
29
-
30
- def test_null
31
- values = [
32
- {"key1" => nil},
33
- nil,
34
- ]
35
- target = build(:null, values)
36
- assert_equal(values, target.values)
37
- end
38
-
39
- def test_boolean
40
- values = [
41
- {"key1" => false, "key2" => nil},
42
- nil,
43
- ]
44
- target = build(:boolean, values)
45
- assert_equal(values, target.values)
46
- end
47
-
48
- def test_int8
49
- values = [
50
- {"key1" => (2 ** 7) - 1, "key2" => nil},
51
- nil,
52
- ]
53
- target = build(:int8, values)
54
- assert_equal(values, target.values)
55
- end
56
-
57
- def test_uint8
58
- values = [
59
- {"key1" => (2 ** 8) - 1, "key2" => nil},
60
- nil,
61
- ]
62
- target = build(:uint8, values)
63
- assert_equal(values, target.values)
64
- end
65
-
66
- def test_uint16
67
- values = [
68
- {"key1" => (2 ** 16) - 1, "key2" => nil},
69
- nil,
70
- ]
71
- target = build(:uint16, values)
72
- assert_equal(values, target.values)
73
- end
74
-
75
- def test_int32
76
- values = [
77
- {"key1" => -(2 ** 31), "key2" => nil},
78
- nil,
79
- ]
80
- target = build(:int32, values)
81
- assert_equal(values, target.values)
82
- end
83
-
84
- def test_uint32
85
- values = [
86
- {"key1" => (2 ** 32) - 1, "key2" => nil},
87
- nil,
88
- ]
89
- target = build(:uint32, values)
90
- assert_equal(values, target.values)
91
- end
92
-
93
- def test_int64
94
- values = [
95
- {"key1" => -(2 ** 63), "key2" => nil},
96
- nil,
97
- ]
98
- target = build(:int64, values)
99
- assert_equal(values, target.values)
100
- end
101
-
102
- def test_uint64
103
- values = [
104
- {"key1" => (2 ** 64) - 1, "key2" => nil},
105
- nil,
106
- ]
107
- target = build(:uint64, values)
108
- assert_equal(values, target.values)
109
- end
110
-
111
- def test_float
112
- values = [
113
- {"key1" => -1.0, "key2" => nil},
114
- nil,
115
- ]
116
- target = build(:float, values)
117
- assert_equal(values, target.values)
118
- end
119
-
120
- def test_double
121
- values = [
122
- {"key1" => -1.0, "key2" => nil},
123
- nil,
124
- ]
125
- target = build(:double, values)
126
- assert_equal(values, target.values)
127
- end
128
-
129
- def test_binary
130
- values = [
131
- {"key1" => "\xff".b, "key2" => nil},
132
- nil,
133
- ]
134
- target = build(:binary, values)
135
- assert_equal(values, target.values)
136
- end
137
-
138
- def test_string
139
- values = [
140
- {"key1" => "Ruby", "key2" => nil},
141
- nil,
142
- ]
143
- target = build(:string, values)
144
- assert_equal(values, target.values)
145
- end
146
-
147
- def test_date32
148
- values = [
149
- {"key1" => Date.new(1960, 1, 1), "key2" => nil},
150
- nil,
151
- ]
152
- target = build(:date32, values)
153
- assert_equal(values, target.values)
154
- end
155
-
156
- def test_date64
157
- values = [
158
- {"key1" => DateTime.new(1960, 1, 1, 2, 9, 30), "key2" => nil},
159
- nil,
160
- ]
161
- target = build(:date64, values)
162
- assert_equal(values, target.values)
163
- end
164
-
165
- def test_timestamp_second
166
- values = [
167
- {"key1" => Time.parse("1960-01-01T02:09:30Z"), "key2" => nil},
168
- nil,
169
- ]
170
- target = build({
171
- type: :timestamp,
172
- unit: :second,
173
- },
174
- values)
175
- assert_equal(values, target.values)
176
- end
177
-
178
- def test_timestamp_milli
179
- values = [
180
- {"key1" => Time.parse("1960-01-01T02:09:30.123Z"), "key2" => nil},
181
- nil,
182
- ]
183
- target = build({
184
- type: :timestamp,
185
- unit: :milli,
186
- },
187
- values)
188
- assert_equal(values, target.values)
189
- end
190
-
191
- def test_timestamp_micro
192
- values = [
193
- {"key1" => Time.parse("1960-01-01T02:09:30.123456Z"), "key2" => nil},
194
- nil,
195
- ]
196
- target = build({
197
- type: :timestamp,
198
- unit: :micro,
199
- },
200
- values)
201
- assert_equal(values, target.values)
202
- end
203
-
204
- def test_timestamp_nano
205
- values = [
206
- {"key1" => Time.parse("1960-01-01T02:09:30.123456789Z"), "key2" => nil},
207
- nil,
208
- ]
209
- target = build({
210
- type: :timestamp,
211
- unit: :nano,
212
- },
213
- values)
214
- assert_equal(values, target.values)
215
- end
216
-
217
- def test_time32_second
218
- unit = Arrow::TimeUnit::SECOND
219
- values = [
220
- # 00:10:00
221
- {"key1" => Arrow::Time.new(unit, 60 * 10), "key2" => nil},
222
- nil,
223
- ]
224
- target = build({
225
- type: :time32,
226
- unit: :second,
227
- },
228
- values)
229
- assert_equal(values, target.values)
230
- end
231
-
232
- def test_time32_milli
233
- unit = Arrow::TimeUnit::MILLI
234
- values = [
235
- # 00:10:00.123
236
- {"key1" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123), "key2" => nil},
237
- nil,
238
- ]
239
- target = build({
240
- type: :time32,
241
- unit: :milli,
242
- },
243
- values)
244
- assert_equal(values, target.values)
245
- end
246
-
247
- def test_time64_micro
248
- unit = Arrow::TimeUnit::MICRO
249
- values = [
250
- # 00:10:00.123456
251
- {"key1" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456), "key2" => nil},
252
- nil,
253
- ]
254
- target = build({
255
- type: :time64,
256
- unit: :micro,
257
- },
258
- values)
259
- assert_equal(values, target.values)
260
- end
261
-
262
- def test_time64_nano
263
- unit = Arrow::TimeUnit::NANO
264
- values = [
265
- # 00:10:00.123456789
266
- {"key1" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789), "key2" => nil},
267
- nil,
268
- ]
269
- target = build({
270
- type: :time64,
271
- unit: :nano,
272
- },
273
- values)
274
- assert_equal(values, target.values)
275
- end
276
-
277
- def test_decimal128
278
- values = [
279
- {"key1" => BigDecimal("92.92"), "key2" => nil},
280
- nil,
281
- ]
282
- target = build({
283
- type: :decimal128,
284
- precision: 8,
285
- scale: 2,
286
- },
287
- values)
288
- assert_equal(values, target.values)
289
- end
290
-
291
- def test_decimal256
292
- values = [
293
- {"key1" => BigDecimal("92.92"), "key2" => nil},
294
- nil,
295
- ]
296
- target = build({
297
- type: :decimal256,
298
- precision: 38,
299
- scale: 2,
300
- },
301
- values)
302
- assert_equal(values, target.values)
303
- end
304
-
305
- def test_month_interval
306
- values = [
307
- {"key1" => 1, "key2" => nil},
308
- nil,
309
- ]
310
- target = build(:month_interval, values)
311
- assert_equal(values, target.values)
312
- end
313
-
314
- def test_day_time_interval
315
- values = [
316
- {
317
- "key1" => {day: 1, millisecond: 100},
318
- "key2" => nil,
319
- },
320
- nil,
321
- ]
322
- target = build(:day_time_interval, values)
323
- assert_equal(values, target.values)
324
- end
325
-
326
- def test_month_day_nano_interval
327
- values = [
328
- {
329
- "key1" => {month: 1, day: 1, nanosecond: 100},
330
- "key2" => nil,
331
- },
332
- nil,
333
- ]
334
- target = build(:month_day_nano_interval, values)
335
- assert_equal(values, target.values)
336
- end
337
-
338
- def test_list
339
- values = [
340
- {"key1" => [true, nil, false], "key2" => nil},
341
- nil,
342
- ]
343
- target = build({
344
- type: :list,
345
- field: {
346
- name: :sub_element,
347
- type: :boolean,
348
- },
349
- },
350
- values)
351
- assert_equal(values, target.values)
352
- end
353
-
354
- def test_struct
355
- values = [
356
- {"key1" => {"field" => true}, "key2" => nil, "key3" => {"field" => nil}},
357
- nil,
358
- ]
359
- target = build({
360
- type: :struct,
361
- fields: [
362
- {
363
- name: :field,
364
- type: :boolean,
365
- },
366
- ],
367
- },
368
- values)
369
- assert_equal(values, target.values)
370
- end
371
-
372
- def test_map
373
- values = [
374
- {"key1" => {"sub_key1" => true, "sub_key2" => nil}, "key2" => nil},
375
- nil,
376
- ]
377
- target = build({
378
- type: :map,
379
- key: :string,
380
- item: :boolean,
381
- },
382
- values)
383
- assert_equal(values, target.values)
384
- end
385
-
386
- def remove_union_field_names(values)
387
- values.collect do |value|
388
- if value.nil?
389
- value
390
- else
391
- val = {}
392
- value.each do |k, v|
393
- v = v.values[0] unless v.nil?
394
- val[k] = v
395
- end
396
- val
397
- end
398
- end
399
- end
400
-
401
- def test_sparse_union
402
- values = [
403
- {
404
- "key1" => {"field1" => true},
405
- "key2" => nil,
406
- "key3" => {"field2" => 29},
407
- "key4" => {"field2" => nil},
408
- },
409
- nil,
410
- ]
411
- target = build({
412
- type: :sparse_union,
413
- fields: [
414
- {
415
- name: :field1,
416
- type: :boolean,
417
- },
418
- {
419
- name: :field2,
420
- type: :uint8,
421
- },
422
- ],
423
- type_codes: [0, 1],
424
- },
425
- values)
426
- assert_equal(remove_union_field_names(values),
427
- target.values)
428
- end
429
-
430
- def test_dense_union
431
- values = [
432
- {
433
- "key1" => {"field1" => true},
434
- "key2" => nil,
435
- "key3" => {"field2" => 29},
436
- "key4" => {"field2" => nil},
437
- },
438
- nil,
439
- ]
440
- target = build({
441
- type: :dense_union,
442
- fields: [
443
- {
444
- name: :field1,
445
- type: :boolean,
446
- },
447
- {
448
- name: :field2,
449
- type: :uint8,
450
- },
451
- ],
452
- type_codes: [0, 1],
453
- },
454
- values)
455
- assert_equal(remove_union_field_names(values),
456
- target.values)
457
- end
458
-
459
- def test_dictionary
460
- values = [
461
- {"key1" => "Ruby", "key2" => nil, "key3" => "GLib"},
462
- nil,
463
- ]
464
- target = build({
465
- type: :dictionary,
466
- index_data_type: :int8,
467
- value_data_type: :string,
468
- ordered: false,
469
- },
470
- values)
471
- assert_equal(values, target.values)
472
- end
473
- end
474
-
475
- class ValuesArrayMapArrayTest < Test::Unit::TestCase
476
- include ValuesMapArrayTests
477
-
478
- def build(item_type, values)
479
- build_array(item_type, values)
480
- end
481
- end
482
-
483
- class ValuesChunkedArrayMapArrayTest < Test::Unit::TestCase
484
- include ValuesMapArrayTests
485
-
486
- def build(item_type, values)
487
- Arrow::ChunkedArray.new([build_array(item_type, values)])
488
- end
489
- end