red-arrow 18.1.0 → 19.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/lib/arrow/array.rb +7 -4
  3. data/lib/arrow/column.rb +4 -4
  4. data/lib/arrow/jruby/array-builder.rb +114 -0
  5. data/lib/arrow/jruby/array.rb +109 -0
  6. data/{test/helper/fixture.rb → lib/arrow/jruby/chunked-array.rb} +14 -6
  7. data/lib/arrow/jruby/compression-type.rb +26 -0
  8. data/{test/test-boolean-scalar.rb → lib/arrow/jruby/csv-read-options.rb} +12 -6
  9. data/{test/test-map-data-type.rb → lib/arrow/jruby/data-type.rb} +24 -12
  10. data/lib/arrow/jruby/decimal128.rb +28 -0
  11. data/lib/arrow/jruby/decimal256.rb +28 -0
  12. data/{test/fixture/null-with-double-quote.csv → lib/arrow/jruby/error.rb} +7 -4
  13. data/lib/arrow/jruby/file-system.rb +24 -0
  14. data/{test/test-null-array.rb → lib/arrow/jruby/function.rb} +5 -4
  15. data/lib/arrow/jruby/record-batch-iterator.rb +24 -0
  16. data/{test/fixture/integer-float.csv → lib/arrow/jruby/record-batch.rb} +8 -4
  17. data/{test/fixture/float-integer.csv → lib/arrow/jruby/sort-key.rb} +8 -4
  18. data/lib/arrow/jruby/sort-options.rb +24 -0
  19. data/lib/arrow/jruby/stream-listener-raw.rb +25 -0
  20. data/{test/test-rolling-window.rb → lib/arrow/jruby/table.rb} +19 -19
  21. data/lib/arrow/jruby/writable.rb +24 -0
  22. data/lib/arrow/jruby.rb +52 -0
  23. data/lib/arrow/libraries.rb +126 -0
  24. data/lib/arrow/list-array-builder.rb +1 -0
  25. data/lib/arrow/loader.rb +3 -111
  26. data/{test/fixture/null-without-double-quote.csv → lib/arrow/ruby.rb} +6 -4
  27. data/lib/arrow/version.rb +1 -1
  28. data/lib/arrow.rb +2 -7
  29. data/red-arrow.gemspec +15 -6
  30. metadata +26 -229
  31. data/test/each-raw-record/test-basic-arrays.rb +0 -411
  32. data/test/each-raw-record/test-dense-union-array.rb +0 -566
  33. data/test/each-raw-record/test-dictionary-array.rb +0 -341
  34. data/test/each-raw-record/test-list-array.rb +0 -628
  35. data/test/each-raw-record/test-map-array.rb +0 -507
  36. data/test/each-raw-record/test-multiple-columns.rb +0 -72
  37. data/test/each-raw-record/test-sparse-union-array.rb +0 -528
  38. data/test/each-raw-record/test-struct-array.rb +0 -529
  39. data/test/each-raw-record/test-table.rb +0 -47
  40. data/test/fixture/TestOrcFile.test1.orc +0 -0
  41. data/test/fixture/with-header-float.csv +0 -20
  42. data/test/fixture/with-header.csv +0 -20
  43. data/test/fixture/without-header-float.csv +0 -19
  44. data/test/fixture/without-header.csv +0 -19
  45. data/test/helper/omittable.rb +0 -49
  46. data/test/helper.rb +0 -31
  47. data/test/raw-records/test-basic-arrays.rb +0 -405
  48. data/test/raw-records/test-dense-union-array.rb +0 -566
  49. data/test/raw-records/test-dictionary-array.rb +0 -341
  50. data/test/raw-records/test-list-array.rb +0 -628
  51. data/test/raw-records/test-map-array.rb +0 -507
  52. data/test/raw-records/test-multiple-columns.rb +0 -65
  53. data/test/raw-records/test-sparse-union-array.rb +0 -556
  54. data/test/raw-records/test-struct-array.rb +0 -529
  55. data/test/raw-records/test-table.rb +0 -47
  56. data/test/run-test.rb +0 -71
  57. data/test/test-array-builder.rb +0 -198
  58. data/test/test-array.rb +0 -332
  59. data/test/test-bigdecimal.rb +0 -40
  60. data/test/test-binary-dictionary-array-builder.rb +0 -103
  61. data/test/test-buffer.rb +0 -49
  62. data/test/test-chunked-array.rb +0 -198
  63. data/test/test-column.rb +0 -123
  64. data/test/test-csv-loader.rb +0 -297
  65. data/test/test-data-type.rb +0 -84
  66. data/test/test-date32-array.rb +0 -24
  67. data/test/test-date64-array.rb +0 -25
  68. data/test/test-decimal128-array-builder.rb +0 -126
  69. data/test/test-decimal128-array.rb +0 -47
  70. data/test/test-decimal128-data-type.rb +0 -31
  71. data/test/test-decimal128.rb +0 -126
  72. data/test/test-decimal256-array-builder.rb +0 -126
  73. data/test/test-decimal256-array.rb +0 -47
  74. data/test/test-decimal256-data-type.rb +0 -31
  75. data/test/test-decimal256.rb +0 -126
  76. data/test/test-dense-union-array.rb +0 -42
  77. data/test/test-dense-union-data-type.rb +0 -41
  78. data/test/test-dictionary-array.rb +0 -41
  79. data/test/test-dictionary-data-type.rb +0 -40
  80. data/test/test-expression.rb +0 -51
  81. data/test/test-feather.rb +0 -49
  82. data/test/test-field.rb +0 -117
  83. data/test/test-file-output-stream.rb +0 -54
  84. data/test/test-fixed-size-binary-array-builder.rb +0 -92
  85. data/test/test-fixed-size-binary-array.rb +0 -36
  86. data/test/test-float-scalar.rb +0 -46
  87. data/test/test-function.rb +0 -210
  88. data/test/test-group.rb +0 -193
  89. data/test/test-half-float-array.rb +0 -43
  90. data/test/test-half-float.rb +0 -130
  91. data/test/test-list-array-builder.rb +0 -79
  92. data/test/test-list-array.rb +0 -32
  93. data/test/test-list-data-type.rb +0 -69
  94. data/test/test-map-array-builder.rb +0 -110
  95. data/test/test-map-array.rb +0 -33
  96. data/test/test-memory-view.rb +0 -434
  97. data/test/test-orc.rb +0 -173
  98. data/test/test-ractor.rb +0 -34
  99. data/test/test-record-batch-builder.rb +0 -125
  100. data/test/test-record-batch-file-reader.rb +0 -136
  101. data/test/test-record-batch-iterator.rb +0 -37
  102. data/test/test-record-batch-reader.rb +0 -46
  103. data/test/test-record-batch-stream-reader.rb +0 -129
  104. data/test/test-record-batch.rb +0 -182
  105. data/test/test-scalar.rb +0 -65
  106. data/test/test-schema.rb +0 -134
  107. data/test/test-slicer.rb +0 -589
  108. data/test/test-sort-indices.rb +0 -40
  109. data/test/test-sort-key.rb +0 -81
  110. data/test/test-sort-options.rb +0 -58
  111. data/test/test-sparse-union-array.rb +0 -38
  112. data/test/test-sparse-union-data-type.rb +0 -41
  113. data/test/test-stream-listener.rb +0 -60
  114. data/test/test-string-dictionary-array-builder.rb +0 -103
  115. data/test/test-struct-array-builder.rb +0 -184
  116. data/test/test-struct-array.rb +0 -94
  117. data/test/test-struct-data-type.rb +0 -112
  118. data/test/test-table.rb +0 -1530
  119. data/test/test-tensor.rb +0 -297
  120. data/test/test-time.rb +0 -288
  121. data/test/test-time32-array.rb +0 -81
  122. data/test/test-time32-data-type.rb +0 -42
  123. data/test/test-time64-array.rb +0 -81
  124. data/test/test-time64-data-type.rb +0 -42
  125. data/test/test-timestamp-array.rb +0 -45
  126. data/test/test-timestamp-data-type.rb +0 -42
  127. data/test/values/test-basic-arrays.rb +0 -335
  128. data/test/values/test-dense-union-array.rb +0 -552
  129. data/test/values/test-dictionary-array.rb +0 -325
  130. data/test/values/test-list-array.rb +0 -587
  131. data/test/values/test-map-array.rb +0 -489
  132. data/test/values/test-sparse-union-array.rb +0 -543
  133. data/test/values/test-struct-array.rb +0 -524
data/test/helper.rb DELETED
@@ -1,31 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- require "arrow"
19
-
20
- require "fiddle"
21
- require "json"
22
- require "pathname"
23
- require "tempfile"
24
- require "timeout"
25
- require "webrick"
26
- require "zlib"
27
-
28
- require "test-unit"
29
-
30
- require_relative "helper/fixture"
31
- require_relative "helper/omittable"
data/test/raw-records/test-basic-arrays.rb DELETED
@@ -1,405 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- module RawRecordsBasicArraysTests
19
- def test_null
20
- records = [
21
- [nil],
22
- [nil],
23
- [nil],
24
- [nil],
25
- ]
26
- target = build({column: :null}, records)
27
- assert_equal(records, target.raw_records)
28
- end
29
-
30
- def test_boolean
31
- records = [
32
- [true],
33
- [nil],
34
- [false],
35
- ]
36
- target = build({column: :boolean}, records)
37
- assert_equal(records, target.raw_records)
38
- end
39
-
40
- def test_int8
41
- records = [
42
- [-(2 ** 7)],
43
- [nil],
44
- [(2 ** 7) - 1],
45
- ]
46
- target = build({column: :int8}, records)
47
- assert_equal(records, target.raw_records)
48
- end
49
-
50
- def test_uint8
51
- records = [
52
- [0],
53
- [nil],
54
- [(2 ** 8) - 1],
55
- ]
56
- target = build({column: :uint8}, records)
57
- assert_equal(records, target.raw_records)
58
- end
59
-
60
- def test_int16
61
- records = [
62
- [-(2 ** 15)],
63
- [nil],
64
- [(2 ** 15) - 1],
65
- ]
66
- target = build({column: :int16}, records)
67
- assert_equal(records, target.raw_records)
68
- end
69
-
70
- def test_uint16
71
- records = [
72
- [0],
73
- [nil],
74
- [(2 ** 16) - 1],
75
- ]
76
- target = build({column: :uint16}, records)
77
- assert_equal(records, target.raw_records)
78
- end
79
-
80
- def test_int32
81
- records = [
82
- [-(2 ** 31)],
83
- [nil],
84
- [(2 ** 31) - 1],
85
- ]
86
- target = build({column: :int32}, records)
87
- assert_equal(records, target.raw_records)
88
- end
89
-
90
- def test_uint32
91
- records = [
92
- [0],
93
- [nil],
94
- [(2 ** 32) - 1],
95
- ]
96
- target = build({column: :uint32}, records)
97
- assert_equal(records, target.raw_records)
98
- end
99
-
100
- def test_int64
101
- records = [
102
- [-(2 ** 63)],
103
- [nil],
104
- [(2 ** 63) - 1],
105
- ]
106
- target = build({column: :int64}, records)
107
- assert_equal(records, target.raw_records)
108
- end
109
-
110
- def test_uint64
111
- records = [
112
- [0],
113
- [nil],
114
- [(2 ** 64) - 1],
115
- ]
116
- target = build({column: :uint64}, records)
117
- assert_equal(records, target.raw_records)
118
- end
119
-
120
- def test_half_float
121
- records = [
122
- [-1.5],
123
- [nil],
124
- [1.5],
125
- ]
126
- target = build({column: :half_float}, records)
127
- assert_equal(records, target.raw_records)
128
- end
129
-
130
- def test_float
131
- records = [
132
- [-1.0],
133
- [nil],
134
- [1.0],
135
- ]
136
- target = build({column: :float}, records)
137
- assert_equal(records, target.raw_records)
138
- end
139
-
140
- def test_double
141
- records = [
142
- [-1.0],
143
- [nil],
144
- [1.0],
145
- ]
146
- target = build({column: :double}, records)
147
- assert_equal(records, target.raw_records)
148
- end
149
-
150
- def test_binary
151
- records = [
152
- ["\x00".b],
153
- [nil],
154
- ["\xff".b],
155
- ]
156
- target = build({column: :binary}, records)
157
- assert_equal(records, target.raw_records)
158
- end
159
-
160
- def test_string
161
- records = [
162
- ["Ruby"],
163
- [nil],
164
- ["\u3042"], # U+3042 HIRAGANA LETTER A
165
- ]
166
- target = build({column: :string}, records)
167
- assert_equal(records, target.raw_records)
168
- end
169
-
170
- def test_date32
171
- records = [
172
- [Date.new(1960, 1, 1)],
173
- [nil],
174
- [Date.new(2017, 8, 23)],
175
- ]
176
- target = build({column: :date32}, records)
177
- assert_equal(records, target.raw_records)
178
- end
179
-
180
- def test_date64
181
- records = [
182
- [DateTime.new(1960, 1, 1, 2, 9, 30)],
183
- [nil],
184
- [DateTime.new(2017, 8, 23, 14, 57, 2)],
185
- ]
186
- target = build({column: :date64}, records)
187
- assert_equal(records, target.raw_records)
188
- end
189
-
190
- def test_timestamp_second
191
- records = [
192
- [Time.parse("1960-01-01T02:09:30Z")],
193
- [nil],
194
- [Time.parse("2017-08-23T14:57:02Z")],
195
- ]
196
- target = build({
197
- column: {
198
- type: :timestamp,
199
- unit: :second,
200
- }
201
- },
202
- records)
203
- assert_equal(records, target.raw_records)
204
- end
205
-
206
- def test_timestamp_milli
207
- records = [
208
- [Time.parse("1960-01-01T02:09:30.123Z")],
209
- [nil],
210
- [Time.parse("2017-08-23T14:57:02.987Z")],
211
- ]
212
- target = build({
213
- column: {
214
- type: :timestamp,
215
- unit: :milli,
216
- }
217
- },
218
- records)
219
- assert_equal(records, target.raw_records)
220
- end
221
-
222
- def test_timestamp_micro
223
- records = [
224
- [Time.parse("1960-01-01T02:09:30.123456Z")],
225
- [nil],
226
- [Time.parse("2017-08-23T14:57:02.987654Z")],
227
- ]
228
- target = build({
229
- column: {
230
- type: :timestamp,
231
- unit: :micro,
232
- }
233
- },
234
- records)
235
- assert_equal(records, target.raw_records)
236
- end
237
-
238
- def test_timestamp_nano
239
- records = [
240
- [Time.parse("1960-01-01T02:09:30.123456789Z")],
241
- [nil],
242
- [Time.parse("2017-08-23T14:57:02.987654321Z")],
243
- ]
244
- target = build({
245
- column: {
246
- type: :timestamp,
247
- unit: :nano,
248
- }
249
- },
250
- records)
251
- assert_equal(records, target.raw_records)
252
- end
253
-
254
- def test_time32_second
255
- unit = Arrow::TimeUnit::SECOND
256
- records = [
257
- [Arrow::Time.new(unit, 60 * 10)], # 00:10:00
258
- [nil],
259
- [Arrow::Time.new(unit, 60 * 60 * 2 + 9)], # 02:00:09
260
- ]
261
- target = build({
262
- column: {
263
- type: :time32,
264
- unit: :second,
265
- }
266
- },
267
- records)
268
- assert_equal(records, target.raw_records)
269
- end
270
-
271
- def test_time32_milli
272
- unit = Arrow::TimeUnit::MILLI
273
- records = [
274
- [Arrow::Time.new(unit, (60 * 10) * 1000 + 123)], # 00:10:00.123
275
- [nil],
276
- [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987)], # 02:00:09.987
277
- ]
278
- target = build({
279
- column: {
280
- type: :time32,
281
- unit: :milli,
282
- }
283
- },
284
- records)
285
- assert_equal(records, target.raw_records)
286
- end
287
-
288
- def test_time64_micro
289
- unit = Arrow::TimeUnit::MICRO
290
- records = [
291
- # 00:10:00.123456
292
- [Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)],
293
- [nil],
294
- # 02:00:09.987654
295
- [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654)],
296
- ]
297
- target = build({
298
- column: {
299
- type: :time64,
300
- unit: :micro,
301
- }
302
- },
303
- records)
304
- assert_equal(records, target.raw_records)
305
- end
306
-
307
- def test_time64_nano
308
- unit = Arrow::TimeUnit::NANO
309
- records = [
310
- # 00:10:00.123456789
311
- [Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)],
312
- [nil],
313
- # 02:00:09.987654321
314
- [Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321)],
315
- ]
316
- target = build({
317
- column: {
318
- type: :time64,
319
- unit: :nano,
320
- }
321
- },
322
- records)
323
- assert_equal(records, target.raw_records)
324
- end
325
-
326
- def test_decimal128
327
- records = [
328
- [BigDecimal("92.92")],
329
- [nil],
330
- [BigDecimal("29.29")],
331
- ]
332
- target = build({
333
- column: {
334
- type: :decimal128,
335
- precision: 8,
336
- scale: 2,
337
- }
338
- },
339
- records)
340
- assert_equal(records, target.raw_records)
341
- end
342
-
343
- def test_decimal256
344
- records = [
345
- [BigDecimal("92.92")],
346
- [nil],
347
- [BigDecimal("29.29")],
348
- ]
349
- target = build({
350
- column: {
351
- type: :decimal256,
352
- precision: 38,
353
- scale: 2,
354
- }
355
- },
356
- records)
357
- assert_equal(records, target.raw_records)
358
- end
359
-
360
- def test_month_interval
361
- records = [
362
- [1],
363
- [nil],
364
- [12],
365
- ]
366
- target = build({column: :month_interval}, records)
367
- assert_equal(records, target.raw_records)
368
- end
369
-
370
- def test_day_time_interval
371
- records = [
372
- [{day: 1, millisecond: 100}],
373
- [nil],
374
- [{day: 2, millisecond: 300}],
375
- ]
376
- target = build({column: :day_time_interval}, records)
377
- assert_equal(records, target.raw_records)
378
- end
379
-
380
- def test_month_day_nano_interval
381
- records = [
382
- [{month: 1, day: 1, nanosecond: 100}],
383
- [nil],
384
- [{month: 2, day: 3, nanosecond: 400}],
385
- ]
386
- target = build({column: :month_day_nano_interval}, records)
387
- assert_equal(records, target.raw_records)
388
- end
389
- end
390
-
391
- class RawRecordsRecordBatchBasicArraysTest < Test::Unit::TestCase
392
- include RawRecordsBasicArraysTests
393
-
394
- def build(schema, records)
395
- Arrow::RecordBatch.new(schema, records)
396
- end
397
- end
398
-
399
- class RawRecordsTableBasicArraysTest < Test::Unit::TestCase
400
- include RawRecordsBasicArraysTests
401
-
402
- def build(schema, records)
403
- Arrow::Table.new(schema, records)
404
- end
405
- end