red-arrow 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of red-arrow might be problematic. Click here for more details.

@@ -0,0 +1,448 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module RawRecordsStructArrayTests
19
+ def build_schema(type)
20
+ field_description = {
21
+ name: :field,
22
+ }
23
+ if type.is_a?(Hash)
24
+ field_description = field_description.merge(type)
25
+ else
26
+ field_description[:type] = type
27
+ end
28
+ {
29
+ column: {
30
+ type: :struct,
31
+ fields: [
32
+ field_description,
33
+ ],
34
+ },
35
+ }
36
+ end
37
+
38
+ def test_null
39
+ records = [
40
+ [{"field" => nil}],
41
+ [nil],
42
+ ]
43
+ target = build(:null, records)
44
+ assert_equal(records, target.raw_records)
45
+ end
46
+
47
+ def test_boolean
48
+ records = [
49
+ [{"field" => true}],
50
+ [nil],
51
+ [{"field" => nil}],
52
+ ]
53
+ target = build(:boolean, records)
54
+ assert_equal(records, target.raw_records)
55
+ end
56
+
57
+ def test_int8
58
+ records = [
59
+ [{"field" => -(2 ** 7)}],
60
+ [nil],
61
+ [{"field" => nil}],
62
+ ]
63
+ target = build(:int8, records)
64
+ assert_equal(records, target.raw_records)
65
+ end
66
+
67
+ def test_uint8
68
+ records = [
69
+ [{"field" => (2 ** 8) - 1}],
70
+ [nil],
71
+ [{"field" => nil}],
72
+ ]
73
+ target = build(:uint8, records)
74
+ assert_equal(records, target.raw_records)
75
+ end
76
+
77
+ def test_int16
78
+ records = [
79
+ [{"field" => -(2 ** 15)}],
80
+ [nil],
81
+ [{"field" => nil}],
82
+ ]
83
+ target = build(:int16, records)
84
+ assert_equal(records, target.raw_records)
85
+ end
86
+
87
+ def test_uint16
88
+ records = [
89
+ [{"field" => (2 ** 16) - 1}],
90
+ [nil],
91
+ [{"field" => nil}],
92
+ ]
93
+ target = build(:uint16, records)
94
+ assert_equal(records, target.raw_records)
95
+ end
96
+
97
+ def test_int32
98
+ records = [
99
+ [{"field" => -(2 ** 31)}],
100
+ [nil],
101
+ [{"field" => nil}],
102
+ ]
103
+ target = build(:int32, records)
104
+ assert_equal(records, target.raw_records)
105
+ end
106
+
107
+ def test_uint32
108
+ records = [
109
+ [{"field" => (2 ** 32) - 1}],
110
+ [nil],
111
+ [{"field" => nil}],
112
+ ]
113
+ target = build(:uint32, records)
114
+ assert_equal(records, target.raw_records)
115
+ end
116
+
117
+ def test_int64
118
+ records = [
119
+ [{"field" => -(2 ** 63)}],
120
+ [nil],
121
+ [{"field" => nil}],
122
+ ]
123
+ target = build(:int64, records)
124
+ assert_equal(records, target.raw_records)
125
+ end
126
+
127
+ def test_uint64
128
+ records = [
129
+ [{"field" => (2 ** 64) - 1}],
130
+ [nil],
131
+ [{"field" => nil}],
132
+ ]
133
+ target = build(:uint64, records)
134
+ assert_equal(records, target.raw_records)
135
+ end
136
+
137
+ def test_float
138
+ records = [
139
+ [{"field" => -1.0}],
140
+ [nil],
141
+ [{"field" => nil}],
142
+ ]
143
+ target = build(:float, records)
144
+ assert_equal(records, target.raw_records)
145
+ end
146
+
147
+ def test_double
148
+ records = [
149
+ [{"field" => -1.0}],
150
+ [nil],
151
+ [{"field" => nil}],
152
+ ]
153
+ target = build(:double, records)
154
+ assert_equal(records, target.raw_records)
155
+ end
156
+
157
+ def test_binary
158
+ records = [
159
+ [{"field" => "\xff".b}],
160
+ [nil],
161
+ [{"field" => nil}],
162
+ ]
163
+ target = build(:binary, records)
164
+ assert_equal(records, target.raw_records)
165
+ end
166
+
167
+ def test_string
168
+ records = [
169
+ [{"field" => "Ruby"}],
170
+ [nil],
171
+ [{"field" => nil}],
172
+ ]
173
+ target = build(:string, records)
174
+ assert_equal(records, target.raw_records)
175
+ end
176
+
177
+ def test_date32
178
+ records = [
179
+ [{"field" => Date.new(1960, 1, 1)}],
180
+ [nil],
181
+ [{"field" => nil}],
182
+ ]
183
+ target = build(:date32, records)
184
+ assert_equal(records, target.raw_records)
185
+ end
186
+
187
+ def test_date64
188
+ records = [
189
+ [{"field" => DateTime.new(1960, 1, 1, 2, 9, 30)}],
190
+ [nil],
191
+ [{"field" => nil}],
192
+ ]
193
+ target = build(:date64, records)
194
+ assert_equal(records, target.raw_records)
195
+ end
196
+
197
+ def test_timestamp_second
198
+ records = [
199
+ [{"field" => Time.parse("1960-01-01T02:09:30Z")}],
200
+ [nil],
201
+ [{"field" => nil}],
202
+ ]
203
+ target = build({
204
+ type: :timestamp,
205
+ unit: :second,
206
+ },
207
+ records)
208
+ assert_equal(records, target.raw_records)
209
+ end
210
+
211
+ def test_timestamp_milli
212
+ records = [
213
+ [{"field" => Time.parse("1960-01-01T02:09:30.123Z")}],
214
+ [nil],
215
+ [{"field" => nil}],
216
+ ]
217
+ target = build({
218
+ type: :timestamp,
219
+ unit: :milli,
220
+ },
221
+ records)
222
+ assert_equal(records, target.raw_records)
223
+ end
224
+
225
+ def test_timestamp_micro
226
+ records = [
227
+ [{"field" => Time.parse("1960-01-01T02:09:30.123456Z")}],
228
+ [nil],
229
+ [{"field" => nil}],
230
+ ]
231
+ target = build({
232
+ type: :timestamp,
233
+ unit: :micro,
234
+ },
235
+ records)
236
+ assert_equal(records, target.raw_records)
237
+ end
238
+
239
+ def test_timestamp_nano
240
+ records = [
241
+ [{"field" => Time.parse("1960-01-01T02:09:30.123456789Z")}],
242
+ [nil],
243
+ [{"field" => nil}],
244
+ ]
245
+ target = build({
246
+ type: :timestamp,
247
+ unit: :nano,
248
+ },
249
+ records)
250
+ assert_equal(records, target.raw_records)
251
+ end
252
+
253
+ def test_time32_second
254
+ records = [
255
+ [{"field" => 60 * 10}], # 00:10:00
256
+ [nil],
257
+ [{"field" => nil}],
258
+ ]
259
+ target = build({
260
+ type: :time32,
261
+ unit: :second,
262
+ },
263
+ records)
264
+ assert_equal(records, target.raw_records)
265
+ end
266
+
267
+ def test_time32_milli
268
+ records = [
269
+ [{"field" => (60 * 10) * 1000 + 123}], # 00:10:00.123
270
+ [nil],
271
+ [{"field" => nil}],
272
+ ]
273
+ target = build({
274
+ type: :time32,
275
+ unit: :milli,
276
+ },
277
+ records)
278
+ assert_equal(records, target.raw_records)
279
+ end
280
+
281
+ def test_time64_micro
282
+ records = [
283
+ [{"field" => (60 * 10) * 1_000_000 + 123_456}], # 00:10:00.123456
284
+ [nil],
285
+ [{"field" => nil}],
286
+ ]
287
+ target = build({
288
+ type: :time64,
289
+ unit: :micro,
290
+ },
291
+ records)
292
+ assert_equal(records, target.raw_records)
293
+ end
294
+
295
+ def test_time64_nano
296
+ records = [
297
+ # 00:10:00.123456789
298
+ [{"field" => (60 * 10) * 1_000_000_000 + 123_456_789}],
299
+ [nil],
300
+ [{"field" => nil}],
301
+ ]
302
+ target = build({
303
+ type: :time64,
304
+ unit: :nano,
305
+ },
306
+ records)
307
+ assert_equal(records, target.raw_records)
308
+ end
309
+
310
+ def test_decimal128
311
+ records = [
312
+ [{"field" => BigDecimal("92.92")}],
313
+ [nil],
314
+ [{"field" => nil}],
315
+ ]
316
+ target = build({
317
+ type: :decimal128,
318
+ precision: 8,
319
+ scale: 2,
320
+ },
321
+ records)
322
+ assert_equal(records, target.raw_records)
323
+ end
324
+
325
+ def test_list
326
+ records = [
327
+ [{"field" => [true, nil, false]}],
328
+ [nil],
329
+ [{"field" => nil}],
330
+ ]
331
+ target = build({
332
+ type: :list,
333
+ field: {
334
+ name: :sub_element,
335
+ type: :boolean,
336
+ },
337
+ },
338
+ records)
339
+ assert_equal(records, target.raw_records)
340
+ end
341
+
342
+ def test_struct
343
+ records = [
344
+ [{"field" => {"sub_field" => true}}],
345
+ [nil],
346
+ [{"field" => nil}],
347
+ [{"field" => {"sub_field" => nil}}],
348
+ ]
349
+ target = build({
350
+ type: :struct,
351
+ fields: [
352
+ {
353
+ name: :sub_field,
354
+ type: :boolean,
355
+ },
356
+ ],
357
+ },
358
+ records)
359
+ assert_equal(records, target.raw_records)
360
+ end
361
+
362
+ def test_sparse_union
363
+ omit("Need to add support for SparseUnionArrayBuilder")
364
+ records = [
365
+ [{"field" => {"field1" => true}}],
366
+ [nil],
367
+ [{"field" => nil}],
368
+ [{"field" => {"field2" => nil}}],
369
+ ]
370
+ target = build({
371
+ type: :sparse_union,
372
+ fields: [
373
+ {
374
+ name: :field1,
375
+ type: :boolean,
376
+ },
377
+ {
378
+ name: :field2,
379
+ type: :uint8,
380
+ },
381
+ ],
382
+ type_codes: [0, 1],
383
+ },
384
+ records)
385
+ assert_equal(records, target.raw_records)
386
+ end
387
+
388
+ def test_dense_union
389
+ omit("Need to add support for DenseUnionArrayBuilder")
390
+ records = [
391
+ [{"field" => {"field1" => true}}],
392
+ [nil],
393
+ [{"field" => nil}],
394
+ [{"field" => {"field2" => nil}}],
395
+ ]
396
+ target = build({
397
+ type: :dense_union,
398
+ fields: [
399
+ {
400
+ name: :field1,
401
+ type: :boolean,
402
+ },
403
+ {
404
+ name: :field2,
405
+ type: :uint8,
406
+ },
407
+ ],
408
+ type_codes: [0, 1],
409
+ },
410
+ records)
411
+ assert_equal(records, target.raw_records)
412
+ end
413
+
414
+ def test_dictionary
415
+ omit("Need to add support for DictionaryArrayBuilder")
416
+ records = [
417
+ [{"field" => "Ruby"}],
418
+ [nil],
419
+ [{"field" => nil}],
420
+ [{"field" => "GLib"}],
421
+ ]
422
+ dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
423
+ target = build({
424
+ type: :dictionary,
425
+ index_data_type: :int8,
426
+ dictionary: dictionary,
427
+ ordered: true,
428
+ },
429
+ records)
430
+ assert_equal(records, target.raw_records)
431
+ end
432
+ end
433
+
434
+ class RawRecordsRecordBatchStructArrayTest < Test::Unit::TestCase
435
+ include RawRecordsStructArrayTests
436
+
437
+ def build(type, records)
438
+ Arrow::RecordBatch.new(build_schema(type), records)
439
+ end
440
+ end
441
+
442
+ class RawRecordsTableStructArrayTest < Test::Unit::TestCase
443
+ include RawRecordsStructArrayTests
444
+
445
+ def build(type, records)
446
+ Arrow::Table.new(build_schema(type), records)
447
+ end
448
+ end
@@ -0,0 +1,47 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ class RawRecordsTableTest < Test::Unit::TestCase
19
+ test("2 arrays") do
20
+ raw_record_batches = [
21
+ [
22
+ [true, nil, "Ruby"],
23
+ [nil, 0, "GLib"],
24
+ [false, 2 ** 8 - 1, nil],
25
+ ],
26
+ [
27
+ [nil, 10, "A"],
28
+ [true, 20, "B"],
29
+ [false, nil, "C"],
30
+ [nil, 40, nil],
31
+ ]
32
+ ]
33
+ raw_records = raw_record_batches.inject do |all_records, record_batch|
34
+ all_records + record_batch
35
+ end
36
+ schema = [
37
+ {name: :column0, type: :boolean},
38
+ {name: :column1, type: :uint8},
39
+ {name: :column2, type: :string},
40
+ ]
41
+ record_batches = raw_record_batches.collect do |record_batch|
42
+ Arrow::RecordBatch.new(schema, record_batch)
43
+ end
44
+ table = Arrow::Table.new(schema, record_batches)
45
+ assert_equal(raw_records, table.raw_records)
46
+ end
47
+ end