red-arrow 0.13.0 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of red-arrow might be problematic. Click here for more details.

@@ -0,0 +1,520 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module RawRecordsListArrayTests
19
+ def build_schema(type)
20
+ field_description = {
21
+ name: :element,
22
+ }
23
+ if type.is_a?(Hash)
24
+ field_description = field_description.merge(type)
25
+ else
26
+ field_description[:type] = type
27
+ end
28
+ {
29
+ column: {
30
+ type: :list,
31
+ field: field_description,
32
+ },
33
+ }
34
+ end
35
+
36
+ def test_null
37
+ records = [
38
+ [[nil, nil, nil]],
39
+ [nil],
40
+ ]
41
+ target = build(:null, records)
42
+ assert_equal(records, target.raw_records)
43
+ end
44
+
45
+ def test_boolean
46
+ records = [
47
+ [[true, nil, false]],
48
+ [nil],
49
+ ]
50
+ target = build(:boolean, records)
51
+ assert_equal(records, target.raw_records)
52
+ end
53
+
54
+ def test_int8
55
+ records = [
56
+ [[-(2 ** 7), nil, (2 ** 7) - 1]],
57
+ [nil],
58
+ ]
59
+ target = build(:int8, records)
60
+ assert_equal(records, target.raw_records)
61
+ end
62
+
63
+ def test_uint8
64
+ records = [
65
+ [[0, nil, (2 ** 8) - 1]],
66
+ [nil],
67
+ ]
68
+ target = build(:uint8, records)
69
+ assert_equal(records, target.raw_records)
70
+ end
71
+
72
+ def test_int16
73
+ records = [
74
+ [[-(2 ** 15), nil, (2 ** 15) - 1]],
75
+ [nil],
76
+ ]
77
+ target = build(:int16, records)
78
+ assert_equal(records, target.raw_records)
79
+ end
80
+
81
+ def test_uint16
82
+ records = [
83
+ [[0, nil, (2 ** 16) - 1]],
84
+ [nil],
85
+ ]
86
+ target = build(:uint16, records)
87
+ assert_equal(records, target.raw_records)
88
+ end
89
+
90
+ def test_int32
91
+ records = [
92
+ [[-(2 ** 31), nil, (2 ** 31) - 1]],
93
+ [nil],
94
+ ]
95
+ target = build(:int32, records)
96
+ assert_equal(records, target.raw_records)
97
+ end
98
+
99
+ def test_uint32
100
+ records = [
101
+ [[0, nil, (2 ** 32) - 1]],
102
+ [nil],
103
+ ]
104
+ target = build(:uint32, records)
105
+ assert_equal(records, target.raw_records)
106
+ end
107
+
108
+ def test_int64
109
+ records = [
110
+ [[-(2 ** 63), nil, (2 ** 63) - 1]],
111
+ [nil],
112
+ ]
113
+ target = build(:int64, records)
114
+ assert_equal(records, target.raw_records)
115
+ end
116
+
117
+ def test_uint64
118
+ records = [
119
+ [[0, nil, (2 ** 64) - 1]],
120
+ [nil],
121
+ ]
122
+ target = build(:uint64, records)
123
+ assert_equal(records, target.raw_records)
124
+ end
125
+
126
+ def test_float
127
+ records = [
128
+ [[-1.0, nil, 1.0]],
129
+ [nil],
130
+ ]
131
+ target = build(:float, records)
132
+ assert_equal(records, target.raw_records)
133
+ end
134
+
135
+ def test_double
136
+ records = [
137
+ [[-1.0, nil, 1.0]],
138
+ [nil],
139
+ ]
140
+ target = build(:double, records)
141
+ assert_equal(records, target.raw_records)
142
+ end
143
+
144
+ def test_binary
145
+ records = [
146
+ [["\x00".b, nil, "\xff".b]],
147
+ [nil],
148
+ ]
149
+ target = build(:binary, records)
150
+ assert_equal(records, target.raw_records)
151
+ end
152
+
153
+ def test_string
154
+ records = [
155
+ [
156
+ [
157
+ "Ruby",
158
+ nil,
159
+ "\u3042", # U+3042 HIRAGANA LETTER A
160
+ ],
161
+ ],
162
+ [nil],
163
+ ]
164
+ target = build(:string, records)
165
+ assert_equal(records, target.raw_records)
166
+ end
167
+
168
+ def test_date32
169
+ records = [
170
+ [
171
+ [
172
+ Date.new(1960, 1, 1),
173
+ nil,
174
+ Date.new(2017, 8, 23),
175
+ ],
176
+ ],
177
+ [nil],
178
+ ]
179
+ target = build(:date32, records)
180
+ assert_equal(records, target.raw_records)
181
+ end
182
+
183
+ def test_date64
184
+ records = [
185
+ [
186
+ [
187
+ DateTime.new(1960, 1, 1, 2, 9, 30),
188
+ nil,
189
+ DateTime.new(2017, 8, 23, 14, 57, 2),
190
+ ],
191
+ ],
192
+ [nil],
193
+ ]
194
+ target = build(:date64, records)
195
+ assert_equal(records, target.raw_records)
196
+ end
197
+
198
+ def test_timestamp_second
199
+ records = [
200
+ [
201
+ [
202
+ Time.parse("1960-01-01T02:09:30Z"),
203
+ nil,
204
+ Time.parse("2017-08-23T14:57:02Z"),
205
+ ],
206
+ ],
207
+ [nil],
208
+ ]
209
+ target = build({
210
+ type: :timestamp,
211
+ unit: :second,
212
+ },
213
+ records)
214
+ assert_equal(records, target.raw_records)
215
+ end
216
+
217
+ def test_timestamp_milli
218
+ records = [
219
+ [
220
+ [
221
+ Time.parse("1960-01-01T02:09:30.123Z"),
222
+ nil,
223
+ Time.parse("2017-08-23T14:57:02.987Z"),
224
+ ],
225
+ ],
226
+ [nil],
227
+ ]
228
+ target = build({
229
+ type: :timestamp,
230
+ unit: :milli,
231
+ },
232
+ records)
233
+ assert_equal(records, target.raw_records)
234
+ end
235
+
236
+ def test_timestamp_micro
237
+ records = [
238
+ [
239
+ [
240
+ Time.parse("1960-01-01T02:09:30.123456Z"),
241
+ nil,
242
+ Time.parse("2017-08-23T14:57:02.987654Z"),
243
+ ],
244
+ ],
245
+ [nil],
246
+ ]
247
+ target = build({
248
+ type: :timestamp,
249
+ unit: :micro,
250
+ },
251
+ records)
252
+ assert_equal(records, target.raw_records)
253
+ end
254
+
255
+ def test_timestamp_nano
256
+ records = [
257
+ [
258
+ [
259
+ Time.parse("1960-01-01T02:09:30.123456789Z"),
260
+ nil,
261
+ Time.parse("2017-08-23T14:57:02.987654321Z"),
262
+ ],
263
+ ],
264
+ [nil],
265
+ ]
266
+ target = build({
267
+ type: :timestamp,
268
+ unit: :nano,
269
+ },
270
+ records)
271
+ assert_equal(records, target.raw_records)
272
+ end
273
+
274
+ def test_time32_test
275
+ records = [
276
+ [
277
+ [
278
+ 60 * 10, # 00:10:00
279
+ nil,
280
+ 60 * 60 * 2 + 9, # 02:00:09
281
+ ],
282
+ ],
283
+ [nil],
284
+ ]
285
+ target = build({
286
+ type: :time32,
287
+ unit: :second,
288
+ },
289
+ records)
290
+ assert_equal(records, target.raw_records)
291
+ end
292
+
293
+ def test_time32_milli
294
+ records = [
295
+ [
296
+ [
297
+ (60 * 10) * 1000 + 123, # 00:10:00.123
298
+ nil,
299
+ (60 * 60 * 2 + 9) * 1000 + 987, # 02:00:09.987
300
+ ],
301
+ ],
302
+ [nil],
303
+ ]
304
+ target = build({
305
+ type: :time32,
306
+ unit: :milli,
307
+ },
308
+ records)
309
+ assert_equal(records, target.raw_records)
310
+ end
311
+
312
+ def test_time64_micro
313
+ records = [
314
+ [
315
+ [
316
+ (60 * 10) * 1_000_000 + 123_456, # 00:10:00.123456
317
+ nil,
318
+ (60 * 60 * 2 + 9) * 1_000_000 + 987_654, # 02:00:09.987654
319
+ ],
320
+ ],
321
+ [nil],
322
+ ]
323
+ target = build({
324
+ type: :time64,
325
+ unit: :micro,
326
+ },
327
+ records)
328
+ assert_equal(records, target.raw_records)
329
+ end
330
+
331
+ def test_time64_nano
332
+ records = [
333
+ [
334
+ [
335
+ (60 * 10) * 1_000_000_000 + 123_456_789, # 00:10:00.123456789
336
+ nil,
337
+ (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321, # 02:00:09.987654321
338
+ ],
339
+ ],
340
+ [nil],
341
+ ]
342
+ target = build({
343
+ type: :time64,
344
+ unit: :nano,
345
+ },
346
+ records)
347
+ assert_equal(records, target.raw_records)
348
+ end
349
+
350
+ def test_decimal128
351
+ records = [
352
+ [
353
+ [
354
+ BigDecimal("92.92"),
355
+ nil,
356
+ BigDecimal("29.29"),
357
+ ],
358
+ ],
359
+ [nil],
360
+ ]
361
+ target = build({
362
+ type: :decimal128,
363
+ precision: 8,
364
+ scale: 2,
365
+ },
366
+ records)
367
+ assert_equal(records, target.raw_records)
368
+ end
369
+
370
+ def test_list
371
+ records = [
372
+ [
373
+ [
374
+ [
375
+ true,
376
+ nil,
377
+ ],
378
+ nil,
379
+ [
380
+ nil,
381
+ false,
382
+ ],
383
+ ],
384
+ ],
385
+ [nil],
386
+ ]
387
+ target = build({
388
+ type: :list,
389
+ field: {
390
+ name: :sub_element,
391
+ type: :boolean,
392
+ },
393
+ },
394
+ records)
395
+ assert_equal(records, target.raw_records)
396
+ end
397
+
398
+ def test_struct
399
+ records = [
400
+ [
401
+ [
402
+ {"field" => true},
403
+ nil,
404
+ {"field" => nil},
405
+ ],
406
+ ],
407
+ [nil],
408
+ ]
409
+ target = build({
410
+ type: :struct,
411
+ fields: [
412
+ {
413
+ name: :field,
414
+ type: :boolean,
415
+ },
416
+ ],
417
+ },
418
+ records)
419
+ assert_equal(records, target.raw_records)
420
+ end
421
+
422
+ def test_sparse
423
+ omit("Need to add support for SparseUnionArrayBuilder")
424
+ records = [
425
+ [
426
+ [
427
+ {"field1" => true},
428
+ nil,
429
+ {"field2" => nil},
430
+ ],
431
+ ],
432
+ [nil],
433
+ ]
434
+ target = build({
435
+ type: :sparse_union,
436
+ fields: [
437
+ {
438
+ name: :field1,
439
+ type: :boolean,
440
+ },
441
+ {
442
+ name: :field2,
443
+ type: :uint8,
444
+ },
445
+ ],
446
+ type_codes: [0, 1],
447
+ },
448
+ records)
449
+ assert_equal(records, target.raw_records)
450
+ end
451
+
452
+ def test_dense
453
+ omit("Need to add support for DenseUnionArrayBuilder")
454
+ records = [
455
+ [
456
+ [
457
+ {"field1" => true},
458
+ nil,
459
+ {"field2" => nil},
460
+ ],
461
+ ],
462
+ [nil],
463
+ ]
464
+ target = build({
465
+ type: :dense_union,
466
+ fields: [
467
+ {
468
+ name: :field1,
469
+ type: :boolean,
470
+ },
471
+ {
472
+ name: :field2,
473
+ type: :uint8,
474
+ },
475
+ ],
476
+ type_codes: [0, 1],
477
+ },
478
+ records)
479
+ assert_equal(records, target.raw_records)
480
+ end
481
+
482
+ def test_dictionary
483
+ omit("Need to add support for DictionaryArrayBuilder")
484
+ records = [
485
+ [
486
+ [
487
+ "Ruby",
488
+ nil,
489
+ "GLib",
490
+ ],
491
+ ],
492
+ [nil],
493
+ ]
494
+ dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
495
+ target = build({
496
+ type: :dictionary,
497
+ index_data_type: :int8,
498
+ dictionary: dictionary,
499
+ ordered: true,
500
+ },
501
+ records)
502
+ assert_equal(records, target.raw_records)
503
+ end
504
+ end
505
+
506
+ class RawRecordsRecordBatchListArrayTest < Test::Unit::TestCase
507
+ include RawRecordsListArrayTests
508
+
509
+ def build(type, records)
510
+ Arrow::RecordBatch.new(build_schema(type), records)
511
+ end
512
+ end
513
+
514
+ class RawRecordsTableListArrayTest < Test::Unit::TestCase
515
+ include RawRecordsListArrayTests
516
+
517
+ def build(type, records)
518
+ Arrow::Table.new(build_schema(type), records)
519
+ end
520
+ end