red-arrow 11.0.0 → 12.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +3 -3
  3. data/ext/arrow/converters.hpp +12 -27
  4. data/lib/arrow/array-computable.rb +13 -0
  5. data/lib/arrow/data-type.rb +9 -0
  6. data/lib/arrow/dense-union-array-builder.rb +49 -0
  7. data/lib/arrow/dense-union-array.rb +26 -0
  8. data/lib/arrow/loader.rb +5 -0
  9. data/lib/arrow/record-batch-file-reader.rb +2 -0
  10. data/lib/arrow/record-batch-stream-reader.rb +2 -0
  11. data/lib/arrow/scalar.rb +67 -0
  12. data/lib/arrow/slicer.rb +61 -0
  13. data/lib/arrow/sparse-union-array-builder.rb +56 -0
  14. data/lib/arrow/sparse-union-array.rb +26 -0
  15. data/lib/arrow/struct-array-builder.rb +0 -5
  16. data/lib/arrow/table.rb +130 -10
  17. data/lib/arrow/union-array-builder.rb +59 -0
  18. data/lib/arrow/version.rb +1 -1
  19. data/test/raw-records/test-dense-union-array.rb +90 -45
  20. data/test/raw-records/test-list-array.rb +28 -10
  21. data/test/raw-records/test-map-array.rb +39 -10
  22. data/test/raw-records/test-sparse-union-array.rb +86 -41
  23. data/test/raw-records/test-struct-array.rb +22 -8
  24. data/test/test-array.rb +7 -0
  25. data/test/test-chunked-array.rb +9 -0
  26. data/test/test-dense-union-array.rb +42 -0
  27. data/test/test-dense-union-data-type.rb +1 -1
  28. data/test/test-function.rb +7 -7
  29. data/test/test-group.rb +58 -58
  30. data/test/test-record-batch-file-reader.rb +21 -0
  31. data/test/test-record-batch-stream-reader.rb +129 -0
  32. data/test/test-scalar.rb +65 -0
  33. data/test/test-slicer.rb +194 -129
  34. data/test/test-sparse-union-array.rb +38 -0
  35. data/test/test-table.rb +200 -38
  36. data/test/values/test-dense-union-array.rb +88 -45
  37. data/test/values/test-list-array.rb +26 -10
  38. data/test/values/test-map-array.rb +33 -10
  39. data/test/values/test-sparse-union-array.rb +84 -41
  40. data/test/values/test-struct-array.rb +20 -8
  41. metadata +20 -7
data/lib/arrow/table.rb CHANGED
@@ -316,8 +316,6 @@ module Arrow
316
316
  end
317
317
  end
318
318
 
319
- filter_options = Arrow::FilterOptions.new
320
- filter_options.null_selection_behavior = :emit_null
321
319
  sliced_tables = []
322
320
  slicers.each do |slicer|
323
321
  slicer = slicer.evaluate if slicer.respond_to?(:evaluate)
@@ -339,7 +337,7 @@ module Arrow
339
337
  to += n_rows if to < 0
340
338
  sliced_tables << slice_by_range(from, to)
341
339
  when ::Array, BooleanArray, ChunkedArray
342
- sliced_tables << filter(slicer, filter_options)
340
+ sliced_tables << filter(slicer)
343
341
  else
344
342
  message = "slicer must be Integer, Range, (from, to), " +
345
343
  "Arrow::ChunkedArray of Arrow::BooleanArray, " +
@@ -472,18 +470,22 @@ module Arrow
472
470
  #
473
471
  # If both of `left_outputs` and `right_outputs` aren't
474
472
  # specified, all columns in `self` and `right` are
475
- # outputted.
473
+ # output.
476
474
  # @param right_outputs [::Array<String, Symbol>] Output columns in
477
475
  # `right`.
478
476
  #
479
477
  # If both of `left_outputs` and `right_outputs` aren't
480
478
  # specified, all columns in `self` and `right` are
481
- # outputted.
479
+ # output.
482
480
  # @return [Arrow::Table]
483
481
  # The joined `Arrow::Table`.
484
482
  #
485
483
  # @overload join(right, type: :inner, left_outputs: nil, right_outputs: nil)
486
- # If key(s) are not supplied, common keys in self and right are used.
484
+ # If key(s) are not supplied, common keys in self and right are used
485
+ # (natural join).
486
+ #
487
+ # Column used as keys are merged and remain in left side
488
+ # when both of `left_outputs` and `right_outputs` are `nil`.
487
489
  #
488
490
  # @macro join_common_before
489
491
  # @macro join_common_after
@@ -493,13 +495,19 @@ module Arrow
493
495
  # @overload join(right, key, type: :inner, left_outputs: nil, right_outputs: nil)
494
496
  # Join right by a key.
495
497
  #
498
+ # Column used as keys are merged and remain in left side
499
+ # when both of `left_outputs` and `right_outputs` are `nil`.
500
+ #
496
501
  # @macro join_common_before
497
502
  # @param key [String, Symbol] A join key.
498
503
  # @macro join_common_after
499
504
  #
500
- # @overload join(right, keys, type: :inner, left_outputs: nil, right_outputs: nil)
505
+ # @overload join(right, keys, type: :inner, left_suffix: "", right_suffix: "",
506
+ # left_outputs: nil, right_outputs: nil)
501
507
  # Join right by keys.
502
508
  #
509
+ # Column name can be renamed by appending `left_suffix` or `right_suffix`.
510
+ #
503
511
  # @macro join_common_before
504
512
  # @param keys [::Array<String, Symbol>] Join keys.
505
513
  # @macro join_common_after
@@ -516,8 +524,16 @@ module Arrow
516
524
  # @macro join_common_after
517
525
  #
518
526
  # @since 7.0.0
519
- def join(right, keys=nil, type: :inner, left_outputs: nil, right_outputs: nil)
527
+ def join(right,
528
+ keys=nil,
529
+ type: :inner,
530
+ left_suffix: "",
531
+ right_suffix: "",
532
+ left_outputs: nil,
533
+ right_outputs: nil)
534
+ is_natural_join = keys.nil?
520
535
  keys ||= (column_names & right.column_names)
536
+ type = JoinType.try_convert(type) || type
521
537
  plan = ExecutePlan.new
522
538
  left_node = plan.build_source_node(self)
523
539
  right_node = plan.build_source_node(right)
@@ -533,21 +549,43 @@ module Arrow
533
549
  hash_join_node_options = HashJoinNodeOptions.new(type,
534
550
  left_keys,
535
551
  right_keys)
552
+ use_manual_outputs = false
536
553
  unless left_outputs.nil?
537
554
  hash_join_node_options.left_outputs = left_outputs
555
+ use_manual_outputs = true
538
556
  end
539
557
  unless right_outputs.nil?
540
558
  hash_join_node_options.right_outputs = right_outputs
559
+ use_manual_outputs = true
541
560
  end
542
561
  hash_join_node = plan.build_hash_join_node(left_node,
543
562
  right_node,
544
563
  hash_join_node_options)
564
+ type_nick = type.nick
565
+ is_filter_join = (type_nick.end_with?("-semi") or
566
+ type_nick.end_with?("-anti"))
567
+ if use_manual_outputs or is_filter_join
568
+ process_node = hash_join_node
569
+ elsif is_natural_join
570
+ process_node = join_merge_keys(plan, hash_join_node, right, keys)
571
+ elsif keys.is_a?(String) or keys.is_a?(Symbol)
572
+ process_node = join_merge_keys(plan, hash_join_node, right, [keys.to_s])
573
+ elsif !keys.is_a?(Hash) and (left_suffix != "" or right_suffix != "")
574
+ process_node = join_rename_keys(plan,
575
+ hash_join_node,
576
+ right,
577
+ keys,
578
+ left_suffix,
579
+ right_suffix)
580
+ else
581
+ process_node = hash_join_node
582
+ end
545
583
  sink_node_options = SinkNodeOptions.new
546
- plan.build_sink_node(hash_join_node, sink_node_options)
584
+ plan.build_sink_node(process_node, sink_node_options)
547
585
  plan.validate
548
586
  plan.start
549
587
  plan.wait
550
- reader = sink_node_options.get_reader(hash_join_node.output_schema)
588
+ reader = sink_node_options.get_reader(process_node.output_schema)
551
589
  table = reader.read_all
552
590
  share_input(table)
553
591
  table
@@ -620,5 +658,87 @@ module Arrow
620
658
  raise ArgumentError, message
621
659
  end
622
660
  end
661
+
662
+ def join_merge_keys(plan, input_node, right, keys)
663
+ expressions = []
664
+ names = []
665
+ normalized_keys = {}
666
+ keys.each do |key|
667
+ normalized_keys[key.to_s] = true
668
+ end
669
+ key_to_outputs = {}
670
+ outputs = []
671
+ left_n_column_names = column_names.size
672
+ column_names.each_with_index do |name, i|
673
+ is_key = normalized_keys.include?(name)
674
+ output = {is_key: is_key, name: name, index: i, direction: :left}
675
+ outputs << output
676
+ key_to_outputs[name] = {left: output} if is_key
677
+ end
678
+ right.column_names.each_with_index do |name, i|
679
+ index = left_n_column_names + i
680
+ is_key = normalized_keys.include?(name)
681
+ output = {is_key: is_key, name: name, index: index, direction: :right}
682
+ outputs << output
683
+ key_to_outputs[name][:right] = output if is_key
684
+ end
685
+
686
+ outputs.each do |output|
687
+ if output[:is_key]
688
+ next if output[:direction] == :right
689
+ left_output = key_to_outputs[output[:name]][:left]
690
+ right_output = key_to_outputs[output[:name]][:right]
691
+ left_field = FieldExpression.new("[#{left_output[:index]}]")
692
+ right_field = FieldExpression.new("[#{right_output[:index]}]")
693
+ is_left_null = CallExpression.new("is_null", [left_field])
694
+ merge_column = CallExpression.new("if_else",
695
+ [
696
+ is_left_null,
697
+ right_field,
698
+ left_field,
699
+ ])
700
+ expressions << merge_column
701
+ else
702
+ expressions << FieldExpression.new("[#{output[:index]}]")
703
+ end
704
+ names << output[:name]
705
+ end
706
+ project_node_options = ProjectNodeOptions.new(expressions, names)
707
+ plan.build_project_node(input_node, project_node_options)
708
+ end
709
+
710
+ def join_rename_keys(plan,
711
+ input_node,
712
+ right,
713
+ keys,
714
+ left_suffix,
715
+ right_suffix)
716
+ expressions = []
717
+ names = []
718
+ normalized_keys = {}
719
+ keys.each do |key|
720
+ normalized_keys[key.to_s] = true
721
+ end
722
+ left_n_column_names = column_names.size
723
+ column_names.each_with_index do |name, i|
724
+ expressions << FieldExpression.new("[#{i}]")
725
+ if normalized_keys.include?(name)
726
+ names << "#{name}#{left_suffix}"
727
+ else
728
+ names << name
729
+ end
730
+ end
731
+ right.column_names.each_with_index do |name, i|
732
+ index = left_n_column_names + i
733
+ expressions << FieldExpression.new("[#{index}]")
734
+ if normalized_keys.include?(name)
735
+ names << "#{name}#{right_suffix}"
736
+ else
737
+ names << name
738
+ end
739
+ end
740
+ project_node_options = ProjectNodeOptions.new(expressions, names)
741
+ plan.build_project_node(input_node, project_node_options)
742
+ end
623
743
  end
624
744
  end
@@ -0,0 +1,59 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class UnionArrayBuilder
20
+ def append_values(values, is_valids=nil)
21
+ if is_valids
22
+ is_valids.each_with_index do |is_valid, i|
23
+ if is_valid
24
+ append_value(values[i])
25
+ else
26
+ append_null
27
+ end
28
+ end
29
+ else
30
+ values.each do |value|
31
+ append_value(value)
32
+ end
33
+ end
34
+ end
35
+
36
+ alias_method :append_child_raw, :append_child
37
+ def append_child(builder, filed_name=nil)
38
+ @child_infos = nil
39
+ append_child_raw(builder, field_name)
40
+ end
41
+
42
+ private
43
+ def child_infos
44
+ @child_infos ||= create_child_infos
45
+ end
46
+
47
+ def create_child_infos
48
+ infos = {}
49
+ type = value_data_type
50
+ type.fields.zip(children, type.type_codes).each do |field, child, id|
51
+ infos[field.name] = {
52
+ builder: child,
53
+ id: id,
54
+ }
55
+ end
56
+ infos
57
+ end
58
+ end
59
+ end
data/lib/arrow/version.rb CHANGED
@@ -16,7 +16,7 @@
16
16
  # under the License.
17
17
 
18
18
  module Arrow
19
- VERSION = "11.0.0"
19
+ VERSION = "12.0.0"
20
20
 
21
21
  module Version
22
22
  numbers, TAG = VERSION.split("-")
@@ -76,12 +76,25 @@ module RawRecordsDenseUnionArrayTests
76
76
  [union_array])
77
77
  end
78
78
 
79
+ def remove_field_names(records)
80
+ records.collect do |record|
81
+ record.collect do |column|
82
+ if column.nil?
83
+ column
84
+ else
85
+ column.values[0]
86
+ end
87
+ end
88
+ end
89
+ end
90
+
79
91
  def test_null
80
92
  records = [
81
93
  [{"0" => nil}],
82
94
  ]
83
95
  target = build(:null, records)
84
- assert_equal(records, target.raw_records)
96
+ assert_equal(remove_field_names(records),
97
+ target.raw_records)
85
98
  end
86
99
 
87
100
  def test_boolean
@@ -90,7 +103,8 @@ module RawRecordsDenseUnionArrayTests
90
103
  [{"1" => nil}],
91
104
  ]
92
105
  target = build(:boolean, records)
93
- assert_equal(records, target.raw_records)
106
+ assert_equal(remove_field_names(records),
107
+ target.raw_records)
94
108
  end
95
109
 
96
110
  def test_int8
@@ -99,7 +113,8 @@ module RawRecordsDenseUnionArrayTests
99
113
  [{"1" => nil}],
100
114
  ]
101
115
  target = build(:int8, records)
102
- assert_equal(records, target.raw_records)
116
+ assert_equal(remove_field_names(records),
117
+ target.raw_records)
103
118
  end
104
119
 
105
120
  def test_uint8
@@ -108,7 +123,8 @@ module RawRecordsDenseUnionArrayTests
108
123
  [{"1" => nil}],
109
124
  ]
110
125
  target = build(:uint8, records)
111
- assert_equal(records, target.raw_records)
126
+ assert_equal(remove_field_names(records),
127
+ target.raw_records)
112
128
  end
113
129
 
114
130
  def test_int16
@@ -117,7 +133,8 @@ module RawRecordsDenseUnionArrayTests
117
133
  [{"1" => nil}],
118
134
  ]
119
135
  target = build(:int16, records)
120
- assert_equal(records, target.raw_records)
136
+ assert_equal(remove_field_names(records),
137
+ target.raw_records)
121
138
  end
122
139
 
123
140
  def test_uint16
@@ -126,7 +143,8 @@ module RawRecordsDenseUnionArrayTests
126
143
  [{"1" => nil}],
127
144
  ]
128
145
  target = build(:uint16, records)
129
- assert_equal(records, target.raw_records)
146
+ assert_equal(remove_field_names(records),
147
+ target.raw_records)
130
148
  end
131
149
 
132
150
  def test_int32
@@ -135,7 +153,8 @@ module RawRecordsDenseUnionArrayTests
135
153
  [{"1" => nil}],
136
154
  ]
137
155
  target = build(:int32, records)
138
- assert_equal(records, target.raw_records)
156
+ assert_equal(remove_field_names(records),
157
+ target.raw_records)
139
158
  end
140
159
 
141
160
  def test_uint32
@@ -144,7 +163,8 @@ module RawRecordsDenseUnionArrayTests
144
163
  [{"1" => nil}],
145
164
  ]
146
165
  target = build(:uint32, records)
147
- assert_equal(records, target.raw_records)
166
+ assert_equal(remove_field_names(records),
167
+ target.raw_records)
148
168
  end
149
169
 
150
170
  def test_int64
@@ -153,7 +173,8 @@ module RawRecordsDenseUnionArrayTests
153
173
  [{"1" => nil}],
154
174
  ]
155
175
  target = build(:int64, records)
156
- assert_equal(records, target.raw_records)
176
+ assert_equal(remove_field_names(records),
177
+ target.raw_records)
157
178
  end
158
179
 
159
180
  def test_uint64
@@ -162,7 +183,8 @@ module RawRecordsDenseUnionArrayTests
162
183
  [{"1" => nil}],
163
184
  ]
164
185
  target = build(:uint64, records)
165
- assert_equal(records, target.raw_records)
186
+ assert_equal(remove_field_names(records),
187
+ target.raw_records)
166
188
  end
167
189
 
168
190
  def test_float
@@ -171,7 +193,8 @@ module RawRecordsDenseUnionArrayTests
171
193
  [{"1" => nil}],
172
194
  ]
173
195
  target = build(:float, records)
174
- assert_equal(records, target.raw_records)
196
+ assert_equal(remove_field_names(records),
197
+ target.raw_records)
175
198
  end
176
199
 
177
200
  def test_double
@@ -180,7 +203,8 @@ module RawRecordsDenseUnionArrayTests
180
203
  [{"1" => nil}],
181
204
  ]
182
205
  target = build(:double, records)
183
- assert_equal(records, target.raw_records)
206
+ assert_equal(remove_field_names(records),
207
+ target.raw_records)
184
208
  end
185
209
 
186
210
  def test_binary
@@ -189,7 +213,8 @@ module RawRecordsDenseUnionArrayTests
189
213
  [{"1" => nil}],
190
214
  ]
191
215
  target = build(:binary, records)
192
- assert_equal(records, target.raw_records)
216
+ assert_equal(remove_field_names(records),
217
+ target.raw_records)
193
218
  end
194
219
 
195
220
  def test_string
@@ -198,7 +223,8 @@ module RawRecordsDenseUnionArrayTests
198
223
  [{"1" => nil}],
199
224
  ]
200
225
  target = build(:string, records)
201
- assert_equal(records, target.raw_records)
226
+ assert_equal(remove_field_names(records),
227
+ target.raw_records)
202
228
  end
203
229
 
204
230
  def test_date32
@@ -207,7 +233,8 @@ module RawRecordsDenseUnionArrayTests
207
233
  [{"1" => nil}],
208
234
  ]
209
235
  target = build(:date32, records)
210
- assert_equal(records, target.raw_records)
236
+ assert_equal(remove_field_names(records),
237
+ target.raw_records)
211
238
  end
212
239
 
213
240
  def test_date64
@@ -216,7 +243,8 @@ module RawRecordsDenseUnionArrayTests
216
243
  [{"1" => nil}],
217
244
  ]
218
245
  target = build(:date64, records)
219
- assert_equal(records, target.raw_records)
246
+ assert_equal(remove_field_names(records),
247
+ target.raw_records)
220
248
  end
221
249
 
222
250
  def test_timestamp_second
@@ -229,7 +257,8 @@ module RawRecordsDenseUnionArrayTests
229
257
  unit: :second,
230
258
  },
231
259
  records)
232
- assert_equal(records, target.raw_records)
260
+ assert_equal(remove_field_names(records),
261
+ target.raw_records)
233
262
  end
234
263
 
235
264
  def test_timestamp_milli
@@ -242,7 +271,8 @@ module RawRecordsDenseUnionArrayTests
242
271
  unit: :milli,
243
272
  },
244
273
  records)
245
- assert_equal(records, target.raw_records)
274
+ assert_equal(remove_field_names(records),
275
+ target.raw_records)
246
276
  end
247
277
 
248
278
  def test_timestamp_micro
@@ -255,7 +285,8 @@ module RawRecordsDenseUnionArrayTests
255
285
  unit: :micro,
256
286
  },
257
287
  records)
258
- assert_equal(records, target.raw_records)
288
+ assert_equal(remove_field_names(records),
289
+ target.raw_records)
259
290
  end
260
291
 
261
292
  def test_timestamp_nano
@@ -268,7 +299,8 @@ module RawRecordsDenseUnionArrayTests
268
299
  unit: :nano,
269
300
  },
270
301
  records)
271
- assert_equal(records, target.raw_records)
302
+ assert_equal(remove_field_names(records),
303
+ target.raw_records)
272
304
  end
273
305
 
274
306
  def test_time32_second
@@ -283,7 +315,8 @@ module RawRecordsDenseUnionArrayTests
283
315
  unit: :second,
284
316
  },
285
317
  records)
286
- assert_equal(records, target.raw_records)
318
+ assert_equal(remove_field_names(records),
319
+ target.raw_records)
287
320
  end
288
321
 
289
322
  def test_time32_milli
@@ -298,7 +331,8 @@ module RawRecordsDenseUnionArrayTests
298
331
  unit: :milli,
299
332
  },
300
333
  records)
301
- assert_equal(records, target.raw_records)
334
+ assert_equal(remove_field_names(records),
335
+ target.raw_records)
302
336
  end
303
337
 
304
338
  def test_time64_micro
@@ -313,7 +347,8 @@ module RawRecordsDenseUnionArrayTests
313
347
  unit: :micro,
314
348
  },
315
349
  records)
316
- assert_equal(records, target.raw_records)
350
+ assert_equal(remove_field_names(records),
351
+ target.raw_records)
317
352
  end
318
353
 
319
354
  def test_time64_nano
@@ -328,7 +363,8 @@ module RawRecordsDenseUnionArrayTests
328
363
  unit: :nano,
329
364
  },
330
365
  records)
331
- assert_equal(records, target.raw_records)
366
+ assert_equal(remove_field_names(records),
367
+ target.raw_records)
332
368
  end
333
369
 
334
370
  def test_decimal128
@@ -342,7 +378,8 @@ module RawRecordsDenseUnionArrayTests
342
378
  scale: 2,
343
379
  },
344
380
  records)
345
- assert_equal(records, target.raw_records)
381
+ assert_equal(remove_field_names(records),
382
+ target.raw_records)
346
383
  end
347
384
 
348
385
  def test_decimal256
@@ -356,7 +393,8 @@ module RawRecordsDenseUnionArrayTests
356
393
  scale: 2,
357
394
  },
358
395
  records)
359
- assert_equal(records, target.raw_records)
396
+ assert_equal(remove_field_names(records),
397
+ target.raw_records)
360
398
  end
361
399
 
362
400
  def test_month_interval
@@ -365,7 +403,8 @@ module RawRecordsDenseUnionArrayTests
365
403
  [{"1" => nil}],
366
404
  ]
367
405
  target = build(:month_interval, records)
368
- assert_equal(records, target.raw_records)
406
+ assert_equal(remove_field_names(records),
407
+ target.raw_records)
369
408
  end
370
409
 
371
410
  def test_day_time_interval
@@ -374,7 +413,8 @@ module RawRecordsDenseUnionArrayTests
374
413
  [{"1" => nil}],
375
414
  ]
376
415
  target = build(:day_time_interval, records)
377
- assert_equal(records, target.raw_records)
416
+ assert_equal(remove_field_names(records),
417
+ target.raw_records)
378
418
  end
379
419
 
380
420
  def test_month_day_nano_interval
@@ -383,7 +423,8 @@ module RawRecordsDenseUnionArrayTests
383
423
  [{"1" => nil}],
384
424
  ]
385
425
  target = build(:month_day_nano_interval, records)
386
- assert_equal(records, target.raw_records)
426
+ assert_equal(remove_field_names(records),
427
+ target.raw_records)
387
428
  end
388
429
 
389
430
  def test_list
@@ -399,7 +440,8 @@ module RawRecordsDenseUnionArrayTests
399
440
  },
400
441
  },
401
442
  records)
402
- assert_equal(records, target.raw_records)
443
+ assert_equal(remove_field_names(records),
444
+ target.raw_records)
403
445
  end
404
446
 
405
447
  def test_struct
@@ -418,7 +460,8 @@ module RawRecordsDenseUnionArrayTests
418
460
  ],
419
461
  },
420
462
  records)
421
- assert_equal(records, target.raw_records)
463
+ assert_equal(remove_field_names(records),
464
+ target.raw_records)
422
465
  end
423
466
 
424
467
  def test_map
@@ -432,14 +475,15 @@ module RawRecordsDenseUnionArrayTests
432
475
  item: :boolean,
433
476
  },
434
477
  records)
435
- assert_equal(records, target.raw_records)
478
+ assert_equal(remove_field_names(records),
479
+ target.raw_records)
436
480
  end
437
481
 
438
482
  def test_sparse_union
439
- omit("Need to add support for SparseUnionArrayBuilder")
440
483
  records = [
441
484
  [{"0" => {"field1" => true}}],
442
485
  [{"1" => nil}],
486
+ [{"0" => {"field2" => 29}}],
443
487
  [{"0" => {"field2" => nil}}],
444
488
  ]
445
489
  target = build({
@@ -457,14 +501,15 @@ module RawRecordsDenseUnionArrayTests
457
501
  type_codes: [0, 1],
458
502
  },
459
503
  records)
460
- assert_equal(records, target.raw_records)
504
+ assert_equal(remove_field_names(remove_field_names(records)),
505
+ target.raw_records)
461
506
  end
462
507
 
463
508
  def test_dense_union
464
- omit("Need to add support for DenseUnionArrayBuilder")
465
509
  records = [
466
510
  [{"0" => {"field1" => true}}],
467
511
  [{"1" => nil}],
512
+ [{"0" => {"field2" => 29}}],
468
513
  [{"0" => {"field2" => nil}}],
469
514
  ]
470
515
  target = build({
@@ -482,25 +527,25 @@ module RawRecordsDenseUnionArrayTests
482
527
  type_codes: [0, 1],
483
528
  },
484
529
  records)
485
- assert_equal(records, target.raw_records)
530
+ assert_equal(remove_field_names(remove_field_names(records)),
531
+ target.raw_records)
486
532
  end
487
533
 
488
534
  def test_dictionary
489
- omit("Need to add support for DictionaryArrayBuilder")
490
535
  records = [
491
536
  [{"0" => "Ruby"}],
492
537
  [{"1" => nil}],
493
538
  [{"0" => "GLib"}],
494
539
  ]
495
- dictionary = Arrow::StringArray.new(["GLib", "Ruby"])
496
540
  target = build({
497
- type: :dictionary,
498
- index_data_type: :int8,
499
- dictionary: dictionary,
500
- ordered: true,
501
- },
502
- records)
503
- assert_equal(records, target.raw_records)
541
+ type: :dictionary,
542
+ index_data_type: :int8,
543
+ value_data_type: :string,
544
+ ordered: false,
545
+ },
546
+ records)
547
+ assert_equal(remove_field_names(records),
548
+ target.raw_records)
504
549
  end
505
550
  end
506
551