red-arrow 8.0.0 → 24.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +15 -7
  3. data/ext/arrow/arrow.cpp +67 -0
  4. data/ext/arrow/converters.cpp +10 -0
  5. data/ext/arrow/converters.hpp +310 -46
  6. data/ext/arrow/extconf.rb +41 -22
  7. data/ext/arrow/raw-records.cpp +165 -2
  8. data/ext/arrow/red-arrow.hpp +2 -0
  9. data/ext/arrow/values.cpp +6 -2
  10. data/lib/arrow/array-builder.rb +89 -14
  11. data/{test/test-time32-data-type.rb → lib/arrow/array-computable.rb} +24 -16
  12. data/{test/test-buffer.rb → lib/arrow/array-statistics.rb} +19 -24
  13. data/lib/arrow/array.rb +40 -4
  14. data/lib/arrow/chunked-array.rb +56 -1
  15. data/lib/arrow/column-containable.rb +9 -0
  16. data/lib/arrow/column.rb +49 -4
  17. data/{test/test-tensor.rb → lib/arrow/csv-write-options.rb} +28 -31
  18. data/lib/arrow/data-type.rb +17 -3
  19. data/lib/arrow/decimal128-array-builder.rb +16 -6
  20. data/lib/arrow/decimal128.rb +14 -0
  21. data/lib/arrow/decimal256-array-builder.rb +16 -6
  22. data/lib/arrow/decimal256.rb +14 -0
  23. data/{test/test-float-scalar.rb → lib/arrow/dense-union-array-builder.rb} +27 -24
  24. data/{test/test-boolean-scalar.rb → lib/arrow/dense-union-array.rb} +7 -7
  25. data/lib/arrow/duration-array-builder.rb +27 -0
  26. data/lib/arrow/duration-array.rb +24 -0
  27. data/lib/arrow/duration-data-type.rb +32 -0
  28. data/lib/arrow/expression.rb +6 -2
  29. data/lib/arrow/field-containable.rb +1 -1
  30. data/lib/arrow/field.rb +44 -3
  31. data/lib/arrow/fixed-size-list-array-builder.rb +29 -0
  32. data/lib/arrow/fixed-size-list-data-type.rb +118 -0
  33. data/lib/arrow/function.rb +0 -1
  34. data/lib/arrow/half-float-array-builder.rb +32 -0
  35. data/lib/arrow/half-float-array.rb +24 -0
  36. data/lib/arrow/half-float.rb +118 -0
  37. data/{test/helper/fixture.rb → lib/arrow/input-referable.rb} +7 -6
  38. data/lib/arrow/jruby/array-builder.rb +114 -0
  39. data/lib/arrow/jruby/array.rb +109 -0
  40. data/lib/arrow/jruby/chunked-array.rb +36 -0
  41. data/lib/arrow/jruby/compression-type.rb +26 -0
  42. data/lib/arrow/jruby/csv-read-options.rb +32 -0
  43. data/{test/test-map-data-type.rb → lib/arrow/jruby/data-type.rb} +24 -12
  44. data/lib/arrow/jruby/decimal128.rb +28 -0
  45. data/lib/arrow/jruby/decimal256.rb +28 -0
  46. data/{test/fixture/float-integer.csv → lib/arrow/jruby/error.rb} +7 -4
  47. data/lib/arrow/jruby/file-system.rb +24 -0
  48. data/{test/test-null-array.rb → lib/arrow/jruby/function.rb} +5 -4
  49. data/lib/arrow/jruby/record-batch-iterator.rb +24 -0
  50. data/{test/fixture/null-with-double-quote.csv → lib/arrow/jruby/record-batch.rb} +8 -4
  51. data/{test/fixture/integer-float.csv → lib/arrow/jruby/sort-key.rb} +8 -4
  52. data/lib/arrow/jruby/sort-options.rb +24 -0
  53. data/lib/arrow/jruby/stream-listener-raw.rb +25 -0
  54. data/{test/test-rolling-window.rb → lib/arrow/jruby/table.rb} +19 -19
  55. data/lib/arrow/jruby/writable.rb +24 -0
  56. data/lib/arrow/jruby.rb +52 -0
  57. data/{test/test-date32-array.rb → lib/arrow/large-list-array-builder.rb} +10 -5
  58. data/lib/arrow/large-list-data-type.rb +83 -0
  59. data/lib/arrow/libraries.rb +140 -0
  60. data/lib/arrow/list-array-builder.rb +1 -68
  61. data/lib/arrow/list-data-type.rb +3 -38
  62. data/{test/test-dictionary-array.rb → lib/arrow/list-field-resolvable.rb} +26 -17
  63. data/lib/arrow/list-slice-options.rb +76 -0
  64. data/lib/arrow/list-values-appendable.rb +88 -0
  65. data/lib/arrow/loader.rb +15 -96
  66. data/{test/test-decimal128-array.rb → lib/arrow/make-struct-options.rb} +18 -18
  67. data/lib/arrow/raw-table-converter.rb +10 -3
  68. data/lib/arrow/raw-tensor-converter.rb +89 -0
  69. data/lib/arrow/record-batch-file-reader.rb +2 -0
  70. data/lib/arrow/record-batch-stream-reader.rb +2 -0
  71. data/lib/arrow/record-batch.rb +6 -2
  72. data/{test/fixture/null-without-double-quote.csv → lib/arrow/ruby.rb} +5 -4
  73. data/lib/arrow/scalar.rb +67 -0
  74. data/lib/arrow/slicer.rb +61 -0
  75. data/lib/arrow/sort-key.rb +3 -3
  76. data/lib/arrow/sparse-union-array-builder.rb +56 -0
  77. data/lib/arrow/sparse-union-array.rb +26 -0
  78. data/lib/arrow/stream-decoder.rb +29 -0
  79. data/{test/test-decimal256-data-type.rb → lib/arrow/stream-listener.rb} +25 -9
  80. data/lib/arrow/string-array-builder.rb +30 -0
  81. data/lib/arrow/struct-array-builder.rb +0 -5
  82. data/lib/arrow/table-formatter.rb +38 -8
  83. data/lib/arrow/table-list-formatter.rb +3 -3
  84. data/lib/arrow/table-loader.rb +11 -5
  85. data/lib/arrow/table-saver.rb +4 -3
  86. data/lib/arrow/table-table-formatter.rb +7 -0
  87. data/lib/arrow/table.rb +180 -33
  88. data/lib/arrow/tensor.rb +144 -0
  89. data/lib/arrow/time-unit.rb +31 -0
  90. data/lib/arrow/time32-array-builder.rb +2 -14
  91. data/lib/arrow/time32-data-type.rb +9 -38
  92. data/lib/arrow/time64-array-builder.rb +2 -14
  93. data/lib/arrow/time64-data-type.rb +9 -38
  94. data/lib/arrow/timestamp-array-builder.rb +3 -15
  95. data/lib/arrow/timestamp-data-type.rb +9 -34
  96. data/{test/test-date64-array.rb → lib/arrow/timestamp-parser.rb} +14 -6
  97. data/lib/arrow/union-array-builder.rb +59 -0
  98. data/lib/arrow/union-array.rb +26 -0
  99. data/lib/arrow/version.rb +1 -1
  100. data/lib/arrow.rb +2 -7
  101. data/red-arrow.gemspec +74 -11
  102. metadata +85 -210
  103. data/test/fixture/TestOrcFile.test1.orc +0 -0
  104. data/test/fixture/with-header-float.csv +0 -20
  105. data/test/fixture/with-header.csv +0 -20
  106. data/test/fixture/without-header-float.csv +0 -19
  107. data/test/fixture/without-header.csv +0 -19
  108. data/test/helper/omittable.rb +0 -36
  109. data/test/helper.rb +0 -30
  110. data/test/raw-records/test-basic-arrays.rb +0 -395
  111. data/test/raw-records/test-dense-union-array.rb +0 -521
  112. data/test/raw-records/test-list-array.rb +0 -610
  113. data/test/raw-records/test-map-array.rb +0 -478
  114. data/test/raw-records/test-multiple-columns.rb +0 -65
  115. data/test/raw-records/test-sparse-union-array.rb +0 -511
  116. data/test/raw-records/test-struct-array.rb +0 -515
  117. data/test/raw-records/test-table.rb +0 -47
  118. data/test/run-test.rb +0 -71
  119. data/test/test-array-builder.rb +0 -136
  120. data/test/test-array.rb +0 -325
  121. data/test/test-bigdecimal.rb +0 -40
  122. data/test/test-binary-dictionary-array-builder.rb +0 -103
  123. data/test/test-chunked-array.rb +0 -183
  124. data/test/test-column.rb +0 -92
  125. data/test/test-csv-loader.rb +0 -250
  126. data/test/test-data-type.rb +0 -83
  127. data/test/test-decimal128-array-builder.rb +0 -112
  128. data/test/test-decimal128-data-type.rb +0 -31
  129. data/test/test-decimal128.rb +0 -102
  130. data/test/test-decimal256-array-builder.rb +0 -112
  131. data/test/test-decimal256-array.rb +0 -38
  132. data/test/test-decimal256.rb +0 -102
  133. data/test/test-dense-union-data-type.rb +0 -41
  134. data/test/test-dictionary-data-type.rb +0 -40
  135. data/test/test-expression.rb +0 -40
  136. data/test/test-feather.rb +0 -49
  137. data/test/test-field.rb +0 -91
  138. data/test/test-file-output-stream.rb +0 -54
  139. data/test/test-fixed-size-binary-array-builder.rb +0 -92
  140. data/test/test-fixed-size-binary-array.rb +0 -36
  141. data/test/test-function.rb +0 -210
  142. data/test/test-group.rb +0 -180
  143. data/test/test-list-array-builder.rb +0 -79
  144. data/test/test-list-array.rb +0 -32
  145. data/test/test-list-data-type.rb +0 -69
  146. data/test/test-map-array-builder.rb +0 -110
  147. data/test/test-map-array.rb +0 -33
  148. data/test/test-memory-view.rb +0 -434
  149. data/test/test-orc.rb +0 -173
  150. data/test/test-record-batch-builder.rb +0 -125
  151. data/test/test-record-batch-file-reader.rb +0 -115
  152. data/test/test-record-batch-iterator.rb +0 -37
  153. data/test/test-record-batch-reader.rb +0 -46
  154. data/test/test-record-batch.rb +0 -182
  155. data/test/test-schema.rb +0 -134
  156. data/test/test-slicer.rb +0 -487
  157. data/test/test-sort-indices.rb +0 -40
  158. data/test/test-sort-key.rb +0 -81
  159. data/test/test-sort-options.rb +0 -58
  160. data/test/test-sparse-union-data-type.rb +0 -41
  161. data/test/test-string-dictionary-array-builder.rb +0 -103
  162. data/test/test-struct-array-builder.rb +0 -184
  163. data/test/test-struct-array.rb +0 -94
  164. data/test/test-struct-data-type.rb +0 -112
  165. data/test/test-table.rb +0 -1123
  166. data/test/test-time.rb +0 -288
  167. data/test/test-time32-array.rb +0 -81
  168. data/test/test-time64-array.rb +0 -81
  169. data/test/test-time64-data-type.rb +0 -42
  170. data/test/test-timestamp-array.rb +0 -45
  171. data/test/test-timestamp-data-type.rb +0 -42
  172. data/test/values/test-basic-arrays.rb +0 -325
  173. data/test/values/test-dense-union-array.rb +0 -509
  174. data/test/values/test-dictionary-array.rb +0 -295
  175. data/test/values/test-list-array.rb +0 -571
  176. data/test/values/test-map-array.rb +0 -466
  177. data/test/values/test-sparse-union-array.rb +0 -500
  178. data/test/values/test-struct-array.rb +0 -512
@@ -1,79 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- class ListArrayBuilderTest < Test::Unit::TestCase
19
- def setup
20
- @data_type = Arrow::ListDataType.new(name: "visible", type: :boolean)
21
- @builder = Arrow::ListArrayBuilder.new(@data_type)
22
- end
23
-
24
- sub_test_case("#append_value") do
25
- test("nil") do
26
- @builder.append_value(nil)
27
- array = @builder.finish
28
- assert_equal(nil, array[0])
29
- end
30
-
31
- test("Array") do
32
- @builder.append_value([true, false, true])
33
- array = @builder.finish
34
- assert_equal([true, false, true], array[0].to_a)
35
- end
36
- end
37
-
38
- sub_test_case("#append_values") do
39
- test("[nil, Array]") do
40
- @builder.append_values([[false], nil, [true, false, true]])
41
- array = @builder.finish
42
- assert_equal([
43
- [false],
44
- nil,
45
- [true, false, true],
46
- ],
47
- array.collect {|list| list ? list.to_a : nil})
48
- end
49
-
50
- test("is_valids") do
51
- @builder.append_values([[false], [true, true], [true, false, true]],
52
- [true, false, true])
53
- array = @builder.finish
54
- assert_equal([
55
- [false],
56
- nil,
57
- [true, false, true],
58
- ],
59
- array.collect {|list| list ? list.to_a : nil})
60
- end
61
- end
62
-
63
- sub_test_case("#append") do
64
- test("backward compatibility") do
65
- @builder.append
66
- @builder.value_builder.append(true)
67
- @builder.value_builder.append(false)
68
- @builder.append
69
- @builder.value_builder.append(true)
70
- array = @builder.finish
71
-
72
- assert_equal([
73
- [true, false],
74
- [true],
75
- ],
76
- array.collect(&:to_a))
77
- end
78
- end
79
- end
@@ -1,32 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- class ListArrayTest < Test::Unit::TestCase
19
- sub_test_case(".new") do
20
- test("build") do
21
- data_type = Arrow::ListDataType.new(name: "visible", type: :boolean)
22
- values = [
23
- [true, false],
24
- nil,
25
- [false, true, false],
26
- ]
27
- array = Arrow::ListArray.new(data_type, values)
28
- assert_equal(values,
29
- array.collect {|value| value ? value.to_a : nil})
30
- end
31
- end
32
- end
@@ -1,69 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- class ListDataTypeTest < Test::Unit::TestCase
19
- sub_test_case(".new") do
20
- test("Arrow::Field") do
21
- field = Arrow::Field.new(:tag, :string)
22
- assert_equal("list<tag: string>",
23
- Arrow::ListDataType.new(field).to_s)
24
- end
25
-
26
- test("name: String") do
27
- assert_equal("list<tag: string>",
28
- Arrow::ListDataType.new(name: "tag", type: :string).to_s)
29
- end
30
-
31
- test("field: Arrow::Field") do
32
- field = Arrow::Field.new(:tag, :string)
33
- assert_equal("list<tag: string>",
34
- Arrow::ListDataType.new(field: field).to_s)
35
- end
36
-
37
- test("field: Hash") do
38
- field_description = {name: "tag", type: :string}
39
- assert_equal("list<tag: string>",
40
- Arrow::ListDataType.new(field: field_description).to_s)
41
- end
42
-
43
- test("Arrow::DataType") do
44
- data_type = Arrow::BooleanDataType.new
45
- assert_equal("list<item: bool>",
46
- Arrow::ListDataType.new(data_type).to_s)
47
- end
48
-
49
- test("String") do
50
- assert_equal("list<item: bool>",
51
- Arrow::ListDataType.new("boolean").to_s)
52
- end
53
-
54
- test("Symbol") do
55
- assert_equal("list<item: bool>",
56
- Arrow::ListDataType.new(:boolean).to_s)
57
- end
58
-
59
- test("[data type name, additional information]") do
60
- assert_equal("list<item: time32[ms]>",
61
- Arrow::ListDataType.new([:time32, :milli]).to_s)
62
- end
63
-
64
- test("type: Symbol") do
65
- assert_equal("list<item: bool>",
66
- Arrow::ListDataType.new(type: :boolean).to_s)
67
- end
68
- end
69
- end
@@ -1,110 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- class MapArrayBuilderTest < Test::Unit::TestCase
19
- def setup
20
- key_type = Arrow::StringDataType.new
21
- item_type = Arrow::Int16DataType.new
22
- data_type = Arrow::MapDataType.new(key_type, item_type)
23
- @builder = Arrow::MapArrayBuilder.new(data_type)
24
- end
25
-
26
- sub_test_case("#append_value") do
27
- test("nil") do
28
- @builder.append_value(nil)
29
- array = @builder.finish
30
- assert_equal([nil], array.collect {|value| value})
31
- end
32
-
33
- test("Hash") do
34
- @builder.append_value({"a" => 0, "b" => 1})
35
- @builder.append_value({"c" => 0, "d" => 1})
36
- array = @builder.finish
37
- assert_equal([
38
- {"a" => 0, "b" => 1},
39
- {"c" => 0, "d" => 1}
40
- ],
41
- array.collect {|value| value})
42
- end
43
-
44
- test("#each") do
45
- @builder.append_value([["a", 0], ["b", 1]])
46
- @builder.append_value([["c", 0], ["d", 1]])
47
- array = @builder.finish
48
- assert_equal([
49
- {"a" => 0, "b" => 1},
50
- {"c" => 0, "d" => 1}
51
- ],
52
- array.collect {|value| value})
53
- end
54
- end
55
-
56
- sub_test_case("#append_values") do
57
- test("[nil]") do
58
- @builder.append_values([nil])
59
- array = @builder.finish
60
- assert_equal([nil], array.collect {|value| value})
61
- end
62
-
63
- test("[Hash]") do
64
- @builder.append_values([{"a" => 0, "b" => 1}, {"c" => 0, "d" => 1}])
65
- array = @builder.finish
66
- assert_equal([
67
- {"a" => 0, "b" => 1},
68
- {"c" => 0, "d" => 1}
69
- ],
70
- array.collect {|value| value})
71
- end
72
-
73
- test("[#each]") do
74
- @builder.append_values([[["a", 0], ["b", 1]], [["c", 0], ["d", 1]]])
75
- array = @builder.finish
76
- assert_equal([
77
- {"a" => 0, "b" => 1},
78
- {"c" => 0, "d" => 1}
79
- ],
80
- array.collect {|value| value})
81
- end
82
-
83
- test("[nil, Hash, #each]") do
84
- @builder.append_values([nil, {"a" => 0, "b" => 1}, [["c", 0], ["d", 1]]])
85
- array = @builder.finish
86
- assert_equal([
87
- nil,
88
- {"a" => 0, "b" => 1},
89
- {"c" => 0, "d" => 1}
90
- ],
91
- array.collect {|value| value})
92
- end
93
-
94
- test("is_valids") do
95
- @builder.append_values([
96
- {"a" => 0, "b" => 1},
97
- {"c" => 0, "d" => 1},
98
- {"e" => 0, "f" => 1}
99
- ],
100
- [true, false, true])
101
- array = @builder.finish
102
- assert_equal([
103
- {"a" => 0, "b" => 1},
104
- nil,
105
- {"e" => 0, "f" => 1}
106
- ],
107
- array.collect {|value| value})
108
- end
109
- end
110
- end
@@ -1,33 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- class MapArrayTest < Test::Unit::TestCase
19
- sub_test_case(".new") do
20
- test("build") do
21
- key_type = Arrow::StringDataType.new
22
- item_type = Arrow::Int16DataType.new
23
- data_type = Arrow::MapDataType.new(key_type, item_type)
24
- values = [
25
- {"a" => 0, "b" => 1},
26
- nil,
27
- {"c" => 0, "d" => 1}
28
- ]
29
- array = Arrow::MapArray.new(data_type, values)
30
- assert_equal(values, array.collect {|value| value})
31
- end
32
- end
33
- end