red-arrow 18.1.0 → 19.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/lib/arrow/array.rb +7 -4
  3. data/lib/arrow/column.rb +4 -4
  4. data/lib/arrow/jruby/array-builder.rb +114 -0
  5. data/lib/arrow/jruby/array.rb +109 -0
  6. data/{test/helper/fixture.rb → lib/arrow/jruby/chunked-array.rb} +14 -6
  7. data/lib/arrow/jruby/compression-type.rb +26 -0
  8. data/{test/test-boolean-scalar.rb → lib/arrow/jruby/csv-read-options.rb} +12 -6
  9. data/{test/test-map-data-type.rb → lib/arrow/jruby/data-type.rb} +24 -12
  10. data/lib/arrow/jruby/decimal128.rb +28 -0
  11. data/lib/arrow/jruby/decimal256.rb +28 -0
  12. data/{test/fixture/null-with-double-quote.csv → lib/arrow/jruby/error.rb} +7 -4
  13. data/lib/arrow/jruby/file-system.rb +24 -0
  14. data/{test/test-null-array.rb → lib/arrow/jruby/function.rb} +5 -4
  15. data/lib/arrow/jruby/record-batch-iterator.rb +24 -0
  16. data/{test/fixture/integer-float.csv → lib/arrow/jruby/record-batch.rb} +8 -4
  17. data/{test/fixture/float-integer.csv → lib/arrow/jruby/sort-key.rb} +8 -4
  18. data/lib/arrow/jruby/sort-options.rb +24 -0
  19. data/lib/arrow/jruby/stream-listener-raw.rb +25 -0
  20. data/{test/test-rolling-window.rb → lib/arrow/jruby/table.rb} +19 -19
  21. data/lib/arrow/jruby/writable.rb +24 -0
  22. data/lib/arrow/jruby.rb +52 -0
  23. data/lib/arrow/libraries.rb +126 -0
  24. data/lib/arrow/list-array-builder.rb +1 -0
  25. data/lib/arrow/loader.rb +3 -111
  26. data/{test/fixture/null-without-double-quote.csv → lib/arrow/ruby.rb} +6 -4
  27. data/lib/arrow/version.rb +1 -1
  28. data/lib/arrow.rb +2 -7
  29. data/red-arrow.gemspec +15 -6
  30. metadata +26 -229
  31. data/test/each-raw-record/test-basic-arrays.rb +0 -411
  32. data/test/each-raw-record/test-dense-union-array.rb +0 -566
  33. data/test/each-raw-record/test-dictionary-array.rb +0 -341
  34. data/test/each-raw-record/test-list-array.rb +0 -628
  35. data/test/each-raw-record/test-map-array.rb +0 -507
  36. data/test/each-raw-record/test-multiple-columns.rb +0 -72
  37. data/test/each-raw-record/test-sparse-union-array.rb +0 -528
  38. data/test/each-raw-record/test-struct-array.rb +0 -529
  39. data/test/each-raw-record/test-table.rb +0 -47
  40. data/test/fixture/TestOrcFile.test1.orc +0 -0
  41. data/test/fixture/with-header-float.csv +0 -20
  42. data/test/fixture/with-header.csv +0 -20
  43. data/test/fixture/without-header-float.csv +0 -19
  44. data/test/fixture/without-header.csv +0 -19
  45. data/test/helper/omittable.rb +0 -49
  46. data/test/helper.rb +0 -31
  47. data/test/raw-records/test-basic-arrays.rb +0 -405
  48. data/test/raw-records/test-dense-union-array.rb +0 -566
  49. data/test/raw-records/test-dictionary-array.rb +0 -341
  50. data/test/raw-records/test-list-array.rb +0 -628
  51. data/test/raw-records/test-map-array.rb +0 -507
  52. data/test/raw-records/test-multiple-columns.rb +0 -65
  53. data/test/raw-records/test-sparse-union-array.rb +0 -556
  54. data/test/raw-records/test-struct-array.rb +0 -529
  55. data/test/raw-records/test-table.rb +0 -47
  56. data/test/run-test.rb +0 -71
  57. data/test/test-array-builder.rb +0 -198
  58. data/test/test-array.rb +0 -332
  59. data/test/test-bigdecimal.rb +0 -40
  60. data/test/test-binary-dictionary-array-builder.rb +0 -103
  61. data/test/test-buffer.rb +0 -49
  62. data/test/test-chunked-array.rb +0 -198
  63. data/test/test-column.rb +0 -123
  64. data/test/test-csv-loader.rb +0 -297
  65. data/test/test-data-type.rb +0 -84
  66. data/test/test-date32-array.rb +0 -24
  67. data/test/test-date64-array.rb +0 -25
  68. data/test/test-decimal128-array-builder.rb +0 -126
  69. data/test/test-decimal128-array.rb +0 -47
  70. data/test/test-decimal128-data-type.rb +0 -31
  71. data/test/test-decimal128.rb +0 -126
  72. data/test/test-decimal256-array-builder.rb +0 -126
  73. data/test/test-decimal256-array.rb +0 -47
  74. data/test/test-decimal256-data-type.rb +0 -31
  75. data/test/test-decimal256.rb +0 -126
  76. data/test/test-dense-union-array.rb +0 -42
  77. data/test/test-dense-union-data-type.rb +0 -41
  78. data/test/test-dictionary-array.rb +0 -41
  79. data/test/test-dictionary-data-type.rb +0 -40
  80. data/test/test-expression.rb +0 -51
  81. data/test/test-feather.rb +0 -49
  82. data/test/test-field.rb +0 -117
  83. data/test/test-file-output-stream.rb +0 -54
  84. data/test/test-fixed-size-binary-array-builder.rb +0 -92
  85. data/test/test-fixed-size-binary-array.rb +0 -36
  86. data/test/test-float-scalar.rb +0 -46
  87. data/test/test-function.rb +0 -210
  88. data/test/test-group.rb +0 -193
  89. data/test/test-half-float-array.rb +0 -43
  90. data/test/test-half-float.rb +0 -130
  91. data/test/test-list-array-builder.rb +0 -79
  92. data/test/test-list-array.rb +0 -32
  93. data/test/test-list-data-type.rb +0 -69
  94. data/test/test-map-array-builder.rb +0 -110
  95. data/test/test-map-array.rb +0 -33
  96. data/test/test-memory-view.rb +0 -434
  97. data/test/test-orc.rb +0 -173
  98. data/test/test-ractor.rb +0 -34
  99. data/test/test-record-batch-builder.rb +0 -125
  100. data/test/test-record-batch-file-reader.rb +0 -136
  101. data/test/test-record-batch-iterator.rb +0 -37
  102. data/test/test-record-batch-reader.rb +0 -46
  103. data/test/test-record-batch-stream-reader.rb +0 -129
  104. data/test/test-record-batch.rb +0 -182
  105. data/test/test-scalar.rb +0 -65
  106. data/test/test-schema.rb +0 -134
  107. data/test/test-slicer.rb +0 -589
  108. data/test/test-sort-indices.rb +0 -40
  109. data/test/test-sort-key.rb +0 -81
  110. data/test/test-sort-options.rb +0 -58
  111. data/test/test-sparse-union-array.rb +0 -38
  112. data/test/test-sparse-union-data-type.rb +0 -41
  113. data/test/test-stream-listener.rb +0 -60
  114. data/test/test-string-dictionary-array-builder.rb +0 -103
  115. data/test/test-struct-array-builder.rb +0 -184
  116. data/test/test-struct-array.rb +0 -94
  117. data/test/test-struct-data-type.rb +0 -112
  118. data/test/test-table.rb +0 -1530
  119. data/test/test-tensor.rb +0 -297
  120. data/test/test-time.rb +0 -288
  121. data/test/test-time32-array.rb +0 -81
  122. data/test/test-time32-data-type.rb +0 -42
  123. data/test/test-time64-array.rb +0 -81
  124. data/test/test-time64-data-type.rb +0 -42
  125. data/test/test-timestamp-array.rb +0 -45
  126. data/test/test-timestamp-data-type.rb +0 -42
  127. data/test/values/test-basic-arrays.rb +0 -335
  128. data/test/values/test-dense-union-array.rb +0 -552
  129. data/test/values/test-dictionary-array.rb +0 -325
  130. data/test/values/test-list-array.rb +0 -587
  131. data/test/values/test-map-array.rb +0 -489
  132. data/test/values/test-sparse-union-array.rb +0 -543
  133. data/test/values/test-struct-array.rb +0 -524
@@ -1,182 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- class RecordBatchTest < Test::Unit::TestCase
19
- sub_test_case(".new") do
20
- def setup
21
- @schema = Arrow::Schema.new(visible: :boolean,
22
- count: :uint32)
23
- end
24
-
25
- test("[raw_table]") do
26
- raw_table = {
27
- visible: [true, nil, false],
28
- count: [1, nil, 3],
29
- }
30
- record_batch = Arrow::RecordBatch.new(raw_table)
31
- assert_equal([
32
- {"visible" => true, "count" => 1},
33
- {"visible" => nil, "count" => nil},
34
- {"visible" => false, "count" => 3},
35
- ],
36
- record_batch.each_record.collect(&:to_h))
37
- end
38
-
39
- test("[Schema, records]") do
40
- records = [
41
- {visible: true, count: 1},
42
- nil,
43
- [false, 3],
44
- ]
45
- record_batch = Arrow::RecordBatch.new(@schema, records)
46
- assert_equal([
47
- {"visible" => true, "count" => 1},
48
- {"visible" => nil, "count" => nil},
49
- {"visible" => false, "count" => 3},
50
- ],
51
- record_batch.each_record.collect(&:to_h))
52
- end
53
-
54
- test("[Schema, columns]") do
55
- columns = {
56
- visible: [true, nil, false],
57
- count: [1, 2, nil],
58
- }
59
- record_batch = Arrow::RecordBatch.new(@schema, columns)
60
- assert_equal([
61
- {"visible" => true, "count" => 1},
62
- {"visible" => nil, "count" => 2},
63
- {"visible" => false, "count" => nil},
64
- ],
65
- record_batch.each_record.collect(&:to_h))
66
- end
67
-
68
- test("[Schema, n_rows, columns]") do
69
- columns = [
70
- Arrow::BooleanArray.new([true, nil, false]),
71
- Arrow::UInt32Array.new([1, 2, nil]),
72
- ]
73
- n_rows = columns[0].length
74
- record_batch = Arrow::RecordBatch.new(@schema, n_rows, columns)
75
- assert_equal([
76
- {"visible" => true, "count" => 1},
77
- {"visible" => nil, "count" => 2},
78
- {"visible" => false, "count" => nil},
79
- ],
80
- record_batch.each_record.collect(&:to_h))
81
- end
82
- end
83
-
84
- sub_test_case("instance methods") do
85
- def setup
86
- @schema = Arrow::Schema.new(count: :uint32)
87
- @counts = Arrow::UInt32Array.new([1, 2, 4, 8])
88
- @record_batch = Arrow::RecordBatch.new(@schema, @counts.length, [@counts])
89
- end
90
-
91
- sub_test_case("#each") do
92
- test("default") do
93
- records = []
94
- @record_batch.each do |record|
95
- records << [record, record.index]
96
- end
97
- assert_equal([
98
- [0, 0],
99
- [1, 1],
100
- [2, 2],
101
- [3, 3],
102
- ],
103
- records.collect {|record, i| [record.index, i]})
104
- end
105
-
106
- test("reuse_record: true") do
107
- records = []
108
- @record_batch.each(reuse_record: true) do |record|
109
- records << [record, record.index]
110
- end
111
- assert_equal([
112
- [3, 0],
113
- [3, 1],
114
- [3, 2],
115
- [3, 3],
116
- ],
117
- records.collect {|record, i| [record.index, i]})
118
- end
119
- end
120
-
121
- test("#to_table") do
122
- assert_equal(Arrow::Table.new(@schema, [@counts]),
123
- @record_batch.to_table)
124
- end
125
-
126
- sub_test_case("#==") do
127
- test("Arrow::RecordBatch") do
128
- assert do
129
- @record_batch == @record_batch
130
- end
131
- end
132
-
133
- test("not Arrow::RecordBatch") do
134
- assert do
135
- not (@record_batch == 29)
136
- end
137
- end
138
- end
139
-
140
- sub_test_case("#[]") do
141
- def setup
142
- @record_batch = Arrow::RecordBatch.new(a: [true],
143
- b: [true],
144
- c: [true],
145
- d: [true],
146
- e: [true],
147
- f: [true],
148
- g: [true])
149
- end
150
-
151
- test("[String]") do
152
- assert_equal(Arrow::Column.new(@record_batch, 0),
153
- @record_batch["a"])
154
- end
155
-
156
- test("[Symbol]") do
157
- assert_equal(Arrow::Column.new(@record_batch, 1),
158
- @record_batch[:b])
159
- end
160
-
161
- test("[Integer]") do
162
- assert_equal(Arrow::Column.new(@record_batch, 6),
163
- @record_batch[-1])
164
- end
165
-
166
- test("[Range]") do
167
- assert_equal(Arrow::RecordBatch.new(d: [true],
168
- e: [true]),
169
- @record_batch[3..4])
170
- end
171
-
172
- test("[[Symbol, String, Integer, Range]]") do
173
- assert_equal(Arrow::RecordBatch.new(c: [true],
174
- a: [true],
175
- g: [true],
176
- d: [true],
177
- e: [true]),
178
- @record_batch[[:c, "a", -1, 3..4]])
179
- end
180
- end
181
- end
182
- end
data/test/test-scalar.rb DELETED
@@ -1,65 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- class ScalarTest < Test::Unit::TestCase
19
- sub_test_case(".resolve") do
20
- test("Scalar") do
21
- assert_equal(Arrow::Int32Scalar.new(29),
22
- Arrow::Scalar.resolve(Arrow::Int32Scalar.new(29)))
23
- end
24
-
25
- test("true") do
26
- assert_equal(Arrow::BooleanScalar.new(true),
27
- Arrow::Scalar.resolve(true))
28
- end
29
-
30
- test("false") do
31
- assert_equal(Arrow::BooleanScalar.new(false),
32
- Arrow::Scalar.resolve(false))
33
- end
34
-
35
- test("Symbol") do
36
- assert_equal(Arrow::StringScalar.new("hello"),
37
- Arrow::Scalar.resolve(:hello))
38
- end
39
-
40
- test("String") do
41
- assert_equal(Arrow::StringScalar.new("hello"),
42
- Arrow::Scalar.resolve("hello"))
43
- end
44
-
45
- test("Integer") do
46
- assert_equal(Arrow::Int64Scalar.new(-29),
47
- Arrow::Scalar.resolve(-29))
48
- end
49
-
50
- test("Float") do
51
- assert_equal(Arrow::DoubleScalar.new(2.9),
52
- Arrow::Scalar.resolve(2.9))
53
- end
54
-
55
- test("Int64Scalar, :int32") do
56
- assert_equal(Arrow::Int32Scalar.new(-29),
57
- Arrow::Scalar.resolve(Arrow::Int64Scalar.new(-29), :int32))
58
- end
59
-
60
- test("Integer, :int32") do
61
- assert_equal(Arrow::Int32Scalar.new(-29),
62
- Arrow::Scalar.resolve(-29, :int32))
63
- end
64
- end
65
- end
data/test/test-schema.rb DELETED
@@ -1,134 +0,0 @@
1
- # Licensed to the Apache Software Foundation (ASF) under one
2
- # or more contributor license agreements. See the NOTICE file
3
- # distributed with this work for additional information
4
- # regarding copyright ownership. The ASF licenses this file
5
- # to you under the Apache License, Version 2.0 (the
6
- # "License"); you may not use this file except in compliance
7
- # with the License. You may obtain a copy of the License at
8
- #
9
- # http://www.apache.org/licenses/LICENSE-2.0
10
- #
11
- # Unless required by applicable law or agreed to in writing,
12
- # software distributed under the License is distributed on an
13
- # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
- # KIND, either express or implied. See the License for the
15
- # specific language governing permissions and limitations
16
- # under the License.
17
-
18
- class SchemaTest < Test::Unit::TestCase
19
- include Helper::Omittable
20
-
21
- def setup
22
- @count_field = Arrow::Field.new("count", :uint32)
23
- @visible_field = Arrow::Field.new("visible", :boolean)
24
- end
25
-
26
- sub_test_case(".new") do
27
- test("[Arrow::Field]") do
28
- fields = [
29
- @count_field,
30
- @visible_field,
31
- ]
32
- assert_equal("count: uint32\n" +
33
- "visible: bool",
34
- Arrow::Schema.new(fields).to_s)
35
- end
36
-
37
- test("[Arrow::Field, Hash]") do
38
- fields = [
39
- @count_field,
40
- {name: "visible", type: :boolean},
41
- ]
42
- assert_equal("count: uint32\n" +
43
- "visible: bool",
44
- Arrow::Schema.new(fields).to_s)
45
- end
46
-
47
- test("{String, Symbol => Arrow::DataType}") do
48
- fields = {
49
- "count" => Arrow::UInt32DataType.new,
50
- :visible => :boolean,
51
- }
52
- assert_equal("count: uint32\n" +
53
- "visible: bool",
54
- Arrow::Schema.new(fields).to_s)
55
- end
56
-
57
- test("{String, Symbol => Hash}") do
58
- fields = {
59
- "count" => {type: :uint32},
60
- :tags => {
61
- type: :list,
62
- field: {
63
- name: "tag",
64
- type: :string,
65
- },
66
- },
67
- }
68
- assert_equal("count: uint32\n" +
69
- "tags: list<tag: string>",
70
- Arrow::Schema.new(fields).to_s)
71
- end
72
- end
73
-
74
- sub_test_case("instance methods") do
75
- def setup
76
- super
77
- @schema = Arrow::Schema.new([@count_field, @visible_field])
78
- end
79
-
80
- sub_test_case("#[]") do
81
- test("[String]") do
82
- assert_equal([@count_field, @visible_field],
83
- [@schema["count"], @schema["visible"]])
84
- end
85
-
86
- test("[Symbol]") do
87
- assert_equal([@count_field, @visible_field],
88
- [@schema[:count], @schema[:visible]])
89
- end
90
-
91
- test("[Integer]") do
92
- assert_equal([@count_field, @visible_field],
93
- [@schema[0], @schema[1]])
94
- end
95
-
96
- test("[invalid]") do
97
- invalid = []
98
- message = +"field name or index must be String, Symbol or Integer"
99
- message << ": <#{invalid.inspect}>"
100
- assert_raise(ArgumentError.new(message)) do
101
- @schema[invalid]
102
- end
103
- end
104
- end
105
-
106
- sub_test_case("#==") do
107
- test("Arrow::Schema") do
108
- assert do
109
- @schema == @schema
110
- end
111
- end
112
-
113
- test("not Arrow::Schema") do
114
- assert do
115
- not (@schema == 29)
116
- end
117
- end
118
- end
119
-
120
- sub_test_case("#to_s") do
121
- test("show_metadata") do
122
- require_gi_bindings(3, 4, 2)
123
-
124
- schema = @schema.with_metadata("key" => "value")
125
- assert_equal(<<-SCHEMA.chomp, schema.to_s(show_metadata: true))
126
- count: uint32
127
- visible: bool
128
- -- metadata --
129
- key: value
130
- SCHEMA
131
- end
132
- end
133
- end
134
- end