red-arrow 0.17.1 → 4.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (72) hide show
  1. checksums.yaml +4 -4
  2. data/ext/arrow/converters.hpp +75 -32
  3. data/ext/arrow/extconf.rb +14 -3
  4. data/ext/arrow/raw-records.cpp +3 -1
  5. data/ext/arrow/values.cpp +3 -1
  6. data/lib/arrow/array-builder.rb +11 -6
  7. data/lib/arrow/array.rb +118 -0
  8. data/lib/arrow/bigdecimal-extension.rb +5 -1
  9. data/lib/arrow/buffer.rb +28 -0
  10. data/lib/arrow/data-type.rb +14 -5
  11. data/lib/arrow/decimal128-array-builder.rb +21 -25
  12. data/lib/arrow/decimal128-data-type.rb +2 -0
  13. data/lib/arrow/decimal128.rb +18 -0
  14. data/lib/arrow/decimal256-array-builder.rb +61 -0
  15. data/lib/arrow/decimal256-array.rb +25 -0
  16. data/lib/arrow/decimal256-data-type.rb +73 -0
  17. data/lib/arrow/decimal256.rb +60 -0
  18. data/lib/arrow/dense-union-data-type.rb +2 -2
  19. data/lib/arrow/dictionary-array.rb +24 -0
  20. data/lib/arrow/dictionary-data-type.rb +2 -2
  21. data/lib/arrow/fixed-size-binary-array-builder.rb +38 -0
  22. data/lib/arrow/fixed-size-binary-array.rb +26 -0
  23. data/lib/arrow/loader.rb +18 -0
  24. data/lib/arrow/raw-table-converter.rb +47 -0
  25. data/lib/arrow/record-batch-iterator.rb +22 -0
  26. data/lib/arrow/record-batch.rb +9 -1
  27. data/lib/arrow/sort-key.rb +193 -0
  28. data/lib/arrow/sort-options.rb +109 -0
  29. data/lib/arrow/sparse-union-data-type.rb +2 -2
  30. data/lib/arrow/struct-array-builder.rb +13 -7
  31. data/lib/arrow/table-saver.rb +6 -6
  32. data/lib/arrow/table.rb +5 -24
  33. data/lib/arrow/time32-data-type.rb +2 -2
  34. data/lib/arrow/time64-data-type.rb +2 -2
  35. data/lib/arrow/timestamp-data-type.rb +2 -2
  36. data/lib/arrow/version.rb +1 -1
  37. data/red-arrow.gemspec +1 -0
  38. data/test/raw-records/test-basic-arrays.rb +17 -0
  39. data/test/raw-records/test-dense-union-array.rb +15 -34
  40. data/test/raw-records/test-list-array.rb +20 -0
  41. data/test/raw-records/test-sparse-union-array.rb +15 -33
  42. data/test/raw-records/test-struct-array.rb +15 -0
  43. data/test/test-array.rb +122 -2
  44. data/test/test-bigdecimal.rb +20 -3
  45. data/test/test-buffer.rb +11 -0
  46. data/test/test-decimal128-array-builder.rb +18 -1
  47. data/test/test-decimal128-data-type.rb +2 -2
  48. data/test/test-decimal128.rb +38 -0
  49. data/test/test-decimal256-array-builder.rb +112 -0
  50. data/test/test-decimal256-array.rb +38 -0
  51. data/test/test-decimal256-data-type.rb +31 -0
  52. data/test/test-decimal256.rb +102 -0
  53. data/test/test-dense-union-data-type.rb +2 -2
  54. data/test/test-dictionary-array.rb +41 -0
  55. data/test/test-feather.rb +1 -1
  56. data/test/test-fixed-size-binary-array-builder.rb +92 -0
  57. data/test/test-fixed-size-binary-array.rb +36 -0
  58. data/test/test-orc.rb +19 -23
  59. data/test/test-record-batch-iterator.rb +37 -0
  60. data/test/test-record-batch.rb +14 -0
  61. data/test/test-sort-indices.rb +40 -0
  62. data/test/test-sort-key.rb +81 -0
  63. data/test/test-sort-options.rb +58 -0
  64. data/test/test-sparse-union-data-type.rb +2 -2
  65. data/test/test-struct-array-builder.rb +16 -12
  66. data/test/test-struct-array.rb +2 -2
  67. data/test/values/test-basic-arrays.rb +11 -0
  68. data/test/values/test-dense-union-array.rb +15 -34
  69. data/test/values/test-list-array.rb +18 -0
  70. data/test/values/test-sparse-union-array.rb +15 -33
  71. data/test/values/test-struct-array.rb +15 -0
  72. metadata +107 -59
data/test/test-buffer.rb CHANGED
@@ -16,6 +16,17 @@
16
16
  # under the License.
17
17
 
18
18
  class BufferTest < Test::Unit::TestCase
19
+ sub_test_case(".new") do
20
+ test("GC") do
21
+ data = "Hello"
22
+ data_id = data.object_id
23
+ _buffer = Arrow::Buffer.new(data)
24
+ data = nil
25
+ GC.start
26
+ assert_equal("Hello", ObjectSpace._id2ref(data_id))
27
+ end
28
+ end
29
+
19
30
  sub_test_case("instance methods") do
20
31
  def setup
21
32
  @buffer = Arrow::Buffer.new("Hello")
@@ -80,8 +80,13 @@ class Decimal128ArrayBuilderTest < Test::Unit::TestCase
80
80
  test("is_valids") do
81
81
  @builder.append_values([
82
82
  Arrow::Decimal128.new("10.1"),
83
- nil,
84
83
  Arrow::Decimal128.new("10.1"),
84
+ Arrow::Decimal128.new("10.1"),
85
+ ],
86
+ [
87
+ true,
88
+ false,
89
+ true,
85
90
  ])
86
91
  array = @builder.finish
87
92
  assert_equal([
@@ -91,5 +96,17 @@ class Decimal128ArrayBuilderTest < Test::Unit::TestCase
91
96
  ],
92
97
  array.to_a)
93
98
  end
99
+
100
+ test("packed") do
101
+ @builder.append_values(Arrow::Decimal128.new("10.1").to_bytes.to_s * 3,
102
+ [true, false, true])
103
+ array = @builder.finish
104
+ assert_equal([
105
+ BigDecimal("10.1"),
106
+ nil,
107
+ BigDecimal("10.1"),
108
+ ],
109
+ array.to_a)
110
+ end
94
111
  end
95
112
  end
@@ -18,12 +18,12 @@
18
18
  class Decimal128DataTypeTest < Test::Unit::TestCase
19
19
  sub_test_case(".new") do
20
20
  test("ordered arguments") do
21
- assert_equal("decimal(8, 2)",
21
+ assert_equal("decimal128(8, 2)",
22
22
  Arrow::Decimal128DataType.new(8, 2).to_s)
23
23
  end
24
24
 
25
25
  test("description") do
26
- assert_equal("decimal(8, 2)",
26
+ assert_equal("decimal128(8, 2)",
27
27
  Arrow::Decimal128DataType.new(precision: 8,
28
28
  scale: 2).to_s)
29
29
  end
@@ -60,5 +60,43 @@ class Decimal128Test < Test::Unit::TestCase
60
60
  @decimal128.to_s(1))
61
61
  end
62
62
  end
63
+
64
+ test("#abs") do
65
+ decimal128 = Arrow::Decimal128.new("-10.1")
66
+ assert_equal([
67
+ Arrow::Decimal128.new("-10.1"),
68
+ Arrow::Decimal128.new("10.1"),
69
+ ],
70
+ [
71
+ decimal128,
72
+ decimal128.abs,
73
+ ])
74
+ end
75
+
76
+ test("#abs!") do
77
+ decimal128 = Arrow::Decimal128.new("-10.1")
78
+ decimal128.abs!
79
+ assert_equal(Arrow::Decimal128.new("10.1"),
80
+ decimal128)
81
+ end
82
+
83
+ test("#negate") do
84
+ decimal128 = Arrow::Decimal128.new("-10.1")
85
+ assert_equal([
86
+ Arrow::Decimal128.new("-10.1"),
87
+ Arrow::Decimal128.new("10.1"),
88
+ ],
89
+ [
90
+ decimal128,
91
+ decimal128.negate,
92
+ ])
93
+ end
94
+
95
+ test("#negate!") do
96
+ decimal128 = Arrow::Decimal128.new("-10.1")
97
+ decimal128.negate!
98
+ assert_equal(Arrow::Decimal128.new("10.1"),
99
+ decimal128)
100
+ end
63
101
  end
64
102
  end
@@ -0,0 +1,112 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ class Decimal256ArrayBuilderTest < Test::Unit::TestCase
19
+ def setup
20
+ @data_type = Arrow::Decimal256DataType.new(3, 1)
21
+ @builder = Arrow::Decimal256ArrayBuilder.new(@data_type)
22
+ end
23
+
24
+ sub_test_case("#append_value") do
25
+ test("nil") do
26
+ @builder.append_value(nil)
27
+ array = @builder.finish
28
+ assert_equal(nil, array[0])
29
+ end
30
+
31
+ test("Arrow::Decimal256") do
32
+ @builder.append_value(Arrow::Decimal256.new("10.1"))
33
+ array = @builder.finish
34
+ assert_equal(BigDecimal("10.1"),
35
+ array[0])
36
+ end
37
+
38
+ test("String") do
39
+ @builder.append_value("10.1")
40
+ array = @builder.finish
41
+ assert_equal(BigDecimal("10.1"),
42
+ array[0])
43
+ end
44
+
45
+ test("Float") do
46
+ @builder.append_value(10.1)
47
+ array = @builder.finish
48
+ assert_equal(BigDecimal("10.1"),
49
+ array[0])
50
+ end
51
+
52
+ test("BigDecimal") do
53
+ @builder.append_value(BigDecimal("10.1"))
54
+ array = @builder.finish
55
+ assert_equal(BigDecimal("10.1"),
56
+ array[0])
57
+ end
58
+ end
59
+
60
+ sub_test_case("#append_values") do
61
+ test("mixed") do
62
+ @builder.append_values([
63
+ Arrow::Decimal256.new("10.1"),
64
+ nil,
65
+ "10.1",
66
+ 10.1,
67
+ BigDecimal("10.1"),
68
+ ])
69
+ array = @builder.finish
70
+ assert_equal([
71
+ BigDecimal("10.1"),
72
+ nil,
73
+ BigDecimal("10.1"),
74
+ BigDecimal("10.1"),
75
+ BigDecimal("10.1"),
76
+ ],
77
+ array.to_a)
78
+ end
79
+
80
+ test("is_valids") do
81
+ @builder.append_values([
82
+ Arrow::Decimal256.new("10.1"),
83
+ Arrow::Decimal256.new("10.1"),
84
+ Arrow::Decimal256.new("10.1"),
85
+ ],
86
+ [
87
+ true,
88
+ false,
89
+ true,
90
+ ])
91
+ array = @builder.finish
92
+ assert_equal([
93
+ BigDecimal("10.1"),
94
+ nil,
95
+ BigDecimal("10.1"),
96
+ ],
97
+ array.to_a)
98
+ end
99
+
100
+ test("packed") do
101
+ @builder.append_values(Arrow::Decimal256.new("10.1").to_bytes.to_s * 3,
102
+ [true, false, true])
103
+ array = @builder.finish
104
+ assert_equal([
105
+ BigDecimal("10.1"),
106
+ nil,
107
+ BigDecimal("10.1"),
108
+ ],
109
+ array.to_a)
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,38 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ class Decimal256ArrayTest < Test::Unit::TestCase
19
+ sub_test_case(".new") do
20
+ test("build") do
21
+ data_type = Arrow::Decimal256DataType.new(3, 1)
22
+ values = [
23
+ 10.1,
24
+ nil,
25
+ "10.1",
26
+ BigDecimal("10.1"),
27
+ ]
28
+ array = Arrow::Decimal256Array.new(data_type, values)
29
+ assert_equal([
30
+ BigDecimal("10.1"),
31
+ nil,
32
+ BigDecimal("10.1"),
33
+ BigDecimal("10.1"),
34
+ ],
35
+ array.to_a)
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,31 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ class Decimal256DataTypeTest < Test::Unit::TestCase
19
+ sub_test_case(".new") do
20
+ test("ordered arguments") do
21
+ assert_equal("decimal256(8, 2)",
22
+ Arrow::Decimal256DataType.new(8, 2).to_s)
23
+ end
24
+
25
+ test("description") do
26
+ assert_equal("decimal256(8, 2)",
27
+ Arrow::Decimal256DataType.new(precision: 8,
28
+ scale: 2).to_s)
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,102 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ class Decimal256Test < Test::Unit::TestCase
19
+ sub_test_case("instance methods") do
20
+ def setup
21
+ @decimal256 = Arrow::Decimal256.new("10.1")
22
+ end
23
+
24
+ sub_test_case("#==") do
25
+ test("Arrow::Decimal256") do
26
+ assert do
27
+ @decimal256 == @decimal256
28
+ end
29
+ end
30
+
31
+ test("not Arrow::Decimal256") do
32
+ assert do
33
+ not (@decimal256 == 10.1)
34
+ end
35
+ end
36
+ end
37
+
38
+ sub_test_case("#!=") do
39
+ test("Arrow::Decimal256") do
40
+ assert do
41
+ not (@decimal256 != @decimal256)
42
+ end
43
+ end
44
+
45
+ test("not Arrow::Decimal256") do
46
+ assert do
47
+ @decimal256 != 10.1
48
+ end
49
+ end
50
+ end
51
+
52
+ sub_test_case("#to_s") do
53
+ test("default") do
54
+ assert_equal("101",
55
+ @decimal256.to_s)
56
+ end
57
+
58
+ test("scale") do
59
+ assert_equal("10.1",
60
+ @decimal256.to_s(1))
61
+ end
62
+ end
63
+
64
+ test("#abs") do
65
+ decimal256 = Arrow::Decimal256.new("-10.1")
66
+ assert_equal([
67
+ Arrow::Decimal256.new("-10.1"),
68
+ Arrow::Decimal256.new("10.1"),
69
+ ],
70
+ [
71
+ decimal256,
72
+ decimal256.abs,
73
+ ])
74
+ end
75
+
76
+ test("#abs!") do
77
+ decimal256 = Arrow::Decimal256.new("-10.1")
78
+ decimal256.abs!
79
+ assert_equal(Arrow::Decimal256.new("10.1"),
80
+ decimal256)
81
+ end
82
+
83
+ test("#negate") do
84
+ decimal256 = Arrow::Decimal256.new("-10.1")
85
+ assert_equal([
86
+ Arrow::Decimal256.new("-10.1"),
87
+ Arrow::Decimal256.new("10.1"),
88
+ ],
89
+ [
90
+ decimal256,
91
+ decimal256.negate,
92
+ ])
93
+ end
94
+
95
+ test("#negate!") do
96
+ decimal256 = Arrow::Decimal256.new("-10.1")
97
+ decimal256.negate!
98
+ assert_equal(Arrow::Decimal256.new("10.1"),
99
+ decimal256)
100
+ end
101
+ end
102
+ end
@@ -28,12 +28,12 @@ class DenseUnionDataTypeTest < Test::Unit::TestCase
28
28
  end
29
29
 
30
30
  test("ordered arguments") do
31
- assert_equal("union[dense]<visible: bool=2, count: int32=9>",
31
+ assert_equal("dense_union<visible: bool=2, count: int32=9>",
32
32
  Arrow::DenseUnionDataType.new(@fields, [2, 9]).to_s)
33
33
  end
34
34
 
35
35
  test("description") do
36
- assert_equal("union[dense]<visible: bool=2, count: int32=9>",
36
+ assert_equal("dense_union<visible: bool=2, count: int32=9>",
37
37
  Arrow::DenseUnionDataType.new(fields: @fields,
38
38
  type_codes: [2, 9]).to_s)
39
39
  end
@@ -0,0 +1,41 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ class DictionaryArrayTest < Test::Unit::TestCase
19
+ sub_test_case("instance methods") do
20
+ def setup
21
+ @values = ["a", "b", "c", "b", "a"]
22
+ @string_array = Arrow::StringArray.new(@values)
23
+ @array = @string_array.dictionary_encode
24
+ end
25
+
26
+ test("#[]") do
27
+ assert_equal(@values, @array.to_a)
28
+ end
29
+
30
+ test("#get_value") do
31
+ assert_equal([
32
+ @values[0],
33
+ @values[3],
34
+ ],
35
+ [
36
+ @array.get_value(0),
37
+ @array.get_value(3),
38
+ ])
39
+ end
40
+ end
41
+ end