red-arrow 0.17.0 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +4 -4
  2. data/ext/arrow/converters.hpp +75 -32
  3. data/ext/arrow/extconf.rb +14 -3
  4. data/ext/arrow/raw-records.cpp +3 -1
  5. data/ext/arrow/values.cpp +3 -1
  6. data/lib/arrow/array-builder.rb +11 -6
  7. data/lib/arrow/bigdecimal-extension.rb +5 -1
  8. data/lib/arrow/buffer.rb +28 -0
  9. data/lib/arrow/decimal128-array-builder.rb +21 -25
  10. data/lib/arrow/decimal128-data-type.rb +2 -0
  11. data/lib/arrow/decimal128.rb +18 -0
  12. data/lib/arrow/decimal256-array-builder.rb +61 -0
  13. data/lib/arrow/decimal256-array.rb +25 -0
  14. data/lib/arrow/decimal256-data-type.rb +73 -0
  15. data/lib/arrow/decimal256.rb +60 -0
  16. data/lib/arrow/dictionary-array.rb +24 -0
  17. data/lib/arrow/fixed-size-binary-array-builder.rb +38 -0
  18. data/lib/arrow/fixed-size-binary-array.rb +26 -0
  19. data/lib/arrow/loader.rb +16 -0
  20. data/lib/arrow/raw-table-converter.rb +47 -0
  21. data/lib/arrow/record-batch-iterator.rb +22 -0
  22. data/lib/arrow/record-batch.rb +9 -1
  23. data/lib/arrow/struct-array-builder.rb +13 -7
  24. data/lib/arrow/table-saver.rb +6 -6
  25. data/lib/arrow/table.rb +5 -24
  26. data/lib/arrow/version.rb +1 -1
  27. data/red-arrow.gemspec +1 -0
  28. data/test/raw-records/test-basic-arrays.rb +17 -0
  29. data/test/raw-records/test-dense-union-array.rb +15 -34
  30. data/test/raw-records/test-list-array.rb +20 -0
  31. data/test/raw-records/test-sparse-union-array.rb +15 -33
  32. data/test/raw-records/test-struct-array.rb +15 -0
  33. data/test/test-array.rb +2 -2
  34. data/test/test-bigdecimal.rb +20 -3
  35. data/test/test-buffer.rb +11 -0
  36. data/test/test-decimal128-array-builder.rb +18 -1
  37. data/test/test-decimal128.rb +38 -0
  38. data/test/test-decimal256-array-builder.rb +112 -0
  39. data/test/test-decimal256-array.rb +38 -0
  40. data/test/test-decimal256-data-type.rb +31 -0
  41. data/test/test-decimal256.rb +102 -0
  42. data/test/test-dense-union-data-type.rb +2 -2
  43. data/test/test-dictionary-array.rb +41 -0
  44. data/test/test-feather.rb +1 -1
  45. data/test/test-fixed-size-binary-array-builder.rb +92 -0
  46. data/test/test-fixed-size-binary-array.rb +36 -0
  47. data/test/test-record-batch-iterator.rb +37 -0
  48. data/test/test-record-batch.rb +14 -0
  49. data/test/test-sparse-union-data-type.rb +2 -2
  50. data/test/test-struct-array-builder.rb +16 -12
  51. data/test/test-struct-array.rb +2 -2
  52. data/test/values/test-basic-arrays.rb +11 -0
  53. data/test/values/test-dense-union-array.rb +15 -34
  54. data/test/values/test-list-array.rb +18 -0
  55. data/test/values/test-sparse-union-array.rb +15 -33
  56. data/test/values/test-struct-array.rb +15 -0
  57. metadata +96 -56
@@ -0,0 +1,61 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class Decimal256ArrayBuilder
20
+ class << self
21
+ # @since 3.0.0
22
+ def build(data_type, values)
23
+ builder = new(data_type)
24
+ builder.build(values)
25
+ end
26
+ end
27
+
28
+ alias_method :append_value_raw, :append_value
29
+ # @since 3.0.0
30
+ def append_value(value)
31
+ append_value_raw(normalize_value(value))
32
+ end
33
+
34
+ alias_method :append_values_raw, :append_values
35
+ # @since 3.0.0
36
+ def append_values(values, is_valids=nil)
37
+ if values.is_a?(::Array)
38
+ values = values.collect do |value|
39
+ normalize_value(value)
40
+ end
41
+ append_values_raw(values, is_valids)
42
+ else
43
+ append_values_packed(values, is_valids)
44
+ end
45
+ end
46
+
47
+ private
48
+ def normalize_value(value)
49
+ case value
50
+ when String
51
+ Decimal256.new(value)
52
+ when Float
53
+ Decimal256.new(value.to_s)
54
+ when BigDecimal
55
+ Decimal256.new(value.to_s)
56
+ else
57
+ value
58
+ end
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,25 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class Decimal256Array
20
+ # @since 3.0.0
21
+ def get_value(i)
22
+ BigDecimal(format_value(i))
23
+ end
24
+ end
25
+ end
@@ -0,0 +1,73 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class Decimal256DataType
20
+ MAX_PRECISION = max_precision
21
+
22
+ alias_method :initialize_raw, :initialize
23
+ private :initialize_raw
24
+
25
+ # Creates a new {Arrow::Decimal256DataType}.
26
+ #
27
+ # @overload initialize(precision, scale)
28
+ #
29
+ # @param precision [Integer] The precision of the decimal data
30
+ # type. It's the number of digits including the number of
31
+ # digits after the decimal point.
32
+ #
33
+ # @param scale [Integer] The scale of the decimal data
34
+ # type. It's the number of digits after the decimal point.
35
+ #
36
+ # @example Create a decimal data type for "XXXXXX.YY" decimal
37
+ # Arrow::Decimal256DataType.new(8, 2)
38
+ #
39
+ # @overload initialize(description)
40
+ #
41
+ # @param description [Hash] The description of the decimal data
42
+ # type. It must have `:precision` and `:scale` values.
43
+ #
44
+ # @option description [Integer] :precision The precision of the
45
+ # decimal data type. It's the number of digits including the
46
+ # number of digits after the decimal point.
47
+ #
48
+ # @option description [Integer] :scale The scale of the decimal
49
+ # data type. It's the number of digits after the decimal
50
+ # point.
51
+ #
52
+ # @example Create a decimal data type for "XXXXXX.YY" decimal
53
+ # Arrow::Decimal256DataType.new(precision: 8,
54
+ # scale: 2)
55
+ #
56
+ # @since 3.0.0
57
+ def initialize(*args)
58
+ n_args = args.size
59
+ case n_args
60
+ when 1
61
+ description = args[0]
62
+ precision = description[:precision]
63
+ scale = description[:scale]
64
+ when 2
65
+ precision, scale = args
66
+ else
67
+ message = "wrong number of arguments (given, #{n_args}, expected 1..2)"
68
+ raise ArgumentError, message
69
+ end
70
+ initialize_raw(precision, scale)
71
+ end
72
+ end
73
+ end
@@ -0,0 +1,60 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class Decimal256
20
+ alias_method :to_s_raw, :to_s
21
+
22
+ # @overload to_s
23
+ #
24
+ # @return [String]
25
+ # The string representation of the decimal.
26
+ #
27
+ # @overload to_s(scale)
28
+ #
29
+ # @param scale [Integer] The scale of the decimal.
30
+ # @return [String]
31
+ # The string representation of the decimal including the scale.
32
+ #
33
+ # @since 3.0.0
34
+ def to_s(scale=nil)
35
+ if scale
36
+ to_string_scale(scale)
37
+ else
38
+ to_s_raw
39
+ end
40
+ end
41
+
42
+ alias_method :abs!, :abs
43
+
44
+ # @since 3.0.0
45
+ def abs
46
+ copied = dup
47
+ copied.abs!
48
+ copied
49
+ end
50
+
51
+ alias_method :negate!, :negate
52
+
53
+ # @since 3.0.0
54
+ def negate
55
+ copied = dup
56
+ copied.negate!
57
+ copied
58
+ end
59
+ end
60
+ end
@@ -0,0 +1,24 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class DictionaryArray
20
+ def get_value(i)
21
+ dictionary[indices[i]]
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,38 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class FixedSizeBinaryArrayBuilder
20
+ class << self
21
+ # @since 3.0.0
22
+ def build(data_type, values)
23
+ builder = new(data_type)
24
+ builder.build(values)
25
+ end
26
+ end
27
+
28
+ alias_method :append_values_raw, :append_values
29
+ # @since 3.0.0
30
+ def append_values(values, is_valids=nil)
31
+ if values.is_a?(::Array)
32
+ append_values_raw(values, is_valids)
33
+ else
34
+ append_values_packed(values, is_valids)
35
+ end
36
+ end
37
+ end
38
+ end
@@ -0,0 +1,26 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class FixedSizeBinaryArray
20
+ alias_method :get_value_raw, :get_value
21
+ # @since 3.0.0
22
+ def get_value(i)
23
+ get_value_raw(i).to_s
24
+ end
25
+ end
26
+ end
@@ -41,6 +41,7 @@ module Arrow
41
41
  require "arrow/array"
42
42
  require "arrow/array-builder"
43
43
  require "arrow/bigdecimal-extension"
44
+ require "arrow/buffer"
44
45
  require "arrow/chunked-array"
45
46
  require "arrow/column"
46
47
  require "arrow/compression-type"
@@ -55,10 +56,17 @@ module Arrow
55
56
  require "arrow/decimal128-array"
56
57
  require "arrow/decimal128-array-builder"
57
58
  require "arrow/decimal128-data-type"
59
+ require "arrow/decimal256"
60
+ require "arrow/decimal256-array"
61
+ require "arrow/decimal256-array-builder"
62
+ require "arrow/decimal256-data-type"
58
63
  require "arrow/dense-union-data-type"
64
+ require "arrow/dictionary-array"
59
65
  require "arrow/dictionary-data-type"
60
66
  require "arrow/field"
61
67
  require "arrow/file-output-stream"
68
+ require "arrow/fixed-size-binary-array"
69
+ require "arrow/fixed-size-binary-array-builder"
62
70
  require "arrow/group"
63
71
  require "arrow/list-array-builder"
64
72
  require "arrow/list-data-type"
@@ -69,6 +77,7 @@ module Arrow
69
77
  require "arrow/record-batch"
70
78
  require "arrow/record-batch-builder"
71
79
  require "arrow/record-batch-file-reader"
80
+ require "arrow/record-batch-iterator"
72
81
  require "arrow/record-batch-stream-reader"
73
82
  require "arrow/rolling-window"
74
83
  require "arrow/schema"
@@ -138,6 +147,7 @@ module Arrow
138
147
  when "Arrow::Date32Array",
139
148
  "Arrow::Date64Array",
140
149
  "Arrow::Decimal128Array",
150
+ "Arrow::Decimal256Array",
141
151
  "Arrow::Time32Array",
142
152
  "Arrow::Time64Array",
143
153
  "Arrow::TimestampArray"
@@ -146,6 +156,12 @@ module Arrow
146
156
  method_name = "get_raw_value"
147
157
  end
148
158
  super(info, klass, method_name)
159
+ when "Arrow::Decimal128", "Arrow::Decimal256"
160
+ case method_name
161
+ when "copy"
162
+ method_name = "dup"
163
+ end
164
+ super(info, klass, method_name)
149
165
  else
150
166
  super
151
167
  end
@@ -0,0 +1,47 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class RawTableConverter
20
+ attr_reader :n_rows
21
+ attr_reader :schema
22
+ attr_reader :values
23
+ def initialize(raw_table)
24
+ @raw_table = raw_table
25
+ convert
26
+ end
27
+
28
+ private
29
+ def convert
30
+ if @raw_table.is_a?(::Array) and @raw_table[0].is_a?(Column)
31
+ fields = @raw_table.collect(&:field)
32
+ @schema = Schema.new(fields)
33
+ @values = @raw_table.collect(&:data)
34
+ else
35
+ fields = []
36
+ @values = []
37
+ @raw_table.each do |name, array|
38
+ array = ArrayBuilder.build(array) if array.is_a?(::Array)
39
+ fields << Field.new(name.to_s, array.value_data_type)
40
+ @values << array
41
+ end
42
+ @schema = Schema.new(fields)
43
+ end
44
+ @n_rows = @values[0].length
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,22 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class RecordBatchIterator
20
+ alias_method :to_a, :to_list
21
+ end
22
+ end
@@ -15,6 +15,8 @@
15
15
  # specific language governing permissions and limitations
16
16
  # under the License.
17
17
 
18
+ require "arrow/raw-table-converter"
19
+
18
20
  module Arrow
19
21
  class RecordBatch
20
22
  include ColumnContainable
@@ -25,13 +27,19 @@ module Arrow
25
27
  def new(*args)
26
28
  n_args = args.size
27
29
  case n_args
30
+ when 1
31
+ raw_table_converter = RawTableConverter.new(args[0])
32
+ n_rows = raw_table_converter.n_rows
33
+ schema = raw_table_converter.schema
34
+ values = raw_table_converter.values
35
+ super(schema, n_rows, values)
28
36
  when 2
29
37
  schema, data = args
30
38
  RecordBatchBuilder.build(schema, data)
31
39
  when 3
32
40
  super
33
41
  else
34
- message = "wrong number of arguments (given #{n_args}, expected 2..3)"
42
+ message = "wrong number of arguments (given #{n_args}, expected 1..3)"
35
43
  raise ArgumentError, message
36
44
  end
37
45
  end