red-arrow 0.17.0 → 3.0.0

data/lib/arrow/decimal256-array-builder.rb

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Decimal256ArrayBuilder
+    class << self
+      # @since 3.0.0
+      def build(data_type, values)
+        builder = new(data_type)
+        builder.build(values)
+      end
+    end
+
+    alias_method :append_value_raw, :append_value
+    # @since 3.0.0
+    def append_value(value)
+      append_value_raw(normalize_value(value))
+    end
+
+    alias_method :append_values_raw, :append_values
+    # @since 3.0.0
+    def append_values(values, is_valids=nil)
+      if values.is_a?(::Array)
+        values = values.collect do |value|
+          normalize_value(value)
+        end
+        append_values_raw(values, is_valids)
+      else
+        append_values_packed(values, is_valids)
+      end
+    end
+
+    private
+    def normalize_value(value)
+      case value
+      when String
+        Decimal256.new(value)
+      when Float
+        Decimal256.new(value.to_s)
+      when BigDecimal
+        Decimal256.new(value.to_s)
+      else
+        value
+      end
+    end
+  end
+end

data/lib/arrow/decimal256-array.rb

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Decimal256Array
+    # @since 3.0.0
+    def get_value(i)
+      BigDecimal(format_value(i))
+    end
+  end
+end

data/lib/arrow/decimal256-data-type.rb

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Decimal256DataType
+    MAX_PRECISION = max_precision
+
+    alias_method :initialize_raw, :initialize
+    private :initialize_raw
+
+    # Creates a new {Arrow::Decimal256DataType}.
+    #
+    # @overload initialize(precision, scale)
+    #
+    #   @param precision [Integer] The precision of the decimal data
+    #     type. It's the number of digits including the number of
+    #     digits after the decimal point.
+    #
+    #   @param scale [Integer] The scale of the decimal data
+    #     type. It's the number of digits after the decimal point.
+    #
+    #   @example Create a decimal data type for "XXXXXX.YY" decimal
+    #     Arrow::Decimal256DataType.new(8, 2)
+    #
+    # @overload initialize(description)
+    #
+    #   @param description [Hash] The description of the decimal data
+    #     type. It must have `:precision` and `:scale` values.
+    #
+    #   @option description [Integer] :precision The precision of the
+    #     decimal data type. It's the number of digits including the
+    #     number of digits after the decimal point.
+    #
+    #   @option description [Integer] :scale The scale of the decimal
+    #     data type. It's the number of digits after the decimal
+    #     point.
+    #
+    #   @example Create a decimal data type for "XXXXXX.YY" decimal
+    #     Arrow::Decimal256DataType.new(precision: 8,
+    #                                   scale: 2)
+    #
+    # @since 3.0.0
+    def initialize(*args)
+      n_args = args.size
+      case n_args
+      when 1
+        description = args[0]
+        precision = description[:precision]
+        scale = description[:scale]
+      when 2
+        precision, scale = args
+      else
+        message = "wrong number of arguments (given, #{n_args}, expected 1..2)"
+        raise ArgumentError, message
+      end
+      initialize_raw(precision, scale)
+    end
+  end
+end

data/lib/arrow/decimal256.rb

@@ -0,0 +1,60 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Decimal256
+    alias_method :to_s_raw, :to_s
+
+    # @overload to_s
+    #
+    #   @return [String]
+    #     The string representation of the decimal.
+    #
+    # @overload to_s(scale)
+    #
+    #   @param scale [Integer] The scale of the decimal.
+    #   @return [String]
+    #      The string representation of the decimal including the scale.
+    #
+    # @since 3.0.0
+    def to_s(scale=nil)
+      if scale
+        to_string_scale(scale)
+      else
+        to_s_raw
+      end
+    end
+
+    alias_method :abs!, :abs
+
+    # @since 3.0.0
+    def abs
+      copied = dup
+      copied.abs!
+      copied
+    end
+
+    alias_method :negate!, :negate
+
+    # @since 3.0.0
+    def negate
+      copied = dup
+      copied.negate!
+      copied
+    end
+  end
+end

data/lib/arrow/dictionary-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DictionaryArray
+    def get_value(i)
+      dictionary[indices[i]]
+    end
+  end
+end

data/lib/arrow/fixed-size-binary-array-builder.rb

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class FixedSizeBinaryArrayBuilder
+    class << self
+      # @since 3.0.0
+      def build(data_type, values)
+        builder = new(data_type)
+        builder.build(values)
+      end
+    end
+
+    alias_method :append_values_raw, :append_values
+    # @since 3.0.0
+    def append_values(values, is_valids=nil)
+      if values.is_a?(::Array)
+        append_values_raw(values, is_valids)
+      else
+        append_values_packed(values, is_valids)
+      end
+    end
+  end
+end

data/lib/arrow/fixed-size-binary-array.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class FixedSizeBinaryArray
+    alias_method :get_value_raw, :get_value
+    # @since 3.0.0
+    def get_value(i)
+      get_value_raw(i).to_s
+    end
+  end
+end

data/lib/arrow/loader.rb

@@ -41,6 +41,7 @@ module Arrow
       require "arrow/array"
       require "arrow/array-builder"
       require "arrow/bigdecimal-extension"
+      require "arrow/buffer"
       require "arrow/chunked-array"
       require "arrow/column"
       require "arrow/compression-type"
@@ -55,10 +56,17 @@ module Arrow
       require "arrow/decimal128-array"
       require "arrow/decimal128-array-builder"
       require "arrow/decimal128-data-type"
+      require "arrow/decimal256"
+      require "arrow/decimal256-array"
+      require "arrow/decimal256-array-builder"
+      require "arrow/decimal256-data-type"
       require "arrow/dense-union-data-type"
+      require "arrow/dictionary-array"
       require "arrow/dictionary-data-type"
       require "arrow/field"
       require "arrow/file-output-stream"
+      require "arrow/fixed-size-binary-array"
+      require "arrow/fixed-size-binary-array-builder"
       require "arrow/group"
       require "arrow/list-array-builder"
       require "arrow/list-data-type"
@@ -69,6 +77,7 @@ module Arrow
       require "arrow/record-batch"
       require "arrow/record-batch-builder"
       require "arrow/record-batch-file-reader"
+      require "arrow/record-batch-iterator"
       require "arrow/record-batch-stream-reader"
       require "arrow/rolling-window"
       require "arrow/schema"
@@ -138,6 +147,7 @@ module Arrow
       when "Arrow::Date32Array",
            "Arrow::Date64Array",
            "Arrow::Decimal128Array",
+           "Arrow::Decimal256Array",
            "Arrow::Time32Array",
            "Arrow::Time64Array",
            "Arrow::TimestampArray"
@@ -146,6 +156,12 @@ module Arrow
           method_name = "get_raw_value"
         end
         super(info, klass, method_name)
+      when "Arrow::Decimal128", "Arrow::Decimal256"
+        case method_name
+        when "copy"
+          method_name = "dup"
+        end
+        super(info, klass, method_name)
       else
         super
       end

data/lib/arrow/raw-table-converter.rb

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class RawTableConverter
+    attr_reader :n_rows
+    attr_reader :schema
+    attr_reader :values
+    def initialize(raw_table)
+      @raw_table = raw_table
+      convert
+    end
+
+    private
+    def convert
+      if @raw_table.is_a?(::Array) and @raw_table[0].is_a?(Column)
+        fields = @raw_table.collect(&:field)
+        @schema = Schema.new(fields)
+        @values = @raw_table.collect(&:data)
+      else
+        fields = []
+        @values = []
+        @raw_table.each do |name, array|
+          array = ArrayBuilder.build(array) if array.is_a?(::Array)
+          fields << Field.new(name.to_s, array.value_data_type)
+          @values << array
+        end
+        @schema = Schema.new(fields)
+      end
+      @n_rows = @values[0].length
+    end
+  end
+end

data/lib/arrow/record-batch-iterator.rb

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class RecordBatchIterator
+    alias_method :to_a, :to_list
+  end
+end

data/lib/arrow/record-batch.rb

@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require "arrow/raw-table-converter"
+
 module Arrow
   class RecordBatch
     include ColumnContainable
@@ -25,13 +27,19 @@ module Arrow
       def new(*args)
         n_args = args.size
         case n_args
+        when 1
+          raw_table_converter = RawTableConverter.new(args[0])
+          n_rows = raw_table_converter.n_rows
+          schema = raw_table_converter.schema
+          values = raw_table_converter.values
+          super(schema, n_rows, values)
         when 2
           schema, data = args
           RecordBatchBuilder.build(schema, data)
         when 3
           super
         else
-          message = "wrong number of arguments (given #{n_args}, expected 2..3)"
+          message = "wrong number of arguments (given #{n_args}, expected 1..3)"
           raise ArgumentError, message
         end
       end