red-arrow 0.14.1 → 0.15.0

data/lib/arrow/date64-array-builder.rb

@@ -19,11 +19,11 @@ module Arrow
   class Date64ArrayBuilder
     private
     def convert_to_arrow_value(value)
-      if value.respond_to?(:to_time) and not value.is_a?(Time)
+      if value.respond_to?(:to_time) and not value.is_a?(::Time)
         value = value.to_time
       end
 
-      if value.is_a?(Time)
+      if value.is_a?(::Time)
         value.to_i * 1_000 + value.usec / 1_000
       else
         value

data/lib/arrow/date64-array.rb

@@ -23,7 +23,7 @@ module Arrow
 
     private
     def to_datetime(raw_value)
-      Time.at(*raw_value.divmod(1_000)).to_datetime
+      ::Time.at(*raw_value.divmod(1_000)).to_datetime
     end
   end
 end

data/lib/arrow/decimal128-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Decimal128Array
+    def get_value(i)
+      BigDecimal(format_value(i))
+    end
+  end
+end

data/lib/arrow/field-containable.rb

@@ -24,6 +24,9 @@ module Arrow
         get_field_by_name(name)
       when Integer
         index = name_or_index
+        raise if index < 0
+        index += n_fields if index < 0
+        return nil if index < 0 or index >= n_fields
         get_field(index)
       else
         message = "field name or index must be String, Symbol or Integer"

data/lib/arrow/group.rb

@@ -152,24 +152,21 @@ module Arrow
       end
 
       grouped_key_arrays_raw = grouped_keys.transpose
-      columns = @keys.collect.with_index do |key, i|
+      fields = []
+      arrays = []
+      @keys.each_with_index do |key, i|
         key_column = @table[key]
-        key_column_array_class = key_column.data.chunks.first.class
-        if key_column_array_class == TimestampArray
-          builder = TimestampArrayBuilder.new(key_column.data_type)
-          key_column_array = builder.build(grouped_key_arrays_raw[i])
-        else
-          key_column_array =
-            key_column_array_class.new(grouped_key_arrays_raw[i])
-        end
-        Column.new(key_column.field, key_column_array)
+        key_column_array_raw = grouped_key_arrays_raw[i]
+        key_column_array = key_column.data_type.build_array(key_column_array_raw)
+        fields << key_column.field
+        arrays << key_column_array
       end
       target_columns.each_with_index do |column, i|
         array = ArrayBuilder.build(aggregated_arrays_raw[i])
-        field = Field.new(column.name, array.value_data_type)
-        columns << Column.new(field, array)
+        arrays << array
+        fields << Field.new(column.field.name, array.value_data_type)
       end
-      Table.new(columns)
+      Table.new(fields, arrays)
     end
   end
 end

data/lib/arrow/loader.rb

@@ -46,6 +46,7 @@ module Arrow
       require "arrow/date64-array"
       require "arrow/date64-array-builder"
       require "arrow/decimal128"
+      require "arrow/decimal128-array"
       require "arrow/decimal128-array-builder"
       require "arrow/decimal128-data-type"
       require "arrow/dense-union-data-type"
@@ -75,7 +76,12 @@ module Arrow
       require "arrow/table-loader"
       require "arrow/table-saver"
       require "arrow/tensor"
+      require "arrow/time"
+      require "arrow/time32-array"
+      require "arrow/time32-array-builder"
       require "arrow/time32-data-type"
+      require "arrow/time64-array"
+      require "arrow/time64-array-builder"
       require "arrow/time64-data-type"
       require "arrow/timestamp-array"
       require "arrow/timestamp-array-builder"
@@ -97,6 +103,14 @@ module Arrow
     end
 
     def load_method_info(info, klass, method_name)
+      case klass.name
+      when /Array\z/
+        case method_name
+        when "values"
+          method_name = "values_raw"
+        end
+      end
+
       case klass.name
       when /Builder\z/
         case method_name
@@ -113,7 +127,12 @@ module Arrow
           method_name = "get_value"
         end
         super(info, klass, method_name)
-      when "Arrow::TimestampArray", "Arrow::Date32Array", "Arrow::Date64Array"
+      when "Arrow::Date32Array",
+           "Arrow::Date64Array",
+           "Arrow::Decimal128Array",
+           "Arrow::Time32Array",
+           "Arrow::Time64Array",
+           "Arrow::TimestampArray"
         case method_name
         when "get_value"
           method_name = "get_raw_value"

data/lib/arrow/record-batch.rb

@@ -15,10 +15,12 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require "arrow/column-containable"
 require "arrow/record-containable"
 
 module Arrow
   class RecordBatch
+    include ColumnContainable
     include RecordContainable
     include Enumerable
 
@@ -40,10 +42,10 @@ module Arrow
 
     alias_method :each, :each_record
 
-    alias_method :columns_raw, :columns
-    def columns
-      @columns ||= columns_raw
-    end
+    alias_method :size, :n_rows
+    alias_method :length, :n_rows
+
+    alias_method :[], :find_column
 
     # Converts the record batch to {Arrow::Table}.
     #

data/lib/arrow/record-containable.rb

@@ -17,12 +17,6 @@
 
 module Arrow
   module RecordContainable
-    def each_column(&block)
-      return to_enum(__method__) unless block_given?
-
-      columns.each(&block)
-    end
-
     def each_record(reuse_record: false)
       unless block_given?
         return to_enum(__method__, reuse_record: reuse_record)
@@ -40,34 +34,5 @@ module Arrow
         end
       end
     end
-
-    def find_column(name_or_index)
-      case name_or_index
-      when String, Symbol
-        name = name_or_index.to_s
-        index = resolve_column_name(name)
-        return nil if index.nil?
-        columns[index]
-      when Integer
-        index = name_or_index
-        columns[index]
-      else
-        message = "column name or index must be String, Symbol or Integer"
-        raise ArgumentError, message
-      end
-    end
-
-    private
-    def resolve_column_name(name)
-      (@column_name_to_index ||= build_column_name_resolve_table)[name]
-    end
-
-    def build_column_name_resolve_table
-      table = {}
-      schema.fields.each_with_index do |field, i|
-        table[field.name] = i
-      end
-      table
-    end
   end
 end

data/lib/arrow/record.rb

@@ -17,38 +17,41 @@
 
 module Arrow
   class Record
+    attr_reader :container
     attr_accessor :index
-    def initialize(record_container, index)
-      @record_container = record_container
+    def initialize(container, index)
+      @container = container
       @index = index
     end
 
     def [](column_name_or_column_index)
-      column = @record_container.find_column(column_name_or_column_index)
+      column = @container.find_column(column_name_or_column_index)
       return nil if column.nil?
       column[@index]
     end
 
-    def columns
-      @record_container.columns
+    def to_a
+      @container.columns.collect do |column|
+        column[@index]
+      end
     end
 
     def to_h
       attributes = {}
-      @record_container.schema.fields.each_with_index do |field, i|
-        attributes[field.name] = self[i]
+      @container.columns.each do |column|
+        attributes[column.name] = column[@index]
       end
       attributes
     end
 
     def respond_to_missing?(name, include_private)
-      return true if @record_container.find_column(name)
+      return true if @container.find_column(name)
       super
     end
 
     def method_missing(name, *args, &block)
       if args.empty?
-        column = @record_container.find_column(name)
+        column = @container.find_column(name)
         return column[@index] if column
       end
       super

data/lib/arrow/slicer.rb

@@ -253,9 +253,9 @@ module Arrow
         case @value
         when nil
           if @column.n_nulls.zero?
-            raw_array = [true] * @column.length
+            raw_array = [true] * @column.n_rows
           else
-            raw_array = @column.length.times.collect do |i|
+            raw_array = @column.n_rows.times.collect do |i|
               @column.valid?(i)
             end
           end

data/lib/arrow/struct-array-builder.rb

@@ -73,11 +73,6 @@ module Arrow
           value.each_with_index do |sub_value, i|
             self[i].append(sub_value)
           end
-        when Arrow::Struct
-          append_value_raw
-          value.values.each_with_index do |sub_value, i|
-            self[i].append(sub_value)
-          end
         when Hash
           append_value_raw
           value.each do |name, sub_value|
@@ -85,8 +80,7 @@ module Arrow
           end
         else
           message =
-            "struct value must be nil, Array, " +
-            "Arrow::Struct or Hash: #{value.inspect}"
+            "struct value must be nil, Array or Hash: #{value.inspect}"
           raise ArgumentError, message
         end
       else

data/lib/arrow/struct-array.rb

@@ -15,8 +15,6 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "arrow/struct"
-
 module Arrow
   class StructArray
     # @param i [Integer]
@@ -24,9 +22,13 @@ module Arrow
     #
     #   You can use {Arrow::Array#[]} for convenient value access.
     #
-    # @return [Arrow::Struct] The `i`-th value.
+    # @return [Hash] The `i`-th struct.
     def get_value(i)
-      Struct.new(self, i)
+      value = {}
+      value_data_type.fields.zip(fields) do |field, field_array|
+        value[field.name] = field_array[i]
+      end
+      value
     end
 
     # @overload find_field(index)
@@ -45,20 +47,20 @@ module Arrow
         (@name_to_field ||= build_name_to_field)[name.to_s]
       else
         index = index_or_name
-        cached_fields[index]
+        fields[index]
       end
     end
 
-    private
-    def cached_fields
-      @fields ||= fields
+    alias_method :fields_raw, :fields
+    def fields
+      @fields ||= fields_raw
     end
 
+    private
     def build_name_to_field
       name_to_field = {}
-      field_arrays = cached_fields
-      value_data_type.fields.each_with_index do |field, i|
-        name_to_field[field.name] = field_arrays[i]
+      value_data_type.fields.zip(fields) do |field, field_array|
+        name_to_field[field.name] = field_array
       end
       name_to_field
     end

data/lib/arrow/table-loader.rb

@@ -88,17 +88,11 @@ module Arrow
 
     def load_raw(input, reader)
       schema = reader.schema
-      chunked_arrays = []
+      record_batches = []
       reader.each do |record_batch|
-        record_batch.columns.each_with_index do |array, i|
-          chunked_array = (chunked_arrays[i] ||= [])
-          chunked_array << array
-        end
-      end
-      columns = schema.fields.collect.with_index do |field, i|
-        Column.new(field, ChunkedArray.new(chunked_arrays[i]))
+        record_batches << record_batch
       end
-      table = Table.new(schema, columns)
+      table = Table.new(schema, record_batches)
       table.instance_variable_set(:@input, input)
       table
     end

data/lib/arrow/table-table-formatter.rb

@@ -33,7 +33,7 @@ module Arrow
     def format_column_name(column)
       case column.data_type
       when TimestampDataType
-        "%*s" % [Time.now.iso8601.size, column.name]
+        "%*s" % [::Time.now.iso8601.size, column.name]
       when FloatDataType, DoubleDataType
         "%*s" % [FLOAT_N_DIGITS, column.name]
       else
@@ -55,7 +55,7 @@ module Arrow
 
     def format_column_value(column, value)
       case value
-      when Time
+      when ::Time
         value.iso8601
       when Float
         "%*f" % [[column.name.size, FLOAT_N_DIGITS].max, value]

data/lib/arrow/table.rb

@@ -15,11 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require "arrow/column-containable"
 require "arrow/group"
 require "arrow/record-containable"
 
 module Arrow
   class Table
+    include ColumnContainable
     include RecordContainable
 
     class << self
@@ -74,6 +76,24 @@ module Arrow
     #     Arrow::Table.new("count" => Arrow::ChunkedArray.new(count_chunks),
     #                      "visible" => Arrow::ChunkedArray.new(visible_chunks))
     #
+    # @overload initialize(raw_table)
+    #
+    #   @param raw_table [Hash<String, ::Array>]
+    #     The pairs of column name and values of the table. Column values is
+    #     `Array`.
+    #
+    #   @example Create a table from column name and values
+    #     count_chunks = [
+    #       Arrow::UInt32Array.new([0, 2]),
+    #       Arrow::UInt32Array.new([nil, 4]),
+    #     ]
+    #     visible_chunks = [
+    #       Arrow::BooleanArray.new([true]),
+    #       Arrow::BooleanArray.new([nil, nil, false]),
+    #     ]
+    #     Arrow::Table.new("count" => [0, 2, nil, 4],
+    #                      "visible" => [true, nil, nil, false])
+    #
     # @overload initialize(schema, columns)
     #
     #   @param schema [Arrow::Schema] The schema of the table.
@@ -152,17 +172,18 @@ module Arrow
       case n_args
       when 1
         if args[0][0].is_a?(Column)
-          values = args[0]
-          fields = values.collect(&:field)
+          columns = args[0]
+          fields = columns.collect(&:field)
+          values = columns.collect(&:data)
           schema = Schema.new(fields)
         else
           raw_table = args[0]
           fields = []
           values = []
           raw_table.each do |name, array|
-            field = Field.new(name.to_s, array.value_data_type)
-            fields << field
-            values << Column.new(field, array)
+            array = ArrayBuilder.build(array) if array.is_a?(::Array)
+            fields << Field.new(name.to_s, array.value_data_type)
+            values << array
           end
           schema = Schema.new(fields)
         end
@@ -170,20 +191,19 @@ module Arrow
         schema = args[0]
         schema = Schema.new(schema) unless schema.is_a?(Schema)
         values = args[1]
-        if values[0].is_a?(::Array)
+        case values[0]
+        when ::Array
           values = [RecordBatch.new(schema, values)]
+        when Column
+          values = values.collect(&:data)
         end
       else
-        message = "wrong number of arguments (given, #{n_args}, expected 1..2)"
+        message = "wrong number of arguments (given #{n_args}, expected 1..2)"
         raise ArgumentError, message
       end
       initialize_raw(schema, values)
     end
 
-    def columns
-      @columns ||= n_columns.times.collect {|i| get_column(i)}
-    end
-
     def each_record_batch
       return to_enum(__method__) unless block_given?
 
@@ -338,7 +358,7 @@ module Arrow
         other.each do |name, value|
           name = name.to_s
           if value
-            added_columns[name] = ensure_column(name, value)
+            added_columns[name] = ensure_raw_column(name, value)
           else
             removed_columns[name] = true
           end
@@ -346,7 +366,8 @@ module Arrow
       when Table
         added_columns = {}
         other.columns.each do |column|
-          added_columns[column.name] = column
+          name = column.name
+          added_columns[name] = ensure_raw_column(name, column)
         end
       else
         message = "merge target must be Hash or Arrow::Table: " +
@@ -363,15 +384,18 @@ module Arrow
           next
         end
         next if removed_columns.key?(column_name)
-        new_columns << column
+        new_columns << ensure_raw_column(column_name, column)
       end
       added_columns.each do |name, new_column|
         new_columns << new_column
       end
-      new_fields = new_columns.collect do |new_column|
-        new_column.field
+      new_fields = []
+      new_arrays = []
+      new_columns.each do |new_column|
+        new_fields << new_column[:field]
+        new_arrays << new_column[:data]
       end
-      self.class.new(Schema.new(new_fields), new_columns)
+      self.class.new(new_fields, new_arrays)
     end
 
     alias_method :remove_column_raw, :remove_column
@@ -447,10 +471,10 @@ module Arrow
     end
 
     def pack
-      packed_columns = columns.collect do |column|
-        column.pack
+      packed_arrays = columns.collect do |column|
+        column.data.pack
       end
-      self.class.new(schema, packed_columns)
+      self.class.new(schema, packed_arrays)
     end
 
     alias_method :to_s_raw, :to_s
@@ -524,13 +548,26 @@ module Arrow
       end
     end
 
-    def ensure_column(name, data)
+    def ensure_raw_column(name, data)
       case data
       when Array
-        field = Field.new(name, data.value_data_type)
-        Column.new(field, data)
+        {
+          field: Field.new(name, data.value_data_type),
+          data: ChunkedArray.new([data]),
+        }
+      when ChunkedArray
+        {
+          field: Field.new(name, data.value_data_type),
+          data: data,
+        }
       when Column
-        data
+        column = data
+        data = column.data
+        data = ChunkedArray.new([data]) unless data.is_a?(ChunkedArray)
+        {
+          field: column.field,
+          data: data,
+        }
       else
         message = "column must be Arrow::Array or Arrow::Column: " +
           "<#{name}>: <#{data.inspect}>: #{inspect}"