red-arrow 0.11.0 → 0.12.0

data/lib/arrow/list-array-builder.rb

@@ -0,0 +1,96 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class ListArrayBuilder
+    class << self
+      def build(data_type, values)
+        builder = new(data_type)
+        builder.build(values)
+      end
+    end
+
+    alias_method :append_value_raw, :append_value
+
+    # @overload append_value
+    #
+    #   Starts appending a list record. You also need to append list
+    #   value by {#value_builder}.
+    #
+    # @overload append_value(list)
+    #
+    #   Appends a list record including list value.
+    #
+    #   @param value [nil, ::Array] The list value of the record.
+    #
+    #     If this is `nil`, the list record is null.
+    #
+    #     If this is `Array`, it's the list value of the record.
+    #
+    # @since 0.12.0
+    def append_value(*args)
+      n_args = args.size
+
+      case n_args
+      when 0
+        append_value_raw
+      when 1
+        value = args[0]
+        case value
+        when nil
+          append_null
+        when ::Array
+          append_value_raw
+          @value_builder ||= value_builder
+          @value_builder.append_values(value, nil)
+        else
+          message = "list value must be nil or Array: #{value.inspect}"
+          raise ArgumentError, message
+        end
+      else
+        message = "wrong number of arguments (given #{n_args}, expected 0..1)"
+        raise ArgumentError, message
+      end
+    end
+
+    def append_values(lists, is_valids=nil)
+      if is_valids
+        is_valids.each_with_index do |is_valid, i|
+          if is_valid
+            append_value(lists[i])
+          else
+            append_null
+          end
+        end
+      else
+        lists.each do |list|
+          append_value(list)
+        end
+      end
+    end
+
+    # @since 0.12.0
+    def append(*values)
+      if values.empty?
+        # For backward compatibility
+        append_value
+      else
+        super
+      end
+    end
+  end
+end

data/lib/arrow/list-data-type.rb

@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class ListDataType
+    alias_method :initialize_raw, :initialize
+    private :initialize_raw
+
+    # Creates a new {Arrow::ListDataType}.
+    #
+    # @overload initialize(field)
+    #
+    #   @param field [Arrow::Field, Hash] The field of the list data
+    #     type. You can also specify field description by `Hash`.
+    #
+    #     See {Arrow::Field.new} how to specify field description.
+    #
+    #   @example Create a list data type with {Arrow::Field}
+    #     visible_field = Arrow::Field.new("visible", :boolean)
+    #     Arrow::ListDataType.new(visible_field)
+    #
+    #   @example Create a list data type with field description
+    #     Arrow::ListDataType.new(name: "visible", type: :boolean)
+    #
+    # @overload initialize(description)
+    #
+    #   @param description [Hash] The description of the list data
+    #     type. It must have `:field` value.
+    #
+    #   @option description [Arrow::Field, Hash] :field The field of
+    #     the list data type. You can also specify field description
+    #     by `Hash`.
+    #
+    #     See {Arrow::Field.new} how to specify field description.
+    #
+    #   @example Create a list data type with {Arrow::Field}
+    #     visible_field = Arrow::Field.new("visible", :boolean)
+    #     Arrow::ListDataType.new(field: visible_field)
+    #
+    #   @example Create a list data type with field description
+    #     Arrow::ListDataType.new(field: {name: "visible", type: :boolean})
+    def initialize(field)
+      if field.is_a?(Hash) and field.key?(:field)
+        description = field
+        field = description[:field]
+      end
+      if field.is_a?(Hash)
+        field_description = field
+        field = Field.new(field_description)
+      end
+      initialize_raw(field)
+    end
+  end
+end

data/lib/arrow/loader.rb

@@ -35,18 +35,35 @@ module Arrow
       require "arrow/array-builder"
       require "arrow/chunked-array"
       require "arrow/column"
+      require "arrow/compression-type"
       require "arrow/csv-loader"
-      require "arrow/csv-reader"
+      require "arrow/csv-read-options"
+      require "arrow/data-type"
       require "arrow/date32-array"
       require "arrow/date32-array-builder"
       require "arrow/date64-array"
       require "arrow/date64-array-builder"
+      require "arrow/decimal128-array-builder"
+      require "arrow/decimal128-data-type"
+      require "arrow/dense-union-data-type"
+      require "arrow/dictionary-data-type"
       require "arrow/field"
+      require "arrow/file-output-stream"
+      require "arrow/list-array-builder"
+      require "arrow/list-data-type"
+      require "arrow/path-extension"
       require "arrow/record"
       require "arrow/record-batch"
+      require "arrow/record-batch-builder"
+      require "arrow/record-batch-file-reader"
+      require "arrow/record-batch-stream-reader"
       require "arrow/rolling-window"
+      require "arrow/schema"
       require "arrow/slicer"
+      require "arrow/sparse-union-data-type"
       require "arrow/struct-array"
+      require "arrow/struct-array-builder"
+      require "arrow/struct-data-type"
       require "arrow/table"
       require "arrow/table-formatter"
       require "arrow/table-list-formatter"
@@ -54,11 +71,12 @@ module Arrow
       require "arrow/table-loader"
       require "arrow/table-saver"
       require "arrow/tensor"
+      require "arrow/time32-data-type"
+      require "arrow/time64-data-type"
       require "arrow/timestamp-array"
       require "arrow/timestamp-array-builder"
-
-      require "arrow/record-batch-file-reader"
-      require "arrow/record-batch-stream-reader"
+      require "arrow/timestamp-data-type"
+      require "arrow/writable"
     end
 
     def load_object_info(info)
@@ -72,6 +90,13 @@ module Arrow
 
     def load_method_info(info, klass, method_name)
       case klass.name
+      when /Builder\z/
+        case method_name
+        when "append"
+          return
+        else
+          super
+        end
       when "Arrow::StringArray"
         case method_name
         when "get_value"
@@ -87,7 +112,7 @@ module Arrow
         end
         super(info, klass, method_name)
       else
-       super
+        super
       end
     end
   end

data/lib/arrow/{csv-reader.rb → path-extension.rb}

@@ -15,40 +15,31 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "csv"
-
 module Arrow
-  class CSVReader
-    def initialize(csv)
-      @csv = csv
+  class PathExtension
+    def initialize(path)
+      @path = path
     end
 
-    def read
-      values_set = []
-      @csv.each do |row|
-        if row.is_a?(CSV::Row)
-          row = row.collect(&:last)
-        end
-        row.each_with_index do |value, i|
-          values = (values_set[i] ||= [])
-          values << value
-        end
-      end
-      return nil if values_set.empty?
+    def extract
+      basename = ::File.basename(@path)
+      components = basename.split(".")
+      return {} if components.size == 1
 
-      arrays = values_set.collect.with_index do |values, i|
-        ArrayBuilder.build(values)
-      end
-      if @csv.headers
-        names = @csv.headers
+      extension = components.last.downcase
+      if components.size > 2
+        compression = CompressionType.resolve_extension(extension)
+        if compression
+          {
+            format: components[-2].downcase,
+            compression: compression,
+          }
+        else
+          {format: extension}
+        end
       else
-        names = arrays.size.times.collect(&:to_s)
-      end
-      raw_table = {}
-      names.each_with_index do |name, i|
-        raw_table[name] = arrays[i]
+        {format: extension}
       end
-      Table.new(raw_table)
     end
   end
 end

data/lib/arrow/record-batch-builder.rb

@@ -0,0 +1,115 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class RecordBatchBuilder
+    class << self
+      # @since 0.12.0
+      def build(schema, data)
+        builder = new(schema)
+        builder.append(data)
+        builder.flush
+      end
+    end
+
+    alias_method :initialize_raw, :initialize
+    private :initialize_raw
+    def initialize(schema)
+      unless schema.is_a?(Schema)
+        schema = Schema.new(schema)
+      end
+      initialize_raw(schema)
+      @name_to_index = {}
+      schema.fields.each_with_index do |field, i|
+        @name_to_index[field.name] = i
+      end
+    end
+
+    # @since 0.12.0
+    def [](name_or_index)
+      case name_or_index
+      when String, Symbol
+        name = name_or_index
+        self[resolve_name(name)]
+      else
+        index = name_or_index
+        column_builders[index]
+      end
+    end
+
+    # @since 0.12.0
+    def append(*values)
+      values.each do |value|
+        case value
+        when Hash
+          append_columns(value)
+        else
+          append_records(value)
+        end
+      end
+    end
+
+    # @since 0.12.0
+    def append_records(records)
+      n = n_fields
+      columns = n.times.collect do
+        []
+      end
+      records.each_with_index do |record, nth_record|
+        case record
+        when nil
+        when Hash
+          record.each do |name, value|
+            nth_column = resolve_name(name)
+            next if nth_column.nil?
+            columns[nth_column] << value
+          end
+        else
+          record.each_with_index do |value, nth_column|
+            columns[nth_column] << value
+          end
+        end
+        columns.each do |column|
+          column << nil if column.size != (nth_record + 1)
+        end
+      end
+      columns.each_with_index do |column, i|
+        self[i].append(*column)
+      end
+    end
+
+    # @since 0.12.0
+    def append_columns(columns)
+      columns.each do |name, values|
+        self[name].append(*values)
+      end
+    end
+
+    private
+    def resolve_name(name)
+      @name_to_index[name.to_s]
+    end
+
+    # TODO: Make public with good name. Is column_builders good enough?
+    # builders? sub_builders?
+    def column_builders
+      @column_builders ||= n_fields.times.collect do |i|
+        get_field(i)
+      end
+    end
+  end
+end

data/lib/arrow/record-batch.rb

@@ -22,6 +22,22 @@ module Arrow
     include RecordContainable
     include Enumerable
 
+    class << self
+      def new(*args)
+        n_args = args.size
+        case n_args
+        when 2
+          schema, data = args
+          RecordBatchBuilder.build(schema, data)
+        when 3
+          super
+        else
+          message = "wrong number of arguments (given #{n_args}, expected 2..3)"
+          raise ArgumentError, message
+        end
+      end
+    end
+
     alias_method :each, :each_record
 
     alias_method :columns_raw, :columns
@@ -29,6 +45,15 @@ module Arrow
       @columns ||= columns_raw
     end
 
+    # Converts the record batch to {Arrow::Table}.
+    #
+    # @return [Arrow::Table]
+    #
+    # @since 0.12.0
+    def to_table
+      Table.new(schema, [self])
+    end
+
     def respond_to_missing?(name, include_private)
       return true if find_column(name)
       super