red-arrow 0.11.0 → 0.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 42b3a66d39aaa8365dcbb141542f4e6d4b0a0a6b6848cd7a386bd8ec3e01585d
-  data.tar.gz: c3b10778cb4cb2d1647d25e600120c38d470cf95f9cc14d6ffb9cb2bbf3fe50f
+  metadata.gz: 847a4994cc15fb50df335c7231c942d49392f1ec528b85647b2bbe6fb4e82f7b
+  data.tar.gz: 4337680e47dea67107a1fef863d66e936b9c5d20b2cb6698e879b774eada3c88
 SHA512:
-  metadata.gz: 625c274985358822083c9be7082badd2581df26bbb0a428e620fc8a4c55bc569f624819064f4f690e5c31b1abf2265c605a48affe1724d0bf3aec3a5ed1a6fd5
-  data.tar.gz: 9934efa27ff209c2fdfa4a47254d1368f06f18a7e7affa694fd94fc26f1ceb8aac7ff61c57228542a6a342c744474d16f42c5aebef95c5706cac370ccee1ba11
+  metadata.gz: c72b26d9b4f488c4d00184ea3243056c69a601fe59fba80a7423075b19034b0db7ad12dbbf5e210ea4b66607acd2ad7123a85057a27266b21665e143961bccb7
+  data.tar.gz: 1fb8c82007a25cac5e99d41f3bb879ed86ad1f9deab320f45470252513fada6c6f972f7ddb0ed3f38ca1dcdf7de3f5662a3fa28536a4981b9e5993dd5bc666a8

data/README.md

@@ -39,7 +39,7 @@ Note that the Apache Arrow GLib packages are "unofficial". "Official" packages w
 
 Install Red Arrow after you install Apache Arrow GLib:
 
-```text
+```console
 % gem install red-arrow
 ```
 

data/Rakefile

@@ -19,6 +19,7 @@
 
 require "rubygems"
 require "bundler/gem_helper"
+require "yard"
 
 base_dir = File.join(__dir__)
 
@@ -37,3 +38,6 @@ task :test do
 end
 
 task default: :test
+
+YARD::Rake::YardocTask.new do |task|
+end

data/lib/arrow/array-builder.rb

@@ -65,6 +65,12 @@ module Arrow
     end
 
     def build(values)
+      append(*values)
+      finish
+    end
+
+    # @since 0.12.0
+    def append(*values)
       value_convertable = respond_to?(:convert_to_arrow_value, true)
       start_index = 0
       current_index = 0
@@ -111,8 +117,6 @@ module Arrow
           append_nulls(current_index - start_index)
         end
       end
-
-      finish
     end
 
     def append_nulls(n)

data/lib/arrow/array.rb

@@ -20,11 +20,15 @@ module Arrow
     include Enumerable
 
     class << self
-      def new(values)
+      def new(*args)
         builder_class_name = "#{name}Builder"
         if const_defined?(builder_class_name)
           builder_class = const_get(builder_class_name)
-          builder_class.build(values)
+          if args.size == builder_class.method(:build).arity
+            builder_class.build(*args)
+          else
+            super
+          end
         else
           super
         end

data/{test/test-csv-reader.rb → lib/arrow/compression-type.rb}

@@ -15,20 +15,23 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class CSVReaderTest < Test::Unit::TestCase
-  include Helper::Fixture
+module Arrow
+  class CompressionType
+    EXTENSIONS = {}
+    values.each do |value|
+      case value
+      when UNCOMPRESSED
+      when GZIP
+        EXTENSIONS["gz"] = value
+      else
+        EXTENSIONS[value.nick] = value
+      end
+    end
 
-  test("#read") do
-    CSV.open(fixture_path("with-header.csv").to_s,
-             headers: true,
-             skip_lines: /^#/) do |csv|
-      reader = Arrow::CSVReader.new(csv)
-      assert_equal(<<-TABLE, reader.read.to_s)
-	name	score
-0	alice	10   
-1	bob 	29   
-2	chris	-1   
-      TABLE
+    class << self
+      def resolve_extension(extension)
+        EXTENSIONS[extension.to_s]
+      end
     end
   end
 end

data/lib/arrow/csv-loader.rb

@@ -30,6 +30,7 @@ module Arrow
     def initialize(path_or_data, **options)
       @path_or_data = path_or_data
       @options = options
+      @compression = @options.delete(:compression)
     end
 
     def load
@@ -60,11 +61,85 @@ module Arrow
     end
 
     def read_csv(csv)
-      reader = CSVReader.new(csv)
-      reader.read
+      values_set = []
+      csv.each do |row|
+        if row.is_a?(CSV::Row)
+          row = row.collect(&:last)
+        end
+        row.each_with_index do |value, i|
+          values = (values_set[i] ||= [])
+          values << value
+        end
+      end
+      return nil if values_set.empty?
+
+      arrays = values_set.collect.with_index do |values, i|
+        ArrayBuilder.build(values)
+      end
+      if csv.headers
+        names = csv.headers
+      else
+        names = arrays.size.times.collect(&:to_s)
+      end
+      raw_table = {}
+      names.each_with_index do |name, i|
+        raw_table[name] = arrays[i]
+      end
+      Table.new(raw_table)
+    end
+
+    def reader_options
+      options = CSVReadOptions.new
+      @options.each do |key, value|
+        case key
+        when :headers
+          if value
+            options.n_header_rows = 1
+          else
+            options.n_header_rows = 0
+          end
+        when :column_types
+          value.each do |name, type|
+            options.add_column_type(name, type)
+          end
+        when :schema
+          options.add_schema(value)
+        else
+          setter = "#{key}="
+          if options.respond_to?(setter)
+            options.__send__(setter, value)
+          else
+            return nil
+          end
+        end
+      end
+      options
+    end
+
+    def open_input(raw_input)
+      if @compression
+        codec = Codec.new(@compression)
+        CompressedInputStream.open(codec, raw_input) do |input|
+          yield(input)
+        end
+      else
+        yield(raw_input)
+      end
     end
 
     def load_from_path(path)
+      options = reader_options
+      if options
+        begin
+          MemoryMappedInputStream.open(path.to_s) do |raw_input|
+            open_input(raw_input) do |input|
+              return CSVReader.new(input, options).read
+            end
+          end
+        rescue Arrow::Error::Invalid
+        end
+      end
+
       options = update_csv_parse_options(@options, :open_csv, path)
       open_csv(path, **options) do |csv|
         read_csv(csv)
@@ -72,6 +147,18 @@ module Arrow
     end
 
     def load_data(data)
+      options = reader_options
+      if options
+        begin
+          BufferInputStream.open(Buffer.new(data)) do |raw_input|
+            open_input(raw_input) do |input|
+              return CSVReader.new(input, options).read
+            end
+          end
+        rescue Arrow::Error::Invalid
+        end
+      end
+
       options = update_csv_parse_options(@options, :parse_csv_data, data)
       parse_csv_data(data, **options) do |csv|
         read_csv(csv)
@@ -119,6 +206,11 @@ module Arrow
       end
     end
 
+    AVAILABLE_CSV_PARSE_OPTIONS = {}
+    CSV.instance_method(:initialize).parameters.each do |type, name|
+      AVAILABLE_CSV_PARSE_OPTIONS[name] = true if type == :key
+    end
+
     def update_csv_parse_options(options, create_csv, *args)
       if options.key?(:converters)
         new_options = options.dup
@@ -127,6 +219,14 @@ module Arrow
         new_options = options.merge(converters: converters)
       end
 
+      # TODO: Support :schema and :column_types
+
+      unless AVAILABLE_CSV_PARSE_OPTIONS.empty?
+        new_options.select! do |key, value|
+          AVAILABLE_CSV_PARSE_OPTIONS.key?(key)
+        end
+      end
+
       unless options.key?(:headers)
         __send__(create_csv, *args, **new_options) do |csv|
           new_options[:headers] = have_header?(csv)

data/lib/arrow/csv-read-options.rb

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class CSVReadOptions
+    alias_method :add_column_type_raw, :add_column_type
+    def add_column_type(name, type)
+      add_column_type_raw(name, DataType.resolve(type))
+    end
+  end
+end

data/lib/arrow/data-type.rb

@@ -0,0 +1,135 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DataType
+    class << self
+      # Creates a new suitable {Arrow::DataType}.
+      #
+      # @overload resolve(data_type)
+      #
+      #   Returns the given data type itself. This is convenient to
+      #   use this method as {Arrow::DataType} converter.
+      #
+      #   @param data_type [Arrow::DataType] The data type.
+      #
+      #   @return [Arrow::DataType] The given data type itself.
+      #
+      # @overload resolve(name, *arguments)
+      #
+      #   Creates a suitable data type from type name. For example,
+      #   you can create {Arrow::BooleanDataType} from `:boolean`.
+      #
+      #   @param name [String, Symbol] The type name of the data type.
+      #
+      #   @param arguments [::Array] The additional information of the
+      #     data type.
+      #
+      #     For example, {Arrow::TimestampDataType} needs unit as
+      #     additional information.
+      #
+      #   @example Create a boolean data type
+      #     Arrow::DataType.resolve(:boolean)
+      #
+      #   @example Create a milliseconds unit timestamp data type
+      #     Arrow::DataType.resolve(:timestamp, :milli)
+      #
+      # @overload resolve(description)
+      #
+      #   Creates a suitable data type from data type description.
+      #
+      #   Data type description is a raw `Hash`. Data type description
+      #   must have `:type` value. `:type` is the type of the data type.
+      #
+      #   If the type needs additional information, you need to
+      #   specify it. See constructor document what information is
+      #   needed. For example, {Arrow::ListDataType#initialize} needs
+      #   `:field` value.
+      #
+      #   @param description [Hash] The description of the data type.
+      #
+      #   @option description [String, Symbol] :type The type name of
+      #     the data type.
+      #
+      #   @example Create a boolean data type
+      #     Arrow::DataType.resolve(type: :boolean)
+      #
+      #   @example Create a list data type
+      #     Arrow::DataType.resolve(type: :list,
+      #                             field: {name: "visible", type: :boolean})
+      def resolve(data_type)
+        case data_type
+        when DataType
+          data_type
+        when String, Symbol
+          resolve_class(data_type).new
+        when ::Array
+          type, *arguments = data_type
+          resolve_class(type).new(*arguments)
+        when Hash
+          type = nil
+          description = {}
+          data_type.each do |key, value|
+            key = key.to_sym
+            case key
+            when :type
+              type = value
+            else
+              description[key] = value
+            end
+          end
+          if type.nil?
+            message =
+              "data type description must have :type value: #{data_type.inspect}"
+            raise ArgumentError, message
+          end
+          data_type_class = resolve_class(type)
+          if description.empty?
+            data_type_class.new
+          else
+            data_type_class.new(description)
+          end
+        else
+          message =
+            "data type must be " +
+            "Arrow::DataType, String, Symbol, [String, ...], [Symbol, ...] " +
+            "{type: String, ...} or {type: Symbol, ...}: #{data_type.inspect}"
+          raise ArgumentError, message
+        end
+      end
+
+      private
+      def resolve_class(data_type)
+        data_type_name = data_type.to_s.capitalize.gsub(/\AUint/, "UInt")
+        data_type_class_name = "#{data_type_name}DataType"
+        unless Arrow.const_defined?(data_type_class_name)
+          available_types = []
+          Arrow.constants.each do |name|
+            if name.to_s.end_with?("DataType")
+              available_types << name.to_s.gsub(/DataType\z/, "").downcase.to_sym
+            end
+          end
+          message =
+            "unknown type: #{data_type.inspect}: " +
+            "available types: #{available_types.inspect}"
+          raise ArgumentError, message
+        end
+        Arrow.const_get(data_type_class_name)
+      end
+    end
+  end
+end

data/lib/arrow/decimal128-array-builder.rb

@@ -0,0 +1,64 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require "bigdecimal"
+
+module Arrow
+  class Decimal128ArrayBuilder
+    class << self
+      def build(data_type, values)
+        builder = new(data_type)
+        builder.build(values)
+      end
+    end
+
+    alias_method :append_value_raw, :append_value
+    def append_value(value)
+      case value
+      when nil
+        return append_null
+      when String
+        value = Decimal128.new(value)
+      when Float
+        value = Decimal128.new(value.to_s)
+      when BigDecimal
+        value = Decimal128.new(value.to_s)
+      end
+      append_value_raw(value)
+    end
+
+    def append_values(values, is_valids=nil)
+      if is_valids
+        is_valids.each_with_index do |is_valid, i|
+          if is_valid
+            append_value(values[i])
+          else
+            append_null
+          end
+        end
+      else
+        values.each do |value|
+          if value.nil?
+            append_null
+          else
+            append_value(value)
+          end
+        end
+      end
+    end
+  end
+end