red-arrow 0.12.0 → 0.13.0

data/lib/arrow/struct-array-builder.rb

@@ -71,17 +71,17 @@ module Arrow
         when ::Array
           append_value_raw
           value.each_with_index do |sub_value, i|
-            self[i].append_value(sub_value)
+            self[i].append(sub_value)
           end
         when Arrow::Struct
           append_value_raw
           value.values.each_with_index do |sub_value, i|
-            self[i].append_value(sub_value)
+            self[i].append(sub_value)
           end
         when Hash
           append_value_raw
           value.each do |name, sub_value|
-            self[name].append_value(sub_value)
+            self[name].append(sub_value)
           end
         else
           message =

data/lib/arrow/struct-array.rb

@@ -19,17 +19,25 @@ require "arrow/struct"
 
 module Arrow
   class StructArray
-    def [](i)
-      warn("Use #{self.class}\#find_field instead. " +
-           "This will returns Arrow::Struct instead of Arrow::Array " +
-           "since 0.13.0.")
-      get_field(i)
-    end
-
+    # @param i [Integer]
+    #   The index of the value to be gotten. You must specify the value index.
+    #
+    #   You can use {Arrow::Array#[]} for convenient value access.
+    #
+    # @return [Arrow::Struct] The `i`-th value.
     def get_value(i)
       Struct.new(self, i)
     end
 
+    # @overload find_field(index)
+    #   @param index [Integer] The index of the field to be found.
+    #   @return [Arrow::Array, nil]
+    #      The `index`-th field or `nil` for out of range.
+    #
+    # @overload find_field(name)
+    #   @param index [String, Symbol] The name of the field to be found.
+    #   @return [Arrow::Array, nil]
+    #      The field that has `name` or `nil` for nonexistent name.
     def find_field(index_or_name)
       case index_or_name
       when String, Symbol

data/lib/arrow/struct.rb

@@ -64,5 +64,16 @@ module Arrow
       end
       super
     end
+
+    def ==(other)
+      other.is_a?(self.class) and
+        @array == other.array and
+        @index == other.index
+    end
+
+    protected
+    def array
+      @array
+    end
   end
 end

data/lib/arrow/table-loader.rb

@@ -18,14 +18,14 @@
 module Arrow
   class TableLoader
     class << self
-      def load(output, options={})
-        new(output, options).load
+      def load(input, options={})
+        new(input, options).load
       end
     end
 
-    def initialize(output, options={})
-      output = output.to_path if output.respond_to?(:to_path)
-      @output = output
+    def initialize(input, options={})
+      input = input.to_path if input.respond_to?(:to_path)
+      @input = input
       @options = options
       fill_options
     end
@@ -50,7 +50,7 @@ module Arrow
         __send__(custom_load_method)
       else
         # For backward compatibility.
-        __send__(custom_load_method, @output)
+        __send__(custom_load_method, @input)
       end
     end
 
@@ -60,10 +60,10 @@ module Arrow
         return
       end
 
-      if @output.is_a?(Buffer)
+      if @input.is_a?(Buffer)
         info = {}
       else
-        extension = PathExtension.new(@output)
+        extension = PathExtension.new(@input)
         info = extension.extract
       end
       format = info[:format]
@@ -79,10 +79,10 @@ module Arrow
     end
 
     def open_input_stream
-      if @output.is_a?(Buffer)
-        BufferInputStream.new(@output)
+      if @input.is_a?(Buffer)
+        BufferInputStream.new(@input)
       else
-        MemoryMappedInputStream.new(@output)
+        MemoryMappedInputStream.new(@input)
       end
     end
 
@@ -152,10 +152,10 @@ module Arrow
     def load_as_csv
       options = @options.dup
       options.delete(:format)
-      if @output.is_a?(Buffer)
-        CSVLoader.load(@output.data.to_s, options)
+      if @input.is_a?(Buffer)
+        CSVLoader.load(@input.data.to_s, options)
       else
-        CSVLoader.load(Pathname.new(@output), options)
+        CSVLoader.load(Pathname.new(@input), options)
       end
     end
 

data/lib/arrow/version.rb

@@ -16,7 +16,7 @@
 # under the License.
 
 module Arrow
-  VERSION = "0.12.0"
+  VERSION = "0.13.0"
 
   module Version
     numbers, TAG = VERSION.split("-")

data/red-arrow.gemspec

@@ -39,21 +39,25 @@ Gem::Specification.new do |spec|
   spec.license = "Apache-2.0"
   spec.files = ["README.md", "Rakefile", "Gemfile", "#{spec.name}.gemspec"]
   spec.files += ["LICENSE.txt", "NOTICE.txt"]
+  spec.files += Dir.glob("ext/**/*.{cpp,hpp,rb}")
   spec.files += Dir.glob("lib/**/*.rb")
   spec.files += Dir.glob("image/*.*")
   spec.files += Dir.glob("doc/text/*")
   spec.test_files += Dir.glob("test/**/*")
-  spec.extensions = ["dependency-check/Rakefile"]
+  spec.extensions = ["ext/arrow/extconf.rb"]
 
-  spec.add_runtime_dependency("gobject-introspection", ">= 3.3.1")
-  spec.add_runtime_dependency("pkg-config")
+  spec.add_runtime_dependency("extpp", ">= 0.0.7")
+  spec.add_runtime_dependency("gio2", ">= 3.3.6")
   spec.add_runtime_dependency("native-package-installer")
+  spec.add_runtime_dependency("pkg-config")
 
+  spec.add_development_dependency("benchmark-driver")
   spec.add_development_dependency("bundler")
+  spec.add_development_dependency("faker")
   spec.add_development_dependency("rake")
   spec.add_development_dependency("redcarpet")
   spec.add_development_dependency("test-unit")
   spec.add_development_dependency("yard")
 
-  spec.metadata["msys2_mingw_dependencies"] = "apache-arrow"
+  spec.metadata["msys2_mingw_dependencies"] = "arrow"
 end

data/test/raw-records/record-batch/test-basic-arrays.rb

@@ -0,0 +1,349 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+class RawRecordsRecordBatchBasicArraysTest < Test::Unit::TestCase
+  test("NullArray") do
+    records = [
+      [nil],
+      [nil],
+      [nil],
+      [nil],
+    ]
+    array = Arrow::NullArray.new(records.size)
+    schema = Arrow::Schema.new(column: :null)
+    record_batch = Arrow::RecordBatch.new(schema,
+                                          records.size,
+                                          [array])
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("BooleanArray") do
+    records = [
+      [true],
+      [nil],
+      [false],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :boolean},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("Int8Array") do
+    records = [
+      [-(2 ** 7)],
+      [nil],
+      [(2 ** 7) - 1],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :int8},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("UInt8Array") do
+    records = [
+      [0],
+      [nil],
+      [(2 ** 8) - 1],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :uint8},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("Int16Array") do
+    records = [
+      [-(2 ** 15)],
+      [nil],
+      [(2 ** 15) - 1],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :int16},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("UInt16Array") do
+    records = [
+      [0],
+      [nil],
+      [(2 ** 16) - 1],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :uint16},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("Int32Array") do
+    records = [
+      [-(2 ** 31)],
+      [nil],
+      [(2 ** 31) - 1],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :int32},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("UInt32Array") do
+    records = [
+      [0],
+      [nil],
+      [(2 ** 32) - 1],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :uint32},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("Int64Array") do
+    records = [
+      [-(2 ** 63)],
+      [nil],
+      [(2 ** 63) - 1],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :int64},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("UInt64Array") do
+    records = [
+      [0],
+      [nil],
+      [(2 ** 64) - 1],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :uint64},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("FloatArray") do
+    records = [
+      [-1.0],
+      [nil],
+      [1.0],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :float},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("DoubleArray") do
+    records = [
+      [-1.0],
+      [nil],
+      [1.0],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :double},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("BinaryArray") do
+    records = [
+      ["\x00".b],
+      [nil],
+      ["\xff".b],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :binary},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("StringArray") do
+    records = [
+      ["Ruby"],
+      [nil],
+      ["\u3042"], # U+3042 HIRAGANA LETTER A
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :string},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("Date32Array") do
+    records = [
+      [Date.new(1960, 1, 1)],
+      [nil],
+      [Date.new(2017, 8, 23)],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :date32},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  test("Date64Array") do
+    records = [
+      [DateTime.new(1960, 1, 1, 2, 9, 30)],
+      [nil],
+      [DateTime.new(2017, 8, 23, 14, 57, 2)],
+    ]
+    record_batch = Arrow::RecordBatch.new({column: :date64},
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+
+  sub_test_case("TimestampArray") do
+    test("second") do
+      records = [
+        [Time.parse("1960-01-01T02:09:30Z")],
+        [nil],
+        [Time.parse("2017-08-23T14:57:02Z")],
+      ]
+      record_batch = Arrow::RecordBatch.new({
+                                              column: {
+                                                type: :timestamp,
+                                                unit: :second,
+                                              }
+                                            },
+                                            records)
+      assert_equal(records, record_batch.raw_records)
+    end
+
+    test("milli") do
+      records = [
+        [Time.parse("1960-01-01T02:09:30.123Z")],
+        [nil],
+        [Time.parse("2017-08-23T14:57:02.987Z")],
+      ]
+      record_batch = Arrow::RecordBatch.new({
+                                              column: {
+                                                type: :timestamp,
+                                                unit: :milli,
+                                              }
+                                            },
+                                            records)
+      assert_equal(records, record_batch.raw_records)
+    end
+
+    test("micro") do
+      records = [
+        [Time.parse("1960-01-01T02:09:30.123456Z")],
+        [nil],
+        [Time.parse("2017-08-23T14:57:02.987654Z")],
+      ]
+      record_batch = Arrow::RecordBatch.new({
+                                              column: {
+                                                type: :timestamp,
+                                                unit: :micro,
+                                              }
+                                            },
+                                            records)
+      assert_equal(records, record_batch.raw_records)
+    end
+
+    test("nano") do
+      records = [
+        [Time.parse("1960-01-01T02:09:30.123456789Z")],
+        [nil],
+        [Time.parse("2017-08-23T14:57:02.987654321Z")],
+      ]
+      record_batch = Arrow::RecordBatch.new({
+                                              column: {
+                                                type: :timestamp,
+                                                unit: :nano,
+                                              }
+                                            },
+                                            records)
+      assert_equal(records, record_batch.raw_records)
+    end
+  end
+
+  sub_test_case("Time32Array") do
+    test("second") do
+      records = [
+        [60 * 10], # 00:10:00
+        [nil],
+        [60 * 60 * 2 + 9], # 02:00:09
+      ]
+      record_batch = Arrow::RecordBatch.new({
+                                              column: {
+                                                type: :time32,
+                                                unit: :second,
+                                              }
+                                            },
+                                            records)
+      assert_equal(records, record_batch.raw_records)
+    end
+
+    test("milli") do
+      records = [
+        [(60 * 10) * 1000 + 123], # 00:10:00.123
+        [nil],
+        [(60 * 60 * 2 + 9) * 1000 + 987], # 02:00:09.987
+      ]
+      record_batch = Arrow::RecordBatch.new({
+                                              column: {
+                                                type: :time32,
+                                                unit: :milli,
+                                              }
+                                            },
+                                            records)
+      assert_equal(records, record_batch.raw_records)
+    end
+  end
+
+  sub_test_case("Time64Array") do
+    test("micro") do
+      records = [
+        [(60 * 10) * 1_000_000 + 123_456], # 00:10:00.123456
+        [nil],
+        [(60 * 60 * 2 + 9) * 1_000_000 + 987_654], # 02:00:09.987654
+      ]
+      record_batch = Arrow::RecordBatch.new({
+                                              column: {
+                                                type: :time64,
+                                                unit: :micro,
+                                              }
+                                            },
+                                            records)
+      assert_equal(records, record_batch.raw_records)
+    end
+
+    test("nano") do
+      records = [
+        [(60 * 10) * 1_000_000_000 + 123_456_789], # 00:10:00.123456789
+        [nil],
+        [(60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321], # 02:00:09.987654321
+      ]
+      record_batch = Arrow::RecordBatch.new({
+                                              column: {
+                                                type: :time64,
+                                                unit: :nano,
+                                              }
+                                            },
+                                            records)
+      assert_equal(records, record_batch.raw_records)
+    end
+  end
+
+  test("Decimal128Array") do
+    records = [
+      [BigDecimal("92.92")],
+      [nil],
+      [BigDecimal("29.29")],
+    ]
+    record_batch = Arrow::RecordBatch.new({
+                                            column: {
+                                              type: :decimal128,
+                                              precision: 8,
+                                              scale: 2,
+                                            }
+                                          },
+                                          records)
+    assert_equal(records, record_batch.raw_records)
+  end
+end