red-arrow 0.16.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 64b14ef4120f4ab290e8161020902ec2a22631c519d5a133a63ce383610e8545
-  data.tar.gz: 2f5850520e2dc69568a454cee0d4246909d52f1d49851221b1b9efd3149bc15c
+  metadata.gz: 171c7e5854ed116c8324153272827255b18f1083d5ae38866903b3cb0faa2977
+  data.tar.gz: 86c21238d83c5e4a8d114464eb8281dca5ddaff7cb6fb3e08b858a67082d4514
 SHA512:
-  metadata.gz: 0e19a4da6182437a51f9dad6212436e70b00881674ee6cd29e2a40910fe711fdef4076c81d2778f3ff9e9dd3f45b573c43e90378244cf08d7550b504ad8b53af
-  data.tar.gz: 547eb8b31fd59d9c1d5fc1163bb25da70154b797d103d3a01f6fda70dddc0b3c2cdb5391b9006c4510be37b8b75988195f809aaead8c91e7fb68f762cc5313de
+  metadata.gz: f46692362251101f0d18782e755dd3e2453a168ddefc564a623cc6f16a336d974fca83079faa63198b201e88e3fc678552bd69adca86bf743daf5ef7cd432ff2
+  data.tar.gz: 887b098b1d9b832a9197a9ef08e7be63e2d12b1772c45054e02b582d47bc8382c6d73645fce3ea2982d9b1f73454a78fe8372a7b01879cb1d865b971167bc2ad

data/Rakefile

@@ -30,36 +30,44 @@ spec = helper.gemspec
 release_task = Rake::Task["release"]
 release_task.prerequisites.replace(["build", "release:rubygem_push"])
 
-def run_extconf(extension_dir, *arguments)
-  cd(extension_dir) do
-    ruby("extconf.rb", *arguments)
+def run_extconf(build_dir, extension_dir, *arguments)
+  cd(build_dir) do
+    ruby(File.join(extension_dir, "extconf.rb"),
+         *arguments)
   end
 end
 
 spec.extensions.each do |extension|
-  extension_dir = File.dirname(extension)
-  CLOBBER << File.join(extension_dir, "Makefile")
-  CLOBBER << File.join(extension_dir, "mkmf.log")
+  extension_dir = File.join(base_dir, File.dirname(extension))
+  build_dir = ENV["BUILD_DIR"]
+  if build_dir
+    build_dir = File.join(build_dir, "red-arrow")
+    directory build_dir
+  else
+    build_dir = extension_dir
+  end
+  CLOBBER << File.join(build_dir, "Makefile")
+  CLOBBER << File.join(build_dir, "mkmf.log")
 
-  makefile = File.join(extension_dir, "Makefile")
-  file makefile do
-    run_extconf(extension_dir)
+  makefile = File.join(build_dir, "Makefile")
+  file makefile => build_dir do
+    run_extconf(build_dir, extension_dir)
   end
 
   desc "Configure"
-  task :configure do
-    run_extconf(extension_dir)
+  task :configure => build_dir do
+    run_extconf(build_dir, extension_dir)
   end
 
   desc "Compile"
   task :compile => makefile do
-    cd(extension_dir) do
+    cd(build_dir) do
       sh("make")
     end
   end
 
   task :clean do
-    cd(extension_dir) do
+    cd(build_dir) do
       sh("make", "clean") if File.exist?("Makefile")
     end
   end
@@ -67,7 +75,9 @@ end
 
 desc "Run tests"
 task :test do
-  ruby("test/run-test.rb")
+  cd(base_dir) do
+    ruby("test/run-test.rb")
+  end
 end
 
 task default: :test
@@ -79,8 +89,10 @@ task :benchmark do
                else
                  FileList["benchmark/{,*/**/}*.yml"]
                end
-  benchmarks.each do |benchmark|
-    sh("benchmark-driver", benchmark)
+  cd(base_dir) do
+    benchmarks.each do |benchmark|
+      sh("benchmark-driver", benchmark)
+    end
   end
 end
 

data/ext/arrow/converters.hpp

@@ -285,7 +285,8 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
     // TODO
@@ -339,9 +340,9 @@ namespace red_arrow {
       index_ = index;
       result_ = rb_hash_new();
       const auto struct_type = array.struct_type();
-      const auto n = struct_type->num_children();
+      const auto n = struct_type->num_fields();
       for (int i = 0; i < n; ++i) {
-        const auto field_type = struct_type->child(i).get();
+        const auto field_type = struct_type->field(i).get();
         const auto& field_name = field_type->name();
         auto key_keep = key_;
         key_ = rb_utf8_str_new(field_name.data(), field_name.length());
@@ -388,7 +389,8 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
     // TODO
@@ -432,10 +434,10 @@ namespace red_arrow {
       index_ = index;
       switch (array.mode()) {
       case arrow::UnionMode::SPARSE:
-        convert_sparse(array);
+        convert_sparse(static_cast<const arrow::SparseUnionArray&>(array));
         break;
       case arrow::UnionMode::DENSE:
-        convert_dense(array);
+        convert_dense(static_cast<const arrow::DenseUnionArray&>(array));
         break;
       default:
         rb_raise(rb_eArgError, "Invalid union mode");
@@ -479,7 +481,8 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
     // TODO
@@ -501,14 +504,14 @@ namespace red_arrow {
       result_ = result;
     }
 
-    uint8_t compute_child_index(const arrow::UnionArray& array,
+    uint8_t compute_field_index(const arrow::UnionArray& array,
                                 arrow::UnionType* type,
                                 const char* tag) {
       const auto type_code = array.raw_type_codes()[index_];
       if (type_code >= 0 && type_code <= arrow::UnionType::kMaxTypeCode) {
-        const auto child_id = type->child_ids()[type_code];
-        if (child_id >= 0) {
-          return child_id;
+        const auto field_id = type->child_ids()[type_code];
+        if (field_id >= 0) {
+          return field_id;
         }
       }
       check_status(arrow::Status::Invalid("Unknown type ID: ", type_code),
@@ -516,33 +519,33 @@ namespace red_arrow {
       return 0;
     }
 
-    void convert_sparse(const arrow::UnionArray& array) {
+    void convert_sparse(const arrow::SparseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-sparse-array]";
-      const auto child_index = compute_child_index(array, type, tag);
-      const auto child_field = type->child(child_index).get();
-      const auto& field_name = child_field->name();
+      const auto index = compute_field_index(array, type, tag);
+      const auto field = type->field(index).get();
+      const auto& field_name = field->name();
       const auto field_name_keep = field_name_;
       field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto child_array = array.child(child_index).get();
-      check_status(child_array->Accept(this), tag);
+      const auto field_array = array.field(index).get();
+      check_status(field_array->Accept(this), tag);
       field_name_ = field_name_keep;
     }
 
-    void convert_dense(const arrow::UnionArray& array) {
+    void convert_dense(const arrow::DenseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-dense-array]";
-      const auto child_index = compute_child_index(array, type, tag);
-      const auto child_field = type->child(child_index).get();
-      const auto& field_name = child_field->name();
+      const auto index = compute_field_index(array, type, tag);
+      const auto field = type->field(index).get();
+      const auto& field_name = field->name();
       const auto field_name_keep = field_name_;
       field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto child_array = array.child(child_index);
+      const auto field_array = array.field(index);
       const auto index_keep = index_;
       index_ = array.value_offset(index_);
-      check_status(child_array->Accept(this), tag);
+      check_status(field_array->Accept(this), tag);
       index_ = index_keep;
       field_name_ = field_name_keep;
     }
@@ -557,30 +560,57 @@ namespace red_arrow {
   public:
     explicit DictionaryArrayValueConverter(ArrayValueConverter* converter)
       : array_value_converter_(converter),
-        index_(0),
+        value_index_(0),
         result_(Qnil) {
     }
 
     VALUE convert(const arrow::DictionaryArray& array,
                   const int64_t index) {
-      index_ = index;
-      auto indices = array.indices().get();
-      check_status(indices->Accept(this),
+      value_index_ = array.GetValueIndex(index);
+      auto dictionary = array.dictionary().get();
+      check_status(dictionary->Accept(this),
                    "[raw-records][dictionary-array]");
       return result_;
     }
 
-    // TODO: Convert to real value.
 #define VISIT(TYPE)                                                     \
     arrow::Status Visit(const arrow::TYPE ## Array& array) override {   \
-      result_ = convert_value(array, index_);                           \
+      result_ = convert_value(array, value_index_);                     \
       return arrow::Status::OK();                                       \
       }
 
+    VISIT(Null)
+    VISIT(Boolean)
     VISIT(Int8)
     VISIT(Int16)
     VISIT(Int32)
     VISIT(Int64)
+    VISIT(UInt8)
+    VISIT(UInt16)
+    VISIT(UInt32)
+    VISIT(UInt64)
+    // TODO
+    // VISIT(HalfFloat)
+    VISIT(Float)
+    VISIT(Double)
+    VISIT(Binary)
+    VISIT(String)
+    VISIT(FixedSizeBinary)
+    VISIT(Date32)
+    VISIT(Date64)
+    VISIT(Time32)
+    VISIT(Time64)
+    VISIT(Timestamp)
+    // TODO
+    // VISIT(Interval)
+    VISIT(List)
+    VISIT(Struct)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
+    VISIT(Dictionary)
+    VISIT(Decimal128)
+    // TODO
+    // VISIT(Extension)
 
 #undef VISIT
 
@@ -592,7 +622,7 @@ namespace red_arrow {
     }
 
     ArrayValueConverter* array_value_converter_;
-    int64_t index_;
+    int64_t value_index_;
     VALUE result_;
   };
 

data/ext/arrow/extconf.rb

@@ -16,7 +16,8 @@
 # under the License.
 
 require "extpp"
-require "mkmf-gnome2"
+require "mkmf-gnome"
+require_relative "../../lib/arrow/version"
 
 arrow_pkg_config_path = ENV["ARROW_PKG_CONFIG_PATH"]
 if arrow_pkg_config_path
@@ -24,7 +25,12 @@ if arrow_pkg_config_path
   ENV["PKG_CONFIG_PATH"] = pkg_config_paths.join(File::PATH_SEPARATOR)
 end
 
-unless required_pkg_config_package("arrow",
+unless required_pkg_config_package([
+                                     "arrow",
+                                     Arrow::Version::MAJOR,
+                                     Arrow::Version::MINOR,
+                                     Arrow::Version::MICRO,
+                                   ],
                                    debian: "libarrow-dev",
                                    redhat: "arrow-devel",
                                    homebrew: "apache-arrow",
@@ -32,7 +38,12 @@ unless required_pkg_config_package("arrow",
   exit(false)
 end
 
-unless required_pkg_config_package("arrow-glib",
+unless required_pkg_config_package([
+                                     "arrow-glib",
+                                     Arrow::Version::MAJOR,
+                                     Arrow::Version::MINOR,
+                                     Arrow::Version::MICRO,
+                                   ],
                                    debian: "libarrow-glib-dev",
                                    redhat: "arrow-glib-devel",
                                    homebrew: "apache-arrow-glib",

data/ext/arrow/raw-records.cpp

@@ -100,7 +100,8 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
-      VISIT(Union)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
       VISIT(Dictionary)
       VISIT(Decimal128)
       // TODO

data/ext/arrow/values.cpp

@@ -81,7 +81,8 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
-      VISIT(Union)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
       VISIT(Dictionary)
       VISIT(Decimal128)
       // TODO

data/lib/arrow/buffer.rb

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Buffer
+    alias_method :initialize_raw, :initialize
+    private :initialize_raw
+
+    def initialize(data)
+      @data = data
+      initialize_raw(data)
+    end
+  end
+end

data/lib/arrow/dictionary-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DictionaryArray
+    def get_value(i)
+      dictionary[indices[i]]
+    end
+  end
+end

data/lib/arrow/generic-filterable.rb

@@ -19,24 +19,24 @@ module Arrow
   module GenericFilterable
     class << self
       def included(base)
-        base.alias_method :filter_raw, :filter
-        base.alias_method :filter, :filter_generic
+        base.__send__(:alias_method, :filter_raw, :filter)
+        base.__send__(:alias_method, :filter, :filter_generic)
       end
     end
 
-    def filter_generic(filter)
+    def filter_generic(filter, options=nil)
       case filter
       when ::Array
-        filter_raw(BooleanArray.new(filter))
+        filter_raw(BooleanArray.new(filter), options)
       when ChunkedArray
         if respond_to?(:filter_chunked_array)
-          filter_chunked_array(filter)
+          filter_chunked_array(filter, options)
         else
           # TODO: Implement this in C++
-          filter_raw(filter.pack)
+          filter_raw(filter.pack, options)
         end
       else
-        filter_raw(filter)
+        filter_raw(filter, options)
       end
     end
   end

data/lib/arrow/generic-takeable.rb

@@ -19,8 +19,8 @@ module Arrow
   module GenericTakeable
     class << self
       def included(base)
-        base.alias_method :take_raw, :take
-        base.alias_method :take, :take_generic
+        base.__send__(:alias_method, :take_raw, :take)
+        base.__send__(:alias_method, :take, :take_generic)
       end
     end
 

data/lib/arrow/loader.rb

@@ -41,6 +41,7 @@ module Arrow
       require "arrow/array"
       require "arrow/array-builder"
       require "arrow/bigdecimal-extension"
+      require "arrow/buffer"
       require "arrow/chunked-array"
       require "arrow/column"
       require "arrow/compression-type"
@@ -56,6 +57,7 @@ module Arrow
       require "arrow/decimal128-array-builder"
       require "arrow/decimal128-data-type"
       require "arrow/dense-union-data-type"
+      require "arrow/dictionary-array"
       require "arrow/dictionary-data-type"
       require "arrow/field"
       require "arrow/file-output-stream"
@@ -69,6 +71,7 @@ module Arrow
       require "arrow/record-batch"
       require "arrow/record-batch-builder"
       require "arrow/record-batch-file-reader"
+      require "arrow/record-batch-iterator"
       require "arrow/record-batch-stream-reader"
       require "arrow/rolling-window"
       require "arrow/schema"