red-arrow 0.15.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 65ddeae926378c460b0945ff6949ecbf3ee911611cdcb95bf21f3cf3765efe6d
-  data.tar.gz: 193bce59b05b836fb5a5d6d8b650ea9adf441bf04beda8a6083633692c796957
+  metadata.gz: a45e003f0a453f175b1dbfc81c1fcf092cbfe964dd43e44d7b16e2087834ef5d
+  data.tar.gz: 5d1dc1d87a821d1ac4c49603d1d92d723230eb1f8a7cdd540dafe3715a53c12e
 SHA512:
-  metadata.gz: b502a0bc9f65b24d04d9f5ee1f58e7eb5db27e885d756158c249fba8be843af7f1ef12c234110cc7136d67f5e998ceb726a7ecf8688b653ba84ec207f7308e78
-  data.tar.gz: 1d8f8798f582fdb8b4e0e566d52407b3303391c4c43deef9471dc0011c9a377f83fca308b230709ef0b332ce83892a1cd967096531b4701382d4ed40066b216c
+  metadata.gz: 85329e05ae20268d44a9ffff5fb3278263e5b77deda6f03fda31595b117252b7970fd17328ce2650c8650895a1813a1fb95799d3331a312bce2c276f3e5d55a4
+  data.tar.gz: fe90b0ff0dfb9b9126765818d54f7d4d203e6c72ab53dfb2680a48dc474fef09b8ed3f0553f3c1fe110d500d625ab2300563c3e4169403326c63c6bf27f33a5f

data/Rakefile

@@ -30,36 +30,44 @@ spec = helper.gemspec
 release_task = Rake::Task["release"]
 release_task.prerequisites.replace(["build", "release:rubygem_push"])
 
-def run_extconf(extension_dir, *arguments)
-  cd(extension_dir) do
-    ruby("extconf.rb", *arguments)
+def run_extconf(build_dir, extension_dir, *arguments)
+  cd(build_dir) do
+    ruby(File.join(extension_dir, "extconf.rb"),
+         *arguments)
   end
 end
 
 spec.extensions.each do |extension|
-  extension_dir = File.dirname(extension)
-  CLOBBER << File.join(extension_dir, "Makefile")
-  CLOBBER << File.join(extension_dir, "mkmf.log")
+  extension_dir = File.join(base_dir, File.dirname(extension))
+  build_dir = ENV["BUILD_DIR"]
+  if build_dir
+    build_dir = File.join(build_dir, "red-arrow")
+    directory build_dir
+  else
+    build_dir = extension_dir
+  end
+  CLOBBER << File.join(build_dir, "Makefile")
+  CLOBBER << File.join(build_dir, "mkmf.log")
 
-  makefile = File.join(extension_dir, "Makefile")
-  file makefile do
-    run_extconf(extension_dir)
+  makefile = File.join(build_dir, "Makefile")
+  file makefile => build_dir do
+    run_extconf(build_dir, extension_dir)
   end
 
   desc "Configure"
-  task :configure do
-    run_extconf(extension_dir)
+  task :configure => build_dir do
+    run_extconf(build_dir, extension_dir)
   end
 
   desc "Compile"
   task :compile => makefile do
-    cd(extension_dir) do
+    cd(build_dir) do
       sh("make")
     end
   end
 
   task :clean do
-    cd(extension_dir) do
+    cd(build_dir) do
       sh("make", "clean") if File.exist?("Makefile")
     end
   end
@@ -67,7 +75,9 @@ end
 
 desc "Run tests"
 task :test do
-  ruby("test/run-test.rb")
+  cd(base_dir) do
+    ruby("test/run-test.rb")
+  end
 end
 
 task default: :test
@@ -79,8 +89,10 @@ task :benchmark do
                else
                  FileList["benchmark/{,*/**/}*.yml"]
                end
-  benchmarks.each do |benchmark|
-    sh("benchmark-driver", benchmark)
+  cd(base_dir) do
+    benchmarks.each do |benchmark|
+      sh("benchmark-driver", benchmark)
+    end
   end
 end
 

data/ext/arrow/converters.hpp

@@ -285,7 +285,8 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
     // TODO
@@ -339,9 +340,9 @@ namespace red_arrow {
       index_ = index;
       result_ = rb_hash_new();
       const auto struct_type = array.struct_type();
-      const auto n = struct_type->num_children();
+      const auto n = struct_type->num_fields();
       for (int i = 0; i < n; ++i) {
-        const auto field_type = struct_type->child(i).get();
+        const auto field_type = struct_type->field(i).get();
         const auto& field_name = field_type->name();
         auto key_keep = key_;
         key_ = rb_utf8_str_new(field_name.data(), field_name.length());
@@ -388,7 +389,8 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
     // TODO
@@ -432,10 +434,10 @@ namespace red_arrow {
       index_ = index;
       switch (array.mode()) {
       case arrow::UnionMode::SPARSE:
-        convert_sparse(array);
+        convert_sparse(static_cast<const arrow::SparseUnionArray&>(array));
         break;
       case arrow::UnionMode::DENSE:
-        convert_dense(array);
+        convert_dense(static_cast<const arrow::DenseUnionArray&>(array));
         break;
       default:
         rb_raise(rb_eArgError, "Invalid union mode");
@@ -479,7 +481,8 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
     // TODO
@@ -501,48 +504,48 @@ namespace red_arrow {
       result_ = result;
     }
 
-    uint8_t compute_child_index(const arrow::UnionArray& array,
+    uint8_t compute_field_index(const arrow::UnionArray& array,
                                 arrow::UnionType* type,
                                 const char* tag) {
-      const auto type_id = array.raw_type_ids()[index_];
-      const auto& type_codes = type->type_codes();
-      for (uint8_t i = 0; i < type_codes.size(); ++i) {
-        if (type_codes[i] == type_id) {
-          return i;
+      const auto type_code = array.raw_type_codes()[index_];
+      if (type_code >= 0 && type_code <= arrow::UnionType::kMaxTypeCode) {
+        const auto field_id = type->child_ids()[type_code];
+        if (field_id >= 0) {
+          return field_id;
         }
       }
-      check_status(arrow::Status::Invalid("Unknown type ID: ", type_id),
+      check_status(arrow::Status::Invalid("Unknown type ID: ", type_code),
                    tag);
       return 0;
     }
 
-    void convert_sparse(const arrow::UnionArray& array) {
+    void convert_sparse(const arrow::SparseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-sparse-array]";
-      const auto child_index = compute_child_index(array, type, tag);
-      const auto child_field = type->child(child_index).get();
-      const auto& field_name = child_field->name();
+      const auto index = compute_field_index(array, type, tag);
+      const auto field = type->field(index).get();
+      const auto& field_name = field->name();
       const auto field_name_keep = field_name_;
       field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto child_array = array.child(child_index).get();
-      check_status(child_array->Accept(this), tag);
+      const auto field_array = array.field(index).get();
+      check_status(field_array->Accept(this), tag);
       field_name_ = field_name_keep;
     }
 
-    void convert_dense(const arrow::UnionArray& array) {
+    void convert_dense(const arrow::DenseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-dense-array]";
-      const auto child_index = compute_child_index(array, type, tag);
-      const auto child_field = type->child(child_index).get();
-      const auto& field_name = child_field->name();
+      const auto index = compute_field_index(array, type, tag);
+      const auto field = type->field(index).get();
+      const auto& field_name = field->name();
       const auto field_name_keep = field_name_;
       field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto child_array = array.child(child_index);
+      const auto field_array = array.field(index);
       const auto index_keep = index_;
       index_ = array.value_offset(index_);
-      check_status(child_array->Accept(this), tag);
+      check_status(field_array->Accept(this), tag);
       index_ = index_keep;
       field_name_ = field_name_keep;
     }
@@ -557,30 +560,57 @@ namespace red_arrow {
   public:
     explicit DictionaryArrayValueConverter(ArrayValueConverter* converter)
       : array_value_converter_(converter),
-        index_(0),
+        value_index_(0),
         result_(Qnil) {
     }
 
     VALUE convert(const arrow::DictionaryArray& array,
                   const int64_t index) {
-      index_ = index;
-      auto indices = array.indices().get();
-      check_status(indices->Accept(this),
+      value_index_ = array.GetValueIndex(index);
+      auto dictionary = array.dictionary().get();
+      check_status(dictionary->Accept(this),
                    "[raw-records][dictionary-array]");
       return result_;
     }
 
-    // TODO: Convert to real value.
 #define VISIT(TYPE)                                                     \
     arrow::Status Visit(const arrow::TYPE ## Array& array) override {   \
-      result_ = convert_value(array, index_);                           \
+      result_ = convert_value(array, value_index_);                     \
       return arrow::Status::OK();                                       \
       }
 
+    VISIT(Null)
+    VISIT(Boolean)
     VISIT(Int8)
     VISIT(Int16)
     VISIT(Int32)
     VISIT(Int64)
+    VISIT(UInt8)
+    VISIT(UInt16)
+    VISIT(UInt32)
+    VISIT(UInt64)
+    // TODO
+    // VISIT(HalfFloat)
+    VISIT(Float)
+    VISIT(Double)
+    VISIT(Binary)
+    VISIT(String)
+    VISIT(FixedSizeBinary)
+    VISIT(Date32)
+    VISIT(Date64)
+    VISIT(Time32)
+    VISIT(Time64)
+    VISIT(Timestamp)
+    // TODO
+    // VISIT(Interval)
+    VISIT(List)
+    VISIT(Struct)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
+    VISIT(Dictionary)
+    VISIT(Decimal128)
+    // TODO
+    // VISIT(Extension)
 
 #undef VISIT
 
@@ -592,7 +622,7 @@ namespace red_arrow {
     }
 
     ArrayValueConverter* array_value_converter_;
-    int64_t index_;
+    int64_t value_index_;
     VALUE result_;
   };
 

data/ext/arrow/raw-records.cpp

@@ -100,7 +100,8 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
-      VISIT(Union)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
       VISIT(Dictionary)
       VISIT(Decimal128)
       // TODO

data/ext/arrow/values.cpp

@@ -81,7 +81,8 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
-      VISIT(Union)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
       VISIT(Dictionary)
       VISIT(Decimal128)
       // TODO

data/lib/arrow/array-builder.rb

@@ -26,60 +26,13 @@ module Arrow
           return builder.build(values)
         end
 
-        builder_class = nil
-        builder_class_arguments = []
+        builder_info = nil
         values.each do |value|
-          case value
-          when nil
-            # Ignore
-          when true, false
-            return BooleanArray.new(values)
-          when String
-            return StringArray.new(values)
-          when Float
-            return DoubleArray.new(values)
-          when Integer
-            if value < 0
-              builder = IntArrayBuilder.new
-              return builder.build(values)
-            else
-              builder_class = UIntArrayBuilder
-              builder_class_arguments = []
-            end
-          when Time
-            data_type = value.data_type
-            case data_type.unit
-            when TimeUnit::SECOND
-              if builder.nil?
-                builder = Time32ArrayBuilder
-                builder_class_arguments = [data_type]
-              end
-            when TimeUnit::MILLI
-              if builder != Time64ArrayBuilder
-                builder = Time32ArrayBuilder
-                builder_class_arguments = [data_type]
-              end
-            when TimeUnit::MICRO
-              builder = Time64ArrayBuilder
-              builder_class_arguments = [data_type]
-            when TimeUnit::NANO
-              builder = Time64ArrayBuilder.new(data_type)
-              return builder.build(values)
-            end
-          when ::Time
-            data_type = TimestampDataType.new(:nano)
-            builder = TimestampArrayBuilder.new(data_type)
-            return builder.build(values)
-          when DateTime
-            return Date64Array.new(values)
-          when Date
-            return Date32Array.new(values)
-          else
-            return StringArray.new(values)
-          end
+          builder_info = detect_builder_info(value, builder_info)
+          break if builder_info and builder_info[:detected]
         end
-        if builder_class
-          builder = builder_class.new(*builder_class_arguments)
+        if builder_info
+          builder = builder_info[:builder]
           builder.build(values)
         else
           Arrow::StringArray.new(values)
@@ -89,6 +42,102 @@ module Arrow
       def buildable?(args)
         args.size == method(:build).arity
       end
+
+      private
+      def detect_builder_info(value, builder_info)
+        case value
+        when nil
+          builder_info
+        when true, false
+          {
+            builder: BooleanArrayBuilder.new,
+            detected: true,
+          }
+        when String
+          {
+            builder: StringArrayBuilder.new,
+            detected: true,
+          }
+        when Float
+          {
+            builder: DoubleArrayBuilder.new,
+            detected: true,
+          }
+        when Integer
+          if value < 0
+            {
+              builder: IntArrayBuilder.new,
+              detected: true,
+            }
+          else
+            {
+              builder: UIntArrayBuilder.new,
+            }
+          end
+        when Time
+          data_type = value.data_type
+          case data_type.unit
+          when TimeUnit::SECOND
+            builder_info || {
+              builder: Time32ArrayBuilder.new(data_type)
+            }
+          when TimeUnit::MILLI
+            if builder_info and builder_info[:builder].is_a?(Time64ArrayBuilder)
+              builder_info
+            else
+              {
+                builder: Time32ArrayBuilder.new(data_type),
+              }
+            end
+          when TimeUnit::MICRO
+            {
+              builder: Time64ArrayBuilder.new(data_type),
+            }
+          when TimeUnit::NANO
+            {
+              builder: Time64ArrayBuilder.new(data_type),
+              detected: true
+            }
+          end
+        when ::Time
+          data_type = TimestampDataType.new(:nano)
+          {
+            builder: TimestampArrayBuilder.new(data_type),
+            detected: true,
+          }
+        when DateTime
+          {
+            builder: Date64ArrayBuilder.new,
+            detected: true,
+          }
+        when Date
+          {
+            builder: Date32ArrayBuilder.new,
+            detected: true,
+          }
+        when ::Array
+          sub_builder_info = nil
+          value.each do |sub_value|
+            sub_builder_info = detect_builder_info(sub_value, sub_builder_info)
+            break if sub_builder_info and sub_builder_info[:detected]
+          end
+          if sub_builder_info and sub_builder_info[:detected]
+            sub_value_data_type = sub_builder_info[:builder].value_data_type
+            field = Field.new("item", sub_value_data_type)
+            {
+              builder: ListArrayBuilder.new(ListDataType.new(field)),
+              detected: true,
+            }
+          else
+            builder_info
+          end
+        else
+          {
+            builder: StringArrayBuilder.new,
+            detected: true,
+          }
+        end
+      end
     end
 
     def build(values)