red-arrow 4.0.1 → 5.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: df66b2dac421ab800bd00510e4a13e250646d1f1c27541f2ca44cd04be38a3ba
-  data.tar.gz: 3c39ec6a1a3ee48708a28583c125d96f9195e9973f6f2f3707cd923eca915329
+  metadata.gz: dcc31d7ec789dfa277963fda5b3710e7eec2451b6fbe425282638e48e684b95f
+  data.tar.gz: 00bfbf2e0357861f7247c42c4faae6a72d9adfca745ad1b3298f549997847158
 SHA512:
-  metadata.gz: 9ebf50ffecfdea79140b158f5aeea0cc3bb63403e405bcbba2b34e288b8575df3df859b8f46eb12e26199e566237016d0e3907acc1d7c0d16e82d638c7bdce2e
-  data.tar.gz: 1ff1389d28db025efd82c87e9f85cfb7e3bce21abe85a1b943ca8536bbc1015b0015c05ab45e5d9a35914034a673da0bbc59f8211bb0b620ee4a7487f5fec413
+  metadata.gz: 8e60e9a4ce4d379f7f1d9387154ae4d9fe4ec91ffe304647966d33d52e380950cc0a8a5f025164466526feb5198638139bb7564f2066fd93b7c21dcd697e07ca
+  data.tar.gz: 140a8f087eeb42baae703eee677c97261c6b30317c0f72ce49231a117b2e32f34175bad71f913bdd6f940abd8097c77f968b27780cecbfca062be60ac87d09ec

data/ext/arrow/arrow.cpp

@@ -18,6 +18,7 @@
  */
 
 #include "red-arrow.hpp"
+#include "memory-view.hpp"
 
 #include <ruby.hpp>
 
@@ -78,4 +79,6 @@ extern "C" void Init_arrow() {
   red_arrow::id_jd = rb_intern("jd");
   red_arrow::id_new = rb_intern("new");
   red_arrow::id_to_datetime = rb_intern("to_datetime");
+
+  red_arrow::memory_view::init(mArrow);
 }

data/ext/arrow/memory-view.cpp

@@ -0,0 +1,311 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include "memory-view.hpp"
+
+#include <arrow-glib/arrow-glib.hpp>
+#include <rbgobject.h>
+
+#include <ruby/version.h>
+
+#if RUBY_API_VERSION_MAJOR >= 3
+#  define HAVE_MEMORY_VIEW
+#  define private memory_view_private
+#  include <ruby/memory_view.h>
+#  undef private
+#endif
+
+#include <sstream>
+
+namespace red_arrow {
+  namespace memory_view {
+#ifdef HAVE_MEMORY_VIEW
+    // This is workaround for the following rb_memory_view_t problems
+    // in C++:
+    //
+    //   * Can't use "private" as member name
+    //   * Can't assign a value to "rb_memory_view_t::private"
+    //
+    // This has compatible layout with rb_memory_view_t.
+    struct memory_view {
+      VALUE obj;
+      void *data;
+      ssize_t byte_size;
+      bool readonly;
+      const char *format;
+      ssize_t item_size;
+      struct {
+        const rb_memory_view_item_component_t *components;
+        size_t length;
+      } item_desc;
+      ssize_t ndim;
+      const ssize_t *shape;
+      const ssize_t *strides;
+      const ssize_t *sub_offsets;
+      void *private_data;
+    };
+
+    struct PrivateData {
+      std::string format;
+    };
+
+    class PrimitiveArrayGetter : public arrow::ArrayVisitor {
+    public:
+      explicit PrimitiveArrayGetter(memory_view *view)
+        : view_(view) {
+      }
+
+      arrow::Status Visit(const arrow::BooleanArray& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        // Memory view doesn't support bit stream. We use one byte
+        // for 8 elements. Users can't calculate the number of
+        // elements from memory view but it's limitation of memory view.
+#ifdef ARROW_LITTLE_ENDIAN
+        view_->format = "b8";
+#else
+        view_->format = "B8";
+#endif
+        view_->item_size = 1;
+        view_->byte_size = (array.length() + 7) / 8;
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::Int8Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "c";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::Int16Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "s";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::Int32Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "l";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::Int64Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "q";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::UInt8Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "C";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::UInt16Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "S";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::UInt32Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "L";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::UInt64Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "Q";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::FloatArray& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "f";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::DoubleArray& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "d";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::FixedSizeBinaryArray& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        auto priv = static_cast<PrivateData *>(view_->private_data);
+        const auto type =
+          std::static_pointer_cast<const arrow::FixedSizeBinaryType>(
+            array.type());
+        std::ostringstream output;
+        output << "C" << type->byte_width();
+        priv->format = output.str();
+        view_->format = priv->format.c_str();
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::Date32Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "l";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::Date64Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "q";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::Time32Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "l";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::Time64Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "q";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::TimestampArray& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "q";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::Decimal128Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "q2";
+        return arrow::Status::OK();
+      }
+
+      arrow::Status Visit(const arrow::Decimal256Array& array) override {
+        fill(static_cast<const arrow::Array&>(array));
+        view_->format = "q4";
+        return arrow::Status::OK();
+      }
+
+      private:
+      void fill(const arrow::Array& array) {
+        const auto array_data = array.data();
+        const auto data = array_data->GetValuesSafe<uint8_t>(1);
+        view_->data = const_cast<void *>(reinterpret_cast<const void *>(data));
+        const auto type =
+          std::static_pointer_cast<const arrow::FixedWidthType>(array.type());
+        view_->item_size = type->bit_width() / 8;
+        view_->byte_size = view_->item_size * array.length();
+      }
+
+      memory_view *view_;
+    };
+
+    bool primitive_array_get(VALUE obj, rb_memory_view_t *view, int flags) {
+      if (flags != RUBY_MEMORY_VIEW_SIMPLE) {
+        return false;
+      }
+      auto view_ = reinterpret_cast<memory_view *>(view);
+      view_->obj = obj;
+      view_->private_data = new PrivateData();
+      auto array = GARROW_ARRAY(RVAL2GOBJ(obj));
+      auto arrow_array = garrow_array_get_raw(array);
+      PrimitiveArrayGetter getter(view_);
+      auto status = arrow_array->Accept(&getter);
+      if (!status.ok()) {
+        return false;
+      }
+      view_->readonly = true;
+      view_->ndim = 1;
+      view_->shape = NULL;
+      view_->strides = NULL;
+      view_->sub_offsets = NULL;
+      return true;
+    }
+
+    bool primitive_array_release(VALUE obj, rb_memory_view_t *view) {
+      auto view_ = reinterpret_cast<memory_view *>(view);
+      delete static_cast<PrivateData *>(view_->private_data);
+      return true;
+    }
+
+    bool primitive_array_available_p(VALUE obj) {
+      return true;
+    }
+
+    rb_memory_view_entry_t primitive_array_entry = {
+      primitive_array_get,
+      primitive_array_release,
+      primitive_array_available_p,
+    };
+
+    bool buffer_get(VALUE obj, rb_memory_view_t *view, int flags) {
+      if (flags != RUBY_MEMORY_VIEW_SIMPLE) {
+        return false;
+      }
+      auto view_ = reinterpret_cast<memory_view *>(view);
+      view_->obj = obj;
+      auto buffer = GARROW_BUFFER(RVAL2GOBJ(obj));
+      auto arrow_buffer = garrow_buffer_get_raw(buffer);
+      view_->data =
+        const_cast<void *>(reinterpret_cast<const void *>(arrow_buffer->data()));
+      // Memory view doesn't support bit stream. We use one byte
+      // for 8 elements. Users can't calculate the number of
+      // elements from memory view but it's limitation of memory view.
+#ifdef ARROW_LITTLE_ENDIAN
+      view_->format = "b8";
+#else
+      view_->format = "B8";
+#endif
+      view_->item_size = 1;
+      view_->byte_size = arrow_buffer->size();
+      view_->readonly = true;
+      view_->ndim = 1;
+      view_->shape = NULL;
+      view_->strides = NULL;
+      view_->sub_offsets = NULL;
+      return true;
+    }
+
+    bool buffer_release(VALUE obj, rb_memory_view_t *view) {
+      return true;
+    }
+
+    bool buffer_available_p(VALUE obj) {
+      return true;
+    }
+
+    rb_memory_view_entry_t buffer_entry = {
+      buffer_get,
+      buffer_release,
+      buffer_available_p,
+    };
+#endif
+
+    void init(VALUE mArrow) {
+#ifdef HAVE_MEMORY_VIEW
+      auto cPrimitiveArray =
+        rb_const_get_at(mArrow, rb_intern("PrimitiveArray"));
+      rb_memory_view_register(cPrimitiveArray,
+                              &(red_arrow::memory_view::primitive_array_entry));
+
+      auto cBuffer = rb_const_get_at(mArrow, rb_intern("Buffer"));
+      rb_memory_view_register(cBuffer, &(red_arrow::memory_view::buffer_entry));
+#endif
+    }
+  }
+}

data/ext/arrow/memory-view.hpp

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <ruby.hpp>
+
+namespace red_arrow {
+  namespace memory_view {
+    void init(VALUE mArrow);
+  }
+}

data/lib/arrow/array.rb

@@ -55,6 +55,18 @@ module Arrow
       end
     end
 
+    # @param other [Arrow::Array] The array to be compared.
+    # @param options [Arrow::EqualOptions, Hash] (nil)
+    #   The options to custom how to compare.
+    #
+    # @return [Boolean]
+    #   `true` if both of them have the same data, `false` otherwise.
+    #
+    # @since 5.0.0
+    def equal_array?(other, options=nil)
+      equal_options(other, options)
+    end
+
     def each
       return to_enum(__method__) unless block_given?
 

data/lib/arrow/buffer.rb

@@ -17,12 +17,16 @@
 
 module Arrow
   class Buffer
-    alias_method :initialize_raw, :initialize
-    private :initialize_raw
-
-    def initialize(data)
-      @data = data
-      initialize_raw(data)
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when String
+          new(value)
+        else
+          nil
+        end
+      end
     end
   end
 end

data/lib/arrow/constructor-arguments-gc-guardable.rb

@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  module ConstructorArgumentsGCGuardable
+    def initialize(*args)
+      super
+      @arguments = args
+    end
+  end
+end

data/lib/arrow/datum.rb

@@ -0,0 +1,98 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Datum
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when Array
+          ArrayDatum.new(value)
+        when ChunkedArray
+          ChunkedArrayDatum.new(value)
+        when Scalar
+          ScalarDatum.new(value)
+        when ::Array
+          ArrayDatum.new(ArrayBuilder.build(value))
+        when Integer
+          case value
+          when (0..((2 ** 8) - 1))
+            try_convert(UInt8Scalar.new(value))
+          when ((-(2 ** 7))..((2 ** 7) - 1))
+            try_convert(Int8Scalar.new(value))
+          when (0..((2 ** 16) - 1))
+            try_convert(UInt16Scalar.new(value))
+          when ((-(2 ** 15))..((2 ** 15) - 1))
+            try_convert(Int16Scalar.new(value))
+          when (0..((2 ** 32) - 1))
+            try_convert(UInt32Scalar.new(value))
+          when ((-(2 ** 31))..((2 ** 31) - 1))
+            try_convert(Int32Scalar.new(value))
+          when (0..((2 ** 64) - 1))
+            try_convert(UInt64Scalar.new(value))
+          when ((-(2 ** 63))..((2 ** 63) - 1))
+            try_convert(Int64Scalar.new(value))
+          else
+            nil
+          end
+        when Float
+          try_convert(DoubleScalar.new(value))
+        when true, false
+          try_convert(BooleanScalar.new(value))
+        when String
+          if value.ascii_only? or value.encoding == Encoding::UTF_8
+            if value.bytesize <= ((2 ** 31) - 1)
+              try_convert(StringScalar.new(value))
+            else
+              try_convert(LargeStringScalar.new(value))
+            end
+          else
+            if value.bytesize <= ((2 ** 31) - 1)
+              try_convert(BinaryScalar.new(value))
+            else
+              try_convert(LargeBinaryScalar.new(value))
+            end
+          end
+        when Date
+          date32_value = (value - Date32ArrayBuilder::UNIX_EPOCH).to_i
+          try_convert(Date32Scalar.new(date32_value))
+        when Time
+          case value.unit
+          when TimeUnit::SECOND, TimeUnit::MILLI
+            data_type = Time32DataType.new(value.unit)
+            scalar_class = Time32Scalar
+          else
+            data_type = Time64DataType.new(value.unit)
+            scalar_class = Time64Scalar
+          end
+          try_convert(scalar_class.new(data_type, value.value))
+        when ::Time
+          data_type = TimestampDataType.new(:nano)
+          timestamp_value = value.to_i * 1_000_000_000 + value.nsec
+          try_convert(TimestampScalar.new(data_type, timestamp_value))
+        when Decimal128
+          data_type = TimestampDataType.new(:nano)
+          timestamp_value = value.to_i * 1_000_000_000 + value.nsec
+          try_convert(Decimal128Scalar.new(data_type, timestamp_value))
+        else
+          nil
+        end
+      end
+    end
+  end
+end