nanoarrow 0.1.0

data/lib/nanoarrow/iterator.rb

@@ -0,0 +1,113 @@
+module Nanoarrow
+  class ArrayViewBaseIterator
+    def initialize(schema)
+      @schema = Utils.c_schema(schema)
+      @schema_view = Utils.c_schema_view(schema)
+      @array_view = CArrayView.from_schema(@schema)
+    end
+
+    def set_array(array)
+      @array_view.set_array(array)
+    end
+  end
+
+  class RbIterator < ArrayViewBaseIterator
+    def self.get_iterator(obj, schema = nil)
+      stream = Utils.c_array_stream(obj, schema)
+      iterator = new(stream.get_cached_schema)
+      Enumerator.new do |yielder|
+        while (array = stream.get_next)
+          iterator.set_array(array)
+          iterator.each(&yielder)
+        end
+      end
+    end
+
+    def initialize(schema)
+      super(schema)
+    end
+
+    def each(&block)
+      type_id = @schema_view.type_id
+      if !ITEMS_ITER_LOOKUP.include?(type_id)
+        raise KeyError, "Can't resolve iterator for type #{@schema_view.type.inspect}"
+      end
+
+      factory = method(ITEMS_ITER_LOOKUP.fetch(type_id))
+      factory.(&block)
+    end
+
+    private
+
+    def each_bool(&block)
+      @array_view.each_bool(&block)
+    end
+
+    def each_int(&block)
+      @array_view.each_int(&block)
+    end
+
+    def each_uint(&block)
+      @array_view.each_uint(&block)
+    end
+
+    def each_double(&block)
+      @array_view.each_double(&block)
+    end
+
+    def each_string(&block)
+      @array_view.each_string(&block)
+    end
+
+    def each_binary(&block)
+      @array_view.each_binary(&block)
+    end
+
+    def each_date(&block)
+      wrap = ->(v) { block.call(v.nil? ? v : EPOCH + v) }
+      @array_view.each_int(&wrap)
+    end
+
+    def each_timestamp(&block)
+      wrap =
+        case @schema_view.time_unit
+        when "s"
+          ->(v) { block.call(v.nil? ? v : Time.at(v)) }
+        when "ms"
+          ->(v) { block.call(v.nil? ? v : Time.at(v / 1_000, v % 1_000, :millisecond)) }
+        when "us"
+          ->(v) { block.call(v.nil? ? v : Time.at(v / 1_000_000, v % 1_000_000, :microsecond)) }
+        when "ns"
+          ->(v) { block.call(v.nil? ? v : Time.at(v / 1_000_000_000, v % 1_000_000_000, :nanosecond)) }
+        else
+          raise Todo
+        end
+      @array_view.each_int(&wrap)
+    end
+
+    ITEMS_ITER_LOOKUP = {
+      Type::BOOL => :each_bool,
+      Type::INT8 => :each_int,
+      Type::UINT8 => :each_uint,
+      Type::INT16 => :each_int,
+      Type::UINT16 => :each_uint,
+      Type::INT32 => :each_int,
+      Type::UINT32 => :each_uint,
+      Type::INT64 => :each_int,
+      Type::UINT64 => :each_uint,
+      Type::HALF_FLOAT => :each_double,
+      Type::FLOAT => :each_double,
+      Type::DOUBLE => :each_double,
+      Type::STRING => :each_string,
+      Type::LARGE_STRING => :each_string,
+      Type::STRING_VIEW => :each_string,
+      Type::BINARY => :each_binary,
+      Type::LARGE_BINARY => :each_binary,
+      Type::BINARY_VIEW => :each_binary,
+      Type::FIXED_SIZE_BINARY => :each_binary,
+      Type::DATE32 => :each_date,
+      Type::DATE64 => :each_date,
+      Type::TIMESTAMP => :each_timestamp,
+    }
+  end
+end

data/lib/nanoarrow/schema.rb

@@ -0,0 +1,85 @@
+module Nanoarrow
+  class Schema
+    def initialize(
+      obj,
+      name: nil,
+      nullable: nil,
+      metadata: nil,
+      fields: nil,
+      **params
+    )
+      if obj.is_a?(Integer)
+        @c_schema = Utils.c_schema_from_type_and_params(obj, params)
+      else
+        if params.any?
+          raise ArgumentError, "params are only supported for obj of class Type"
+        end
+        @c_schema = Utils.c_schema(obj)
+      end
+
+      if !name.nil? || !nullable.nil? || !metadata.nil? || !fields.nil?
+        @c_schema = @c_schema.modify(nil, name, nil, nullable, metadata, fields, nil, true)
+      end
+
+      @c_schema_view = CSchemaView.new(@c_schema)
+    end
+
+    def type
+      @c_schema_view.type
+    end
+
+    def name
+      @c_schema.name
+    end
+
+    def nullable
+      @c_schema_view.nullable
+    end
+
+    def metadata
+      @c_schema.metadata
+    end
+
+    def byte_width
+      if @c_schema_view.type_id == Type::FIXED_SIZE_BINARY
+        @c_schema_view.fixed_size
+      end
+    end
+
+    def unit
+      @c_schema_view.time_unit
+    end
+
+    def timezone
+      @c_schema_view.timezone
+    end
+
+    def precision
+      @c_schema_view.decimal_precision
+    end
+
+    def scale
+      @c_schema_view.decimal_scale
+    end
+
+    def n_fields
+      @c_schema.n_children
+    end
+
+    def field(i)
+      Schema.new(@c_schema.child(i).deep_dup)
+    end
+
+    def fields
+      n_fields.times.map { |i| field(i) }
+    end
+
+    def inspect
+      @c_schema.to_s
+    end
+
+    def arrow_c_schema
+      @c_schema.arrow_c_schema
+    end
+  end
+end

data/lib/nanoarrow/types.rb

@@ -0,0 +1,157 @@
+module Nanoarrow
+  def self.null(nullable: true)
+    Schema.new(Type::NULL, nullable:)
+  end
+
+  def self.bool(nullable: true)
+    Schema.new(Type::BOOL, nullable:)
+  end
+
+  def self.int8(nullable: true)
+    Schema.new(Type::INT8, nullable:)
+  end
+
+  def self.uint8(nullable: true)
+    Schema.new(Type::UINT8, nullable:)
+  end
+
+  def self.int16(nullable: true)
+    Schema.new(Type::INT16, nullable:)
+  end
+
+  def self.uint16(nullable: true)
+    Schema.new(Type::UINT16, nullable:)
+  end
+
+  def self.int32(nullable: true)
+    Schema.new(Type::INT32, nullable:)
+  end
+
+  def self.uint32(nullable: true)
+    Schema.new(Type::UINT32, nullable:)
+  end
+
+  def self.int64(nullable: true)
+    Schema.new(Type::INT64, nullable:)
+  end
+
+  def self.uint64(nullable: true)
+    Schema.new(Type::UINT64, nullable:)
+  end
+
+  def self.half_float(nullable: true)
+    Schema.new(Type::HALF_FLOAT, nullable:)
+  end
+
+  def self.float(nullable: true)
+    Schema.new(Type::FLOAT, nullable:)
+  end
+
+  def self.double(nullable: true)
+    Schema.new(Type::DOUBLE, nullable:)
+  end
+
+  def self.string(nullable: true)
+    Schema.new(Type::STRING, nullable:)
+  end
+
+  def self.large_string(nullable: true)
+    Schema.new(Type::LARGE_STRING, nullable:)
+  end
+
+  def self.string_view(nullable: true)
+    Schema.new(Type::STRING_VIEW, nullable:)
+  end
+
+  def self.binary(nullable: true)
+    Schema.new(Type::BINARY, nullable:)
+  end
+
+  def self.large_binary(nullable: true)
+    Schema.new(Type::LARGE_BINARY, nullable:)
+  end
+
+  def self.binary_view(nullable: true)
+    Schema.new(Type::BINARY_VIEW, nullable:)
+  end
+
+  def self.fixed_size_binary(byte_width, nullable: true)
+    Schema.new(Type::FIXED_SIZE_BINARY, byte_width:, nullable:)
+  end
+
+  def self.date32(nullable: true)
+    Schema.new(Type::DATE32, nullable:)
+  end
+
+  def self.date64(nullable: true)
+    Schema.new(Type::DATE64, nullable:)
+  end
+
+  def self.time32(unit, nullable: true)
+    Schema.new(Type::TIME32, unit:, nullable:)
+  end
+
+  def self.time64(unit, nullable: true)
+    Schema.new(Type::TIME64, unit:, nullable:)
+  end
+
+  def self.timestamp(unit, timezone: nil, nullable: true)
+    Schema.new(Type::TIMESTAMP, unit:, timezone:, nullable:)
+  end
+
+  def self.duration(unit, nullable: true)
+    Schema.new(Type::DURATION, unit:, nullable:)
+  end
+
+  def self.interval_months(nullable: true)
+    Schema.new(Type::INTERVAL_MONTHS, nullable:)
+  end
+
+  def self.interval_day_time(nullable: true)
+    Schema.new(Type::INTERVAL_DAY_TIME, nullable:)
+  end
+
+  def self.interval_month_day_nano(nullable: true)
+    Schema.new(Type::INTERVAL_MONTH_DAY_NANO, nullable:)
+  end
+
+  def self.decimal128(precision, scale, nullable: true)
+    Schema.new(Type::DECIMAL128, precision:, scale:, nullable:)
+  end
+
+  def self.decimal256(precision, scale, nullable: true)
+    Schema.new(Type::DECIMAL256, precision:, scale:, nullable:)
+  end
+
+  def self.struct(fields, nullable: true)
+    Schema.new(Type::STRUCT, fields:, nullable:)
+  end
+
+  def self.list(value_type, nullable: true)
+    Schema.new(Type::LIST, value_type:, nullable:)
+  end
+
+  def self.large_list(value_type, nullable: true)
+    Schema.new(Type::LARGE_LIST, value_type:, nullable:)
+  end
+
+  def self.fixed_size_list(value_type, list_size, nullable: true)
+    Schema.new(Type::FIXED_SIZE_LIST, value_type:, list_size:, nullable:)
+  end
+
+  def self.map(key_type, value_type, keys_sorted: false, nullable: true)
+    Schema.new(Type::MAP, key_type:, value_type:, keys_sorted:, nullable:)
+  end
+
+  def self.dictionary(index_type, value_type, dictionary_ordered: false)
+    Schema.new(Type::DICTIONARY, index_type:, value_type:, dictionary_ordered:)
+  end
+
+  def self.sparse_union(fields, nullable: true)
+    Schema.new(Type::SPARSE_UNION, fields:, nullable:)
+  end
+
+  def self.dense_union(fields, nullable: true)
+    Schema.new(Type::DENSE_UNION, fields:, nullable:)
+  end
+end

data/lib/nanoarrow/utils.rb

@@ -0,0 +1,127 @@
+module Nanoarrow
+  module Utils
+    def self.c_array(obj, schema)
+      if !schema.nil?
+        schema = c_schema(schema)
+      end
+
+      if obj.is_a?(CArray) && schema.nil?
+        return obj
+      end
+
+      if obj.respond_to?(:arrow_c_array)
+        raise Todo if schema
+        return CArray.import_from_c_capsule(obj.arrow_c_array)
+      end
+
+      if obj.is_a?(Capsule) && obj.name == "arrow_array"
+        raise Todo if schema
+        return CArray.import_from_c_capsule(obj)
+      end
+
+      if schema.nil?
+        raise Todo
+      end
+      builder = ArrayFromIterableBuilder.new(schema)
+      builder.build_c_array(obj)
+    end
+
+    def self.c_array_stream(obj, schema)
+      if !schema.nil?
+        schema = c_schema(schema)
+      end
+
+      if obj.is_a?(CArrayStream) && schema.nil?
+        return obj
+      end
+
+      if obj.respond_to?(:arrow_c_stream)
+        raise Todo if schema
+        return CArrayStream.import_from_c_capsule(obj.arrow_c_stream)
+      end
+
+      if obj.is_a?(Capsule) && obj.name == "arrow_array_stream"
+        raise Todo if schema
+        return CArrayStream.import_from_c_capsule(obj)
+      end
+
+      array = c_array(obj, schema)
+      CArrayStream.from_c_arrays([array], array.schema, false)
+    end
+
+    def self.c_schema(obj)
+      if obj.is_a?(CSchema)
+        return obj
+      end
+
+      if obj.respond_to?(:arrow_c_schema)
+        return CSchema.import_from_c_capsule(obj.arrow_c_schema)
+      end
+
+      if obj.is_a?(Capsule) && obj.name == "arrow_schema"
+        return CSchema.import_from_c_capsule(obj)
+      end
+
+      raise TypeError, "Can't convert object of type #{obj.class.name} to CSchema"
+    end
+
+    def self.c_schema_view(obj)
+      if obj.is_a?(CSchemaView)
+        return obj
+      end
+
+      CSchemaView.new(c_schema(obj))
+    end
+
+    def self.c_schema_from_type_and_params(type, params)
+      factory = CSchemaBuilder.new
+
+      if [Type::DECIMAL128, Type::DECIMAL256].include?(type)
+        precision = Integer(params.delete(:precision))
+        scale = Integer(params.delete(:scale))
+        factory.set_type_decimal(type, precision, scale)
+
+      elsif [Type::TIME32, Type::TIME64, Type::TIMESTAMP, Type::DURATION].include?(type)
+        time_unit = params.delete(:unit)
+        timezone = params.delete(:timezone)
+        factory.set_type_date_time(type, TIME_UNITS.fetch(time_unit), timezone)
+
+      elsif type == Type::FIXED_SIZE_BINARY
+        factory.set_type_fixed_size(type, Integer(params.delete(:byte_width)))
+
+      elsif type == Type::LIST
+        raise Todo
+
+      elsif type == Type::LARGE_LIST
+        raise Todo
+
+      elsif type == Type::FIXED_SIZE_LIST
+        raise Todo
+
+      elsif type == Type::MAP
+        raise Todo
+
+      elsif type == Type::DICTIONARY
+        raise Todo
+
+      elsif type == Type::SPARSE_UNION
+        raise Todo
+
+      elsif type == Type::DENSE_UNION
+        raise Todo
+
+      else
+        factory.set_type(type)
+      end
+
+      if params.any?
+        unused = params.keys.map(&:inspect).join(", ")
+        raise ArgumentError, "Unused parameters whilst constructing Schema: #{unused}"
+      end
+
+      factory.set_name("")
+
+      factory.finish
+    end
+  end
+end

data/lib/nanoarrow/version.rb

@@ -0,0 +1,3 @@
+module Nanoarrow
+  VERSION = "0.1.0"
+end

data/lib/nanoarrow.rb

@@ -0,0 +1,28 @@
+# ext
+require "nanoarrow/ext"
+
+# stdlib
+require "date"
+
+# modules
+require_relative "nanoarrow/array"
+require_relative "nanoarrow/array_builder"
+require_relative "nanoarrow/array_stream"
+require_relative "nanoarrow/iterator"
+require_relative "nanoarrow/schema"
+require_relative "nanoarrow/types"
+require_relative "nanoarrow/utils"
+require_relative "nanoarrow/version"
+
+module Nanoarrow
+  class Error < StandardError; end
+
+  class Todo < Error
+    def initialize(message = "not implemented yet")
+      super(message)
+    end
+  end
+
+  EPOCH = Date.new(1970, 1, 1)
+  TIME_UNITS = {"s" => 0, "ms" => 1, "us" => 2, "ns" => 3}
+end

metadata

@@ -0,0 +1,70 @@
+--- !ruby/object:Gem::Specification
+name: nanoarrow
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Andrew Kane
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+email: andrew@ankane.org
+executables: []
+extensions:
+- ext/nanoarrow/extconf.rb
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- NOTICE.txt
+- README.md
+- ext/nanoarrow/array.c
+- ext/nanoarrow/array_builder.c
+- ext/nanoarrow/array_stream.c
+- ext/nanoarrow/array_view.c
+- ext/nanoarrow/buffer.c
+- ext/nanoarrow/capsule.c
+- ext/nanoarrow/ext.c
+- ext/nanoarrow/ext.h
+- ext/nanoarrow/extconf.rb
+- ext/nanoarrow/mat_array_stream.c
+- ext/nanoarrow/nanoarrow.c
+- ext/nanoarrow/nanoarrow/nanoarrow.h
+- ext/nanoarrow/rc.c
+- ext/nanoarrow/schema.c
+- ext/nanoarrow/schema_builder.c
+- ext/nanoarrow/schema_view.c
+- ext/nanoarrow/types.c
+- ext/nanoarrow/utils.c
+- lib/nanoarrow.rb
+- lib/nanoarrow/array.rb
+- lib/nanoarrow/array_builder.rb
+- lib/nanoarrow/array_stream.rb
+- lib/nanoarrow/iterator.rb
+- lib/nanoarrow/schema.rb
+- lib/nanoarrow/types.rb
+- lib/nanoarrow/utils.rb
+- lib/nanoarrow/version.rb
+homepage: https://github.com/ankane/nanoarrow-ruby
+licenses:
+- Apache-2.0
+metadata: {}
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.14
+specification_version: 4
+summary: Zero-dependency Arrow library for Ruby
+test_files: []