galaaz 2.1.0 → 2.1.2

data/lib/galaaz/arrow_ipc/java_arrow_backend.rb

@@ -0,0 +1,250 @@
+# frozen_string_literal: true
+
+##########################################################################################
+# JRuby Arrow IPC writer via Apache Arrow Java.
+#
+# JAR loading order:
+#   1. GALAAZ_ARROW_JARS directory (all *.jar)
+#   2. ~/arrow_jars if present
+#   3. jar-dependencies require_jar for pinned Arrow 18.1.0 artifacts
+##########################################################################################
+
+module Galaaz
+  module ArrowIpc
+    module JavaArrowBackend
+      ARROW_JAVA_VERSION = '18.1.0'
+
+      module_function
+
+      def available?
+        return false unless RUBY_ENGINE == 'jruby'
+
+        load_jars!
+        true
+      rescue LoadError, StandardError
+        false
+      end
+
+      def write_columns(columns_hash, path)
+        load_jars!
+        lengths = columns_hash.values.map(&:length).uniq
+        raise ArgumentError, "all columns must have the same length (got #{lengths.inspect})" if lengths.size != 1
+
+        n = lengths.first
+        alloc = org.apache.arrow.memory.RootAllocator.new(java.lang.Long::MAX_VALUE)
+        fields = []
+        kinds = {}
+
+        columns_hash.each do |name, values|
+          kind = infer_kind(values)
+          kinds[name] = kind
+          fields << org.apache.arrow.vector.types.pojo.Field.nullable(name, arrow_type(kind))
+        end
+
+        schema = org.apache.arrow.vector.types.pojo.Schema.new(fields)
+        root = org.apache.arrow.vector.VectorSchemaRoot.create(schema, alloc)
+
+        begin
+          columns_hash.each do |name, values|
+            fill_vector(root.getVector(name), kinds[name], values, n)
+          end
+          root.setRowCount(n)
+
+          fos = java.io.FileOutputStream.new(path)
+          begin
+            channel = java.nio.channels.Channels.newChannel(fos)
+            writer = org.apache.arrow.vector.ipc.ArrowFileWriter.new(root, nil, channel)
+            begin
+              writer.start
+              writer.writeBatch
+              writer.end
+            ensure
+              writer.close
+            end
+          ensure
+            fos.close
+          end
+        ensure
+          root.close
+          alloc.close
+        end
+      end
+
+      def read_columns(path)
+        load_jars!
+        alloc = org.apache.arrow.memory.RootAllocator.new(java.lang.Long::MAX_VALUE)
+        fis = java.io.FileInputStream.new(path)
+        columns = nil
+        begin
+          reader = org.apache.arrow.vector.ipc.ArrowFileReader.new(fis.getChannel, alloc)
+          begin
+            loop do
+              break unless reader.loadNextBatch
+
+              root = reader.getVectorSchemaRoot
+              columns ||= root.getSchema.getFields.map { |f| f.getName }.each_with_object({}) { |n, h| h[n] = [] }
+              n = root.getRowCount
+              root.getFieldVectors.each do |vec|
+                name = vec.getName
+                n.times { |i| columns[name] << java_value(vec, i) }
+              end
+            end
+          ensure
+            reader.close
+          end
+        ensure
+          fis.close
+          alloc.close
+        end
+        raise ArgumentError, "no record batches in #{path}" if columns.nil?
+
+        columns
+      end
+
+      def load_jars!
+        return if defined?(@jars_loaded) && @jars_loaded
+
+        unless RUBY_ENGINE == 'jruby'
+          raise LoadError, 'Galaaz::ArrowIpc::JavaArrowBackend requires JRuby'
+        end
+
+        loaded = load_jars_from_dir(ENV['GALAAZ_ARROW_JARS'])
+        loaded ||= load_jars_from_dir(File.expand_path('~/arrow_jars'))
+        loaded ||= load_jars_via_jar_dependencies
+
+        unless loaded
+          raise LoadError,
+                'Apache Arrow Java JARs not found. Set GALAAZ_ARROW_JARS to a directory of ' \
+                "Arrow #{ARROW_JAVA_VERSION} JARs, place them in ~/arrow_jars, or install " \
+                'jar-dependencies so require_jar can resolve them.'
+        end
+
+        @jars_loaded = true
+      end
+      private_class_method :load_jars!
+
+      def load_jars_from_dir(dir)
+        return false if dir.nil? || dir.empty?
+        return false unless File.directory?(dir)
+
+        jars = Dir[File.join(dir, '*.jar')]
+        return false if jars.empty?
+
+        jars.each { |jar| require jar }
+        true
+      end
+      private_class_method :load_jars_from_dir
+
+      def load_jars_via_jar_dependencies
+        require 'jar-dependencies'
+        require_jar 'org.apache.arrow', 'arrow-format', ARROW_JAVA_VERSION
+        require_jar 'org.apache.arrow', 'arrow-memory-core', ARROW_JAVA_VERSION
+        require_jar 'org.apache.arrow', 'arrow-memory-netty', ARROW_JAVA_VERSION
+        require_jar 'org.apache.arrow', 'arrow-memory-unsafe', ARROW_JAVA_VERSION
+        require_jar 'org.apache.arrow', 'arrow-vector', ARROW_JAVA_VERSION
+        true
+      rescue LoadError
+        false
+      end
+      private_class_method :load_jars_via_jar_dependencies
+
+      def arrow_type(kind)
+        case kind
+        when :float64
+          org.apache.arrow.vector.types.pojo.ArrowType::FloatingPoint.new(
+            org.apache.arrow.vector.types.FloatingPointPrecision::DOUBLE
+          )
+        when :int32
+          org.apache.arrow.vector.types.pojo.ArrowType::Int.new(32, true)
+        when :utf8
+          org.apache.arrow.vector.types.pojo.ArrowType::Utf8.new
+        else
+          raise ArgumentError, "unsupported kind #{kind.inspect}"
+        end
+      end
+      private_class_method :arrow_type
+
+      def fill_vector(vector, kind, values, n)
+        vector.allocateNew
+        case kind
+        when :float64
+          values.each_with_index do |v, i|
+            if v.nil?
+              vector.setNull(i)
+            else
+              vector.setSafe(i, v.to_f)
+            end
+          end
+        when :int32
+          values.each_with_index do |v, i|
+            if v.nil?
+              vector.setNull(i)
+            else
+              vector.setSafe(i, Integer(v))
+            end
+          end
+        when :utf8
+          values.each_with_index do |v, i|
+            if v.nil?
+              vector.setNull(i)
+            else
+              bytes = v.to_s.to_java_bytes
+              vector.setSafe(i, bytes)
+            end
+          end
+        end
+        vector.setValueCount(n)
+      end
+      private_class_method :fill_vector
+
+      def infer_kind(values)
+        sample = values.find { |v| !v.nil? }
+        return :float64 if sample.nil?
+
+        case sample
+        when Float
+          :float64
+        when Integer
+          values.any? { |v| v.is_a?(Float) } ? :float64 : :int32
+        when String, Symbol
+          :utf8
+        when TrueClass, FalseClass
+          raise ArgumentError, 'boolean columns are not supported in B1 (use int32/float64/utf8)'
+        else
+          if sample.is_a?(Numeric)
+            sample.is_a?(Integer) ? :int32 : :float64
+          else
+            :utf8
+          end
+        end
+      end
+      private_class_method :infer_kind
+
+      def java_value(vector, i)
+        return nil if vector.isNull(i)
+
+        minor = vector.getMinorType.toString
+        case minor
+        when 'FLOAT8', 'FLOAT4'
+          vector.get(i).to_f
+        when 'INT', 'SMALLINT', 'TINYINT', 'UINT1', 'UINT2', 'UINT4'
+          vector.get(i).to_i
+        when 'BIGINT', 'UINT8'
+          vector.get(i).to_i
+        when 'VARCHAR', 'VARBINARY'
+          bytes = vector.get(i)
+          bytes.nil? ? nil : String.from_java_bytes(bytes)
+        else
+          obj = vector.respond_to?(:getObject) ? vector.getObject(i) : vector.get(i)
+          return obj if obj.nil?
+          return obj.to_s if obj.is_a?(String) || obj.java_kind_of?(java.lang.CharSequence)
+          return obj.to_f if obj.is_a?(Float) || obj.java_kind_of?(java.lang.Double) || obj.java_kind_of?(java.lang.Float)
+          return obj.to_i if obj.is_a?(Integer) || obj.java_kind_of?(java.lang.Number)
+
+          obj.to_s
+        end
+      end
+      private_class_method :java_value
+    end
+  end
+end

data/lib/galaaz/arrow_ipc/red_arrow_backend.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require 'rbconfig'
+
+##########################################################################################
+# CRuby Arrow IPC writer via red-arrow (gem "arrow").
+##########################################################################################
+
+module Galaaz
+  module ArrowIpc
+    module RedArrowBackend
+      module_function
+
+      def available?
+        load!
+        true
+      rescue LoadError, StandardError
+        false
+      end
+
+      def write_columns(columns_hash, path)
+        load!
+        lengths = columns_hash.values.map(&:length).uniq
+        raise ArgumentError, "all columns must have the same length (got #{lengths.inspect})" if lengths.size != 1
+
+        table_hash = {}
+        columns_hash.each do |name, values|
+          table_hash[name] = build_array(name, values)
+        end
+        table = ::Arrow::Table.new(table_hash)
+        table.save(path)
+      end
+
+      def read_columns(path)
+        load!
+        table = ::Arrow::Table.load(path)
+        table.schema.fields.each_with_object({}) do |field, out|
+          col = table[field.name]
+          values = col.respond_to?(:to_a) ? col.to_a : col.data.to_a
+          out[field.name] = values
+        end
+      end
+
+      def load!
+        return if defined?(@loaded) && @loaded
+
+        begin
+          require 'arrow'
+        rescue LoadError
+          prepend_user_red_arrow_load_path!
+          begin
+            require 'arrow'
+          rescue LoadError => e
+            raise LoadError,
+                  "Galaaz::ArrowIpc on CRuby requires the red-arrow gem " \
+                  "(gem install red-arrow, matching pkg-config arrow-glib). #{e.message}"
+          end
+        end
+        @loaded = true
+      end
+      private_class_method :load!
+
+      # bundle exec only exposes Gemfile gems; red-arrow is optional and often
+      # user-installed. Put its lib + native extension on $LOAD_PATH.
+      def prepend_user_red_arrow_load_path!
+        names = '{red-arrow,gobject-introspection,glib2,gio2,native-package-installer,pkg-config,extpp}'
+        roots = []
+        roots << Gem.user_dir if Gem.respond_to?(:user_dir)
+        rubylibdir = RbConfig::CONFIG['rubylibdir']
+        if rubylibdir
+          roots << File.join(File.dirname(rubylibdir), 'gems', RbConfig::CONFIG['ruby_version'])
+        end
+        roots.concat(Array(Gem.path))
+        roots.compact.uniq.each do |root|
+          next unless File.directory?(root)
+
+          Dir.glob(File.join(root, 'gems', "#{names}-*", 'lib')).sort.each do |lib|
+            $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+          end
+          Dir.glob(File.join(root, 'extensions', '**', "#{names}-*")).select { |p| File.directory?(p) }.sort.each do |ext|
+            $LOAD_PATH.unshift(ext) unless $LOAD_PATH.include?(ext)
+          end
+        end
+      end
+      private_class_method :prepend_user_red_arrow_load_path!
+
+      def build_array(name, values)
+        kind = infer_kind(values)
+        case kind
+        when :float64
+          ::Arrow::DoubleArray.new(values.map { |v| v.nil? ? nil : v.to_f })
+        when :int32
+          ::Arrow::Int32Array.new(values.map { |v| v.nil? ? nil : Integer(v) })
+        when :utf8
+          ::Arrow::StringArray.new(values.map { |v| v.nil? ? nil : v.to_s })
+        else
+          raise ArgumentError, "unsupported column type for #{name.inspect}"
+        end
+      end
+      private_class_method :build_array
+
+      def infer_kind(values)
+        sample = values.find { |v| !v.nil? }
+        return :float64 if sample.nil?
+
+        case sample
+        when Float
+          :float64
+        when Integer
+          values.any? { |v| v.is_a?(Float) } ? :float64 : :int32
+        when String, Symbol
+          :utf8
+        when TrueClass, FalseClass
+          raise ArgumentError, 'boolean columns are not supported in B1 (use int32/float64/utf8)'
+        else
+          if sample.respond_to?(:to_f) && sample.is_a?(Numeric)
+            sample.is_a?(Integer) ? :int32 : :float64
+          else
+            :utf8
+          end
+        end
+      end
+      private_class_method :infer_kind
+    end
+  end
+end

data/lib/galaaz/arrow_ipc.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'securerandom'
+require 'tmpdir'
+
+##########################################################################################
+# Stage B: Ruby ↔ R Arrow IPC file handoff (B1 ingest, B2 export).
+#
+# NewBridge carries only the path. This is IPC/mmap file handoff — not zero-copy
+# shared heap between Ruby and R.
+##########################################################################################
+
+module Galaaz
+  class ArrowIpcError < StandardError; end
+
+  module ArrowIpc
+    module_function
+
+    # @return [Boolean] true when the current Ruby engine's Arrow backend can load
+    def available?
+      backend.available?
+    rescue StandardError
+      false
+    end
+
+    # Write columnar data to an Arrow IPC file.
+    #
+    # @param columns_hash [Hash{String,Symbol => Array}] column name → values
+    # @return [String] absolute path to the IPC file (fsync'd / closed)
+    def write(columns_hash)
+      raise ArgumentError, 'columns_hash must be a Hash' unless columns_hash.is_a?(Hash)
+      raise ArgumentError, 'columns_hash must not be empty' if columns_hash.empty?
+
+      normalized = normalize_columns(columns_hash)
+      path = next_path
+      backend.write_columns(normalized, path)
+      fsync_path(path)
+      path
+    end
+
+    # Convenience: Array of row Hashes → columnar write.
+    #
+    # @param row_hashes [Enumerable<Hash>]
+    # @return [String] path
+    def write_batches(row_hashes)
+      rows = []
+      row_hashes.each do |row|
+        raise ArgumentError, 'each row must be a Hash' unless row.is_a?(Hash)
+
+        rows << row
+      end
+      raise ArgumentError, 'write_batches requires at least one row' if rows.empty?
+
+      keys = rows.flat_map(&:keys).map(&:to_s).uniq
+      columns = keys.each_with_object({}) { |k, h| h[k] = [] }
+      rows.each do |row|
+        key_map = row.each_with_object({}) { |(k, v), h| h[k.to_s] = v }
+        keys.each { |k| columns[k] << key_map[k] }
+      end
+      write(columns)
+    end
+
+    # Read an Arrow IPC file into a column hash (String keys → Arrays).
+    # B2 types: float64, int32/int64, utf8 (nulls preserved).
+    #
+    # @param path [String]
+    # @return [Hash{String => Array}]
+    def read(path)
+      path = File.expand_path(path.to_s)
+      raise ArgumentError, "Arrow IPC file not found: #{path}" unless File.file?(path)
+
+      backend.read_columns(path)
+    end
+
+    # Read an Arrow IPC file as an Array of row Hashes (symbol keys).
+    #
+    # @param path [String]
+    # @return [Array<Hash>]
+    def read_batches(path)
+      columns = read(path)
+      raise ArgumentError, 'IPC file has no columns' if columns.empty?
+
+      n = columns.values.map(&:length).uniq
+      raise ArgumentError, "ragged columns after read: #{n.inspect}" if n.size != 1
+
+      n.first.times.map do |i|
+        columns.each_with_object({}) { |(name, values), row| row[name.to_sym] = values[i] }
+      end
+    end
+
+    # Unique scratch path for a new IPC file (used by R::Arrow.write_ipc when path omitted).
+    #
+    # @return [String]
+    def allocate_path
+      File.join(scratch_dir, "galaaz_ipc_#{Process.pid}_#{SecureRandom.hex(8)}.arrow")
+    end
+
+    # Unlink an IPC scratch file if present.
+    #
+    # @param path [String]
+    # @return [void]
+    def release(path)
+      return if path.nil? || path.to_s.empty?
+
+      File.unlink(path) if File.exist?(path)
+    rescue Errno::ENOENT
+      nil
+    end
+
+    # Prefer /dev/shm when writable; else Dir.tmpdir / gem tmp.
+    #
+    # @return [String]
+    def scratch_dir
+      @scratch_dir ||= begin
+        candidates = []
+        candidates << '/dev/shm' if File.directory?('/dev/shm')
+        candidates << File.join(Dir.tmpdir, 'galaaz_arrow_ipc')
+        candidates << File.expand_path('../../../tmp/galaaz_arrow_ipc', __dir__)
+
+        chosen = candidates.find do |dir|
+          begin
+            FileUtils.mkdir_p(dir)
+            File.writable?(dir)
+          rescue StandardError
+            false
+          end
+        end
+        raise ArrowIpcError, 'no writable scratch directory for Arrow IPC' unless chosen
+
+        chosen
+      end
+    end
+
+    # @return [Module] RedArrowBackend or JavaArrowBackend
+    def backend
+      @backend ||=
+        if RUBY_ENGINE == 'jruby'
+          require_relative 'arrow_ipc/java_arrow_backend'
+          JavaArrowBackend
+        else
+          require_relative 'arrow_ipc/red_arrow_backend'
+          RedArrowBackend
+        end
+    end
+
+    def normalize_columns(columns_hash)
+      columns_hash.each_with_object({}) do |(name, values), out|
+        key = name.to_s
+        raise ArgumentError, "column #{key.inspect} values must be an Array" unless values.is_a?(Array)
+
+        out[key] = values
+      end
+    end
+    private_class_method :normalize_columns
+
+    def next_path
+      allocate_path
+    end
+    private_class_method :next_path
+
+    def fsync_path(path)
+      File.open(path, 'rb') { |f| f.fsync }
+    end
+    private_class_method :fsync_path
+  end
+end