galaaz 0.5.0 → 2.1.0

data/lib/R_interface/rvector.rb

@@ -30,8 +30,12 @@ module R
     include UnaryOperators
     include ExecUniOp
     include LogicalOperators
-    include Enumerable
-    
+    include ::Enumerable
+
+    def class
+      ::R::Vector
+    end
+
     #--------------------------------------------------------------------------------------
     #
     #--------------------------------------------------------------------------------------
@@ -41,17 +45,62 @@ module R
     end
     
     #--------------------------------------------------------------------------------------
-    # When indexing with '[' or '[[' an R object is returned.  Sometimes we need to have
-    # access to an umboxed Ruby element, for instance, in an numeric array, we might want
-    # to receive the actual number that can be used in a Ruby method.  In this case, we
-    # use the '<<' operator.
-    # @return the Ruby element at the given index in the vector
+    # Unbox vector to Ruby. Atomic vectors: length 1 → scalar, length > 1 → Array of scalars.
+    # Recursion depth is checked; vectors do not recurse into R::Object (elements are scalars).
     #--------------------------------------------------------------------------------------
+    def unboxed_get(index = nil, depth = 0)
+      if depth >= ::R::Support::MAX_UNBOX_DEPTH
+        ::Kernel.raise(::R::UnboxDepthError, "unbox: list too deep (max depth #{::R::Support::MAX_UNBOX_DEPTH} exceeded)")
+      end
+      if index.nil?
+        arr = ::R.bridge.pull_vector(@r_interop)
+        return (arr.length == 1 ? arr[0] : arr)
+      end
+
+      # For indexed unboxing, use binary transport + result protocol (no printed-text parsing).
+      idx = index
+      len = atomic_vector_length
+      ::Kernel.raise(::IndexError.new("index #{idx} out of array bounds: 0...#{len - 1}")) if idx >= len
+
+      type = atomic_vector_typeof
 
-    def >>(index)
-      raise IndexError.new("index #{index} out of array bounds: -#{index - 1}...#{index - 1}") if
-        (index >= @r_interop.size) 
-      @r_interop[index]
+      case type
+      when 'integer'
+        data = ::R.bridge.pull_integer_vector(@r_interop, 1, idx, 1)
+        data[0]
+      when 'character'
+        var_name = ::R::Support.generate_var_name
+        assignment = "#{var_name} <- #{@r_interop}[[#{idx + 1}]]"
+        envelope = ::R.bridge.eval_r_with_result(assignment)
+        raise "Result protocol: no envelope for character element #{@r_interop}[[#{idx + 1}]]" unless envelope
+        raise "Result protocol: expected scalar_character, got #{envelope[:type]}" unless envelope[:type] == :scalar_character
+        v = envelope[:value]
+        (v.is_a?(::String) && v =~ /^rb_obj_\d+$/) ? ::R::Support.get_ruby_object(v) : v
+      when 'logical'
+        var_name = ::R::Support.generate_var_name
+        assignment = "#{var_name} <- #{@r_interop}[[#{idx + 1}]]"
+        envelope = ::R.bridge.eval_r_with_result(assignment)
+        raise "Result protocol: no envelope for logical element #{@r_interop}[[#{idx + 1}]]" unless envelope
+        raise "Result protocol: expected scalar_logical, got #{envelope[:type]}" unless envelope[:type] == :scalar_logical
+        case envelope[:value]
+        when true then true
+        when false then false
+        else nil
+        end
+      else
+        data = ::R.bridge.pull_double_vector(@r_interop, 1, idx, 1)
+        data[0]
+      end
+    end
+
+    alias_method :>>, :unboxed_get
+    alias_method :<<, :unboxed_get
+
+    # Return Ruby array so RSpec/eq and array conversion don't forward to_ary to R.
+    # Must always return an Array (length-1 vector unboxes to scalar via >> nil, so wrap in [v]).
+    def to_ary
+      v = self >> nil
+      v.is_a?(::Array) ? v : [v]
     end
 
     #--------------------------------------------------------------------------------------
@@ -59,32 +108,88 @@ module R
     #--------------------------------------------------------------------------------------
 
     def pop
-      self >> 0
+      # Return unboxed first element as a Ruby scalar
+      unboxed_get(0)
     end
     
-    #--------------------------------------------------------------------------------------
-    # @bug
-    # Each cannot return a Enumerator because R is single threaded.  When this restriction
-    # is removed, make each return self.to_enum
-    #--------------------------------------------------------------------------------------
+    CHUNK_SIZE = 1_000_000 # 1M rows per page
 
-    def each(result = :vec)
+    def stitch(halo: 0, &block)
+      len = atomic_vector_length
 
-      case result
+      # 2. Allocate result vector in R
+      res_name = ::R::Support.generate_var_name
+      ::R.bridge.eval_r("#{res_name} <- numeric(#{len})")
+
+      # 3. Process in chunks with Halo
+      offset = 0
+      while offset < len
+        current_chunk_size = [CHUNK_SIZE, len - offset].min
+        
+        # Calculate extended range for Halo
+        # R indices start at 1, but pull_double_vector uses 0-based offset
+        pull_offset = [0, offset - halo].max
+        pull_end = [len, offset + current_chunk_size + halo].min
+        actual_pull_size = pull_end - pull_offset
+        
+        # 3.1 Pull chunk with Halo
+        data = ::R.bridge.pull_double_vector(@r_interop, len, pull_offset, actual_pull_size)
+        
+        # 3.2 Process in Ruby
+        # The block receives the data with halo. 
+        # It's up to the block to handle the context, 
+        # but the result should be the same size as data.
+        result_data_with_halo = block.call(data)
+        
+        # 3.3 Extract the "inner" part (discard halo results)
+        # The offset of our chunk within the pulled data is (offset - pull_offset)
+        inner_start = offset - pull_offset
+        inner_result = result_data_with_halo[inner_start, current_chunk_size]
+        
+        # 3.4 Push only the inner chunk back to R
+        ::R.bridge.push_double_vector(inner_result, res_name, offset, len)
+        
+        offset += current_chunk_size
+      end
+      
+      # 4. Return as R::Object (wrapped as Vector)
+      ::R::Object.build(res_name)
+    end
+
+    def map(&block)
+      stitch(halo: 0) do |data|
+        data.map(&block)
+      end
+    end
+
+    def each(mode = :vec)
+      case mode
       when :vec
-        # length is a R::Vector, in order to extract its size as a Numeric we need to
-        # use the >> operator
-        (1..length >> 0).each do |i|
+        (1..length.unboxed_get(0)).each do |i|
           yield self[i]
         end
       when :native
-        (0...length >> 0).each do |i|
-          yield self >> i
+        (0...length.unboxed_get(0)).each do |i|
+          yield unboxed_get(i)
         end
       else
-        raise "Type #{result} is unknown for method :each"
+        ::Kernel.raise("Type #{mode.inspect} is unknown for method :each")
+      end
+    end
+
+    # CRuby Enumerable#sum probes begin/end (range fast-path). R::Object#respond_to?
+    # always returns true, so that probe would hit method_missing and call R's begin.
+    # Implement sum in Ruby so MRI matches JRuby Enumerable behavior for blocks;
+    # without a block, forward to R's sum via method_missing.
+    def sum(*args, &block)
+      if block
+        ::Kernel.raise(::ArgumentError, "wrong number of arguments (given #{args.size}, expected 0..1)") if args.size > 1
+        acc = args.empty? ? 0 : args[0]
+        each { |el| acc = acc + block.call(el) }
+        acc
+      else
+        method_missing(:sum, *args)
       end
-      
     end
 
     #--------------------------------------------------------------------------------------
@@ -94,15 +199,15 @@ module R
     def each_with_index(result = :vec)
       case result
       when :vec
-        (1..length >> 0).each do |i|
+        (1..length.unboxed_get(0)).each do |i|
           yield self[i], i
         end
       when :native
-        (0...length >> 0).each do |i|
-          yield self >> i, i
+        (0...length.unboxed_get(0)).each do |i|
+          yield unboxed_get(i), i
         end
       else
-        raise "Type #{result} is unknown for method :each"
+        ::Kernel.raise("Type #{result} is unknown for method :each")
       end
       
     end
@@ -112,7 +217,33 @@ module R
     #--------------------------------------------------------------------------------------
 
     def <=>(other_vector)
-      puts "comparison called"
+      ::Kernel.raise(::NotImplementedError, "R::Vector#<=> is not implemented")
+    end
+
+    private
+
+    # Length via result protocol (scalar assignment envelope), not printed `[1] n` text.
+    def atomic_vector_length
+      var_name = ::R::Support.generate_var_name
+      env = ::R.bridge.eval_r_with_result("#{var_name} <- length(#{@r_interop})")
+      raise "Result protocol: no envelope for length(#{@r_interop})" unless env
+      case env[:type]
+      when :scalar_integer
+        env[:value].to_i
+      when :scalar_double
+        env[:value].to_i
+      else
+        raise "Result protocol: expected scalar length, got #{env[:type].inspect}"
+      end
+    end
+
+    # R typeof() as string via result protocol.
+    def atomic_vector_typeof
+      var_name = ::R::Support.generate_var_name
+      env = ::R.bridge.eval_r_with_result("#{var_name} <- typeof(#{@r_interop})")
+      raise "Result protocol: no envelope for typeof(#{@r_interop})" unless env
+      raise "Result protocol: expected scalar_character for typeof, got #{env[:type]}" unless env[:type] == :scalar_character
+      env[:value].to_s
     end
 
   end

data/lib/galaaz_jruby.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+# JVM arguments required for every JRuby process that loads Galaaz (Apache Arrow memory
+# on Java 9+). Rake and Ruby launchers should use GalaazJRuby.shell_j_arg_string.
+#
+# Optional extra flags (e.g. -J-Xmx4g): set environment variable GALAAZ_JRUBY_OPTS
+# (space-separated; passed through to the shell unquoted).
+#
+# Bash entrypoints (bin/run_rspec, etc.) source bin/galaaz_jruby_env.inc.sh — keep the
+# -J list there identical to REQUIRED_JRUBY_J_ARGS below.
+module GalaazJRuby
+  REQUIRED_JRUBY_J_ARGS = %w[
+    -J--add-opens=java.base/java.nio=ALL-UNNAMED
+  ].freeze
+
+  OPTIONAL_ENV = 'GALAAZ_JRUBY_OPTS'
+
+  def self.shell_j_arg_string
+    extra = ENV[OPTIONAL_ENV].to_s.strip
+    [REQUIRED_JRUBY_J_ARGS.join(' '), extra].reject(&:empty?).join(' ')
+  end
+end

data/lib/galaaz_ruby.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'shellwords'
+
+# Interpreter selection for Ruby launchers (bin/gknit, bin/gbookdown, bin/gstudio, Rake).
+# Default is `ruby` on PATH (JRuby or CRuby). Override with GALAAZ_RUBY=jruby / ruby / path.
+# JVM flags are applied only when the selected interpreter is JRuby (see galaaz_jruby.rb).
+module GalaazRuby
+  def self.interpreter
+    bin = ENV['GALAAZ_RUBY'].to_s.strip
+    bin.empty? ? 'ruby' : bin
+  end
+
+  def self.jruby?(bin = interpreter)
+    base = File.basename(bin.to_s)
+    return true if base == 'jruby' || base.start_with?('jruby.')
+
+    engine = `#{Shellwords.escape(bin)} -e 'print RUBY_ENGINE' 2>/dev/null`.to_s.strip
+    engine == 'jruby'
+  rescue StandardError
+    false
+  end
+
+  # Shell prefix: "<bin> [jvm-args] -I<lib_path>"
+  def self.shell_invocation(lib_path)
+    bin = interpreter
+    if jruby?(bin)
+      require_relative 'galaaz_jruby'
+      "#{bin} #{GalaazJRuby.shell_j_arg_string} -I#{lib_path}"
+    else
+      "#{bin} -I#{lib_path}"
+    end
+  end
+end

data/lib/gknit/diagnostics.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module GknitDiagnostics
+  @internal_errors = []
+
+  INTERNAL_MESSAGE_PATTERNS = [
+    /undefined method .*R::NewBridgeAdapter/i,
+    /Result protocol:/i,
+    /invalid connection/i,
+    /galaaz_callback_call timeout/i,
+    /NewBridge::SessionClient::/i,
+    /Unsupported graphics device/i
+  ].freeze
+
+  def self.reset!
+    @internal_errors = []
+  end
+
+  def self.internal_errors
+    @internal_errors
+  end
+
+  def self.internal_error?(error)
+    msg = error.message.to_s
+    INTERNAL_MESSAGE_PATTERNS.any? { |rx| rx.match?(msg) }
+  end
+
+  def self.record(chunk_label:, error:)
+    return unless internal_error?(error)
+
+    @internal_errors << {
+      chunk: chunk_label.to_s,
+      klass: error.class.to_s,
+      message: error.message.to_s
+    }
+  rescue StandardError
+    nil
+  end
+
+  def self.print_report(io = $stderr)
+    return if @internal_errors.nil? || @internal_errors.empty?
+
+    io.puts
+    io.puts 'gknit internal errors detected:'
+    @internal_errors.each_with_index do |entry, idx|
+      io.puts "  #{idx + 1}) chunk=#{entry[:chunk]} | #{entry[:klass]}: #{entry[:message]}"
+    end
+  end
+end
+

data/lib/gknit/draft.rb

@@ -27,26 +27,32 @@ module GKnit
   #
   #--------------------------------------------------------------------------------------
   
-  Polyglot.eval("R", <<-R)
+  def self.template_path(gem, family, template)
+    full_filename = Dir.glob("gknit-templates*", base: "#{Gem.default_dir}/gems")
+    template_path = "#{Gem.default_dir}/gems/#{full_filename[0]}/#{family}/#{template}"
+  end
+  
+  # Register the template_path method as a callback
+  @template_path_cb = R::Support.register_callback(GKnit.singleton_method(:template_path))
+
+  R.bridge.eval_r(<<-R)
     ruby_template = function(gem, family, template, type, ...) {
     print(gem)
     print(family)
     print(template)
     print(type)
-    data_dir = eval.polyglot("ruby", "GKnit")$template_path(gem, family, template)
+    
+    # Use the new callback mechanism
+    cat('--G_CALLBACK--#{@template_path_cb}--', gem, '|', family, '|', template, '--\\n', sep='')
+    flush.console()
+    data_dir <- readLines('/dev/shm/galaaz_callback_fifo', n=1)
+    # The returned value from Ruby will be quoted, we might need to eval/parse it
+    data_dir <- eval(parse(text=data_dir))
+
     print(data_dir)
     invoke(type, list(template = "template.tex", pandoc_args = paste0("--data-dir=", data_dir), ...))
   }
   R
-
-  #--------------------------------------------------------------------------------------
-  #
-  #--------------------------------------------------------------------------------------
-
-  def self.template_path(gem, family, template)
-    full_filename = Dir.glob("gknit-templates*", base: "#{Gem.default_dir}/gems")
-    template_path = "#{Gem.default_dir}/gems/#{full_filename[0]}/#{family}/#{template}"
-  end
   
   #--------------------------------------------------------------------------------------
   #
@@ -63,9 +69,9 @@ module GKnit
     # TODO: if package is a rubygem, then look there somehow
     if (is_package)
       template_path =
-        R.system__file("rmarkdown", "templates", template, package: package) >> 0
+        R.system__file("rmarkdown", "templates", template, package: package).unboxed_get(0)
       raise "The template '#{template}' was not found in the package '#{package}'" if
-        !(R.nzchar(template_path) >> 0)
+        !(R.nzchar(template_path).unboxed_get(0))
     else
       full_filename = Dir.glob("gknit-templates*", base: "#{Gem.default_dir}/gems")
       template_path = "#{Gem.default_dir}/gems/#{full_filename[0]}/#{template}"
@@ -80,14 +86,14 @@ module GKnit
 
     template_meta = R::Yaml::yaml__load(R.read_utf8(template_yaml))
     raise "template.yaml must contain 'name' and 'description' fields" if
-      (template_meta.name.is__null || template_meta.description.is__null) >> 0
+      (template_meta.name.is__null || template_meta.description.is__null).unboxed_get(0)
     puts "Creating template:"
-    puts "Template name: #{template_meta.name >> 0}"
-    puts "Description: #{template_meta.description >> 0}"
+    puts "Template name: #{template_meta.name.unboxed_get(0)}"
+    puts "Description: #{template_meta.description.unboxed_get(0)}"
 
     if (create_dir == 'default')
       # check if template asks for new directory
-      create_dir = template_meta.create_dir.isTRUE >> 0
+      create_dir = template_meta.create_dir.isTRUE.unboxed_get(0)
 
       if (create_dir)
         raise "The directory '#{file_basename}' already exists" if Dir.exist?(file_basename)

data/lib/gknit/include_engine.rb

@@ -36,13 +36,21 @@ class IncludeEngine < KnitrEngine
     super
          
     @engine = Proc.new do |options|
-
-      # check if require should be relative or not
-      req = (options[['relative']].is__null | options[['relative']].isTRUE) >> 0
-      
-      # load the content of the file in options.code
-      options.code = GalaazUtil.inline_file(options.label >> 0, req)
-
+      # Ensure include chunks always load file content into options['code'].
+      # This keeps include behavior working even if R-side wrappers are replaced.
+      begin
+        label = options['label'].unboxed_get(0).to_s
+        relative_opt = options['relative']
+        relative =
+          if relative_opt.respond_to?(:unboxed_get)
+            !!relative_opt.unboxed_get(0)
+          else
+            false
+          end
+        options.code = GalaazUtil.inline_file(label, relative)
+      rescue StandardError => e
+        options.code = "# Include failed: #{e.message}"
+      end
       @base_engine.call(options)
     end