galaaz 2.1.0 → 2.1.1

data/lib/galaaz_jruby.rb

@@ -3,14 +3,20 @@
 # JVM arguments required for every JRuby process that loads Galaaz (Apache Arrow memory
 # on Java 9+). Rake and Ruby launchers should use GalaazJRuby.shell_j_arg_string.
 #
+# The opens flag must be on the *JVM at startup*. `jruby -J--add-opens=... -S bundle exec`
+# does not pass it to the child rspec JVM — export JAVA_OPTS instead (see
+# +apply_java_opts_env!+ and bin/galaaz_jruby_env.inc.sh).
+#
 # Optional extra flags (e.g. -J-Xmx4g): set environment variable GALAAZ_JRUBY_OPTS
 # (space-separated; passed through to the shell unquoted).
 #
 # Bash entrypoints (bin/run_rspec, etc.) source bin/galaaz_jruby_env.inc.sh — keep the
 # -J list there identical to REQUIRED_JRUBY_J_ARGS below.
 module GalaazJRuby
-  REQUIRED_JRUBY_J_ARGS = %w[
-    -J--add-opens=java.base/java.nio=ALL-UNNAMED
+  JAVA_NIO_ADD_OPENS = '--add-opens=java.base/java.nio=ALL-UNNAMED'
+
+  REQUIRED_JRUBY_J_ARGS = %W[
+    -J#{JAVA_NIO_ADD_OPENS}
   ].freeze
 
   OPTIONAL_ENV = 'GALAAZ_JRUBY_OPTS'
@@ -19,4 +25,13 @@ module GalaazJRuby
     extra = ENV[OPTIONAL_ENV].to_s.strip
     [REQUIRED_JRUBY_J_ARGS.join(' '), extra].reject(&:empty?).join(' ')
   end
+
+  # For subprocess JVMs (and so launchers that exec JRuby after requiring this file
+  # pick up the flag). Does not change the already-running JVM.
+  def self.apply_java_opts_env!
+    opts = ENV['JAVA_OPTS'].to_s
+    return if opts.split.include?(JAVA_NIO_ADD_OPENS)
+
+    ENV['JAVA_OPTS'] = [JAVA_NIO_ADD_OPENS, opts].reject(&:empty?).join(' ')
+  end
 end

data/new_bridge_specs/arrow_ipc_async_spec.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+# Async synchronization for Stage B1 Arrow IPC handoff.
+# Run: bin/run_all_rspec new_bridge_specs/arrow_ipc_async_spec.rb
+
+require 'thread'
+require 'galaaz'
+
+RSpec.describe 'Arrow IPC async handoff' do
+  before(:all) do
+    skip 'R not on PATH' unless system('command -v R >/dev/null 2>&1')
+    skip 'Galaaz::ArrowIpc writer backend not available' unless Galaaz::ArrowIpc.available?
+    ok = R::Support.eval("requireNamespace('arrow', quietly=TRUE) && requireNamespace('dplyr', quietly=TRUE)")
+    skip 'R packages arrow/dplyr are not available' unless ok == true
+    @paths = []
+  end
+
+  after(:each) do
+    Array(@paths).each { |p| Galaaz::ArrowIpc.release(p) }
+    @paths = []
+  end
+
+  def track(path)
+    @paths << path
+    path
+  end
+
+  it 'happens-before: write+fsync then async open returns correct nrow' do
+    values = [1.0, 2.0, 3.0, 4.0]
+    path = track(Galaaz::ArrowIpc.write(value: values))
+
+    done = Queue.new
+    # Path-only over the bridge; R reads the IPC file.
+    R.eval_r_async("nrow(arrow::read_ipc_file(#{path.inspect}))", timeout: 30) do |result|
+      done.push(result)
+    end
+    r = done.pop
+    expect(r).to be_ok
+    expect(r.value).to match(/4/)
+  end
+
+  it 'async open then sync R reduction on the proxy' do
+    path = track(Galaaz::ArrowIpc.write(value: [10.0, 20.0, 30.0]))
+
+    done = Queue.new
+    R::Async.arrow___read_ipc_file(path, as_data_frame: false, timeout: 30) do |result|
+      done.push(result)
+    end
+    r = done.pop
+    expect(r).to be_ok
+    expect(r.value).to be_a(R::Object)
+
+    tbl = r.value
+    mean = R.mean(R.dplyr___pull(tbl, :value))
+    mean_v = mean.respond_to?(:>>) ? (mean >> 0) : mean
+    expect(mean_v).to eq(20.0)
+  end
+
+  it 'two concurrent handoffs do not cross-talk' do
+    path_a = track(Galaaz::ArrowIpc.write(tag: ['A'], n: [1]))
+    path_b = track(Galaaz::ArrowIpc.write(tag: ['B', 'B'], n: [2, 2]))
+
+    done = Queue.new
+    R.eval_r_async("nrow(arrow::read_ipc_file(#{path_a.inspect}))", timeout: 30) do |result|
+      done.push([:a, result])
+    end
+    R.eval_r_async("nrow(arrow::read_ipc_file(#{path_b.inspect}))", timeout: 30) do |result|
+      done.push([:b, result])
+    end
+
+    results = {}
+    2.times do
+      key, res = done.pop
+      expect(res).to be_ok
+      results[key] = res.value
+    end
+    expect(results[:a]).to match(/1/)
+    expect(results[:b]).to match(/2/)
+  end
+
+  it 'missing path fails without hanging' do
+    done = Queue.new
+    missing = File.join(Galaaz::ArrowIpc.scratch_dir, 'galaaz_missing_async.arrow')
+    R.eval_r_async("arrow::read_ipc_file(#{missing.inspect})", timeout: 10) do |result|
+      done.push(result)
+    end
+    r = done.pop
+    expect(r).not_to be_ok
+  end
+end

data/new_bridge_specs/arrow_ipc_export_async_spec.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+# Async synchronization for Stage B2 R→Ruby Arrow IPC export.
+# Run: bin/run_all_rspec new_bridge_specs/arrow_ipc_export_async_spec.rb
+
+require 'thread'
+require 'galaaz'
+
+RSpec.describe 'Arrow IPC export async (Stage B2)' do
+  before(:all) do
+    skip 'R not on PATH' unless system('command -v R >/dev/null 2>&1')
+    skip 'Galaaz::ArrowIpc backend not available' unless Galaaz::ArrowIpc.available?
+    ok = R::Support.eval("requireNamespace('arrow', quietly=TRUE)")
+    skip 'R package arrow is not available' unless ok == true
+    @paths = []
+  end
+
+  after(:each) do
+    Array(@paths).each { |p| Galaaz::ArrowIpc.release(p) }
+    @paths = []
+  end
+
+  def track(path)
+    @paths << path
+    path
+  end
+
+  it 'happens-before: async R write_ipc_file then Ruby read matches' do
+    in_path = track(Galaaz::ArrowIpc.write(n: [1, 2, 3]))
+    out_path = track(Galaaz::ArrowIpc.allocate_path)
+
+    done = Queue.new
+    R.eval_r_async(
+      "({ arrow::write_ipc_file(arrow::read_ipc_file(#{in_path.inspect}, as_data_frame=FALSE), #{out_path.inspect}, compression='uncompressed'); 1L })",
+      timeout: 30
+    ) { |result| done.push(result) }
+    r = done.pop
+    expect(r).to be_ok
+
+    cols = Galaaz::ArrowIpc.read(out_path)
+    expect(cols['n'].map(&:to_i)).to eq([1, 2, 3])
+  end
+
+  it 'two concurrent R exports do not cross-talk' do
+    in_a = track(Galaaz::ArrowIpc.write(tag: ['A']))
+    in_b = track(Galaaz::ArrowIpc.write(tag: ['B']))
+    out_a = track(Galaaz::ArrowIpc.allocate_path)
+    out_b = track(Galaaz::ArrowIpc.allocate_path)
+
+    done = Queue.new
+    R.eval_r_async(
+      "({ arrow::write_ipc_file(arrow::read_ipc_file(#{in_a.inspect}, as_data_frame=FALSE), #{out_a.inspect}, compression='uncompressed'); 1L })",
+      timeout: 30
+    ) { |res| done.push([:a, res]) }
+    R.eval_r_async(
+      "({ arrow::write_ipc_file(arrow::read_ipc_file(#{in_b.inspect}, as_data_frame=FALSE), #{out_b.inspect}, compression='uncompressed'); 1L })",
+      timeout: 30
+    ) { |res| done.push([:b, res]) }
+
+    2.times do
+      _key, res = done.pop
+      expect(res).to be_ok
+    end
+
+    expect(Galaaz::ArrowIpc.read(out_a)['tag']).to eq(['A'])
+    expect(Galaaz::ArrowIpc.read(out_b)['tag']).to eq(['B'])
+  end
+end

data/r_requires/arrow.txt

@@ -0,0 +1,3 @@
+# CRAN packages for `galaaz add arrow`.
+dplyr
+arrow

data/r_requires/knit-extras.txt

@@ -0,0 +1,4 @@
+# Best-effort CRAN extras for `galaaz add knit` (warn if missing).
+ggplot2
+kableExtra
+RColorBrewer

data/r_requires/knit.txt

@@ -0,0 +1,4 @@
+# Hard CRAN set for `galaaz add knit` (must succeed).
+dplyr
+knitr
+rmarkdown

data/specs/arrow_ipc_export_spec.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require 'galaaz'
+
+describe 'Arrow IPC export (Stage B2)' do
+  before(:all) do
+    skip 'Galaaz::ArrowIpc backend not available' unless Galaaz::ArrowIpc.available?
+    ok = R::Support.eval("requireNamespace('arrow', quietly=TRUE) && requireNamespace('dplyr', quietly=TRUE)")
+    skip 'R packages arrow/dplyr are not available' unless ok == true
+    @paths = []
+  end
+
+  after(:each) do
+    Array(@paths).each { |p| Galaaz::ArrowIpc.release(p) }
+    @paths = []
+  end
+
+  def track(path)
+    @paths << path
+    path
+  end
+
+  it 'reads columns R wrote via write_ipc (R→Ruby)' do
+    in_path = track(Galaaz::ArrowIpc.write(value: [1.5, 2.5], grp: %w[a b]))
+    tbl = R::Arrow.open_ipc(in_path)
+    out_path = track(R::Arrow.write_ipc(tbl))
+    expect(File.exist?(out_path)).to be true
+
+    cols = Galaaz::ArrowIpc.read(out_path)
+    expect(cols['value']).to eq([1.5, 2.5])
+    expect(cols['grp']).to eq(%w[a b])
+
+    rows = Galaaz::ArrowIpc.read_batches(out_path)
+    expect(rows).to eq([{ value: 1.5, grp: 'a' }, { value: 2.5, grp: 'b' }])
+  end
+
+  it 'proves DB-style loop: Ruby ingest → R summarise → Ruby reads result table' do
+    in_path = track(Galaaz::ArrowIpc.write_batches([
+      { id: 1, grp: 'a', value: 10.0 },
+      { id: 2, grp: 'b', value: 20.5 },
+      { id: 3, grp: 'a', value: 30.0 }
+    ]))
+    tbl = R::Arrow.open_ipc(in_path)
+    grouped = R.dplyr___group_by(tbl, :grp)
+    summed = R.dplyr___summarise(grouped, total: E.sum(:value))
+    out_path = track(R::Arrow.write_ipc(summed))
+
+    rows = Galaaz::ArrowIpc.read_batches(out_path)
+    by_grp = rows.each_with_object({}) { |r, h| h[r[:grp].to_s] = r[:total].to_f }
+    expect(by_grp['a']).to eq(40.0)
+    expect(by_grp['b']).to eq(20.5)
+
+    Galaaz::ArrowIpc.release(out_path)
+    expect(File.exist?(out_path)).to be false
+  end
+
+  it 'raises when reading a missing IPC file' do
+    expect {
+      Galaaz::ArrowIpc.read('/tmp/galaaz_missing_b2.arrow')
+    }.to raise_error(ArgumentError, /not found/)
+  end
+end

data/specs/arrow_ipc_handoff_spec.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'galaaz'
+
+describe 'Arrow IPC handoff (Stage B1)' do
+  before(:all) do
+    skip 'Galaaz::ArrowIpc writer backend not available' unless Galaaz::ArrowIpc.available?
+    ok = R::Support.eval("requireNamespace('arrow', quietly=TRUE) && requireNamespace('dplyr', quietly=TRUE)")
+    skip 'R packages arrow/dplyr are not available' unless ok == true
+    @paths = []
+  end
+
+  after(:each) do
+    Array(@paths).each { |p| Galaaz::ArrowIpc.release(p) }
+    @paths = []
+  end
+
+  def track(path)
+    @paths << path
+    path
+  end
+
+  it 'round-trips a float64 column through open_ipc' do
+    values = [1.5, 2.5, -3.0, 0.0]
+    path = track(Galaaz::ArrowIpc.write(value: values))
+    expect(File.exist?(path)).to be true
+
+    tbl = R::Arrow.open_ipc(path)
+    expect(tbl.rclass.split(' ')).to include('Table')
+    expect((R.nrow(tbl) >> 0)).to eq(values.length)
+
+    collected = R.dplyr___collect(tbl)
+    col = collected[['value']]
+    got = (1..(col.length >> 0)).map do |i|
+      v = col[i]
+      v.respond_to?(:>>) ? (v >> 0) : v
+    end
+    expect(got).to eq(values)
+
+    Galaaz::ArrowIpc.release(path)
+    expect(File.exist?(path)).to be false
+  end
+
+  it 'round-trips multi-column id/grp/value via write_batches' do
+    rows = [
+      { id: 1, grp: 'a', value: 10.0 },
+      { id: 2, grp: 'b', value: 20.5 },
+      { id: 3, grp: 'a', value: 30.0 }
+    ]
+    path = track(Galaaz::ArrowIpc.write_batches(rows))
+    tbl = R::Arrow.open_ipc(path)
+
+    expect((R.nrow(tbl) >> 0)).to eq(3)
+    summed = R.dplyr___summarise(tbl, total: E.sum(:value))
+    out = R.dplyr___collect(summed)
+    total = out[['total']]
+    tot = total[1]
+    tot = tot.respond_to?(:>>) ? (tot >> 0) : tot
+    expect(tot).to eq(60.5)
+  end
+
+  it 'raises on missing R arrow package path for open_ipc with bad path' do
+    expect {
+      R::Arrow.open_ipc('/tmp/galaaz_missing_ipc_file_does_not_exist.arrow')
+    }.to raise_error(StandardError)
+  end
+
+  it 'raises when write is called without a usable backend message path' do
+    # available? already gated the suite; still check empty columns
+    expect { Galaaz::ArrowIpc.write({}) }.to raise_error(ArgumentError)
+  end
+
+  it 'optional timing smoke vs from_ruby_batches for N≈50k' do
+    n = 50_000
+    cols = {
+      id: (0...n).to_a,
+      grp: (0...n).map { |i| "g#{i % 5}" },
+      value: (0...n).map { |i| (i % 10) + 1.0 }
+    }
+
+    t0 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    path = track(Galaaz::ArrowIpc.write(cols))
+    tbl = R::Arrow.open_ipc(path)
+    ipc_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t0) * 1000.0
+    expect((R.nrow(tbl) >> 0)).to eq(n)
+
+    rows = (0...n).map { |i| { id: i, grp: "g#{i % 5}", value: (i % 10) + 1.0 } }
+    t1 = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    tbl2 = R::Arrow.from_ruby_batches(rows)
+    batch_ms = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - t1) * 1000.0
+    expect((R.nrow(tbl2) >> 0)).to eq(n)
+
+    # Not a flaky perf gate — only ensure both paths produce a table.
+    expect(ipc_ms).to be > 0
+    expect(batch_ms).to be > 0
+  end
+end

data/version.rb

@@ -1,2 +1,2 @@
 $gem_name = "galaaz"
-$version="2.1.0"
+$version="2.1.1"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: galaaz
 version: !ruby/object:Gem::Version
-  version: 2.1.0
+  version: 2.1.1
 platform: ruby
 authors:
 - Rodrigo Botafogo
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-09-03 00:00:00.000000000 Z
+date: 2026-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: msgpack
@@ -337,6 +337,10 @@ files:
 - lib/R_interface/runary_operators.rb
 - lib/R_interface/rvector.rb
 - lib/galaaz.rb
+- lib/galaaz/arrow_ipc.rb
+- lib/galaaz/arrow_ipc/java_arrow_backend.rb
+- lib/galaaz/arrow_ipc/red_arrow_backend.rb
+- lib/galaaz/cli.rb
 - lib/galaaz_jruby.rb
 - lib/galaaz_ruby.rb
 - lib/gknit.rb
@@ -357,6 +361,8 @@ files:
 - lib/new_bridge/tcp_framed.rb
 - lib/util/exec_ruby.rb
 - lib/util/inline_file.rb
+- new_bridge_specs/arrow_ipc_async_spec.rb
+- new_bridge_specs/arrow_ipc_export_async_spec.rb
 - new_bridge_specs/benchmark_phase5_5_unboxing_spec.rb
 - new_bridge_specs/eval_r_async_spec.rb
 - new_bridge_specs/integration_phase5_1_concurrent_spec.rb
@@ -383,10 +389,15 @@ files:
 - new_bridge_specs/phase4_2_hardening_spec.rb
 - new_bridge_specs/phase4_3_r_instance_manager_spec.rb
 - new_bridge_specs/phase4_nested_callbacks_spec.rb
+- r_requires/arrow.txt
 - r_requires/ggplot.rb
+- r_requires/knit-extras.txt
+- r_requires/knit.txt
 - r_requires/knitr.rb
 - specs/all.rb
 - specs/arrow_from_ruby_batches_spec.rb
+- specs/arrow_ipc_export_spec.rb
+- specs/arrow_ipc_handoff_spec.rb
 - specs/arrow_semantics_spec.rb
 - specs/bridge_concurrent_spec.rb
 - specs/bridge_nested_spec.rb