lexer_kit 0.5.0

data/ext/lexer_kit_rust/src/fast_stream.rs

@@ -0,0 +1,468 @@
+//! LexStream - High-performance streaming lexer
+//!
+//! This module implements the performance-critical parts of LexStream in Rust.
+//! Higher-level operations like token_name, text, and line_col are implemented in Ruby.
+
+use crate::types::CompiledProgram;
+use crate::vm;
+use rb_sys::*;
+use std::ptr;
+
+/// Define a Ruby method with no arguments
+macro_rules! define_ruby_method_noarg {
+    ($class:expr, $name:expr, $fn:expr) => {
+        rb_define_method(
+            $class,
+            $name.as_ptr(),
+            Some(std::mem::transmute::<
+                unsafe extern "C" fn(VALUE) -> VALUE,
+                unsafe extern "C" fn() -> VALUE,
+            >($fn)),
+            0,
+        )
+    };
+}
+
+/// Define a Ruby method with variable arguments
+macro_rules! define_ruby_method_vararg {
+    ($class:expr, $name:expr, $fn:expr) => {
+        rb_define_method(
+            $class,
+            $name.as_ptr(),
+            Some(std::mem::transmute::<
+                unsafe extern "C" fn(libc::c_int, *const VALUE, VALUE) -> VALUE,
+                unsafe extern "C" fn() -> VALUE,
+            >($fn)),
+            -1,
+        )
+    };
+}
+
+const STRIDE: usize = 3;
+
+/// Token buffer using Vec for safe memory management
+///
+/// Memory allocation and deallocation is handled safely by Vec.
+pub(crate) struct FastTokenBuffer {
+    /// Flat array: [token_id, start, len] * count
+    data: Vec<i32>,
+    /// Number of tokens (data.len() / 3)
+    count: usize,
+    /// Current position
+    index: usize,
+}
+
+impl FastTokenBuffer {
+    /// Create from collected tokens
+    pub(crate) fn from_tokens(tokens: Vec<(u16, usize, usize)>) -> Result<Self, &'static str> {
+        let count = tokens.len();
+        let mut data = Vec::with_capacity(count * STRIDE);
+
+        for (token_id, start, length) in tokens {
+            if start > i32::MAX as usize || length > i32::MAX as usize {
+                return Err("token position exceeds maximum value (input too large)");
+            }
+            data.push(token_id as i32);
+            data.push(start as i32);
+            data.push(length as i32);
+        }
+
+        Ok(Self { data, count, index: 0 })
+    }
+
+    #[inline(always)]
+    pub(crate) fn eof(&self) -> bool {
+        self.index >= self.count
+    }
+
+    #[inline(always)]
+    pub(crate) fn token_id(&self) -> i32 {
+        if self.index >= self.count {
+            -1
+        } else {
+            self.data[self.index * STRIDE]
+        }
+    }
+
+    #[inline(always)]
+    pub(crate) fn start(&self) -> i32 {
+        if self.index >= self.count {
+            -1
+        } else {
+            self.data[self.index * STRIDE + 1]
+        }
+    }
+
+    #[inline(always)]
+    pub(crate) fn len(&self) -> i32 {
+        if self.index >= self.count {
+            -1
+        } else {
+            self.data[self.index * STRIDE + 2]
+        }
+    }
+
+    #[inline(always)]
+    pub(crate) fn peek_token_id(&self, n: usize) -> i32 {
+        let i = self.index + n;
+        if i >= self.count {
+            -1
+        } else {
+            self.data[i * STRIDE]
+        }
+    }
+
+    #[inline(always)]
+    pub(crate) fn peek_start(&self, n: usize) -> i32 {
+        let i = self.index + n;
+        if i >= self.count {
+            -1
+        } else {
+            self.data[i * STRIDE + 1]
+        }
+    }
+
+    #[inline(always)]
+    pub(crate) fn peek_len(&self, n: usize) -> i32 {
+        let i = self.index + n;
+        if i >= self.count {
+            -1
+        } else {
+            self.data[i * STRIDE + 2]
+        }
+    }
+
+    #[inline(always)]
+    pub(crate) fn advance(&mut self) {
+        if self.index < self.count {
+            self.index += 1;
+        }
+    }
+
+    /// Get total number of tokens (for testing)
+    #[cfg(test)]
+    pub(crate) fn count(&self) -> usize {
+        self.count
+    }
+
+    /// Get current index (for testing)
+    #[cfg(test)]
+    pub(crate) fn index(&self) -> usize {
+        self.index
+    }
+}
+
+// No manual Drop needed - Vec handles memory deallocation automatically
+
+// ============================================================================
+// LexStream - Minimal Rust structure for fast VM execution
+// ============================================================================
+
+/// LexStream holds token buffer. Higher-level state is managed in Ruby.
+#[repr(C)]
+pub struct LexStream {
+    buffer: FastTokenBuffer,
+}
+
+// ============================================================================
+// Ruby TypedData integration
+// ============================================================================
+
+static mut LEX_STREAM_CLASS: VALUE = 0;
+static mut LEX_STREAM_DATA_TYPE: *const rb_data_type_t = ptr::null();
+
+// No mark function needed - LexStream no longer holds Ruby VALUEs
+
+unsafe extern "C" fn lex_stream_free(ptr: *mut libc::c_void) {
+    if !ptr.is_null() {
+        let _ = Box::from_raw(ptr as *mut LexStream);
+    }
+}
+
+unsafe extern "C" fn lex_stream_size(ptr: *const libc::c_void) -> size_t {
+    if ptr.is_null() {
+        0
+    } else {
+        let stream = &*(ptr as *const LexStream);
+        (std::mem::size_of::<LexStream>()
+            + stream.buffer.data.capacity() * std::mem::size_of::<i32>()) as size_t
+    }
+}
+
+#[inline(always)]
+unsafe fn get_stream(obj: VALUE) -> *mut LexStream {
+    RTYPEDDATA_GET_DATA(obj) as *mut LexStream
+}
+
+// ============================================================================
+// Thread-local storage for passing program between magnus and rb-sys
+// ============================================================================
+
+thread_local! {
+    static CURRENT_PROGRAM: std::cell::RefCell<*const CompiledProgram> = const { std::cell::RefCell::new(ptr::null()) };
+}
+
+pub fn set_current_program(prog: *const CompiledProgram) {
+    CURRENT_PROGRAM.with(|p| {
+        *p.borrow_mut() = prog;
+    });
+}
+
+fn take_current_program() -> *const CompiledProgram {
+    CURRENT_PROGRAM.with(|p| {
+        let ptr = *p.borrow();
+        *p.borrow_mut() = ptr::null(); // Clear after taking
+        ptr
+    })
+}
+
+// ============================================================================
+// Ruby method implementations
+// ============================================================================
+
+/// LexStream.create(program, input) -> LexStream
+///
+/// SAFETY: This method requires CURRENT_PROGRAM to be set via set_current_program()
+/// before calling. It is intended to be called only from CompiledProgram#stream.
+/// Direct calls will fail with an error.
+unsafe extern "C" fn lex_stream_create(
+    _klass: VALUE,
+    program: VALUE,
+    input: VALUE,
+) -> VALUE {
+    // Take the compiled program from thread-local (clears it to prevent reuse)
+    let prog_ptr = take_current_program();
+    if prog_ptr.is_null() {
+        rb_raise(
+            rb_eArgError,
+            c"LexStream.create must be called via CompiledProgram#stream".as_ptr(),
+        );
+    }
+
+    // Get input bytes
+    let input_ptr = rb_string_value_ptr(&input as *const _ as *mut _);
+    let input_len = RSTRING_LEN(input) as usize;
+    let bytes = std::slice::from_raw_parts(input_ptr as *const u8, input_len);
+
+    // Collect tokens using VM
+    let prog = &*prog_ptr;
+    let tokens = match vm::collect_tokens(prog, bytes) {
+        Ok(t) => t,
+        Err(msg) => {
+            let err_msg = std::ffi::CString::new(msg)
+                .unwrap_or_else(|_| std::ffi::CString::new("VM error").unwrap());
+            rb_raise(rb_eRuntimeError, err_msg.as_ptr());
+            unreachable!()
+        }
+    };
+
+    // Create buffer
+    let buffer = match FastTokenBuffer::from_tokens(tokens) {
+        Ok(b) => b,
+        Err(_msg) => {
+            rb_raise(rb_eRuntimeError, c"Lexer execution failed".as_ptr());
+            unreachable!()
+        }
+    };
+
+    // Create LexStream (minimal Rust struct)
+    let stream = Box::new(LexStream { buffer });
+    let stream_ptr = Box::into_raw(stream);
+
+    let obj = rb_data_typed_object_wrap(
+        LEX_STREAM_CLASS,
+        stream_ptr as *mut libc::c_void,
+        LEX_STREAM_DATA_TYPE,
+    );
+
+    // Get token_names from program
+    let token_names = rb_funcall(program, rb_intern(c"token_names".as_ptr()), 0);
+
+    // Set instance variables directly (avoids rb_funcall overhead)
+    rb_ivar_set(obj, rb_intern(c"@input".as_ptr()), input);
+    rb_ivar_set(obj, rb_intern(c"@token_names".as_ptr()), token_names);
+
+    obj
+}
+
+/// eof? -> bool
+unsafe extern "C" fn lex_stream_eof(slf: VALUE) -> VALUE {
+    let stream = get_stream(slf);
+    if (*stream).buffer.eof() {
+        Qtrue as VALUE
+    } else {
+        Qfalse as VALUE
+    }
+}
+
+/// advance -> self
+unsafe extern "C" fn lex_stream_advance(slf: VALUE) -> VALUE {
+    let stream = get_stream(slf);
+    (*stream).buffer.advance();
+    slf
+}
+
+/// token_id -> Integer or nil
+unsafe extern "C" fn lex_stream_token_id(slf: VALUE) -> VALUE {
+    let stream = get_stream(slf);
+    let id = (*stream).buffer.token_id();
+    if id < 0 {
+        Qnil as VALUE
+    } else {
+        rb_int2inum(id as isize)
+    }
+}
+
+/// input -> String
+unsafe extern "C" fn lex_stream_input(slf: VALUE) -> VALUE {
+    rb_ivar_get(slf, rb_intern(c"@input".as_ptr()))
+}
+
+/// start -> Integer or nil
+unsafe extern "C" fn lex_stream_start(slf: VALUE) -> VALUE {
+    let stream = get_stream(slf);
+    let s = (*stream).buffer.start();
+    if s < 0 {
+        Qnil as VALUE
+    } else {
+        rb_int2inum(s as isize)
+    }
+}
+
+/// len -> Integer or nil
+unsafe extern "C" fn lex_stream_len(slf: VALUE) -> VALUE {
+    let stream = get_stream(slf);
+    let l = (*stream).buffer.len();
+    if l < 0 {
+        Qnil as VALUE
+    } else {
+        rb_int2inum(l as isize)
+    }
+}
+
+/// peek_token_id(n = 1) -> Integer or nil
+unsafe extern "C" fn lex_stream_peek_token_id(argc: libc::c_int, argv: *const VALUE, slf: VALUE) -> VALUE {
+    let n = if argc == 0 {
+        1
+    } else {
+        let n_val = rb_num2long(*argv);
+        if n_val < 0 {
+            rb_raise(rb_eArgError, c"offset must be non-negative".as_ptr());
+            unreachable!()
+        }
+        n_val as usize
+    };
+
+    let stream = get_stream(slf);
+    let id = (*stream).buffer.peek_token_id(n);
+    if id < 0 {
+        Qnil as VALUE
+    } else {
+        rb_int2inum(id as isize)
+    }
+}
+
+/// peek_start(n = 1) -> Integer or nil
+unsafe extern "C" fn lex_stream_peek_start(argc: libc::c_int, argv: *const VALUE, slf: VALUE) -> VALUE {
+    let n = if argc == 0 {
+        1
+    } else {
+        let n_val = rb_num2long(*argv);
+        if n_val < 0 {
+            rb_raise(rb_eArgError, c"offset must be non-negative".as_ptr());
+            unreachable!()
+        }
+        n_val as usize
+    };
+
+    let stream = get_stream(slf);
+    let s = (*stream).buffer.peek_start(n);
+    if s < 0 {
+        Qnil as VALUE
+    } else {
+        rb_int2inum(s as isize)
+    }
+}
+
+/// peek_len(n = 1) -> Integer or nil
+unsafe extern "C" fn lex_stream_peek_len(argc: libc::c_int, argv: *const VALUE, slf: VALUE) -> VALUE {
+    let n = if argc == 0 {
+        1
+    } else {
+        let n_val = rb_num2long(*argv);
+        if n_val < 0 {
+            rb_raise(rb_eArgError, c"offset must be non-negative".as_ptr());
+            unreachable!()
+        }
+        n_val as usize
+    };
+
+    let stream = get_stream(slf);
+    let l = (*stream).buffer.peek_len(n);
+    if l < 0 {
+        Qnil as VALUE
+    } else {
+        rb_int2inum(l as isize)
+    }
+}
+
+// ============================================================================
+// Initialization
+// ============================================================================
+
+pub fn init_lex_stream() {
+    unsafe {
+        let data_type = Box::new(rb_data_type_t {
+            wrap_struct_name: c"LexerKit::LexStream".as_ptr(),
+            function: rb_data_type_struct__bindgen_ty_1 {
+                dmark: None, // No Ruby VALUEs to mark
+                dfree: Some(lex_stream_free),
+                dsize: Some(lex_stream_size),
+                dcompact: None,
+                reserved: [ptr::null_mut(); 1],
+            },
+            parent: ptr::null(),
+            data: ptr::null_mut(),
+            flags: 0,
+        });
+        LEX_STREAM_DATA_TYPE = Box::into_raw(data_type);
+
+        let lexer_kit_id = rb_intern(c"LexerKit".as_ptr());
+        let lexer_kit = if rb_const_defined(rb_cObject, lexer_kit_id) != 0 {
+            rb_const_get(rb_cObject, lexer_kit_id)
+        } else {
+            rb_define_module(c"LexerKit".as_ptr())
+        };
+
+        LEX_STREAM_CLASS = rb_define_class_under(
+            lexer_kit,
+            c"LexStream".as_ptr(),
+            rb_cObject,
+        );
+
+        rb_undef_alloc_func(LEX_STREAM_CLASS);
+
+        // Class method
+        rb_define_singleton_method(
+            LEX_STREAM_CLASS,
+            c"create".as_ptr(),
+            Some(std::mem::transmute::<
+                unsafe extern "C" fn(VALUE, VALUE, VALUE) -> VALUE,
+                unsafe extern "C" fn() -> VALUE,
+            >(lex_stream_create)),
+            2,
+        );
+
+        // Instance methods (performance-critical, kept in Rust)
+        define_ruby_method_noarg!(LEX_STREAM_CLASS, c"eof?", lex_stream_eof);
+        define_ruby_method_noarg!(LEX_STREAM_CLASS, c"advance", lex_stream_advance);
+        define_ruby_method_noarg!(LEX_STREAM_CLASS, c"token_id", lex_stream_token_id);
+        define_ruby_method_noarg!(LEX_STREAM_CLASS, c"start", lex_stream_start);
+        define_ruby_method_noarg!(LEX_STREAM_CLASS, c"len", lex_stream_len);
+        define_ruby_method_noarg!(LEX_STREAM_CLASS, c"input", lex_stream_input);
+
+        define_ruby_method_vararg!(LEX_STREAM_CLASS, c"peek_token_id", lex_stream_peek_token_id);
+        define_ruby_method_vararg!(LEX_STREAM_CLASS, c"peek_start", lex_stream_peek_start);
+        define_ruby_method_vararg!(LEX_STREAM_CLASS, c"peek_len", lex_stream_peek_len);
+    }
+}

data/ext/lexer_kit_rust/src/lib.rs

@@ -0,0 +1,248 @@
+//! LexerKit Rust Extension
+//!
+//! High-performance VM for executing compiled lexer programs.
+
+mod deserializer;
+#[cfg(not(test))]
+mod dfa;
+#[cfg(test)]
+pub mod dfa; // テスト時は公開
+#[cfg(not(test))]
+mod fast_stream;
+#[cfg(test)]
+pub mod fast_stream; // テスト時は公開
+#[cfg(not(test))]
+mod opcodes;
+#[cfg(test)]
+pub mod opcodes; // テスト時は公開
+#[cfg(not(test))]
+mod trie;
+#[cfg(test)]
+pub mod trie; // テスト時は公開
+#[cfg(not(test))]
+mod types;
+#[cfg(test)]
+pub mod types; // テスト時は公開
+#[cfg(not(test))]
+mod vm;
+#[cfg(test)]
+pub mod vm; // テスト時は公開
+
+#[cfg(test)]
+mod safety_test;
+
+use magnus::{
+    function, method, prelude::*, Error, IntoValue, RArray, RHash, RString, Ruby, TryConvert, Value,
+};
+use std::cell::RefCell;
+use types::CompiledProgram;
+
+/// Wrapper for CompiledProgram to make it shareable with Ruby
+#[magnus::wrap(class = "LexerKit::IR::RustHandle", free_immediately, size)]
+struct RustHandle {
+    prog: RefCell<CompiledProgram>,
+}
+
+impl RustHandle {
+    /// Load program from Ruby data hash
+    fn load_from_ruby(rb_data: RHash) -> Result<Self, Error> {
+        let prog = deserializer::parse_ruby_data(rb_data)?;
+        Ok(Self {
+            prog: RefCell::new(prog),
+        })
+    }
+
+    /// Tokenize input using the VM and yield tokens
+    fn lex_native(&self, rb_bytes: RString) -> Result<RArray, Error> {
+        let ruby = Ruby::get().unwrap();
+        let bytes = unsafe { rb_bytes.as_slice() };
+        let prog = self.prog.borrow();
+
+        let tokens = vm::collect_tokens(&prog, bytes)
+            .map_err(|e| Error::new(ruby.exception_runtime_error(), e))?;
+
+        let result = ruby.ary_new_capa(tokens.len());
+        for (token_id, start, length) in tokens {
+            let token = ruby.ary_new_from_values(&[
+                ruby.into_value(token_id),
+                ruby.into_value(start),
+                ruby.into_value(length),
+            ]);
+            result.push(token)?;
+        }
+
+        Ok(result)
+    }
+
+    /// Tokenize input and yield each token to a block
+    fn lex_native_block(&self, rb_bytes: RString) -> Result<Value, Error> {
+        let ruby = Ruby::get().unwrap();
+        let bytes = unsafe { rb_bytes.as_slice() };
+        let prog = self.prog.borrow();
+
+        let tokens = vm::collect_tokens(&prog, bytes)
+            .map_err(|e| Error::new(ruby.exception_runtime_error(), e))?;
+
+        for (token_id, start, length) in tokens {
+            let args = (
+                ruby.into_value(token_id),
+                ruby.into_value(start),
+                ruby.into_value(length),
+            );
+            let _: Value = ruby.yield_values(args)?;
+        }
+
+        Ok(ruby.qnil().as_value())
+    }
+
+}
+
+/// Internal variable name for Rust handle (non-@ name, inaccessible from Ruby)
+static RUST_HANDLE_IVAR: &std::ffi::CStr = c"__lkt_rust__";
+
+/// RustNative module methods - added to CompiledProgram
+struct RustNative;
+
+impl RustNative {
+    /// Load Rust native handle from Ruby data
+    /// compiled_program.load_rust_native(data) -> self
+    fn load_rust_native(rb_self: Value, rb_data: RHash) -> Result<Value, Error> {
+        let ruby = Ruby::get().unwrap();
+        let handle = RustHandle::load_from_ruby(rb_data)?;
+
+        // Store handle as internal variable (non-@ name)
+        unsafe {
+            let ivar_id = rb_sys::rb_intern(RUST_HANDLE_IVAR.as_ptr());
+            let handle_value: rb_sys::VALUE = std::mem::transmute::<Value, rb_sys::VALUE>(handle.into_value_with(&ruby));
+            rb_sys::rb_ivar_set(std::mem::transmute::<Value, rb_sys::VALUE>(rb_self), ivar_id, handle_value);
+        }
+
+        Ok(rb_self)
+    }
+
+    /// Check if Rust native handle is loaded
+    fn rust_native_loaded(rb_self: Value) -> bool {
+        let handle = unsafe {
+            let ivar_id = rb_sys::rb_intern(RUST_HANDLE_IVAR.as_ptr());
+            rb_sys::rb_ivar_get(std::mem::transmute::<Value, rb_sys::VALUE>(rb_self), ivar_id)
+        };
+        handle != rb_sys::Qnil as rb_sys::VALUE
+    }
+
+    /// Tokenize using Rust VM and yield to block
+    fn lex_rust_native(rb_self: Value, rb_bytes: RString) -> Result<Value, Error> {
+        let handle = Self::get_handle(rb_self)?;
+        handle.lex_native_block(rb_bytes)
+    }
+
+    /// Create a LexStream
+    fn create_rust_stream(rb_self: Value, rb_bytes: RString) -> Result<Value, Error> {
+        use magnus::IntoValue;
+        let handle = Self::get_handle(rb_self)?;
+        let prog = handle.prog.borrow();
+        let ruby = Ruby::get().unwrap();
+
+        // Set current program for LexStream to use
+        fast_stream::set_current_program(&*prog as *const CompiledProgram);
+
+        // Create LexStream directly using rb-sys
+        let result = unsafe {
+            let klass = rb_sys::rb_const_get(
+                rb_sys::rb_const_get(
+                    rb_sys::rb_cObject,
+                    rb_sys::rb_intern(c"LexerKit".as_ptr()),
+                ),
+                rb_sys::rb_intern(c"LexStream".as_ptr()),
+            );
+            let program_value: rb_sys::VALUE = std::mem::transmute(rb_self);
+            let input_value: rb_sys::VALUE = std::mem::transmute(rb_bytes.into_value_with(&ruby));
+            rb_sys::rb_funcall(
+                klass,
+                rb_sys::rb_intern(c"create".as_ptr()),
+                2,
+                program_value,
+                input_value,
+            )
+        };
+
+        Ok(unsafe { std::mem::transmute::<rb_sys::VALUE, Value>(result) })
+    }
+
+    /// Get handle from internal variable
+    ///
+    /// # Safety
+    /// The returned reference is valid as long as `rb_self` is kept alive by Ruby's GC.
+    /// Since this is called within Ruby method invocations where `rb_self` is rooted,
+    /// the reference is guaranteed to be valid for the duration of the method call.
+    ///
+    /// Note: The lifetime is tied to magnus's TypedData wrapper, not truly 'static.
+    /// Callers must not store this reference beyond the current method call.
+    fn get_handle<'a>(rb_self: Value) -> Result<&'a RustHandle, Error> {
+        let ruby = Ruby::get().unwrap();
+        let handle_value = unsafe {
+            let ivar_id = rb_sys::rb_intern(RUST_HANDLE_IVAR.as_ptr());
+            rb_sys::rb_ivar_get(std::mem::transmute::<Value, rb_sys::VALUE>(rb_self), ivar_id)
+        };
+
+        if handle_value == rb_sys::Qnil as rb_sys::VALUE {
+            return Err(Error::new(
+                ruby.exception_runtime_error(),
+                "Rust native not loaded. Call load_rust_native first.",
+            ));
+        }
+
+        // Convert VALUE back to RustHandle reference.
+        // Safety: The handle is stored as an instance variable of rb_self.
+        // Ruby's GC keeps rb_self alive during method calls, so the handle
+        // is valid for the duration of the caller's use.
+        let handle: &RustHandle = unsafe {
+            let value: Value = std::mem::transmute(handle_value);
+            TryConvert::try_convert(value)?
+        };
+
+        Ok(handle)
+    }
+}
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    // Define LexerKit module and IR submodule
+    let lexer_kit = ruby.define_module("LexerKit")?;
+    let ir = lexer_kit.define_module("IR")?;
+
+    // Define RustHandle class (internal, not meant to be used directly)
+    let rust_handle = ir.define_class("RustHandle", ruby.class_object())?;
+    rust_handle
+        .define_singleton_method("load_from_ruby", function!(RustHandle::load_from_ruby, 1))?;
+    rust_handle.define_method("lex_native", method!(RustHandle::lex_native, 1))?;
+    rust_handle.define_method("lex_native_block", method!(RustHandle::lex_native_block, 1))?;
+
+    // Define RustNative module (to be included in CompiledProgram)
+    let rust_native = ir.define_module("RustNative")?;
+    rust_native.define_method("load_rust_native", method!(RustNative::load_rust_native, 1))?;
+    rust_native.define_method(
+        "rust_native_loaded?",
+        method!(RustNative::rust_native_loaded, 0),
+    )?;
+    rust_native.define_method("lex_rust_native", method!(RustNative::lex_rust_native, 1))?;
+    rust_native.define_method(
+        "create_rust_stream",
+        method!(RustNative::create_rust_stream, 1),
+    )?;
+
+    // Include RustNative into CompiledProgram if it exists
+    let compiled_program = ir.const_get::<_, Value>("CompiledProgram");
+    if let Ok(cp) = compiled_program {
+        unsafe {
+            rb_sys::rb_include_module(
+                std::mem::transmute::<Value, rb_sys::VALUE>(cp),
+                std::mem::transmute::<Value, rb_sys::VALUE>(rust_native.as_value()),
+            );
+        }
+    }
+
+    // Initialize LexStream class (rb-sys based, no RefCell overhead)
+    fast_stream::init_lex_stream();
+
+    Ok(())
+}