lexer_kit 0.5.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 9461615637319b9b54724de2d2b7597bd6e8351157b893460aac1039942e3a36
+  data.tar.gz: dda19a6134d27d3524ffccc44b63eef24bd5ca60a905d0ba36ef38cf7b872699
+SHA512:
+  metadata.gz: bc120d92fe97d14f9dcd1522770c8872ad4722e6b8d40b471edffce25662507b5add3afb1a3424b73d5d2b4d30eaed3ef9d4a514d58fb2dbe761165fcfd876a8
+  data.tar.gz: d5c052b7e48111cc23fa5383fc37a9366368f8057c68d65293e421b1f933465b59de4a8f9ca3360d4eeffcb6efbdd15499669c488551a7e572f012bf9fc8676d

data/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Masayoshi Takahashi (takahashim)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,157 @@
+# LexerKit
+
+A high-performance lexer toolkit for Ruby.
+Define tokenizers with a Ruby DSL and run them through a Rust native extension.
+
+## Features
+
+- DSL-based lexer definition
+- Fast stream lexing with minimal allocation
+- On-demand token object creation for diagnostics
+- Compiled lexer serialization
+- Regex-based token patterns compiled to DFA
+
+## Installation
+
+```ruby
+# Gemfile
+gem "lexer_kit"
+```
+
+```bash
+bundle install
+```
+
+## Quick Start
+
+```ruby
+require "lexer_kit"
+
+lexer = LexerKit.build do
+  token :NUMBER, /[0-9]+/
+  token :PLUS,   "+"
+  token :MINUS,  "-"
+  token :SPACE,  /[ \t\r\n]+/, skip: true
+end.compile
+
+stream = lexer.stream("12 + 34 - 5")
+until stream.eof?
+  puts "#{stream.token_name}: #{stream.text.inspect}"
+  stream.advance
+end
+```
+
+## Core DSL
+
+### `token`
+
+```ruby
+token :IDENT,  /[a-zA-Z_][a-zA-Z0-9_]*/
+token :ARROW,  "->"
+token :SPACE,  /[ \t]+/, skip: true
+token :DQUOTE, '"', push: :string
+token :END_Q,  '"', pop: true
+```
+
+Options:
+
+- `skip: true` skips emitting the token
+- `push: :mode_name` pushes a mode
+- `pop: true` pops the current mode
+
+### `keyword` / `define_keywords`
+
+```ruby
+token :IDENT, /[a-z_]+/
+keyword :IF, "if"
+define_keywords :else, :while, :return
+```
+
+### `mode`
+
+```ruby
+LexerKit.build do
+  token :DQUOTE, '"', push: :string
+  token :IDENT,  /[a-z]+/
+
+  mode :string do
+    token :CONTENT, /[^"\\]+/
+    token :ESCAPE,  /\\./
+    token :DQUOTE,  '"', pop: true
+  end
+end
+```
+
+### `scan_until` / `delimited`
+
+```ruby
+scan_until :BLOCK_COMMENT, open: "/*", close: "*/", skip: true
+
+delimited :TEXT, delimiter: "{{" do
+  token :IDENT, /[a-zA-Z_]+/
+  token :DOT,   "."
+  token :CLOSE, "}}", pop: true
+end
+```
+
+### `utf8_range`
+
+```ruby
+token :HIRAGANA, LexerKit.utf8_range("ぁ".."ん")
+token :CJK,      LexerKit.utf8_range(0x4E00..0x9FFF)
+```
+
+## Regex Notes
+
+- Most common regex syntax is supported (`[]`, quantifiers, groups, alternation, escapes, `/.../i`)
+- Backtracking-dependent features are not supported (lookaround, backreference, etc.)
+- Anchors and word-boundary assertions are not used in lexer matching
+- `*?`, `+?`, `??` are parsed but behave as longest-match (DFA behavior)
+
+## Stream API and Error Handling
+
+`stream.start` and `stream.len` are byte offsets.
+
+```ruby
+stream = lexer.stream(input)
+until stream.eof?
+  if stream.error?
+    token = stream.make_token
+    puts token.render_diagnostic("unexpected character")
+  end
+  stream.advance
+end
+```
+
+`LexerKit` always falls back to `:INVALID` for unmatched input.
+
+## Serialization
+
+Pre-compile lexers for faster startup:
+
+```ruby
+lexer = builder.compile
+LexerKit::Format::LKT1.save(lexer, path: "lexer.lkt1")
+LexerKit::Format::LKB1.save(lexer, path: "lexer.lkb1")
+```
+
+```bash
+lexer_kit compile lexer.rb -o lexer.lkt1
+```
+
+Load later:
+
+```ruby
+lexer = LexerKit.load_lexer(File.expand_path("data/lexer.lkt1", __dir__))
+```
+
+## Performance Snapshot
+
+JSON benchmark (600KB input, project benchmark script):
+
+- LexerKit: `95.2 i/s`
+- StringScanner: `4.8 i/s` (about `20x` slower)
+
+## License
+
+MIT License

data/exe/lexer_kit

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require "lexer_kit"
+require "lexer_kit/cli"
+
+exit(LexerKit::CLI.run(ARGV))

data/ext/lexer_kit_rust/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "lexer_kit_rust"
+version = "0.1.0"
+edition = "2021"
+authors = ["LexerKit Authors"]
+license = "MIT"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = "0.8"
+rb-sys = "0.9"
+libc = "0.2"
+
+# Profile settings are in workspace root Cargo.toml

data/ext/lexer_kit_rust/extconf.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "mkmf"
+require "rb_sys/mkmf"
+
+create_rust_makefile("lexer_kit_rust/lexer_kit_rust")

data/ext/lexer_kit_rust/src/deserializer.rs

@@ -0,0 +1,213 @@
+//! Deserializer for CompiledProgram from Ruby data structures
+//!
+//! This module handles the conversion of Ruby Hash/Array data into
+//! the Rust CompiledProgram structure used by the VM.
+
+use magnus::{Error, RArray, RHash, RString, Ruby};
+
+use crate::types::{
+    CompiledProgram, ConstantEntry, DfaTable, Instruction, JumpTable, KeywordEntry, KeywordTable,
+    Mode,
+};
+
+// =============================================================================
+// Safety Helpers for Ruby String Access
+// =============================================================================
+
+/// Extract bytes from a Ruby String as a slice.
+///
+/// # Safety Contract
+/// The returned slice is valid only for the duration of the current function call.
+/// The caller must:
+/// - Use the slice immediately without storing it
+/// - Not call any Ruby methods that could trigger GC while using the slice
+/// - Not return the slice to the caller
+///
+/// This is safe within deserialization functions because:
+/// - The RString is rooted by Ruby's stack during the function call
+/// - No Ruby code runs between obtaining the slice and consuming it
+#[inline]
+fn rstring_as_bytes(rb_str: &RString) -> &[u8] {
+    // SAFETY: See function documentation. The slice is used immediately
+    // within deserialization and no GC can occur.
+    unsafe { rb_str.as_slice() }
+}
+
+/// Copy bytes from a Ruby String into an owned Vec.
+///
+/// This is the preferred method when the data needs to outlive the current scope.
+/// The data is immediately copied, avoiding any lifetime concerns.
+#[inline]
+fn rstring_to_vec(rb_str: &RString) -> Vec<u8> {
+    // SAFETY: We immediately copy the data into an owned Vec,
+    // so the temporary slice lifetime is not an issue.
+    unsafe { rb_str.as_slice() }.to_vec()
+}
+
+/// Parse Ruby data hash into CompiledProgram
+pub fn parse_ruby_data(rb_data: RHash) -> Result<CompiledProgram, Error> {
+    let ruby = Ruby::get().unwrap();
+    let mut prog = CompiledProgram::new();
+
+    // Instructions
+    let rb_instructions: RString = rb_data.fetch::<_, RString>(ruby.sym_new("instructions"))?;
+    let instr_bytes = rstring_as_bytes(&rb_instructions);
+    let instr_count = instr_bytes.len() / 4;
+    prog.instructions = (0..instr_count)
+        .map(|i| Instruction::from_bytes(&instr_bytes[i * 4..(i + 1) * 4]))
+        .collect();
+
+    // DFA tables
+    let rb_dfas: RArray = rb_data.fetch(ruby.sym_new("dfa_tables"))?;
+    for i in 0..rb_dfas.len() {
+        let rb_dfa: RHash = rb_dfas.entry(i as isize)?;
+        let dfa = parse_dfa_table(&ruby, rb_dfa)?;
+        prog.dfa_tables.push(dfa);
+    }
+
+    // Jump tables
+    let rb_jumps: RArray = rb_data.fetch(ruby.sym_new("jump_tables"))?;
+    for i in 0..rb_jumps.len() {
+        let rb_jt: RHash = rb_jumps.entry(i as isize)?;
+        let jt = parse_jump_table(&ruby, rb_jt)?;
+        prog.jump_tables.push(jt);
+    }
+
+    // Keyword tables
+    let rb_keywords: RArray = rb_data.fetch(ruby.sym_new("keyword_tables"))?;
+    for i in 0..rb_keywords.len() {
+        let rb_kt: RHash = rb_keywords.entry(i as isize)?;
+        let kt = parse_keyword_table(&ruby, rb_kt)?;
+        prog.keyword_tables.push(kt);
+    }
+
+    // Constant pool
+    let rb_pool: RArray = rb_data.fetch(ruby.sym_new("constant_pool"))?;
+    for i in 0..rb_pool.len() {
+        let rb_entry: RString = rb_pool.entry(i as isize)?;
+        let data = rstring_to_vec(&rb_entry);
+        prog.constant_pool.entries.push(ConstantEntry { data });
+    }
+
+    // Modes
+    let rb_modes: RArray = rb_data.fetch(ruby.sym_new("modes"))?;
+    for i in 0..rb_modes.len() {
+        let rb_mode: RArray = rb_modes.entry(i as isize)?;
+        let rb_name: RString = rb_mode.entry(0)?;
+        let rb_offset: u32 = rb_mode.entry(1)?;
+
+        let name = rstring_as_bytes(&rb_name);
+        prog.modes.push(Mode {
+            start_offset: rb_offset,
+        });
+
+        // Check for default mode
+        if name == b"default" {
+            prog.default_mode_offset = rb_offset;
+        }
+    }
+
+    Ok(prog)
+}
+
+/// Parse DFA table from Ruby hash
+fn parse_dfa_table(ruby: &Ruby, rb_dfa: RHash) -> Result<DfaTable, Error> {
+    let mut dfa = DfaTable::new();
+
+    dfa.state_count = rb_dfa.fetch::<_, u16>(ruby.sym_new("state_count"))?;
+    dfa.class_count = rb_dfa.fetch::<_, u16>(ruby.sym_new("class_count"))?;
+
+    // Byte class
+    let rb_byte_class: RString = rb_dfa.fetch(ruby.sym_new("byte_class"))?;
+    let byte_class_data = rstring_as_bytes(&rb_byte_class);
+    if byte_class_data.len() < 256 {
+        return Err(Error::new(
+            ruby.exception_arg_error(),
+            "byte_class must be 256 bytes",
+        ));
+    }
+    dfa.byte_class = byte_class_data[..256].to_vec();
+
+    // Transitions (big-endian u16 packed) - validate size
+    let rb_trans: RString = rb_dfa.fetch(ruby.sym_new("transitions"))?;
+    let trans_data = rstring_as_bytes(&rb_trans);
+    let trans_size = (dfa.state_count as usize) * (dfa.class_count as usize);
+    let required_trans_bytes = trans_size * 2;
+    if trans_data.len() < required_trans_bytes {
+        return Err(Error::new(
+            ruby.exception_arg_error(),
+            "transitions data too short",
+        ));
+    }
+    dfa.transitions = (0..trans_size)
+        .map(|i| {
+            let off = i * 2;
+            ((trans_data[off] as u16) << 8) | (trans_data[off + 1] as u16)
+        })
+        .collect();
+
+    // Accept tokens (big-endian u16 packed) - validate size
+    let rb_accept: RString = rb_dfa.fetch(ruby.sym_new("accept_tokens"))?;
+    let accept_data = rstring_as_bytes(&rb_accept);
+    let required_accept_bytes = (dfa.state_count as usize) * 2;
+    if accept_data.len() < required_accept_bytes {
+        return Err(Error::new(
+            ruby.exception_arg_error(),
+            "accept_tokens must have state_count entries",
+        ));
+    }
+    dfa.accept_tokens = (0..dfa.state_count as usize)
+        .map(|i| {
+            let off = i * 2;
+            ((accept_data[off] as u16) << 8) | (accept_data[off + 1] as u16)
+        })
+        .collect();
+
+    Ok(dfa)
+}
+
+/// Parse jump table from Ruby hash
+fn parse_jump_table(ruby: &Ruby, rb_jt: RHash) -> Result<JumpTable, Error> {
+    let mut jt = JumpTable::new();
+
+    // Lookup table (big-endian u32 packed, 256 entries) - validate size
+    let rb_lookup: RString = rb_jt.fetch(ruby.sym_new("lookup"))?;
+    let lookup_data = rstring_as_bytes(&rb_lookup);
+    if lookup_data.len() < 256 * 4 {
+        return Err(Error::new(
+            ruby.exception_arg_error(),
+            "jump table lookup must be 1024 bytes",
+        ));
+    }
+    for i in 0..256 {
+        let off = i * 4;
+        jt.lookup[i] = ((lookup_data[off] as u32) << 24)
+            | ((lookup_data[off + 1] as u32) << 16)
+            | ((lookup_data[off + 2] as u32) << 8)
+            | (lookup_data[off + 3] as u32);
+    }
+
+    // Default offset
+    let rb_default: Option<u32> = rb_jt.fetch(ruby.sym_new("default_offset")).ok();
+    jt.default_offset = rb_default.unwrap_or(0);
+
+    Ok(jt)
+}
+
+/// Parse keyword table from Ruby hash
+fn parse_keyword_table(ruby: &Ruby, rb_kt: RHash) -> Result<KeywordTable, Error> {
+    let base_token_id: u16 = rb_kt.fetch(ruby.sym_new("base_token_id"))?;
+    let mut kt = KeywordTable::new(base_token_id);
+
+    let rb_keywords: RArray = rb_kt.fetch(ruby.sym_new("keywords"))?;
+    for i in 0..rb_keywords.len() {
+        let rb_entry: RArray = rb_keywords.entry(i as isize)?;
+        let rb_key: RString = rb_entry.entry(0)?;
+        let token_id: u16 = rb_entry.entry(1)?;
+
+        let keyword = rstring_to_vec(&rb_key);
+        kt.entries.push(KeywordEntry { keyword, token_id });
+    }
+
+    Ok(kt)
+}

data/ext/lexer_kit_rust/src/dfa.rs

@@ -0,0 +1,217 @@
+//! DFA (Deterministic Finite Automaton) execution
+//!
+//! This module implements DFA-based pattern matching for the lexer VM.
+
+use crate::types::{DfaTable, DFA_DEAD_STATE, DFA_NO_ACCEPT};
+
+/// DFA match result (length of the match)
+#[derive(Debug, Clone, Copy)]
+pub struct DfaMatch {
+    pub length: usize,
+}
+
+impl DfaTable {
+    /// Run DFA on input bytes, returning the match length and token ID.
+    ///
+    /// Returns None if no match, Some((length, token_id)) on match.
+    /// Supports zero-length matches (e.g., "a*" matching empty string).
+    pub fn run(&self, bytes: &[u8]) -> Option<DfaMatch> {
+        // Safety check: verify DFA has valid dimensions
+        if self.state_count == 0 || self.class_count == 0 {
+            return None;
+        }
+        if self.byte_class.len() < 256 {
+            return None;
+        }
+        if self.transitions.is_empty() || self.accept_tokens.is_empty() {
+            return None;
+        }
+
+        let mut state: u16 = 1; // Start state (0 is dead state)
+        let mut last_accept: Option<DfaMatch> = None;
+
+        // Validate start state is within bounds
+        if state >= self.state_count {
+            return None;
+        }
+
+        // Check if start state is accepting (handles zero-length matches like "a*")
+        if self.accept_tokens[state as usize] != DFA_NO_ACCEPT {
+            last_accept = Some(DfaMatch { length: 0 });
+        }
+
+        for (i, &byte) in bytes.iter().enumerate() {
+            let cls = self.byte_class[byte as usize];
+            let idx = (state as usize) * (self.class_count as usize) + (cls as usize);
+
+            if idx >= self.transitions.len() {
+                break;
+            }
+
+            state = self.transitions[idx];
+
+            if state == DFA_DEAD_STATE || state >= self.state_count {
+                break;
+            }
+
+            // Check if accepting state
+            if self.accept_tokens[state as usize] != DFA_NO_ACCEPT {
+                last_accept = Some(DfaMatch { length: i + 1 });
+            }
+        }
+
+        last_accept
+    }
+}
+
+/// Scan until delimiter found (using memchr for optimization)
+///
+/// Returns the position where delimiter starts, or input length if not found.
+pub fn scan_until(bytes: &[u8], delim: &[u8]) -> usize {
+    if delim.is_empty() {
+        return bytes.len();
+    }
+
+    if delim.len() == 1 {
+        // Single-byte delimiter: use memchr for SIMD optimization
+        return memchr_single(bytes, delim[0]).unwrap_or(bytes.len());
+    }
+
+    // Multi-byte delimiter: search for first byte, then check rest
+    let first = delim[0];
+    let mut pos = 0;
+
+    while pos < bytes.len() {
+        match memchr_single(&bytes[pos..], first) {
+            Some(idx) => {
+                let abs_idx = pos + idx;
+                if abs_idx + delim.len() > bytes.len() {
+                    return bytes.len();
+                }
+                if &bytes[abs_idx..abs_idx + delim.len()] == delim {
+                    return abs_idx;
+                }
+                pos = abs_idx + 1;
+            }
+            None => return bytes.len(),
+        }
+    }
+
+    bytes.len()
+}
+
+/// Simple memchr implementation
+#[inline]
+fn memchr_single(haystack: &[u8], needle: u8) -> Option<usize> {
+    haystack.iter().position(|&b| b == needle)
+}
+
+/// Match literal string at current position
+///
+/// Returns true if the bytes at the start match the literal.
+#[inline]
+pub fn match_literal(bytes: &[u8], literal: &[u8]) -> bool {
+    if bytes.len() < literal.len() {
+        return false;
+    }
+    &bytes[..literal.len()] == literal
+}
+
+/// Scan until delimiter found, skipping escape sequences
+///
+/// Returns (found, position) where found is true if delimiter was found.
+pub fn scan_until_escape(bytes: &[u8], close: &[u8], escape: &[u8]) -> (bool, usize) {
+    let mut pos = 0;
+
+    while pos < bytes.len() {
+        let remaining = &bytes[pos..];
+        let scanned = scan_until(remaining, close);
+
+        if scanned >= remaining.len() {
+            // Delimiter not found - reached EOF
+            return (false, bytes.len());
+        }
+
+        let hit = pos + scanned;
+
+        // Check if this delimiter is part of an escape sequence
+        if !escape.is_empty() {
+            // Case 1: escape sequence starts with the close delimiter
+            // e.g., escape = "{[{]}" with close = "{["
+            if escape.starts_with(close)
+                && hit + escape.len() <= bytes.len()
+                && &bytes[hit..hit + escape.len()] == escape
+            {
+                // Escape sequence found - skip past it and continue
+                pos = hit + escape.len();
+                continue;
+            }
+
+            // Case 2: escape sequence ends with the close delimiter
+            // e.g., escape = "\\\"" with close = "\""
+            if escape.len() > close.len() && escape.ends_with(close) {
+                let prefix_len = escape.len() - close.len();
+                if hit >= prefix_len && &bytes[hit - prefix_len..hit + close.len()] == escape {
+                    // Escape sequence found - skip past the close delimiter
+                    pos = hit + close.len();
+                    continue;
+                }
+            }
+        }
+
+        // Delimiter found without escape
+        return (true, hit);
+    }
+
+    (false, bytes.len())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_scan_until_single_byte() {
+        assert_eq!(scan_until(b"hello world", b" "), 5);
+        assert_eq!(scan_until(b"hello", b" "), 5);
+        assert_eq!(scan_until(b" hello", b" "), 0);
+    }
+
+    #[test]
+    fn test_scan_until_multi_byte() {
+        assert_eq!(scan_until(b"hello-->world", b"-->"), 5);
+        assert_eq!(scan_until(b"-->world", b"-->"), 0);
+        assert_eq!(scan_until(b"hello world", b"-->"), 11);
+    }
+
+    #[test]
+    fn test_match_literal() {
+        assert!(match_literal(b"hello world", b"hello"));
+        assert!(!match_literal(b"hello world", b"world"));
+        assert!(!match_literal(b"hi", b"hello"));
+    }
+
+    #[test]
+    fn test_scan_until_escape() {
+        // Without escape - finds delimiter at position 5
+        assert_eq!(
+            scan_until_escape(b"hello\"world", b"\"", b"\\\""),
+            (true, 5)
+        );
+        // With escape - escape starts at position 5, skips past it, finds real delimiter at 12
+        assert_eq!(
+            scan_until_escape(b"hello\\\"world\"end", b"\"", b"\\\""),
+            (true, 12)
+        );
+        // No delimiter
+        assert_eq!(
+            scan_until_escape(b"hello world", b"\"", b"\\\""),
+            (false, 11)
+        );
+        // Multi-byte delimiter with escape
+        assert_eq!(
+            scan_until_escape(b"hello{[{]}world{[test", b"{[", b"{[{]}"),
+            (true, 15)
+        );
+    }
+}