lexer_kit 0.5.0

data/ext/lexer_kit_rust/src/safety_test.rs

@@ -0,0 +1,498 @@
+//! Safety tests that can be run under Miri
+//!
+//! Run with: cargo +nightly miri test
+//!
+//! These tests verify memory safety for:
+//! - FastTokenBuffer (Vec-based token buffer)
+//! - DFA execution (boundary conditions)
+//! - Trie execution (malformed data handling)
+//! - VM execution (infinite loop detection, stack overflow)
+
+#[cfg(test)]
+mod tests {
+    use crate::dfa::{match_literal, scan_until, scan_until_escape};
+    use crate::fast_stream::FastTokenBuffer;
+    use crate::trie;
+    use crate::types::{
+        CompiledProgram, DfaTable as DfaTableType, Instruction, KeywordEntry, KeywordTable, Mode,
+        DFA_NO_ACCEPT,
+    };
+    use crate::vm;
+
+    // =========================================================================
+    // FastTokenBuffer tests
+    // =========================================================================
+
+    #[test]
+    fn test_token_buffer_basic() {
+        let tokens = vec![
+            (1u16, 0usize, 5usize),
+            (2u16, 5usize, 3usize),
+            (3u16, 8usize, 2usize),
+        ];
+        let mut buffer = FastTokenBuffer::from_tokens(tokens).unwrap();
+
+        assert!(!buffer.eof());
+        assert_eq!(buffer.token_id(), 1);
+        assert_eq!(buffer.start(), 0);
+        assert_eq!(buffer.len(), 5);
+
+        buffer.advance();
+        assert_eq!(buffer.token_id(), 2);
+
+        buffer.advance();
+        assert_eq!(buffer.token_id(), 3);
+
+        buffer.advance();
+        assert!(buffer.eof());
+        assert_eq!(buffer.token_id(), -1);
+    }
+
+    #[test]
+    fn test_empty_buffer() {
+        let buffer = FastTokenBuffer::from_tokens(vec![]).unwrap();
+        assert!(buffer.eof());
+        assert_eq!(buffer.token_id(), -1);
+        assert_eq!(buffer.start(), -1);
+        assert_eq!(buffer.len(), -1);
+    }
+
+    #[test]
+    fn test_peek() {
+        let tokens = vec![(1u16, 0usize, 5usize), (2u16, 5usize, 3usize)];
+        let buffer = FastTokenBuffer::from_tokens(tokens).unwrap();
+
+        assert_eq!(buffer.peek_token_id(0), 1);
+        assert_eq!(buffer.peek_token_id(1), 2);
+        assert_eq!(buffer.peek_token_id(2), -1); // out of bounds
+        assert_eq!(buffer.peek_start(0), 0);
+        assert_eq!(buffer.peek_start(1), 5);
+        assert_eq!(buffer.peek_start(2), -1);
+        assert_eq!(buffer.peek_len(0), 5);
+        assert_eq!(buffer.peek_len(1), 3);
+        assert_eq!(buffer.peek_len(2), -1);
+    }
+
+    #[test]
+    fn test_single_token() {
+        let tokens = vec![(42u16, 100usize, 50usize)];
+        let mut buffer = FastTokenBuffer::from_tokens(tokens).unwrap();
+
+        assert_eq!(buffer.count(), 1);
+        assert_eq!(buffer.index(), 0);
+        assert_eq!(buffer.token_id(), 42);
+        assert_eq!(buffer.start(), 100);
+        assert_eq!(buffer.len(), 50);
+
+        buffer.advance();
+        assert!(buffer.eof());
+        assert_eq!(buffer.index(), 1);
+    }
+
+    #[test]
+    fn test_many_tokens() {
+        let tokens: Vec<_> = (0..1000)
+            .map(|i| (i as u16, i * 10, 10usize))
+            .collect();
+        let mut buffer = FastTokenBuffer::from_tokens(tokens).unwrap();
+
+        assert_eq!(buffer.count(), 1000);
+
+        for i in 0..1000 {
+            assert_eq!(buffer.token_id(), i as i32);
+            assert_eq!(buffer.start(), (i * 10) as i32);
+            buffer.advance();
+        }
+
+        assert!(buffer.eof());
+    }
+
+    #[test]
+    fn test_advance_past_end() {
+        let tokens = vec![(1u16, 0usize, 1usize)];
+        let mut buffer = FastTokenBuffer::from_tokens(tokens).unwrap();
+
+        buffer.advance();
+        assert!(buffer.eof());
+
+        // Advancing past end should be safe
+        buffer.advance();
+        buffer.advance();
+        buffer.advance();
+        assert!(buffer.eof());
+        assert_eq!(buffer.token_id(), -1);
+    }
+
+    // =========================================================================
+    // DFA tests
+    // =========================================================================
+
+    #[test]
+    fn test_dfa_empty_input() {
+        let dfa = create_simple_dfa();
+        let result = dfa.run(b"");
+        // Empty input may or may not match depending on DFA (zero-length match)
+        assert!(result.is_none() || result.unwrap().length == 0);
+    }
+
+    #[test]
+    fn test_dfa_empty_dfa() {
+        let dfa = DfaTableType::new();
+        let result = dfa.run(b"hello");
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_dfa_single_byte_match() {
+        let dfa = create_simple_dfa();
+        let result = dfa.run(b"a");
+        assert!(result.is_some());
+        assert_eq!(result.unwrap().length, 1);
+    }
+
+    #[test]
+    fn test_dfa_no_match() {
+        let dfa = create_simple_dfa();
+        let result = dfa.run(b"xyz");
+        assert!(result.is_none());
+    }
+
+    fn create_simple_dfa() -> DfaTableType {
+        // DFA that matches 'a'
+        let mut dfa = DfaTableType::new();
+        dfa.state_count = 3;
+        dfa.class_count = 2;
+        dfa.byte_class = vec![0; 256];
+        dfa.byte_class[b'a' as usize] = 1;
+        dfa.transitions = vec![
+            0, 0, // state 0 (dead)
+            0, 2, // state 1: class 0 -> dead, class 1 ('a') -> state 2
+            0, 0, // state 2 (accept)
+        ];
+        dfa.accept_tokens = vec![
+            DFA_NO_ACCEPT, // state 0
+            DFA_NO_ACCEPT, // state 1
+            1,             // state 2: token 1
+        ];
+        dfa
+    }
+
+    // =========================================================================
+    // scan_until / match_literal tests
+    // =========================================================================
+
+    #[test]
+    fn test_scan_until_empty() {
+        assert_eq!(scan_until(b"", b"x"), 0);
+        assert_eq!(scan_until(b"hello", b""), 5);
+    }
+
+    #[test]
+    fn test_scan_until_found() {
+        assert_eq!(scan_until(b"hello world", b" "), 5);
+        assert_eq!(scan_until(b"hello-->world", b"-->"), 5);
+    }
+
+    #[test]
+    fn test_scan_until_not_found() {
+        assert_eq!(scan_until(b"hello", b"x"), 5);
+    }
+
+    #[test]
+    fn test_scan_until_escape_basic() {
+        let (found, pos) = scan_until_escape(b"hello\"world", b"\"", b"\\\"");
+        assert!(found);
+        assert_eq!(pos, 5);
+    }
+
+    #[test]
+    fn test_scan_until_escape_with_escape() {
+        let (found, pos) = scan_until_escape(b"hello\\\"world\"end", b"\"", b"\\\"");
+        assert!(found);
+        assert_eq!(pos, 12);
+    }
+
+    #[test]
+    fn test_match_literal_basic() {
+        assert!(match_literal(b"hello", b"hel"));
+        assert!(!match_literal(b"hello", b"world"));
+        assert!(!match_literal(b"hi", b"hello"));
+    }
+
+    // =========================================================================
+    // Trie tests
+    // =========================================================================
+
+    #[test]
+    fn test_trie_empty_data() {
+        let result = trie::run(&[], b"hello");
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_trie_empty_input() {
+        let trie_data = build_simple_trie();
+        let result = trie::run(&trie_data, b"");
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_trie_match() {
+        let trie_data = build_simple_trie();
+        let result = trie::run(&trie_data, b"a");
+        assert!(result.is_some());
+        let m = result.unwrap();
+        assert_eq!(m.length, 1);
+    }
+
+    #[test]
+    fn test_trie_longer_match() {
+        let trie_data = build_simple_trie();
+        let result = trie::run(&trie_data, b"ab");
+        assert!(result.is_some());
+        let m = result.unwrap();
+        assert_eq!(m.length, 2);
+    }
+
+    #[test]
+    fn test_trie_no_match() {
+        let trie_data = build_simple_trie();
+        let result = trie::run(&trie_data, b"xyz");
+        assert!(result.is_none());
+    }
+
+    #[test]
+    fn test_trie_malformed_short() {
+        // Too short to be valid
+        let result = trie::run(&[0, 0, 0, 1], b"a");
+        assert!(result.is_none());
+    }
+
+    fn build_simple_trie() -> Vec<u8> {
+        // Trie for "a" (action=10) and "ab" (action=20)
+        let mut trie = Vec::new();
+
+        // Header: 3 nodes, 2 edges
+        trie.extend_from_slice(&3u32.to_be_bytes());
+        trie.extend_from_slice(&2u32.to_be_bytes());
+
+        // Node 0: root
+        trie.extend_from_slice(&0u32.to_be_bytes()); // edge_start
+        trie.extend_from_slice(&1u16.to_be_bytes()); // edge_len
+        trie.extend_from_slice(&0xFFFFu16.to_be_bytes()); // order (not accepting)
+        trie.extend_from_slice(&0xFFFFFFFFu32.to_be_bytes()); // action (not accepting)
+
+        // Node 1: after 'a'
+        trie.extend_from_slice(&1u32.to_be_bytes()); // edge_start
+        trie.extend_from_slice(&1u16.to_be_bytes()); // edge_len
+        trie.extend_from_slice(&1u16.to_be_bytes()); // order
+        trie.extend_from_slice(&10u32.to_be_bytes()); // action
+
+        // Node 2: after 'ab'
+        trie.extend_from_slice(&0u32.to_be_bytes()); // edge_start
+        trie.extend_from_slice(&0u16.to_be_bytes()); // edge_len
+        trie.extend_from_slice(&0u16.to_be_bytes()); // order
+        trie.extend_from_slice(&20u32.to_be_bytes()); // action
+
+        // Edge 0: 'a' -> node 1
+        trie.push(b'a');
+        trie.extend_from_slice(&1u32.to_be_bytes());
+
+        // Edge 1: 'b' -> node 2
+        trie.push(b'b');
+        trie.extend_from_slice(&2u32.to_be_bytes());
+
+        trie
+    }
+
+    // =========================================================================
+    // VM tests
+    // =========================================================================
+
+    #[test]
+    fn test_vm_empty_program() {
+        let prog = CompiledProgram::new();
+        let result = vm::collect_tokens(&prog, b"hello");
+        assert!(result.is_ok());
+        assert!(result.unwrap().is_empty());
+    }
+
+    #[test]
+    fn test_vm_empty_input() {
+        let prog = create_halt_program();
+        let result = vm::collect_tokens(&prog, b"");
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_vm_single_token() {
+        let prog = create_simple_vm_program();
+        let result = vm::collect_tokens(&prog, b"a");
+        assert!(result.is_ok());
+        let tokens = result.unwrap();
+        assert_eq!(tokens.len(), 1);
+        assert_eq!(tokens[0], (1, 0, 1));
+    }
+
+    #[test]
+    fn test_vm_multiple_tokens() {
+        let prog = create_simple_vm_program();
+        let result = vm::collect_tokens(&prog, b"aaa");
+        assert!(result.is_ok());
+        let tokens = result.unwrap();
+        assert_eq!(tokens.len(), 3);
+    }
+
+    #[test]
+    fn test_vm_mode_stack() {
+        // Test that mode operations don't cause memory issues
+        let mut prog = CompiledProgram::new();
+
+        // Add modes
+        prog.modes.push(Mode { start_offset: 0 });
+        prog.modes.push(Mode { start_offset: 2 });
+
+        // Instructions: PUSH_MODE 1, POP_MODE, HALT
+        prog.instructions = vec![
+            Instruction {
+                opcode: 0x30, // PUSH_MODE
+                arg_hi: 0,
+                arg_mid: 0,
+                arg_lo: 1,
+            },
+            Instruction {
+                opcode: 0xFF, // HALT (at mode 1 offset)
+                arg_hi: 0,
+                arg_mid: 0,
+                arg_lo: 0,
+            },
+            Instruction {
+                opcode: 0xFF, // HALT
+                arg_hi: 0,
+                arg_mid: 0,
+                arg_lo: 0,
+            },
+        ];
+
+        let result = vm::collect_tokens(&prog, b"x");
+        assert!(result.is_ok());
+    }
+
+    #[test]
+    fn test_vm_keyword_lookup() {
+        let mut prog = CompiledProgram::new();
+
+        // Add a keyword table
+        let mut kt = KeywordTable::new(1); // base token = 1
+        kt.entries.push(KeywordEntry {
+            keyword: b"if".to_vec(),
+            token_id: 2,
+        });
+        prog.keyword_tables.push(kt);
+
+        // Simple program that just halts
+        prog.instructions = vec![Instruction {
+            opcode: 0xFF, // HALT
+            arg_hi: 0,
+            arg_mid: 0,
+            arg_lo: 0,
+        }];
+
+        let result = vm::collect_tokens(&prog, b"if");
+        assert!(result.is_ok());
+    }
+
+    fn create_halt_program() -> CompiledProgram {
+        let mut prog = CompiledProgram::new();
+        prog.instructions = vec![Instruction {
+            opcode: 0xFF, // HALT
+            arg_hi: 0,
+            arg_mid: 0,
+            arg_lo: 0,
+        }];
+        prog.modes.push(Mode { start_offset: 0 });
+        prog
+    }
+
+    fn create_simple_vm_program() -> CompiledProgram {
+        use crate::opcodes::*;
+
+        let mut prog = CompiledProgram::new();
+
+        // DFA that matches 'a'
+        let mut dfa = DfaTableType::new();
+        dfa.state_count = 3;
+        dfa.class_count = 2;
+        dfa.byte_class = vec![0; 256];
+        dfa.byte_class[b'a' as usize] = 1;
+        dfa.transitions = vec![0, 0, 0, 2, 0, 0];
+        dfa.accept_tokens = vec![DFA_NO_ACCEPT, DFA_NO_ACCEPT, 1];
+        prog.dfa_tables.push(dfa);
+
+        // Instructions:
+        // 0: MARK
+        // 1: DFA_RUN_IF_MATCH (dfa=0, fail->4)
+        // 2: EMIT 1
+        // 3: JUMP 0
+        // 4: HALT
+        prog.instructions = vec![
+            Instruction {
+                opcode: OP_MARK,
+                arg_hi: 0,
+                arg_mid: 0,
+                arg_lo: 0,
+            },
+            Instruction {
+                opcode: OP_DFA_RUN_IF_MATCH,
+                arg_hi: 0,
+                arg_mid: 0,
+                arg_lo: 4,
+            },
+            Instruction {
+                opcode: OP_EMIT,
+                arg_hi: 0,
+                arg_mid: 0,
+                arg_lo: 1,
+            },
+            Instruction {
+                opcode: OP_JUMP,
+                arg_hi: 0,
+                arg_mid: 0,
+                arg_lo: 0,
+            },
+            Instruction {
+                opcode: OP_HALT,
+                arg_hi: 0,
+                arg_mid: 0,
+                arg_lo: 0,
+            },
+        ];
+
+        prog.modes.push(Mode { start_offset: 0 });
+        prog.default_mode_offset = 0;
+
+        prog
+    }
+
+    // =========================================================================
+    // Stress tests (smaller under Miri)
+    // =========================================================================
+
+    #[test]
+    fn test_stress_token_buffer() {
+        // Miri is slow, so keep this reasonable
+        let count = if cfg!(miri) { 100 } else { 10000 };
+
+        let tokens: Vec<_> = (0..count).map(|i| (i as u16, i, 1usize)).collect();
+
+        let mut buffer = FastTokenBuffer::from_tokens(tokens).unwrap();
+
+        for i in 0..count {
+            assert_eq!(buffer.token_id(), i as i32);
+            buffer.advance();
+        }
+
+        assert!(buffer.eof());
+    }
+
+}

data/ext/lexer_kit_rust/src/trie.rs

@@ -0,0 +1,206 @@
+//! Literal Trie matching
+//!
+//! This module implements trie-based literal matching for the lexer VM.
+//! The trie format is a binary serialization format:
+//!
+//! Header (8 bytes):
+//!   - node_count: u32 (big-endian)
+//!   - edge_count: u32 (big-endian)
+//!
+//! Nodes (12 bytes each):
+//!   - edge_start: u32 (big-endian) - index into edges array
+//!   - edge_len: u16 (big-endian) - number of outgoing edges
+//!   - order: u16 (big-endian) - match priority
+//!   - action: u32 (big-endian) - action IP (0xFFFFFFFF = no action)
+//!
+//! Edges (5 bytes each):
+//!   - byte: u8 - the byte to match
+//!   - child_idx: u32 (big-endian) - index of child node
+
+/// Trie match result
+#[derive(Debug, Clone, Copy)]
+pub struct TrieMatch {
+    pub length: usize,
+    pub order: u16,
+    pub action_ip: u32,
+}
+
+/// Read big-endian u32 from bytes
+#[inline]
+fn read_u32(bytes: &[u8]) -> u32 {
+    ((bytes[0] as u32) << 24)
+        | ((bytes[1] as u32) << 16)
+        | ((bytes[2] as u32) << 8)
+        | (bytes[3] as u32)
+}
+
+/// Read big-endian u16 from bytes
+#[inline]
+fn read_u16(bytes: &[u8]) -> u16 {
+    ((bytes[0] as u16) << 8) | (bytes[1] as u16)
+}
+
+/// Run literal trie and return best match
+///
+/// Returns None if no match, Some(TrieMatch) on match.
+pub fn run(trie: &[u8], input: &[u8]) -> Option<TrieMatch> {
+    // Minimum trie size: header (8) + at least one node (12)
+    if trie.len() < 8 {
+        return None;
+    }
+
+    let node_count = read_u32(&trie[0..4]);
+    let edge_count = read_u32(&trie[4..8]);
+
+    let nodes_offset = 8;
+    let nodes_size = (node_count as usize) * 12;
+    let edges_offset = nodes_offset + nodes_size;
+    let edges_size = (edge_count as usize) * 5;
+
+    if edges_offset + edges_size > trie.len() {
+        return None;
+    }
+
+    let mut best: Option<TrieMatch> = None;
+    let mut node_idx: u32 = 0;
+    let mut pos = 0;
+
+    while pos < input.len() {
+        // Read current node
+        let node_ptr = nodes_offset + (node_idx as usize) * 12;
+        if node_ptr + 12 > trie.len() {
+            break;
+        }
+
+        let edge_start = read_u32(&trie[node_ptr..node_ptr + 4]);
+        let edge_len = read_u16(&trie[node_ptr + 4..node_ptr + 6]);
+
+        let byte = input[pos];
+        let mut child_idx: Option<u32> = None;
+
+        // Search for matching edge
+        let edge_base = edges_offset + (edge_start as usize) * 5;
+        for i in 0..edge_len {
+            let edge_ptr = edge_base + (i as usize) * 5;
+            if edge_ptr + 5 > trie.len() {
+                return best;
+            }
+            if trie[edge_ptr] == byte {
+                child_idx = Some(read_u32(&trie[edge_ptr + 1..edge_ptr + 5]));
+                break;
+            }
+        }
+
+        match child_idx {
+            Some(idx) if idx < node_count => {
+                node_idx = idx;
+                pos += 1;
+
+                // Check if this node has an action
+                let child_ptr = nodes_offset + (node_idx as usize) * 12;
+                let order = read_u16(&trie[child_ptr + 6..child_ptr + 8]);
+                let action = read_u32(&trie[child_ptr + 8..child_ptr + 12]);
+
+                if action != 0xFFFFFFFF {
+                    // Update best if longer or same length with earlier order
+                    let should_update = match &best {
+                        None => true,
+                        Some(b) => pos > b.length || (pos == b.length && order < b.order),
+                    };
+                    if should_update {
+                        best = Some(TrieMatch {
+                            length: pos,
+                            order,
+                            action_ip: action,
+                        });
+                    }
+                }
+            }
+            _ => break,
+        }
+    }
+
+    best
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn build_simple_trie() -> Vec<u8> {
+        // Build a simple trie for "a" and "ab"
+        // Node 0: root (edges to 'a' -> node 1)
+        // Node 1: after 'a' (action=10, edges to 'b' -> node 2)
+        // Node 2: after 'ab' (action=20)
+        let mut trie = Vec::new();
+
+        // Header: 3 nodes, 2 edges
+        trie.extend_from_slice(&3u32.to_be_bytes());
+        trie.extend_from_slice(&2u32.to_be_bytes());
+
+        // Node 0: edge_start=0, edge_len=1, order=0xFFFF, action=0xFFFFFFFF
+        trie.extend_from_slice(&0u32.to_be_bytes());
+        trie.extend_from_slice(&1u16.to_be_bytes());
+        trie.extend_from_slice(&0xFFFFu16.to_be_bytes());
+        trie.extend_from_slice(&0xFFFFFFFFu32.to_be_bytes());
+
+        // Node 1: edge_start=1, edge_len=1, order=1, action=10
+        trie.extend_from_slice(&1u32.to_be_bytes());
+        trie.extend_from_slice(&1u16.to_be_bytes());
+        trie.extend_from_slice(&1u16.to_be_bytes());
+        trie.extend_from_slice(&10u32.to_be_bytes());
+
+        // Node 2: edge_start=0, edge_len=0, order=0, action=20
+        trie.extend_from_slice(&0u32.to_be_bytes());
+        trie.extend_from_slice(&0u16.to_be_bytes());
+        trie.extend_from_slice(&0u16.to_be_bytes());
+        trie.extend_from_slice(&20u32.to_be_bytes());
+
+        // Edge 0: byte='a', child=1
+        trie.push(b'a');
+        trie.extend_from_slice(&1u32.to_be_bytes());
+
+        // Edge 1: byte='b', child=2
+        trie.push(b'b');
+        trie.extend_from_slice(&2u32.to_be_bytes());
+
+        trie
+    }
+
+    #[test]
+    fn test_trie_match_single() {
+        let trie = build_simple_trie();
+        let result = run(&trie, b"a");
+        assert!(result.is_some());
+        let m = result.unwrap();
+        assert_eq!(m.length, 1);
+        assert_eq!(m.action_ip, 10);
+    }
+
+    #[test]
+    fn test_trie_match_longer() {
+        let trie = build_simple_trie();
+        let result = run(&trie, b"ab");
+        assert!(result.is_some());
+        let m = result.unwrap();
+        assert_eq!(m.length, 2);
+        assert_eq!(m.action_ip, 20);
+    }
+
+    #[test]
+    fn test_trie_match_prefix() {
+        let trie = build_simple_trie();
+        let result = run(&trie, b"abc");
+        assert!(result.is_some());
+        let m = result.unwrap();
+        assert_eq!(m.length, 2);
+        assert_eq!(m.action_ip, 20);
+    }
+
+    #[test]
+    fn test_trie_no_match() {
+        let trie = build_simple_trie();
+        let result = run(&trie, b"xyz");
+        assert!(result.is_none());
+    }
+}