lexer_kit 0.5.0

data/lib/lexer_kit/ir/constant_pool.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module IR
+    # ConstantPool stores string constants (delimiters, keywords, etc.)
+    class ConstantPool
+      attr_reader :entries
+
+      def initialize
+        @entries = []
+        @index = {}
+      end
+
+      # Add a constant and return its ID
+      # @param value [String] constant value
+      # @return [Integer] constant ID
+      def add(value)
+        value = value.b.freeze
+        return @index[value] if @index.key?(value)
+
+        id = @entries.size
+        @entries << value
+        @index[value] = id
+        id
+      end
+
+      # Add a constant without interning
+      # @param value [String] constant value
+      # @return [Integer] constant ID
+      def add_uninterned(value)
+        value = value.b.freeze
+        id = @entries.size
+        @entries << value
+        id
+      end
+
+      # Replace an existing constant by ID
+      # @param id [Integer] constant ID
+      # @param value [String] new value
+      def replace(id, value)
+        value = value.b.freeze
+        old = @entries[id]
+        @entries[id] = value
+        if old && @index[old] == id
+          @index.delete(old)
+        end
+        @index[value] = id unless @index.key?(value)
+      end
+
+      # Get constant by ID
+      # @param id [Integer]
+      # @return [String]
+      def get(id)
+        @entries[id]
+      end
+
+      # Number of constants
+      # @return [Integer]
+      def size
+        @entries.size
+      end
+
+      # Encode to binary
+      # @return [String]
+      def to_binary
+        parts = []
+
+        # Count (u16)
+        parts << [@entries.size].pack("S>")
+
+        # Entries: [length (u16), bytes...]
+        @entries.each do |entry|
+          parts << [entry.bytesize].pack("S>")
+          parts << entry
+        end
+
+        parts.join
+      end
+
+      # Decode from binary
+      # @param bytes [String]
+      # @return [Array(ConstantPool, Integer)] [pool, bytes_consumed]
+      def self.from_binary(bytes)
+        pos = 0
+
+        count = bytes.byteslice(pos, 2).unpack1("S>")
+        pos += 2
+
+        pool = new
+        count.times do
+          len = bytes.byteslice(pos, 2).unpack1("S>")
+          pos += 2
+
+          value = bytes.byteslice(pos, len)
+          pool.add(value)
+          pos += len
+        end
+
+        [pool, pos]
+      end
+
+      def inspect
+        "#<ConstantPool size=#{@entries.size}>"
+      end
+    end
+  end
+end

data/lib/lexer_kit/ir/dfa_table.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module IR
+    # DFATable represents a compiled DFA for pattern matching.
+    # Uses byte class compression to reduce table size.
+    class DFATable
+      # State 0 indicates no valid transition (must match C DFA_DEAD_STATE)
+      DEAD_STATE = 0
+      # Token ID indicating non-accepting state (must match C DFA_NO_ACCEPT)
+      NO_ACCEPT = 0xFFFF
+
+      attr_reader :state_count, :class_count, :byte_class, :transitions, :accept_states
+
+      # @param state_count [Integer] number of states
+      # @param byte_class [Array<Integer>] 256-element array mapping bytes to classes
+      # @param transitions [Array<Integer>] state × class → next_state
+      # @param accept_states [Hash<Integer, Integer>] state → token_id for accepting states
+      def initialize(state_count:, byte_class:, transitions:, accept_states:)
+        @state_count = state_count
+        @byte_class = byte_class.freeze
+        @class_count = byte_class.max + 1
+        @transitions = transitions.freeze
+        @accept_states = accept_states.freeze
+      end
+
+      # Get next state for current state and input byte
+      # @param state [Integer] current state
+      # @param byte [Integer] input byte
+      # @return [Integer] next state (0 = dead state)
+      def transition(state, byte)
+        cls = @byte_class[byte]
+        @transitions[(state * @class_count) + cls]
+      end
+
+      # Check if state is accepting
+      # @param state [Integer]
+      # @return [Integer, nil] token ID if accepting, nil otherwise
+      def accept(state)
+        @accept_states[state]
+      end
+
+      # Encode to binary
+      # @return [String]
+      def to_binary
+        parts = []
+
+        # Header: state_count (u16), class_count (u16)
+        parts << [@state_count, @class_count].pack("S>S>")
+
+        # Byte class table (256 bytes)
+        parts << @byte_class.pack("C256")
+
+        # Transitions (state_count × class_count × u16)
+        parts << @transitions.pack("S>*")
+
+        # Accept states count (u16)
+        parts << [@accept_states.size].pack("S>")
+
+        # Accept states: [state (u16), token_id (u16)] pairs
+        @accept_states.each do |state, token_id|
+          parts << [state, token_id].pack("S>S>")
+        end
+
+        parts.join
+      end
+
+      # Decode from binary
+      # @param bytes [String]
+      # @return [Array(DFATable, Integer)] [table, bytes_consumed]
+      def self.from_binary(bytes)
+        pos = 0
+
+        state_count, class_count = bytes.byteslice(pos, 4).unpack("S>S>")
+        pos += 4
+
+        byte_class = bytes.byteslice(pos, 256).unpack("C256")
+        pos += 256
+
+        trans_size = state_count * class_count
+        transitions = bytes.byteslice(pos, trans_size * 2).unpack("S>*")
+        pos += trans_size * 2
+
+        accept_count = bytes.byteslice(pos, 2).unpack1("S>")
+        pos += 2
+
+        accept_states = {}
+        accept_count.times do
+          state, token_id = bytes.byteslice(pos, 4).unpack("S>S>")
+          accept_states[state] = token_id
+          pos += 4
+        end
+
+        table = new(
+          state_count: state_count,
+          byte_class: byte_class,
+          transitions: transitions,
+          accept_states: accept_states
+        )
+
+        [table, pos]
+      end
+
+      def inspect
+        "#<DFATable states=#{@state_count} classes=#{@class_count} accepts=#{@accept_states.size}>"
+      end
+
+      # Convert to format suitable for C native loading
+      # Returns dense accept_tokens array for O(1) lookup
+      # @return [Hash] data for C extension
+      def to_native_format
+        # Build dense array: accept_tokens[state] = token_id
+        accept_tokens = Array.new(@state_count, NO_ACCEPT)
+        @accept_states.each { |state, token_id| accept_tokens[state] = token_id }
+        {
+          state_count: @state_count,
+          class_count: @class_count,
+          byte_class: @byte_class.pack("C*"),
+          transitions: @transitions.pack("S>*"),
+          accept_tokens: accept_tokens.pack("S>*")
+        }
+      end
+    end
+  end
+end

data/lib/lexer_kit/ir/instruction.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module IR
+    # Instruction represents a single VM instruction.
+    # Each instruction has an opcode and an optional argument.
+    class Instruction
+      attr_reader :opcode, :arg
+
+      # @param opcode [Integer] opcode from Opcode module
+      # @param arg [Integer] argument (interpretation depends on opcode)
+      def initialize(opcode, arg = 0)
+        @opcode = opcode
+        @arg = arg
+      end
+
+      # Encode to binary (4 bytes: 1 opcode + 3 arg)
+      # @return [String] binary representation
+      def to_binary
+        [
+          @opcode,
+          (@arg >> 16) & 0xFF,
+          (@arg >> 8) & 0xFF,
+          @arg & 0xFF
+        ].pack("C4")
+      end
+
+      # Decode from binary
+      # @param bytes [String] 4 bytes
+      # @return [Instruction]
+      def self.from_binary(bytes)
+        opcode, a1, a2, a3 = bytes.unpack("C4")
+        arg = (a1 << 16) | (a2 << 8) | a3
+        new(opcode, arg)
+      end
+
+      def to_s
+        "#{Opcode.name(@opcode)} #{@arg}"
+      end
+
+      def inspect
+        "#<Instruction #{self}>"
+      end
+
+      def ==(other)
+        other.is_a?(Instruction) && @opcode == other.opcode && @arg == other.arg
+      end
+    end
+  end
+end

data/lib/lexer_kit/ir/jump_table.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module IR
+    # JumpTable for SWITCH_BYTE instruction.
+    # Maps bytes to instruction offsets.
+    class JumpTable
+      attr_reader :entries, :default_offset
+
+      # @param entries [Hash<Integer, Integer>] byte → offset mapping
+      # @param default_offset [Integer] offset for unmatched bytes
+      def initialize(entries:, default_offset:)
+        @entries = entries.freeze
+        @default_offset = default_offset
+      end
+
+      # Get offset for a byte
+      # @param byte [Integer]
+      # @return [Integer] instruction offset
+      def lookup(byte)
+        @entries[byte] || @default_offset
+      end
+
+      # Encode to binary
+      # @return [String]
+      def to_binary
+        parts = []
+
+        # Default offset (u24)
+        parts << [
+          (@default_offset >> 16) & 0xFF,
+          (@default_offset >> 8) & 0xFF,
+          @default_offset & 0xFF
+        ].pack("C3")
+
+        # Entry count (u16)
+        parts << [@entries.size].pack("S>")
+
+        # Entries: [byte (u8), offset (u24)] pairs
+        @entries.each do |byte, offset|
+          parts << [
+            byte,
+            (offset >> 16) & 0xFF,
+            (offset >> 8) & 0xFF,
+            offset & 0xFF
+          ].pack("C4")
+        end
+
+        parts.join
+      end
+
+      # Decode from binary
+      # @param bytes [String]
+      # @return [Array(JumpTable, Integer)] [table, bytes_consumed]
+      def self.from_binary(bytes)
+        pos = 0
+
+        d1, d2, d3 = bytes.byteslice(pos, 3).unpack("C3")
+        default_offset = (d1 << 16) | (d2 << 8) | d3
+        pos += 3
+
+        entry_count = bytes.byteslice(pos, 2).unpack1("S>")
+        pos += 2
+
+        entries = {}
+        entry_count.times do
+          byte, o1, o2, o3 = bytes.byteslice(pos, 4).unpack("C4")
+          offset = (o1 << 16) | (o2 << 8) | o3
+          entries[byte] = offset
+          pos += 4
+        end
+
+        [new(entries: entries, default_offset: default_offset), pos]
+      end
+
+      def inspect
+        "#<JumpTable entries=#{@entries.size} default=#{@default_offset}>"
+      end
+
+      # Convert to format suitable for C native loading
+      # Returns dense 256-entry lookup table for O(1) access
+      # @return [Hash] data for C extension
+      def to_native_format
+        # Build dense lookup table: all 256 bytes → offset
+        lookup = Array.new(256, @default_offset)
+        @entries.each { |byte, offset| lookup[byte] = offset }
+        {
+          lookup: lookup.pack("L>*"), # 256 * 4 = 1024 bytes
+          default_offset: @default_offset # For EOF case
+        }
+      end
+    end
+  end
+end

data/lib/lexer_kit/ir/keyword_table.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module IR
+    # KeywordTable maps keyword strings to token IDs.
+    # Used for efficient O(1) keyword lookup after identifier match.
+    #
+    # Keywords must be UTF-8 encoded strings. This is consistent with
+    # LexerKit's token specification which requires UTF-8 encoding.
+    class KeywordTable
+      attr_reader :base_token_id, :keywords
+
+      # @param base_token_id [Integer] identifier token ID (used when not a keyword)
+      # @param keywords [Hash{String => Integer}] UTF-8 keyword string → token ID
+      def initialize(base_token_id:, keywords:)
+        validate_base_token_id(base_token_id)
+        validate_keywords(keywords)
+
+        @base_token_id = base_token_id
+        @keywords = keywords.freeze
+      end
+
+      # Encode to binary format
+      # Format:
+      #   base_token_id: u16
+      #   keyword_count: u16
+      #   for each keyword:
+      #     key_len: u16
+      #     key: bytes
+      #     token_id: u16
+      # @return [String]
+      # @raise [ArgumentError] if keyword count or any keyword length exceeds u16 limit
+      def to_binary
+        # Validate keyword count fits in u16
+        if @keywords.size > 0xFFFF
+          raise ArgumentError, "Too many keywords (#{@keywords.size}): maximum is 65535"
+        end
+
+        parts = []
+
+        # Base token ID (u16)
+        parts << [@base_token_id].pack("S>")
+
+        # Keyword count (u16)
+        parts << [@keywords.size].pack("S>")
+
+        # Keywords
+        @keywords.each do |key, token_id|
+          key_bytes = key.b
+
+          # Validate keyword length fits in u16
+          if key_bytes.bytesize > 0xFFFF
+            raise ArgumentError, "Keyword too long (#{key_bytes.bytesize} bytes): maximum is 65535 bytes"
+          end
+
+          parts << [key_bytes.bytesize].pack("S>")
+          parts << key_bytes
+          parts << [token_id].pack("S>")
+        end
+
+        parts.join
+      end
+
+      # Convert to format suitable for C native loading
+      # @return [Hash] data for C extension
+      def to_native_format
+        {
+          base_token_id: @base_token_id,
+          keywords: @keywords.map { |key, token_id| [key.b, token_id] }
+        }
+      end
+
+      # Decode from binary
+      # @param bytes [String]
+      # @return [Array(KeywordTable, Integer)] table and consumed bytes
+      # @raise [LexerKit::InvalidBinaryError] if binary data is invalid
+      def self.from_binary(bytes)
+        raise LexerKit::InvalidBinaryError, "Binary data too short (expected at least 4 bytes, got #{bytes.bytesize})" if bytes.bytesize < 4
+
+        pos = 0
+
+        # Base token ID
+        base_token_id = bytes.byteslice(pos, 2)&.unpack1("S>")
+        raise LexerKit::InvalidBinaryError, "Invalid header data" if base_token_id.nil?
+
+        pos += 2
+
+        # Keyword count
+        keyword_count = bytes.byteslice(pos, 2)&.unpack1("S>")
+        raise LexerKit::InvalidBinaryError, "Invalid header data" if keyword_count.nil?
+
+        pos += 2
+
+        # Keywords
+        keywords = {}
+        keyword_count.times do
+          raise LexerKit::InvalidBinaryError, "Unexpected end of data while reading keyword entry" if pos + 2 > bytes.bytesize
+
+          key_len = bytes.byteslice(pos, 2)&.unpack1("S>")
+          raise LexerKit::InvalidBinaryError, "Invalid key length" if key_len.nil?
+
+          pos += 2
+
+          raise LexerKit::InvalidBinaryError, "Unexpected end of data while reading keyword" if pos + key_len + 2 > bytes.bytesize
+
+          key = bytes.byteslice(pos, key_len).force_encoding(Encoding::UTF_8)
+          pos += key_len
+
+          token_id = bytes.byteslice(pos, 2)&.unpack1("S>")
+          raise LexerKit::InvalidBinaryError, "Invalid token ID" if token_id.nil?
+
+          pos += 2
+
+          keywords[key] = token_id
+        end
+
+        [new(base_token_id: base_token_id, keywords: keywords), pos]
+      rescue LexerKit::InvalidBinaryError
+        raise
+      rescue StandardError => e
+        raise LexerKit::InvalidBinaryError, "Failed to parse binary data: #{e.message}"
+      end
+
+      private
+
+      def validate_base_token_id(id)
+        return if id.is_a?(Integer) && id >= 0 && id <= 0xFFFF
+
+        raise ArgumentError, "base_token_id must be a 16-bit unsigned integer (0..65535), got: #{id.inspect}"
+      end
+
+      def validate_keywords(keywords)
+        unless keywords.is_a?(Hash)
+          raise ArgumentError, "keywords must be a Hash, got: #{keywords.class}"
+        end
+
+        keywords.each do |keyword, token_id|
+          validate_keyword(keyword)
+          validate_token_id(token_id)
+        end
+      end
+
+      def validate_keyword(keyword)
+        unless keyword.is_a?(String)
+          raise ArgumentError, "keyword must be a String, got: #{keyword.inspect}"
+        end
+
+        if keyword.empty?
+          raise ArgumentError, "keyword must not be empty"
+        end
+
+        unless keyword.encoding == Encoding::UTF_8 || keyword.encoding == Encoding::US_ASCII
+          raise ArgumentError, "keyword must be UTF-8 encoded, got: #{keyword.encoding}"
+        end
+
+        return if keyword.valid_encoding?
+
+        raise ArgumentError, "keyword contains invalid UTF-8 byte sequence"
+      end
+
+      def validate_token_id(token_id)
+        return if token_id.is_a?(Integer) && token_id >= 0 && token_id <= 0xFFFF
+
+        raise ArgumentError, "token_id must be a 16-bit unsigned integer (0..65535), got: #{token_id.inspect}"
+      end
+    end
+  end
+end

data/lib/lexer_kit/ir/opcode.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module IR
+    # Opcode definitions for the LexerKit VM
+    module Opcode
+      # DFA operations
+      DFA_RUN = 0x01 # Run DFA table, arg = table_id
+      # Optimized DFA run for single-candidate regex tokens.
+      # Argument encoding: (dfa_id << 14) | fail_target (10+14 bits)
+      # On success: advance position, move to next instruction
+      # On failure: jump directly to fail_target
+      DFA_RUN_IF_MATCH = 0x02
+
+      # Delimiter/literal operations
+      SCAN_UNTIL = 0x10 # Scan until delimiter, arg = const_id
+      MATCH_LITERAL = 0x12 # Match fixed string, arg = const_id
+      SCAN_UNTIL_ESCAPE = 0x13 # Scan until delimiter with escape, arg = config_id
+      MATCH_RANGE = 0x14 # Match byte range, arg = (min << 8) | max
+      # Optimized literal matching with embedded failure jump target.
+      # Argument encoding: (const_id << 14) | fail_target (10+14 bits)
+      # On success: advance position, move to next instruction
+      # On failure: jump directly to fail_target
+      MATCH_LITERAL_OR_JUMP = 0x16
+
+      # Branch/control operations
+      SWITCH_BYTE     = 0x20  # Branch on next byte, arg = jump_table_id
+      JUMP            = 0x21  # Unconditional jump, arg = offset
+      JUMP_IF_EOF     = 0x24  # Jump if at EOF, arg = offset
+
+      # Mode operations
+      PUSH_MODE       = 0x30  # Push mode to stack, arg = mode_id
+      POP_MODE        = 0x31  # Pop mode from stack
+
+      # Token operations
+      EMIT            = 0x40  # Emit token, arg = token_id
+      EMIT_SKIP       = 0x41  # Skip (advance without emit)
+      EMIT_ERROR      = 0x42  # Emit error token, arg = error_id
+      MARK            = 0x43  # Mark current position
+      # Optimized EMIT + JUMP for common token emission pattern.
+      # Argument encoding: (token_id << 14) | jump_target (10+14 bits)
+      EMIT_AND_JUMP   = 0x44
+      KEYWORD_LOOKUP  = 0x45  # Keyword lookup, arg = keyword_table_id
+      LITERAL_TRIE_RUN = 0x46 # Run literal trie, arg = const_id
+      CLEAR_BEST      = 0x49  # Clear best match tracking
+      COMMIT_BEST     = 0x4C  # Commit best match, arg = default offset
+      # Optimized regex candidate matching with embedded action target.
+      # Argument encoding: (order << 14) | action_ip (10+14 bits)
+      SET_MATCH       = 0x4D
+      # Optimized CLEAR_BEST + LITERAL_TRIE_RUN + COMMIT_BEST for literal-only matching.
+      # Argument encoding: (const_id << 14) | fail_target (10+14 bits)
+      # On match: jump to best match action IP
+      # On no match: jump to fail_target
+      LITERAL_TRIE_COMMIT = 0x4E
+      # Optimized EMIT_SKIP + JUMP for skipped token pattern.
+      # Argument encoding: jump_target (24 bits)
+      EMIT_SKIP_AND_JUMP = 0x4F
+
+      # Special
+      HALT = 0xFF # Stop execution
+
+      # Opcode metadata
+      NAMES = {
+        DFA_RUN => "DFA_RUN",
+        DFA_RUN_IF_MATCH => "DFA_RUN_IF_MATCH",
+        SCAN_UNTIL => "SCAN_UNTIL",
+        MATCH_LITERAL => "MATCH_LITERAL",
+        SCAN_UNTIL_ESCAPE => "SCAN_UNTIL_ESCAPE",
+        MATCH_RANGE => "MATCH_RANGE",
+        MATCH_LITERAL_OR_JUMP => "MATCH_LITERAL_OR_JUMP",
+        SWITCH_BYTE => "SWITCH_BYTE",
+        JUMP => "JUMP",
+        JUMP_IF_EOF => "JUMP_IF_EOF",
+        PUSH_MODE => "PUSH_MODE",
+        POP_MODE => "POP_MODE",
+        EMIT => "EMIT",
+        EMIT_SKIP => "EMIT_SKIP",
+        EMIT_ERROR => "EMIT_ERROR",
+        MARK => "MARK",
+        EMIT_AND_JUMP => "EMIT_AND_JUMP",
+        KEYWORD_LOOKUP => "KEYWORD_LOOKUP",
+        LITERAL_TRIE_RUN => "LITERAL_TRIE_RUN",
+        CLEAR_BEST => "CLEAR_BEST",
+        COMMIT_BEST => "COMMIT_BEST",
+        SET_MATCH => "SET_MATCH",
+        LITERAL_TRIE_COMMIT => "LITERAL_TRIE_COMMIT",
+        EMIT_SKIP_AND_JUMP => "EMIT_SKIP_AND_JUMP",
+        HALT => "HALT"
+      }.freeze
+
+      def self.name(opcode)
+        NAMES[opcode] || "UNKNOWN(0x#{opcode.to_s(16)})"
+      end
+    end
+  end
+end