lexer_kit 0.5.0

data/lib/lexer_kit/ir/serializer.rb

@@ -0,0 +1,249 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module IR
+    # Serializer handles binary encoding/decoding of CompiledProgram.
+    # Extracted from CompiledProgram to separate serialization concerns.
+    class Serializer
+      MAGIC = "LKT1"
+      FORMAT_VERSION = 3 # v3: added token_meta
+
+      # Encode a CompiledProgram to binary format.
+      # @param program [CompiledProgram]
+      # @return [String] binary data
+      def self.to_binary(program)
+        new.encode(program)
+      end
+
+      # Decode binary data to a CompiledProgram.
+      # @param bytes [String] binary data
+      # @return [CompiledProgram]
+      def self.from_binary(bytes)
+        new.decode(bytes)
+      end
+
+      # Encode CompiledProgram to binary.
+      # @param program [CompiledProgram]
+      # @return [String]
+      def encode(program)
+        parts = []
+
+        # Magic (4 bytes)
+        parts << MAGIC
+
+        # Format version (u16)
+        parts << [FORMAT_VERSION].pack("S>")
+
+        # User version (u32)
+        parts << [program.version].pack("L>")
+
+        # Token names
+        encode_string_array(parts, program.token_names)
+
+        # Mode names
+        encode_string_array(parts, program.mode_names)
+
+        # Modes (name → offset mapping)
+        modes = program.mode_offsets
+        parts << [modes.size].pack("S>")
+        modes.each do |name, offset|
+          name_bytes = name.to_s.b
+          parts << [name_bytes.bytesize].pack("S>")
+          parts << name_bytes
+          parts << [offset].pack("L>")
+        end
+
+        # Constant pool
+        pool_binary = program.constant_pool.to_binary
+        parts << [pool_binary.bytesize].pack("L>")
+        parts << pool_binary
+
+        # DFA tables
+        encode_table_array(parts, program.dfa_tables)
+
+        # Jump tables
+        encode_table_array(parts, program.jump_tables)
+
+        # Keyword tables
+        encode_table_array(parts, program.keyword_tables)
+
+        # Token metadata
+        encode_token_meta(parts, program.token_meta)
+
+        # Instructions
+        instructions = program.instructions
+        parts << [instructions.size].pack("L>")
+        instructions.each do |instr|
+          parts << instr.to_binary
+        end
+
+        parts.join
+      end
+
+      # Decode binary data to CompiledProgram.
+      # @param bytes [String]
+      # @return [CompiledProgram]
+      def decode(bytes)
+        @bytes = bytes
+        @pos = 0
+
+        # Magic
+        magic = read_bytes(4)
+        raise ArgumentError, "invalid magic: #{magic.inspect}" unless magic == MAGIC
+
+        # Format version
+        format_version = read_uint16
+        raise ArgumentError, "unsupported format version: #{format_version}" unless format_version == FORMAT_VERSION
+
+        # User version
+        version = read_uint32
+
+        # Token names
+        token_names = decode_symbol_array
+
+        # Mode names
+        mode_names = decode_symbol_array
+
+        # Modes
+        mode_count = read_uint16
+        modes = {}
+        mode_count.times do
+          name = read_length_prefixed_string.to_sym
+          offset = read_uint32
+          modes[name] = offset
+        end
+
+        # Constant pool
+        pool_len = read_uint32
+        constant_pool, = ConstantPool.from_binary(read_bytes(pool_len))
+
+        # DFA tables
+        dfa_tables = decode_table_array(DFATable)
+
+        # Jump tables
+        jump_tables = decode_table_array(JumpTable)
+
+        # Keyword tables
+        keyword_tables = decode_table_array(KeywordTable)
+
+        # Token metadata
+        token_meta = format_version >= 3 ? decode_token_meta : {}
+
+        # Instructions
+        instr_count = read_uint32
+        instructions = []
+        instr_count.times do
+          instructions << Instruction.from_binary(read_bytes(4))
+        end
+
+        CompiledProgram.new(
+          instructions: instructions,
+          dfa_tables: dfa_tables,
+          jump_tables: jump_tables,
+          constant_pool: constant_pool,
+          modes: modes,
+          token_names: token_names,
+          mode_names: mode_names,
+          keyword_tables: keyword_tables,
+          token_meta: token_meta,
+          version: version
+        )
+      end
+
+      private
+
+      # Encode an array of symbol names as length-prefixed strings.
+      def encode_string_array(parts, names)
+        parts << [names.size].pack("S>")
+        names.each do |name|
+          name_bytes = name.to_s.b
+          parts << [name_bytes.bytesize].pack("S>")
+          parts << name_bytes
+        end
+      end
+
+      # Encode an array of tables (DFA, Jump, Keyword).
+      def encode_table_array(parts, tables)
+        parts << [tables.size].pack("S>")
+        tables.each do |table|
+          table_binary = table.to_binary
+          parts << [table_binary.bytesize].pack("L>")
+          parts << table_binary
+        end
+      end
+
+      # Decode an array of symbols.
+      def decode_symbol_array
+        count = read_uint16
+        symbols = []
+        count.times do
+          symbols << read_length_prefixed_string.to_sym
+        end
+        symbols
+      end
+
+      # Decode an array of tables.
+      def decode_table_array(table_class)
+        count = read_uint16
+        tables = []
+        count.times do
+          table_len = read_uint32
+          table, = table_class.from_binary(read_bytes(table_len))
+          tables << table
+        end
+        tables
+      end
+
+      # Read raw bytes.
+      def read_bytes(len)
+        data = @bytes.byteslice(@pos, len)
+        @pos += len
+        data
+      end
+
+      # Read unsigned 16-bit big-endian.
+      def read_uint16
+        read_bytes(2).unpack1("S>")
+      end
+
+      # Read unsigned 32-bit big-endian.
+      def read_uint32
+        read_bytes(4).unpack1("L>")
+      end
+
+      # Read length-prefixed string (u16 length + bytes).
+      def read_length_prefixed_string
+        len = read_uint16
+        read_bytes(len)
+      end
+
+      # Encode token metadata hash.
+      # Format: [count: u16] [token_id: u16, json_len: u16, json: bytes]×n
+      def encode_token_meta(parts, token_meta)
+        require "json"
+        parts << [token_meta.size].pack("S>")
+        token_meta.each do |token_id, meta|
+          json = JSON.generate(meta)
+          json_bytes = json.b
+          parts << [token_id, json_bytes.bytesize].pack("S>S>")
+          parts << json_bytes
+        end
+      end
+
+      # Decode token metadata hash.
+      def decode_token_meta
+        require "json"
+        count = read_uint16
+        token_meta = {}
+        count.times do
+          token_id = read_uint16
+          json_len = read_uint16
+          json_bytes = read_bytes(json_len)
+          meta = JSON.parse(json_bytes, symbolize_names: true)
+          token_meta[token_id] = meta
+        end
+        token_meta
+      end
+    end
+  end
+end

data/lib/lexer_kit/ir.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require_relative "ir/opcode"
+require_relative "ir/instruction"
+require_relative "ir/dfa_table"
+require_relative "ir/jump_table"
+require_relative "ir/constant_pool"
+require_relative "ir/keyword_table"
+require_relative "ir/serializer"
+require_relative "ir/compiled_program"
+
+module LexerKit
+  # IR module contains intermediate representation types for the lexer VM.
+  module IR
+  end
+end

data/lib/lexer_kit/runner.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+module LexerKit
+  # Runner coordinates program, stream, and source for token processing.
+  # It provides a high-level interface for lexing with full Token support.
+  #
+  # Runner has the same interface as LexStream (duck-typing compatible)
+  # plus additional methods for Token creation.
+  #
+  # Created via CompiledProgram#stream(input, filename:).
+  class Runner
+    attr_reader :program, :stream
+
+    # @api private
+    # Use CompiledProgram#stream instead
+    def initialize(program, lex_stream, filename: nil)
+      @program = program
+      @stream = lex_stream
+      @filename = filename
+      @source = nil # lazy
+    end
+
+    # Delegate basic stream operations
+    def eof?     = @stream.eof?
+    def token_id = @stream.token_id
+    def start    = @stream.start
+    def len      = @stream.len
+    def input    = @stream.input
+
+    # Advance to next token, returns self for chaining
+    def advance
+      @stream.advance
+      self
+    end
+
+    # Token name lookup (via program for correctness)
+    def token_name(id = nil)
+      id ||= token_id
+      return nil if id.nil? || id < 0
+
+      @program.token_name(id)
+    end
+
+    # Text extraction
+    def text
+      return nil if eof?
+
+      s = start
+      l = len
+      return nil if s.nil? || l.nil?
+
+      input.byteslice(s, l)
+    end
+
+    # Error detection (uses constant, no @program needed)
+    def error?(id = nil)
+      id ||= token_id
+      return false if id.nil? || id < 0
+
+      id == LexerKit::INVALID_TOKEN_ID
+    end
+
+    # Create Token object (lazy Source creation)
+    def make_token
+      return nil if eof?
+
+      tok_id = token_id
+      s = start
+      l = len
+      return nil if tok_id.nil? || s.nil? || l.nil?
+
+      @source ||= Core::Source.new(input, filename: @filename)
+      Core::Token.new(
+        id: tok_id,
+        name: @program.token_name(tok_id),
+        start: s,
+        len: l,
+        source: @source,
+        meta: @program.token_meta[tok_id]
+      )
+    end
+
+    # Line/col for current position
+    def line_col
+      return nil if eof?
+
+      s = start
+      return nil if s.nil?
+
+      @source ||= Core::Source.new(input, filename: @filename)
+      @source.line_col(s)
+    end
+
+    # Peek operations (delegate to stream)
+    def peek_token_id(n = 1) = @stream.peek_token_id(n)
+    def peek_start(n = 1)    = @stream.peek_start(n)
+    def peek_len(n = 1)      = @stream.peek_len(n)
+
+    def peek_token_name(n = 1)
+      id = peek_token_id(n)
+      return nil if id.nil? || id < 0
+
+      @program.token_name(id)
+    end
+
+    def peek_text(n = 1)
+      s = peek_start(n)
+      l = peek_len(n)
+      return nil if s.nil? || l.nil?
+
+      input.byteslice(s, l)
+    end
+  end
+end

data/lib/lexer_kit/trie.rb

@@ -0,0 +1,170 @@
+# frozen_string_literal: true
+
+module LexerKit
+  # Builds and encodes trie data structures for literal matching.
+  #
+  # ## Value Constraints
+  #
+  # - order: 0..MAX_ORDER (ORDER_UNSET is reserved for unset values)
+  # - action_ip: 0..0xFFFFFFFF (ACTION_UNSET used for non-terminal nodes)
+  #
+  # ## Label Resolution
+  #
+  # labels hash must have Symbol keys only (Integer keys are ambiguous with direct offsets).
+  # Resolution order:
+  # 1. If action_ref is in labels hash, use mapped value
+  # 2. If action_ref is Symbol not in labels, raise ArgumentError
+  # 3. If action_ref is Integer, use directly as offset
+  #
+  class Trie
+    ORDER_UNSET = 0xFFFF
+    MAX_ORDER = 0xFFFE
+    ACTION_UNSET = 0xFFFFFFFF
+    MAX_EDGE_COUNT = 0xFFFF
+
+    def initialize(entries)
+      @trie = build_trie(entries)
+    end
+
+    def encode(labels: nil)
+      encode_trie(@trie, labels || {})
+    end
+
+    def self.build(entries)
+      trie_entries = entries.map.with_index do |(literal, action_ip), index|
+        [literal, index, action_ip]
+      end
+      new(trie_entries).encode
+    end
+
+    Node = Struct.new(:edges, :order, :action_ref, keyword_init: true) do
+      def initialize(edges: {}, order: nil, action_ref: nil)
+        super
+      end
+    end
+
+    NodeEntry = Struct.new(:edge_start, :edge_count, :order, :action_ref, keyword_init: true)
+    TrieData = Struct.new(:nodes, :edges, keyword_init: true)
+
+    private
+
+    def build_trie(entries)
+      nodes = [Node.new]
+
+      entries.each do |literal, order, action_ref|
+        validate_literal(literal)
+        validate_order(order)
+
+        bytes = literal.b
+        node_idx = 0
+
+        bytes.each_byte do |byte|
+          node = nodes[node_idx]
+          child = node.edges[byte]
+
+          unless child
+            child = nodes.size
+            node.edges[byte] = child
+            nodes << Node.new
+          end
+
+          node_idx = child
+        end
+
+        node = nodes[node_idx]
+        if node.order.nil? || order < node.order
+          node.order = order
+          node.action_ref = action_ref
+        end
+      end
+
+      # NOTE: O(E log E) per node due to Hash#sort, but acceptable for typical
+      # literal counts. For very large trie (>10k literals), consider optimization.
+      edge_list = []
+      node_entries = nodes.map do |node|
+        edge_start = edge_list.size
+        edge_count = node.edges.size
+
+        validate_edge_count(edge_count)
+
+        node.edges.sort.each { |byte, child| edge_list << [byte, child] }
+        NodeEntry.new(
+          edge_start: edge_start,
+          edge_count: edge_count,
+          order: node.order,
+          action_ref: node.action_ref
+        )
+      end
+
+      TrieData.new(nodes: node_entries, edges: edge_list)
+    end
+
+    def encode_trie(trie, labels)
+      validate_labels(labels)
+
+      nodes = trie.nodes
+      edges = trie.edges
+
+      parts = []
+      parts << [nodes.size].pack("L>")
+      parts << [edges.size].pack("L>")
+
+      nodes.each do |node|
+        order = node.order || ORDER_UNSET
+
+        action = if node.action_ref.nil?
+                   ACTION_UNSET
+                 elsif node.action_ref.is_a?(Symbol)
+                   labels.fetch(node.action_ref) do
+                     raise ArgumentError, "Unknown label: #{node.action_ref}"
+                   end
+                 else
+                   node.action_ref
+                 end
+
+        validate_action_ip(action)
+
+        parts << [node.edge_start, node.edge_count, order, action].pack("L>S>S>L>")
+      end
+
+      edges.each do |byte, child_idx|
+        parts << [byte, child_idx].pack("CL>")
+      end
+
+      parts.join.force_encoding(Encoding::ASCII_8BIT)
+    end
+
+    def validate_literal(literal)
+      raise ArgumentError, "literal must be a non-empty string, got: #{literal.inspect}" if literal.nil? || literal.empty?
+    end
+
+    def validate_order(order)
+      return if order.is_a?(Integer) && order >= 0 && order <= MAX_ORDER
+
+      raise ArgumentError, "order must be in range 0..#{format('0x%X', MAX_ORDER)} (#{format('0x%X', ORDER_UNSET)} reserved), got: #{order.inspect}"
+    end
+
+    def validate_edge_count(count)
+      return if count.between?(0, MAX_EDGE_COUNT)
+
+      raise ArgumentError, "edge_count must fit in 16-bit unsigned integer, got: #{count}"
+    end
+
+    def validate_action_ip(action_ip)
+      # ACTION_UNSET is allowed as the reserved "unset" value for non-terminal nodes
+      return if action_ip.is_a?(Integer) && action_ip >= 0 && action_ip <= ACTION_UNSET
+
+      raise ArgumentError, "action_ip must be in range 0..0x#{ACTION_UNSET.to_s(16).upcase} (32-bit unsigned), got: #{action_ip.inspect}"
+    end
+
+    def validate_labels(labels)
+      return if labels.empty?
+
+      # Check for Integer keys (ambiguous with direct offsets)
+      integer_keys = labels.keys.select { |k| k.is_a?(Integer) }
+      return if integer_keys.empty?
+
+      raise ArgumentError, "labels must have Symbol keys only (Integer keys are ambiguous), found Integer keys: #{integer_keys.inspect}"
+    end
+  end
+end

data/lib/lexer_kit/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module LexerKit
+  VERSION = "0.5.0"
+end

data/lib/lexer_kit.rb

@@ -0,0 +1,155 @@
+# frozen_string_literal: true
+
+require_relative "lexer_kit/version"
+require_relative "lexer_kit/core"      # Load first (errors depend on Core::Source, Core::Diagnostic)
+require_relative "lexer_kit/errors"    # Load after core
+require_relative "lexer_kit/ir"
+require_relative "lexer_kit/dfa"
+require_relative "lexer_kit/builder"
+require_relative "lexer_kit/format"
+
+# Load Trie class (for testing and internal use)
+require_relative "lexer_kit/trie"
+
+# Try to load Rust extension
+begin
+  require "lexer_kit_rust/lexer_kit_rust"
+  LEXER_KIT_NATIVE = true
+
+  # Runner wraps LexStream (defined in Rust) with high-level methods
+  require_relative "lexer_kit/runner"
+rescue LoadError
+  LEXER_KIT_NATIVE = false
+end
+
+module LexerKit
+  # Reserved token IDs
+  # 0:   Internal sentinel (never emitted)
+  # 1:   INVALID (error token)
+  # 2-7: Reserved for future use
+  # 8+:  User-defined tokens
+  #
+  # Note: The VM only emits tokens with valid IDs:
+  # - INVALID_TOKEN_ID (1) for error tokens
+  # - User tokens (>= FIRST_USER_TOKEN_ID)
+  # Tokens with sentinel/reserved IDs (0, 2-7) or zero length are filtered out.
+  RESERVED_TOKEN_ID = 0
+  INVALID_TOKEN_ID = 1
+  FIRST_USER_TOKEN_ID = 8
+
+  module RegexAstProvider
+    def to_ast
+      raise NotImplementedError
+    end
+
+    def to_regex
+      DFA::RegexAST::Regex.new(ast: to_ast, case_insensitive: false)
+    end
+  end
+
+  # Create a UTF-8 range pattern for the LexerKit regex engine.
+  #
+  # Accepted inputs:
+  # - "あ" (single character)
+  # - "あ".."ん" (Range, inclusive)
+  # - Integer codepoint ranges (e.g., 0x3041..0x3096)
+  #
+  # Notes:
+  # - Exclusive ranges (e.g., "a"..."z") are not supported.
+  # - Multi-character strings like "abc" are not supported.
+  # - Range endpoints must be single characters or integers.
+  def self.utf8_range(*ranges)
+    require_relative "lexer_kit/dfa/utf8_range_pattern"
+    parsed = ranges.map { |range| parse_range_codepoints(range) }
+    DFA::Utf8RangePattern.new(parsed)
+  end
+
+  def self.parse_range_codepoints(range)
+    if range.is_a?(Range)
+      raise ArgumentError, "utf8_range does not support exclusive ranges" if range.exclude_end?
+
+      start_cp = range.begin.is_a?(Integer) ? range.begin : single_char_ord(range.begin.to_s)
+      end_cp = range.end.is_a?(Integer) ? range.end : single_char_ord(range.end.to_s)
+      return [start_cp, end_cp]
+    end
+
+    str = range.to_s
+
+    cp = single_char_ord(str)
+    [cp, cp]
+  end
+
+  def self.single_char_ord(str)
+    raise ArgumentError, "utf8_range expects a single character, got #{str.inspect}" unless str.length == 1
+
+    str.ord
+  end
+
+  private_class_method :parse_range_codepoints
+  private_class_method :single_char_ord
+
+  # Check if native Rust extension is available
+  # @return [Boolean]
+  def self.native?
+    LEXER_KIT_NATIVE
+  end
+
+  # Build a lexer from DSL
+  # @yield [Builder] DSL block
+  # @return [Builder] configured builder
+  def self.build(&block)
+    Builder.new.tap { |b| b.instance_eval(&block) if block }
+  end
+
+  # Load a compiled lexer from .lkt1 or .lkb1 file
+  #
+  # @param path [String] path to lexer file (relative or absolute)
+  # @return [IR::CompiledProgram] compiled lexer program
+  # @raise [ArgumentError] if file not found or invalid extension
+  #
+  # @example Load from relative path
+  #   LexerKit.load_lexer("lexers/json.lkt1")
+  #   LexerKit.load_lexer(File.expand_path("../data/json.lkt1", __dir__))
+  #
+  # @example Load from absolute path
+  #   LexerKit.load_lexer("/path/to/json.lkt1")
+  def self.load_lexer(path)
+    # Expand relative/absolute paths from current directory
+    path = File.expand_path(path)
+
+    raise ArgumentError, "Lexer not found: #{path}" unless File.exist?(path)
+
+    if path.end_with?(".lkt1")
+      Format::LKT1.load(path).program
+    elsif path.end_with?(".lkb1")
+      Format::LKB1.load(path).program
+    else
+      raise ArgumentError, "Expected .lkt1 or .lkb1 file: #{path}"
+    end
+  end
+
+  # Load a builder from DSL source file
+  #
+  # @param path [String] path to DSL source file (relative or absolute)
+  # @return [Builder] builder instance
+  # @raise [ArgumentError] if file doesn't return Builder instance
+  #
+  # @example Load from relative path
+  #   LexerKit.load_builder("examples/languages/json.rb")
+  #
+  # @example Load from absolute path
+  #   LexerKit.load_builder("/path/to/json.rb")
+  def self.load_builder(path)
+    # Expand relative/absolute paths from current directory
+    path = File.expand_path(path)
+
+    raise ArgumentError, "Builder source not found: #{path}" unless File.exist?(path)
+
+    content = File.read(path)
+    result = eval(content, TOPLEVEL_BINDING, path) # rubocop:disable Security/Eval
+
+    return result if result.is_a?(Builder)
+
+    raise ArgumentError, "DSL file must return LexerKit::Builder instance"
+  end
+end