lexer_kit 0.5.0

data/lib/lexer_kit/dfa/regex_parser.rb

@@ -0,0 +1,385 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module DFA
+    # RegexParser parses a subset of regular expressions into an AST.
+    # Supported features:
+    # - Literal characters
+    # - Character classes: [a-z], [^abc], ., \d, \w, \s, \D, \W, \S
+    # - Quantifiers: *, +, ?, {n}, {n,}, {n,m}
+    # - Alternation: |
+    # - Grouping: (), (?:...)
+    # - Escapes: \\, \., \[, etc.
+    class RegexParser
+      # Include AST types from RegexAST module
+      include RegexAST
+
+      def initialize(source)
+        @source = source.is_a?(Regexp) ? source.source : source
+        @pos = 0
+      end
+
+      def parse
+        result = parse_alternation
+        error("unexpected character at position #{@pos}") unless eof?
+
+        result
+      end
+
+      private
+
+      def error(message)
+        source = Core::Source.new(@source)
+        raise LexerKit::ParseError.new(message, source: source, position: @pos)
+      end
+
+      def parse_alternation
+        start_pos = @pos
+        children = [parse_concat]
+
+        while peek == "|"
+          advance
+          children << parse_concat
+        end
+
+        return children[0] if children.length == 1
+
+        Alternation.new(children: children, meta: meta_for_children(children, start_pos, @pos))
+      end
+
+      def parse_concat
+        start_pos = @pos
+        children = []
+
+        while !eof? && peek != "|" && peek != ")"
+          children << parse_quantified
+        end
+
+        case children.length
+        when 0 then Literal.new(byte: nil, meta: nil) # empty
+        when 1 then children[0]
+        else Concat.new(children: children, meta: meta_for_children(children, start_pos, @pos))
+        end
+      end
+
+      def parse_quantified
+        start_pos = @pos
+        child = parse_atom
+
+        case peek
+        when "*"
+          advance
+          greedy = peek != "?"
+          advance if peek == "?"
+          Quantifier.new(child: child, min: 0, max: nil, greedy: greedy, meta: meta_for_span(start_pos, @pos))
+        when "+"
+          advance
+          greedy = peek != "?"
+          advance if peek == "?"
+          Quantifier.new(child: child, min: 1, max: nil, greedy: greedy, meta: meta_for_span(start_pos, @pos))
+        when "?"
+          advance
+          greedy = peek != "?"
+          advance if peek == "?"
+          Quantifier.new(child: child, min: 0, max: 1, greedy: greedy, meta: meta_for_span(start_pos, @pos))
+        when "{"
+          parse_counted_quantifier(child, start_pos)
+        else
+          child
+        end
+      end
+
+      def parse_counted_quantifier(child, start_pos)
+        advance # skip {
+        min = parse_number
+        max = min
+
+        if peek == ","
+          advance
+          max = peek == "}" ? nil : parse_number
+        end
+
+        expect("}")
+        greedy = peek != "?"
+        advance if peek == "?"
+
+        Quantifier.new(child: child, min: min, max: max, greedy: greedy, meta: meta_for_span(start_pos, @pos))
+      end
+
+      def parse_number
+        start = @pos
+        advance while peek =~ /\d/
+        @source[start...@pos].to_i
+      end
+
+      def parse_atom
+        start_pos = @pos
+        case peek
+        when "("
+          advance
+          # Check for non-capturing group (?:...) or other extensions
+          if peek == "?"
+            advance
+            case peek
+            when ":"
+              # Non-capturing group (?:...) - same as regular group for DFA
+              advance
+              result = parse_alternation
+              expect(")")
+              Group.new(child: result, meta: meta_for_span(start_pos, @pos))
+            else
+              # Other extensions like (?=...), (?!...) etc - not supported
+              error("unsupported group extension (?#{peek}) at position #{@pos}")
+            end
+          else
+            # Regular capturing group
+            result = parse_alternation
+            expect(")")
+            Group.new(child: result, meta: meta_for_span(start_pos, @pos))
+          end
+        when "["
+          parse_char_class
+        when "\\"
+          parse_escape(start_pos)
+        when "."
+          advance
+          Any.new(meta: meta_for_span(start_pos, @pos, literal_text: "."))
+        when "^", "$"
+          # Anchors - not supported in DFA, just skip
+          advance
+          Literal.new(byte: nil, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos]))
+        when nil, "|", ")", "*", "+", "?", "{"
+          error("unexpected '#{peek}' at position #{@pos}")
+        else
+          # Handle multi-byte UTF-8 characters
+          char = peek
+          advance
+          bytes = char.bytes
+          if bytes.size == 1
+            make_literal_node(bytes[0], meta_for_span(start_pos, @pos, literal_text: char, codepoint: bytes[0], bytes: bytes))
+          else
+            # Multi-byte UTF-8: create concatenation of literals
+            # Note: case folding only applies to ASCII letters, so UTF-8 bytes are unchanged
+            meta = meta_for_span(start_pos, @pos, literal_text: char, codepoint: char.ord, bytes: bytes)
+            Concat.new(children: bytes.map { |b| Literal.new(byte: b, meta: nil) }, meta: meta)
+          end
+        end
+      end
+
+      def parse_char_class
+        start_pos = @pos
+        advance # skip [
+        negated = peek == "^"
+        advance if negated
+
+        collector = CharClassCollector.new
+
+        while peek && peek != "]"
+          start_item = parse_char_class_item
+
+          if range_follows?
+            advance # skip -
+            end_item = parse_char_class_item
+            collector.add_range(start_item, end_item)
+          else
+            collector.add_item(start_item)
+          end
+        end
+
+        expect("]")
+        meta = meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos])
+        collector.to_ast(negated: negated, meta: meta)
+      rescue ArgumentError => e
+        error(e.message)
+      end
+
+      def range_follows?
+        peek == "-" && @source[@pos + 1] != "]"
+      end
+
+      def parse_char_class_item
+        if peek == "\\"
+          advance
+          bytes = parse_escape_char
+          if bytes.length == 1
+            return { type: :byte, value: bytes[0] }
+          end
+
+          codepoint = bytes.pack("C*").force_encoding("UTF-8").ord
+          return { type: :codepoint, value: codepoint }
+        end
+
+        char = peek
+        advance
+        bytes = char.bytes
+        if bytes.length == 1
+          { type: :byte, value: bytes[0] }
+        else
+          { type: :codepoint, value: char.ord }
+        end
+      end
+
+      def parse_escape(start_pos)
+        advance # skip \
+
+        case peek
+        when "d"
+          advance
+          CharClass.new(ranges: [[0x30, 0x39]], negated: false, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos]))
+        when "D"
+          advance
+          CharClass.new(ranges: [[0x30, 0x39]], negated: true, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos]))
+        when "w"
+          advance
+          CharClass.new(ranges: [[0x30, 0x39], [0x41, 0x5A], [0x61, 0x7A], [0x5F, 0x5F]], negated: false,
+                        meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos]))
+        when "W"
+          advance
+          CharClass.new(ranges: [[0x30, 0x39], [0x41, 0x5A], [0x61, 0x7A], [0x5F, 0x5F]], negated: true,
+                        meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos]))
+        when "s"
+          advance
+          CharClass.new(ranges: [[0x09, 0x0D], [0x20, 0x20]], negated: false,
+                        meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos])) # \t\n\v\f\r and space
+        when "S"
+          advance
+          CharClass.new(ranges: [[0x09, 0x0D], [0x20, 0x20]], negated: true,
+                        meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos]))
+        when "n"
+          advance
+          Literal.new(byte: 0x0A, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: 0x0A, bytes: [0x0A])) # newline
+        when "r"
+          advance
+          Literal.new(byte: 0x0D, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: 0x0D, bytes: [0x0D])) # carriage return
+        when "t"
+          advance
+          Literal.new(byte: 0x09, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: 0x09, bytes: [0x09])) # tab
+        when "v"
+          advance
+          Literal.new(byte: 0x0B, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: 0x0B, bytes: [0x0B])) # vertical tab
+        when "f"
+          advance
+          Literal.new(byte: 0x0C, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: 0x0C, bytes: [0x0C])) # form feed
+        when "0"
+          advance
+          Literal.new(byte: 0x00, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: 0x00, bytes: [0x00])) # null
+        when "b"
+          advance
+          Literal.new(byte: 0x08, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: 0x08, bytes: [0x08])) # backspace
+        when "x"
+          advance
+          hex = consume_hex_digits(2)
+          Literal.new(byte: hex, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: hex, bytes: [hex]))
+        when "u"
+          advance
+          codepoint = consume_hex_digits(4)
+          # Convert Unicode codepoint to UTF-8 byte sequence
+          utf8_bytes = [codepoint].pack("U").bytes
+          if utf8_bytes.size == 1
+            Literal.new(byte: utf8_bytes[0], meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: codepoint, bytes: utf8_bytes))
+          else
+            # Multi-byte UTF-8: create concatenation of literals
+            meta = meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: codepoint, bytes: utf8_bytes)
+            Concat.new(children: utf8_bytes.map { |b| Literal.new(byte: b, meta: nil) }, meta: meta)
+          end
+        when nil
+          error("unexpected end of pattern after \\")
+        else
+          byte = peek.ord
+          advance
+          Literal.new(byte: byte, meta: meta_for_span(start_pos, @pos, literal_text: @source[start_pos...@pos], codepoint: byte, bytes: [byte]))
+        end
+      end
+
+      def consume_hex_digits(count)
+        digits = +""
+        count.times do
+          error("expected hex digit at position #{@pos}") unless peek =~ /[0-9a-fA-F]/
+
+          digits << peek
+          advance
+        end
+        digits.to_i(16)
+      end
+
+      # Returns an array of bytes (for multi-byte UTF-8 support)
+      def parse_escape_char
+        case peek
+        when "n"
+          advance
+          [0x0A] # newline
+        when "r"
+          advance
+          [0x0D] # carriage return
+        when "t"
+          advance
+          [0x09] # tab
+        when "v"
+          advance
+          [0x0B] # vertical tab
+        when "f"
+          advance
+          [0x0C] # form feed
+        when "0"
+          advance
+          [0x00] # null
+        when "b"
+          advance
+          [0x08] # backspace
+        when "x"
+          advance
+          [consume_hex_digits(2)]
+        when "u"
+          advance
+          codepoint = consume_hex_digits(4)
+          [codepoint].pack("U").bytes
+        when nil then error("unexpected end of pattern after \\")
+        else
+          byte = peek.ord
+          advance
+          [byte]
+        end
+      end
+
+      def peek
+        @source[@pos]
+      end
+
+      def advance
+        @pos += 1
+      end
+
+      def expect(char)
+        if peek != char
+          error("expected '#{char}' at position #{@pos}, got '#{peek}'")
+        end
+
+        advance
+      end
+
+      def eof?
+        @pos >= @source.length
+      end
+
+      # Create a literal node for a single byte
+      def make_literal_node(byte, meta)
+        Literal.new(byte: byte, meta: meta)
+      end
+
+      def meta_for_span(start_pos, end_pos, literal_text: nil, codepoint: nil, bytes: nil)
+        Meta.new(
+          span: [@source[0...start_pos].bytesize, @source[0...end_pos].bytesize],
+          literal_text: literal_text,
+          codepoint: codepoint,
+          bytes: bytes
+        )
+      end
+
+      def meta_for_children(children, start_pos, end_pos)
+        return nil if children.empty?
+
+        meta_for_span(start_pos, end_pos)
+      end
+    end
+  end
+end

data/lib/lexer_kit/dfa/utf8_range.rb

@@ -0,0 +1,175 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module DFA
+    # Build regex AST from UTF-8 codepoint ranges.
+    module Utf8Range
+      def self.ast_for_ranges(ranges)
+        parts = ranges.flat_map do |start_cp, end_cp|
+          build_range(start_cp, end_cp)
+        end
+
+        return RegexAST::Literal.new(byte: nil, meta: nil) if parts.empty?
+        return parts.first if parts.size == 1
+
+        RegexAST::Alternation.new(children: parts, meta: nil)
+      end
+
+      def self.build_range(start_cp, end_cp)
+        raise ArgumentError, "invalid range" if start_cp > end_cp
+
+        segments = []
+        segments.concat(build_for_length(start_cp, end_cp, 1))
+        segments.concat(build_for_length(start_cp, end_cp, 2))
+        segments.concat(build_for_length(start_cp, end_cp, 3))
+        segments.concat(build_for_length(start_cp, end_cp, 4))
+        segments
+      end
+
+      def self.build_for_length(start_cp, end_cp, length)
+        if length == 3 && overlaps_surrogates?(start_cp, end_cp)
+          segments = []
+          segments.concat(build_for_length(start_cp, 0xD7FF, length)) if start_cp <= 0xD7FF
+          segments.concat(build_for_length(0xE000, end_cp, length)) if end_cp >= 0xE000
+          return segments
+        end
+
+        min_cp, max_cp = codepoint_bounds(length)
+        return [] if end_cp < min_cp || start_cp > max_cp
+
+        s = [start_cp, min_cp].max
+        e = [end_cp, max_cp].min
+        return [] if s > e
+
+        segments = []
+        lead_bytes_for_length(length).each do |lead|
+          lead_min_cp, lead_max_cp = codepoint_range_for_lead(lead, length)
+          next if e < lead_min_cp || s > lead_max_cp
+
+          seg_start = [s, lead_min_cp].max
+          seg_end = [e, lead_max_cp].min
+          start_bytes = utf8_bytes(seg_start)
+          end_bytes = utf8_bytes(seg_end)
+          min_bytes, max_bytes = byte_bounds_for_lead(lead, length)
+          segments.concat(build_sequences(start_bytes, end_bytes, min_bytes, max_bytes, 1).map do |seq|
+            RegexAST::Concat.new(children: [byte_node(lead)] + seq, meta: nil)
+          end)
+        end
+
+        segments
+      end
+
+      def self.build_sequences(start_bytes, end_bytes, min_bytes, max_bytes, idx)
+        return [[]] if idx >= start_bytes.size
+
+        if start_bytes[idx] == end_bytes[idx]
+          tails = build_sequences(start_bytes, end_bytes, min_bytes, max_bytes, idx + 1)
+          return tails.map { |t| [byte_node(start_bytes[idx])] + t }
+        end
+
+        sequences = []
+
+        end_first = max_bytes.dup
+        end_first[0...idx] = start_bytes[0...idx]
+        end_first[idx] = start_bytes[idx]
+        first_tails = build_sequences(start_bytes, end_first, min_bytes, max_bytes, idx + 1)
+        sequences.concat(first_tails.map { |t| [byte_node(start_bytes[idx])] + t })
+
+        if start_bytes[idx] + 1 <= end_bytes[idx] - 1
+          middle = byte_range_node(start_bytes[idx] + 1, end_bytes[idx] - 1)
+          sequences << ([middle] + full_range_nodes(min_bytes, max_bytes, idx + 1))
+        end
+
+        start_last = min_bytes.dup
+        start_last[0...idx] = end_bytes[0...idx]
+        start_last[idx] = end_bytes[idx]
+        last_tails = build_sequences(start_last, end_bytes, min_bytes, max_bytes, idx + 1)
+        sequences.concat(last_tails.map { |t| [byte_node(end_bytes[idx])] + t })
+
+        sequences
+      end
+
+      def self.full_range_nodes(min_bytes, max_bytes, idx)
+        (idx...min_bytes.size).map do |pos|
+          byte_range_node(min_bytes[pos], max_bytes[pos])
+        end
+      end
+
+      def self.byte_node(byte)
+        RegexAST::Literal.new(byte: byte, meta: nil)
+      end
+
+      def self.byte_range_node(min_byte, max_byte)
+        if min_byte == max_byte
+          RegexAST::Literal.new(byte: min_byte, meta: nil)
+        else
+          RegexAST::CharClass.new(ranges: [[min_byte, max_byte]], negated: false, meta: nil)
+        end
+      end
+
+      def self.utf8_bytes(codepoint)
+        [codepoint].pack("U").bytes
+      end
+
+      def self.codepoint_bounds(length)
+        case length
+        when 1 then [0x0, 0x7F]
+        when 2 then [0x80, 0x7FF]
+        when 3 then [0x800, 0xFFFF]
+        when 4 then [0x10000, 0x10FFFF]
+        else [1, 0]
+        end
+      end
+
+      def self.overlaps_surrogates?(start_cp, end_cp)
+        start_cp <= 0xDFFF && end_cp >= 0xD800
+      end
+
+      def self.lead_bytes_for_length(length)
+        case length
+        when 1 then (0x00..0x7F).to_a
+        when 2 then (0xC2..0xDF).to_a
+        when 3 then (0xE0..0xEF).to_a
+        when 4 then (0xF0..0xF4).to_a
+        else []
+        end
+      end
+
+      def self.byte_bounds_for_lead(lead, length)
+        case length
+        when 1
+          [[lead], [lead]]
+        when 2
+          [[lead, 0x80], [lead, 0xBF]]
+        when 3
+          min_second, max_second = if lead == 0xE0
+                                     [0xA0, 0xBF]
+                                   elsif lead == 0xED
+                                     [0x80, 0x9F]
+                                   else
+                                     [0x80, 0xBF]
+                                   end
+          [[lead, min_second, 0x80], [lead, max_second, 0xBF]]
+        when 4
+          min_second, max_second = if lead == 0xF0
+                                     [0x90, 0xBF]
+                                   elsif lead == 0xF4
+                                     [0x80, 0x8F]
+                                   else
+                                     [0x80, 0xBF]
+                                   end
+          [[lead, min_second, 0x80, 0x80], [lead, max_second, 0xBF, 0xBF]]
+        else
+          [[], []]
+        end
+      end
+
+      def self.codepoint_range_for_lead(lead, length)
+        min_bytes, max_bytes = byte_bounds_for_lead(lead, length)
+        min_cp = min_bytes.pack("C*").force_encoding("UTF-8").ord
+        max_cp = max_bytes.pack("C*").force_encoding("UTF-8").ord
+        [min_cp, max_cp]
+      end
+    end
+  end
+end

data/lib/lexer_kit/dfa/utf8_range_pattern.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module DFA
+    class Utf8RangePattern
+      include LexerKit::RegexAstProvider
+
+      def initialize(ranges)
+        @ranges = ranges
+      end
+
+      def to_ast
+        Utf8Range.ast_for_ranges(@ranges)
+      end
+    end
+  end
+end

data/lib/lexer_kit/dfa.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "dfa/regex_ast"
+require_relative "dfa/utf8_range"
+require_relative "dfa/case_folding"
+require_relative "dfa/char_class_collector"
+require_relative "dfa/regex_parser"
+require_relative "dfa/nfa"
+require_relative "dfa/dfa_minimizer"
+require_relative "dfa/byte_class_builder"
+require_relative "dfa/dfa_builder"
+
+module LexerKit
+  # DFA module provides regex to DFA compilation.
+  module DFA
+    # Compile a Regex to DFA
+    # @param regex [RegexAST::Regex] regex pattern
+    # @param token_id [Integer] token ID for accepting state
+    # @return [IR::DFATable]
+    def self.compile_regex(regex, token_id = 0)
+      raise ArgumentError, "expected Regex, got #{regex.class}" unless regex.is_a?(RegexAST::Regex)
+
+      nfa = NFA.from_regex(regex, token_id)
+      DFABuilder.new(nfa).build
+    end
+
+    # Compute the set of possible first bytes for a Regex
+    # @param regex [RegexAST::Regex] regex pattern
+    # @return [Set<Integer>] set of bytes (0-255)
+    def self.first_byte_set(regex)
+      raise ArgumentError, "expected Regex, got #{regex.class}" unless regex.is_a?(RegexAST::Regex)
+
+      nfa = NFA.from_regex(regex, 0)
+      nfa.first_byte_set
+    end
+  end
+end

data/lib/lexer_kit/errors.rb

@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module LexerKit
+  # Base error for all LexerKit errors
+  class Error < StandardError; end
+
+  # ===========================================================================
+  # Binary/File Errors
+  # ===========================================================================
+
+  # Raised when binary deserialization fails
+  class InvalidBinaryError < Error; end
+
+  # Raised when file integrity check fails (SHA256 mismatch, etc.)
+  class IntegrityError < Error; end
+
+  # ===========================================================================
+  # User Input Errors (with diagnostic information)
+  # ===========================================================================
+
+  # Base class for errors with diagnostic information (source location, span, etc.)
+  # Provides rich error messages with source code context.
+  class DiagnosticError < Error
+    attr_reader :diagnostic, :source
+
+    def initialize(message, source: nil, span: nil, level: :error, notes: nil)
+      @source = source
+      @diagnostic = Core::Diagnostic.new(level: level, message: message, span: span, notes: notes) if source && span
+      super(message)
+    end
+
+    def render(color: $stderr.tty?)
+      return message unless @diagnostic && @source
+
+      @diagnostic.render(@source, color: color)
+    end
+
+    def self.from_location(location, message, level: :error, notes: nil)
+      return new(message) unless location&.path && File.file?(location.path)
+
+      bytes = File.binread(location.path)
+      source = Core::Source.new(bytes, filename: location.path)
+      span = source.span_for_line(location.lineno)
+      new(message, source: source, span: span, level: level, notes: notes)
+    rescue Errno::ENOENT, Errno::EACCES
+      new(message)
+    end
+  end
+
+  # Raised during Builder DSL evaluation (invalid token definitions, etc.)
+  class BuildError < DiagnosticError; end
+
+  # Raised during compilation (IR generation, optimization, etc.)
+  class CompileError < DiagnosticError; end
+
+  # Raised during regex parsing
+  class ParseError < DiagnosticError
+    attr_reader :position
+
+    def initialize(message, source: nil, position: nil)
+      @position = position # character position (codepoint-based)
+      span = source.span_for_char_index(position) if source && position
+      super(message, source: source, span: span)
+    end
+  end
+
+  # ===========================================================================
+  # Runtime Errors
+  # ===========================================================================
+
+  # Raised when C native extension is required but not loaded
+  class NativeExtensionError < Error; end
+
+  # Raised during VM execution (stack overflow, infinite loop, etc.)
+  class VMError < Error; end
+end