lexer_kit 0.5.0

data/lib/lexer_kit/dfa/nfa.rb

@@ -0,0 +1,304 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module DFA
+    # NFA represents a Non-deterministic Finite Automaton.
+    # Built using Thompson's construction from regex AST.
+    # Case folding is applied during NFA construction, not parsing.
+    class NFA
+      include CaseFolding
+
+      # Epsilon transition marker
+      EPSILON = :epsilon
+
+      attr_reader :start_state, :accept_state, :transitions, :token_id
+
+      def initialize(case_insensitive: false)
+        @case_insensitive = case_insensitive
+        @states = 0
+        @transitions = Hash.new { |h, k| h[k] = [] }
+        @start_state = nil
+        @accept_state = nil
+        @token_id = nil
+      end
+
+      # Create a new state and return its ID
+      def new_state
+        state = @states
+        @states += 1
+        state
+      end
+
+      # Add a transition
+      def add_transition(from, to, input)
+        @transitions[from] << [input, to]
+      end
+
+      # Build NFA from regex AST
+      # @param ast [RegexAST::*] AST node
+      # @param token_id [Integer] token ID for accept state
+      # @param case_insensitive [Boolean] whether to apply case folding
+      # @return [NFA]
+      def self.from_ast(ast, token_id = 0, case_insensitive: false)
+        nfa = new(case_insensitive: case_insensitive)
+        start, accept = nfa.build(ast)
+        nfa.finalize!(start_state: start, accept_state: accept, token_id: token_id)
+        nfa
+      end
+
+      # Build NFA from Regex (AST with flags)
+      # @param regex [RegexAST::Regex] regex pattern with flags
+      # @param token_id [Integer] token ID for accept state
+      # @return [NFA]
+      def self.from_regex(regex, token_id = 0)
+        from_ast(regex.ast, token_id, case_insensitive: regex.case_insensitive)
+      end
+
+      def finalize!(start_state:, accept_state:, token_id:)
+        @start_state = start_state
+        @accept_state = accept_state
+        @token_id = token_id
+      end
+
+      # Build NFA fragment from AST node
+      # Returns [start_state, accept_state]
+      def build(node)
+        case node
+        when RegexAST::Literal
+          build_literal(node)
+        when RegexAST::CharClass
+          build_char_class(node)
+        when RegexAST::Any
+          build_any
+        when RegexAST::Concat
+          build_concat(node)
+        when RegexAST::Alternation
+          build_alternation(node)
+        when RegexAST::Quantifier
+          build_quantifier(node)
+        when RegexAST::Group
+          build(node.child)
+        else
+          raise "unknown AST node: #{node.class}"
+        end
+      end
+
+      # Compute epsilon closure of a set of states
+      # @param states [Set<Integer>] initial states
+      # @return [Set<Integer>] epsilon closure
+      def epsilon_closure(states)
+        closure = Set.new(states)
+        worklist = states.to_a
+
+        while (state = worklist.pop)
+          @transitions[state].each do |input, target|
+            if input == EPSILON && !closure.include?(target)
+              closure << target
+              worklist << target
+            end
+          end
+        end
+
+        closure
+      end
+
+      # Get all states reachable from a set of states on a given input
+      # @param states [Set<Integer>] current states
+      # @param input [Integer] input byte
+      # @return [Set<Integer>] reachable states
+      def move(states, input)
+        result = Set.new
+        states.each do |state|
+          @transitions[state].each do |trans_input, target|
+            result << target if trans_input == input
+          end
+        end
+        result
+      end
+
+      # Compute the set of possible first bytes this NFA can match
+      # @return [Set<Integer>] set of bytes (0-255)
+      def first_byte_set
+        closure = epsilon_closure(Set[@start_state])
+        result = Set.new
+
+        closure.each do |state|
+          @transitions[state].each do |input, _target|
+            result << input if input.is_a?(Integer)
+          end
+        end
+
+        result
+      end
+
+      private
+
+      def build_literal(node)
+        return [new_state, new_state] if node.byte.nil? # empty
+
+        start = new_state
+        accept = new_state
+
+        # Apply case folding if enabled
+        case_fold_byte(node.byte).each do |byte|
+          add_transition(start, accept, byte)
+        end
+
+        [start, accept]
+      end
+
+      def build_char_class(node)
+        start = new_state
+        accept = new_state
+
+        # Expand ranges with case folding
+        expanded_ranges = node.ranges.flat_map do |from, to|
+          case_fold_range(from, to)
+        end
+
+        if node.negated
+          # Negated: add transitions for all bytes NOT in the class
+          included = Set.new
+          expanded_ranges.each do |from, to|
+            (from..to).each { |b| included << b }
+          end
+          (0..255).each do |byte|
+            add_transition(start, accept, byte) unless included.include?(byte)
+          end
+        else
+          # Normal: add transitions for all bytes in the class
+          expanded_ranges.each do |from, to|
+            (from..to).each do |byte|
+              add_transition(start, accept, byte)
+            end
+          end
+        end
+
+        [start, accept]
+      end
+
+      def build_any
+        start = new_state
+        accept = new_state
+        # Match any byte except newline (0x0A)
+        (0..255).each do |byte|
+          add_transition(start, accept, byte) unless byte == 0x0A
+        end
+        [start, accept]
+      end
+
+      def build_concat(node)
+        return [new_state, new_state] if node.children.empty?
+
+        fragments = node.children.map { |child| build(child) }
+
+        # Chain fragments together with epsilon transitions
+        fragments.each_cons(2) do |(_, accept), (next_start, _)|
+          add_transition(accept, next_start, EPSILON)
+        end
+
+        [fragments.first[0], fragments.last[1]]
+      end
+
+      def build_alternation(node)
+        start = new_state
+        accept = new_state
+
+        node.children.each do |child|
+          child_start, child_accept = build(child)
+          add_transition(start, child_start, EPSILON)
+          add_transition(child_accept, accept, EPSILON)
+        end
+
+        [start, accept]
+      end
+
+      def build_quantifier(node)
+        case [node.min, node.max] # rubocop:disable Style/MinMax
+        when [0, nil]  # *
+          build_star(node.child)
+        when [1, nil]  # +
+          build_plus(node.child)
+        when [0, 1]    # ?
+          build_optional(node.child)
+        else
+          build_counted(node.child, node.min, node.max)
+        end
+      end
+
+      def build_star(child_ast)
+        start = new_state
+        accept = new_state
+
+        child_start, child_accept = build(child_ast)
+
+        add_transition(start, child_start, EPSILON)
+        add_transition(start, accept, EPSILON)
+        add_transition(child_accept, child_start, EPSILON)
+        add_transition(child_accept, accept, EPSILON)
+
+        [start, accept]
+      end
+
+      def build_plus(child_ast)
+        start = new_state
+        accept = new_state
+
+        child_start, child_accept = build(child_ast)
+
+        add_transition(start, child_start, EPSILON)
+        add_transition(child_accept, child_start, EPSILON)
+        add_transition(child_accept, accept, EPSILON)
+
+        [start, accept]
+      end
+
+      def build_optional(child_ast)
+        start = new_state
+        accept = new_state
+
+        child_start, child_accept = build(child_ast)
+
+        add_transition(start, child_start, EPSILON)
+        add_transition(start, accept, EPSILON)
+        add_transition(child_accept, accept, EPSILON)
+
+        [start, accept]
+      end
+
+      def build_counted(child_ast, min, max)
+        start = new_state
+        current = start
+
+        # Build min required copies
+        min.times do
+          child_start, child_accept = build(child_ast)
+          add_transition(current, child_start, EPSILON)
+          current = child_accept
+        end
+
+        accept = new_state
+        add_transition(current, accept, EPSILON)
+
+        if max.nil?
+          # {n,} - min copies followed by star
+          child_start, child_accept = build(child_ast)
+          add_transition(current, child_start, EPSILON)
+          add_transition(child_accept, child_start, EPSILON)
+          add_transition(child_accept, accept, EPSILON)
+        elsif max > min
+          # {n,m} - min copies followed by (max-min) optional copies
+          (max - min).times do
+            child_start, child_accept = build(child_ast)
+            add_transition(current, child_start, EPSILON)
+            add_transition(current, accept, EPSILON)
+            current = child_accept
+            add_transition(current, accept, EPSILON)
+          end
+        end
+
+        [start, accept]
+      end
+    end
+  end
+end

data/lib/lexer_kit/dfa/regex_ast.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module DFA
+    # RegexAST contains immutable AST node types for parsed regular expressions.
+    # Uses Ruby's Data class for immutability and value semantics.
+    module RegexAST
+      # Metadata for AST nodes (source location, original text, etc.)
+      Meta = Data.define(:span, :literal_text, :codepoint, :bytes) do
+        def to_s
+          parts = []
+          parts << "span=#{span.inspect}" if span
+          parts << "text=#{literal_text.inspect}" if literal_text
+          parts << "cp=U+#{codepoint.to_s(16).upcase}" if codepoint
+          "{#{parts.join(', ')}}"
+        end
+      end
+
+      # Literal byte match
+      Literal = Data.define(:byte, :meta)
+
+      # Character class (e.g., [a-z], [^0-9])
+      # ranges: [[from, to], ...]
+      CharClass = Data.define(:ranges, :negated, :meta)
+
+      # Concatenation of patterns
+      Concat = Data.define(:children, :meta)
+
+      # Alternation (e.g., a|b|c)
+      Alternation = Data.define(:children, :meta)
+
+      # Quantifier (*, +, ?, {n,m})
+      # max = nil means unlimited
+      Quantifier = Data.define(:child, :min, :max, :greedy, :meta)
+
+      # Grouping (capturing or non-capturing)
+      Group = Data.define(:child, :meta)
+
+      # Any character (. pattern) - matches any byte except newline
+      Any = Data.define(:meta)
+
+      # Complete regex pattern with flags
+      # Wraps an AST node with pattern-level settings
+      Regex = Data.define(:ast, :case_insensitive) do
+        # Parse a pattern into Regex
+        # @param pattern [Regexp, String, Regex, RegexAstProvider]
+        # @return [Regex]
+        def self.parse(pattern)
+          case pattern
+          when self
+            pattern
+          when LexerKit::RegexAstProvider
+            pattern.to_regex
+          else
+            source = pattern.is_a?(Regexp) ? pattern.source : pattern
+            case_insensitive = pattern.is_a?(Regexp) && pattern.casefold?
+            ast = RegexParser.new(source).parse
+            new(ast: ast, case_insensitive: case_insensitive)
+          end
+        end
+      end
+    end
+  end
+end