lexer_kit 0.5.0

data/lib/lexer_kit/debug/visualizer.rb

@@ -0,0 +1,203 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+module LexerKit
+  module Debug
+    # Visualizer for DFA tables and Jump tables
+    module Visualizer
+      # Format a DFA table as a readable string
+      # @param dfa [IR::DFATable] the DFA table
+      # @param program [IR::CompiledProgram, nil] optional program for token names
+      # @return [String]
+      def self.format_dfa(dfa, program: nil)
+        output = StringIO.new
+
+        output.puts "DFA Table"
+        output.puts "  States: #{dfa.state_count}"
+        output.puts "  Classes: #{dfa.class_count}"
+        output.puts
+
+        # Byte class mapping (show non-zero classes)
+        output.puts "  Byte Classes:"
+        classes_by_value = Hash.new { |h, k| h[k] = [] }
+        dfa.byte_class.each_with_index do |cls, byte|
+          classes_by_value[cls] << byte if cls > 0
+        end
+        classes_by_value.sort.each do |cls, bytes|
+          byte_ranges = compress_ranges(bytes)
+          output.puts "    Class #{cls}: #{byte_ranges}"
+        end
+        output.puts
+
+        # Transitions (non-dead state transitions)
+        output.puts "  Transitions:"
+        (1...dfa.state_count).each do |state|
+          transitions = []
+          (0...dfa.class_count).each do |cls|
+            # Access transitions array directly: transitions[state * class_count + cls]
+            next_state = dfa.transitions[(state * dfa.class_count) + cls]
+            if next_state && next_state != 0 # Not dead state
+              transitions << "c#{cls}->s#{next_state}"
+            end
+          end
+          next unless transitions.any?
+
+          accept = dfa.accept(state)
+          accept_str = accept ? " [accept: #{format_token(accept, program)}]" : ""
+          output.puts "    State #{state}: #{transitions.join(', ')}#{accept_str}"
+        end
+        output.puts
+
+        # Accept states
+        output.puts "  Accept States:"
+        (1...dfa.state_count).each do |state|
+          token = dfa.accept(state)
+          if token
+            output.puts "    State #{state}: #{format_token(token, program)}"
+          end
+        end
+
+        output.string
+      end
+
+      # Format a DFA table as Graphviz DOT format
+      # @param dfa [IR::DFATable] the DFA table
+      # @param program [IR::CompiledProgram, nil] optional program for token names
+      # @return [String]
+      def self.dfa_to_dot(dfa, program: nil)
+        output = StringIO.new
+
+        output.puts "digraph DFA {"
+        output.puts "  rankdir=LR;"
+        output.puts "  node [shape=circle];"
+        output.puts
+
+        # Mark accept states with double circle
+        (1...dfa.state_count).each do |state|
+          token = dfa.accept(state)
+          if token
+            label = format_token(token, program)
+            output.puts "  s#{state} [shape=doublecircle, label=\"#{state}\\n#{label}\"];"
+          end
+        end
+        output.puts
+
+        # Build class to byte mapping for edge labels
+        bytes_by_class = Hash.new { |h, k| h[k] = [] }
+        dfa.byte_class.each_with_index do |cls, byte|
+          bytes_by_class[cls] << byte
+        end
+
+        # Transitions
+        (1...dfa.state_count).each do |state|
+          (0...dfa.class_count).each do |cls|
+            next_state = dfa.transitions[(state * dfa.class_count) + cls]
+            next if next_state.nil? || next_state == 0 # Skip dead state
+
+            bytes = bytes_by_class[cls]
+            label = compress_ranges(bytes)
+            output.puts "  s#{state} -> s#{next_state} [label=\"#{escape_dot(label)}\"];"
+          end
+        end
+
+        output.puts "}"
+        output.string
+      end
+
+      # Format a JumpTable as a readable string
+      # @param table [IR::JumpTable] the jump table
+      # @return [String]
+      def self.format_jump_table(table)
+        output = StringIO.new
+
+        output.puts "Jump Table"
+        output.puts "  Default: -> #{format('%04d', table.default_offset)}"
+        output.puts "  Entries:"
+
+        # Group entries by target offset
+        by_offset = Hash.new { |h, k| h[k] = [] }
+        table.entries.each do |byte, offset|
+          by_offset[offset] << byte
+        end
+
+        by_offset.sort.each do |offset, bytes|
+          byte_str = compress_ranges(bytes)
+          output.puts "    #{byte_str} -> #{format('%04d', offset)}"
+        end
+
+        output.string
+      end
+
+      # Format a KeywordTable as a readable string
+      # @param table [IR::KeywordTable] the keyword table
+      # @param program [IR::CompiledProgram, nil] optional program for token names
+      # @return [String]
+      def self.format_keyword_table(table, program: nil)
+        output = StringIO.new
+
+        output.puts "Keyword Table"
+        output.puts "  Base Token: #{format_token(table.base_token_id, program)}"
+        output.puts "  Keywords:"
+
+        table.keywords.each do |keyword, token_id|
+          output.puts "    #{keyword.inspect} -> #{format_token(token_id, program)}"
+        end
+
+        output.string
+      end
+
+      private_class_method def self.format_token(token_id, program)
+        if program
+          name = program.token_name(token_id)
+          name ? "#{name} (#{token_id})" : token_id.to_s
+        else
+          token_id.to_s
+        end
+      end
+
+      private_class_method def self.compress_ranges(bytes)
+        return "" if bytes.empty?
+
+        bytes = bytes.sort.uniq
+        ranges = []
+        start = bytes.first
+        last = start
+
+        bytes[1..].each do |byte|
+          if byte == last + 1
+            last = byte
+          else
+            ranges << format_range(start, last)
+            start = byte
+            last = byte
+          end
+        end
+        ranges << format_range(start, last)
+
+        ranges.join(", ")
+      end
+
+      private_class_method def self.format_range(start, last)
+        if start == last
+          format_byte(start)
+        else
+          "#{format_byte(start)}-#{format_byte(last)}"
+        end
+      end
+
+      private_class_method def self.format_byte(byte)
+        case byte
+        when 0x22 then "'\"'"
+        when 0x5C then "'\\'"
+        when 0x20...0x7F then "'#{byte.chr}'"
+        else format("0x%02X", byte)
+        end
+      end
+
+      private_class_method def self.escape_dot(str)
+        str.gsub("\\", "\\\\\\\\").gsub("\"", "\\\"")
+      end
+    end
+  end
+end

data/lib/lexer_kit/debug.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require_relative "debug/disassembler"
+require_relative "debug/visualizer"
+
+module LexerKit
+  # Debug tools for LexerKit
+  # Provides disassembly and visualization utilities
+  module Debug
+  end
+end

data/lib/lexer_kit/dfa/byte_class_builder.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module DFA
+    # ByteClassBuilder compresses DFA transitions by grouping bytes
+    # with identical behavior into equivalence classes.
+    class ByteClassBuilder
+      # @param dfa_states [Hash] NFA state set → DFA state ID mapping
+      # @param dfa_transitions [Hash] [state, byte] → target state mapping
+      # @param state_count [Integer] number of DFA states
+      def initialize(dfa_states:, dfa_transitions:, state_count:)
+        @dfa_states = dfa_states
+        @dfa_transitions = dfa_transitions
+        @state_count = state_count
+      end
+
+      # Build byte classes and transition table
+      # @return [Hash] { byte_class:, class_count:, transitions: }
+      def build
+        byte_class, class_count = build_byte_classes
+        transitions = build_transition_table(byte_class, class_count)
+
+        {
+          byte_class: byte_class,
+          class_count: class_count,
+          transitions: transitions
+        }
+      end
+
+      private
+
+      def build_byte_classes
+        signatures = {}
+
+        (0..255).each do |byte|
+          sig = @dfa_states.keys.map do |nfa_set|
+            state = @dfa_states[nfa_set]
+            @dfa_transitions[[state, byte]] || 0
+          end
+          signatures[byte] = sig
+        end
+
+        sig_to_class = {}
+        class_count = 0
+        byte_class = Array.new(256)
+
+        (0..255).each do |byte|
+          sig = signatures[byte]
+          unless sig_to_class.key?(sig)
+            sig_to_class[sig] = class_count
+            class_count += 1
+          end
+          byte_class[byte] = sig_to_class[sig]
+        end
+
+        [byte_class, class_count]
+      end
+
+      def build_transition_table(byte_class, class_count)
+        transitions = Array.new(@state_count * class_count, 0)
+        @dfa_transitions.each do |(state, byte), target|
+          cls = byte_class[byte]
+          transitions[(state * class_count) + cls] = target
+        end
+        transitions
+      end
+    end
+  end
+end

data/lib/lexer_kit/dfa/case_folding.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module DFA
+    # Case folding helpers for ASCII letters.
+    # Include this module and set @case_insensitive to use.
+    module CaseFolding
+      private
+
+      def ascii_letter?(byte)
+        byte.between?(0x41, 0x5A) || byte.between?(0x61, 0x7A)
+      end
+
+      def case_fold_byte(byte)
+        return [byte] unless @case_insensitive && ascii_letter?(byte)
+
+        if byte.between?(0x41, 0x5A) # A-Z
+          [byte, byte + 32]
+        else # a-z
+          [byte - 32, byte]
+        end
+      end
+
+      def case_fold_range(from, to)
+        return [[from, to]] unless @case_insensitive
+
+        ranges = [[from, to]]
+
+        if from <= 0x5A && to >= 0x41
+          upper_start = [from, 0x41].max
+          upper_end = [to, 0x5A].min
+          ranges << [upper_start + 32, upper_end + 32]
+        end
+
+        if from <= 0x7A && to >= 0x61
+          lower_start = [from, 0x61].max
+          lower_end = [to, 0x7A].min
+          ranges << [lower_start - 32, lower_end - 32]
+        end
+
+        ranges.uniq
+      end
+    end
+  end
+end

data/lib/lexer_kit/dfa/char_class_collector.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module DFA
+    # CharClassCollector collects character class items and builds appropriate AST.
+    # Separates byte/codepoint handling from parsing control flow.
+    # Case folding is handled at the NFA layer, not here.
+    class CharClassCollector
+      include RegexAST
+
+      def initialize
+        @byte_ranges = []
+        @codepoint_ranges = []
+      end
+
+      # Add a single item (byte or codepoint)
+      def add_item(item)
+        if item[:type] == :byte
+          @byte_ranges << [item[:value], item[:value]]
+        else
+          @codepoint_ranges << [item[:value], item[:value]]
+        end
+      end
+
+      # Add a range of items
+      def add_range(start_item, end_item)
+        raise ArgumentError, "mixed byte and multibyte range in char class" if start_item[:type] != end_item[:type]
+
+        if start_item[:type] == :byte
+          @byte_ranges << [start_item[:value], end_item[:value]]
+        else
+          start_cp = start_item[:value]
+          end_cp = end_item[:value]
+          raise ArgumentError, "invalid multibyte range" if start_cp > end_cp
+
+          @codepoint_ranges << [start_cp, end_cp]
+        end
+      end
+
+      # Build the final AST
+      def to_ast(negated:, meta:)
+        validate_negated_multibyte!(negated)
+        ascii_ast = build_ascii_ast(negated, meta)
+        utf8_ast = build_utf8_ast(meta)
+        combine_asts(ascii_ast, utf8_ast, negated, meta)
+      end
+
+      private
+
+      def validate_negated_multibyte!(negated)
+        return unless negated && @codepoint_ranges.any?
+
+        raise ArgumentError, "negated char class with multibyte is not supported"
+      end
+
+      def build_ascii_ast(negated, meta)
+        return nil unless @byte_ranges.any? || negated
+
+        CharClass.new(ranges: @byte_ranges, negated: negated, meta: meta)
+      end
+
+      def build_utf8_ast(meta)
+        return nil unless @codepoint_ranges.any?
+
+        utf8_ast = Utf8Range.ast_for_ranges(@codepoint_ranges)
+        utf8_ast = utf8_ast.with(meta: meta) if utf8_ast.respond_to?(:with)
+        utf8_ast
+      end
+
+      def combine_asts(ascii_ast, utf8_ast, negated, meta)
+        if utf8_ast
+          return utf8_ast unless ascii_ast
+
+          return Alternation.new(children: [ascii_ast, utf8_ast], meta: meta)
+        end
+
+        ascii_ast || CharClass.new(ranges: [], negated: negated, meta: meta)
+      end
+    end
+  end
+end

data/lib/lexer_kit/dfa/dfa_builder.rb

@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module DFA
+    # DFABuilder converts NFA to DFA using subset construction,
+    # then compresses using byte classes.
+    class DFABuilder
+      def initialize(nfa)
+        @nfa = nfa
+        @dfa_states = {}
+        @dfa_transitions = {}
+        @dfa_accept = {}
+        @state_count = 1
+      end
+
+      # Build DFA from NFA
+      # @return [IR::DFATable]
+      def build
+        initial_set = @nfa.epsilon_closure(Set[@nfa.start_state])
+        run_subset_construction(initial_set)
+        create_dfa_table
+      end
+
+      private
+
+      def run_subset_construction(initial_set)
+        get_or_create_state(initial_set)
+
+        worklist = [initial_set]
+        processed = Set.new
+
+        while (current_set = worklist.shift)
+          next if processed.include?(current_set)
+
+          processed << current_set
+          current_state = @dfa_states[current_set]
+
+          (0..255).each do |byte|
+            moved = @nfa.move(current_set, byte)
+            next if moved.empty?
+
+            target_set = @nfa.epsilon_closure(moved)
+            target_state = get_or_create_state(target_set)
+            @dfa_transitions[[current_state, byte]] = target_state
+
+            worklist << target_set unless processed.include?(target_set)
+          end
+        end
+      end
+
+      def get_or_create_state(nfa_state_set)
+        return @dfa_states[nfa_state_set] if @dfa_states.key?(nfa_state_set)
+
+        state_id = @state_count
+        @state_count += 1
+        @dfa_states[nfa_state_set] = state_id
+
+        if nfa_state_set.include?(@nfa.accept_state)
+          @dfa_accept[state_id] = @nfa.token_id
+        end
+
+        state_id
+      end
+
+      def create_dfa_table
+        # Build byte classes and transition table
+        byte_class_result = ByteClassBuilder.new(
+          dfa_states: @dfa_states,
+          dfa_transitions: @dfa_transitions,
+          state_count: @state_count
+        ).build
+
+        byte_class = byte_class_result[:byte_class]
+        class_count = byte_class_result[:class_count]
+        transitions = byte_class_result[:transitions]
+
+        # Minimize DFA using Hopcroft's algorithm
+        minimizer = DFAMinimizer.new(
+          state_count: @state_count,
+          transitions: transitions,
+          accept_states: @dfa_accept,
+          class_count: class_count
+        )
+        minimized = minimizer.minimize
+
+        IR::DFATable.new(
+          state_count: minimized[:state_count],
+          byte_class: byte_class,
+          transitions: minimized[:transitions],
+          accept_states: minimized[:accept_states]
+        )
+      end
+    end
+  end
+end

data/lib/lexer_kit/dfa/dfa_minimizer.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module DFA
+    # DFAMinimizer reduces DFA state count using Hopcroft's algorithm.
+    # Merges equivalent states while preserving language recognition.
+    class DFAMinimizer
+      # @param state_count [Integer] number of states (including dead state 0)
+      # @param transitions [Array<Integer>] state × class_count → next_state
+      # @param accept_states [Hash<Integer, Integer>] state → token_id
+      # @param class_count [Integer] number of byte classes
+      def initialize(state_count:, transitions:, accept_states:, class_count:)
+        @state_count = state_count
+        @transitions = transitions
+        @accept_states = accept_states
+        @class_count = class_count
+      end
+
+      # Minimize the DFA
+      # @return [Hash] { state_count:, transitions:, accept_states: }
+      def minimize
+        return trivial_result if @state_count <= 2
+
+        # 1. Initial partition: group by acceptance status and token_id
+        partitions = build_initial_partitions
+
+        # 2. Refine partitions using Hopcroft's algorithm
+        partitions = refine_partitions(partitions)
+
+        # 3. Build minimized DFA
+        build_minimized_dfa(partitions)
+      end
+
+      private
+
+      def trivial_result
+        {
+          state_count: @state_count,
+          transitions: @transitions,
+          accept_states: @accept_states
+        }
+      end
+
+      def build_initial_partitions
+        # Non-accepting states (excluding dead state 0)
+        non_accept = (1...@state_count).reject { |s| @accept_states.key?(s) }
+
+        # Accepting states grouped by token_id
+        accept_groups = @accept_states.group_by { |_, tid| tid }
+                                      .transform_values { |pairs| pairs.map(&:first) }
+
+        partitions = []
+        partitions << non_accept unless non_accept.empty?
+        accept_groups.each_value { |states| partitions << states }
+
+        partitions
+      end
+
+      def refine_partitions(partitions)
+        # Build state → partition index mapping
+        state_to_partition = build_state_mapping(partitions)
+
+        loop do
+          new_partitions = []
+          changed = false
+
+          partitions.each do |partition|
+            split = split_partition(partition, state_to_partition)
+            new_partitions.concat(split)
+            changed = true if split.size > 1
+          end
+
+          break unless changed
+
+          partitions = new_partitions
+          state_to_partition = build_state_mapping(partitions)
+        end
+
+        partitions
+      end
+
+      def build_state_mapping(partitions)
+        mapping = {}
+        partitions.each_with_index do |states, idx|
+          states.each { |s| mapping[s] = idx }
+        end
+        mapping
+      end
+
+      def split_partition(partition, state_to_partition)
+        return [partition] if partition.size <= 1
+
+        # Group states by their transition signature
+        groups = partition.group_by do |state|
+          compute_signature(state, state_to_partition)
+        end
+
+        groups.values
+      end
+
+      def compute_signature(state, state_to_partition)
+        (0...@class_count).map do |cls|
+          target = @transitions[(state * @class_count) + cls]
+          if target.zero?
+            -1 # Dead state
+          else
+            state_to_partition[target] || -1
+          end
+        end
+      end
+
+      def build_minimized_dfa(partitions)
+        # Find which partition contains the start state (state 1)
+        state_to_partition = build_state_mapping(partitions)
+        start_partition = state_to_partition[1]
+
+        # Reorder partitions so start state's partition comes first
+        if start_partition && start_partition != 0
+          partitions[0], partitions[start_partition] = partitions[start_partition], partitions[0]
+          state_to_partition = build_state_mapping(partitions)
+        end
+
+        # Build partition → new state mapping (partition 0 → state 1, etc.)
+        partition_to_new_state = {}
+        partitions.each_with_index { |_, idx| partition_to_new_state[idx] = idx + 1 }
+
+        new_state_count = partitions.size + 1 # +1 for dead state 0
+        new_transitions = Array.new(new_state_count * @class_count, 0)
+        new_accept = {}
+
+        partitions.each_with_index do |states, idx|
+          new_state = idx + 1
+          representative = states.first
+
+          # Copy transitions using representative
+          (0...@class_count).each do |cls|
+            target = @transitions[(representative * @class_count) + cls]
+            if target.zero?
+              new_transitions[(new_state * @class_count) + cls] = 0
+            else
+              target_partition = state_to_partition[target]
+              new_transitions[(new_state * @class_count) + cls] = partition_to_new_state[target_partition]
+            end
+          end
+
+          # Copy accept status if representative is accepting
+          new_accept[new_state] = @accept_states[representative] if @accept_states.key?(representative)
+        end
+
+        {
+          state_count: new_state_count,
+          transitions: new_transitions,
+          accept_states: new_accept
+        }
+      end
+    end
+  end
+end