lexer_kit 0.5.0

data/lib/lexer_kit/builder/conflict_detector.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+module LexerKit
+  class Builder
+    # ConflictDetector analyzes token patterns to detect potential conflicts.
+    # A conflict occurs when multiple patterns can match the same input.
+    class ConflictDetector
+      Conflict = Struct.new(:type, :token1, :token2, :description, keyword_init: true)
+
+      def initialize(builder)
+        @builder = builder
+        @conflicts = []
+      end
+
+      # Detect all conflicts in the lexer definition
+      # @return [Array<Conflict>]
+      def detect
+        @conflicts = []
+
+        @builder.mode_defs.each do |_mode_name, mode_def|
+          detect_mode_conflicts(mode_def)
+        end
+
+        @conflicts
+      end
+
+      private
+
+      def detect_mode_conflicts(mode_def)
+        tokens = mode_def.tokens
+
+        # Check each pair of tokens
+        tokens.each_with_index do |token1, i|
+          tokens[(i + 1)..].each do |token2|
+            check_pair(token1, token2)
+          end
+        end
+      end
+
+      def check_pair(token1, token2)
+        # Skip if both have the same name (same token defined multiple times is ok)
+        return if token1.name == token2.name
+
+        pattern1 = token1.pattern
+        pattern2 = token2.pattern
+
+        if pattern1.is_a?(String) && pattern2.is_a?(String)
+          check_literal_pair(token1, token2)
+        elsif pattern1.is_a?(String) && pattern2.is_a?(Regexp)
+          check_literal_regex_pair(token1, token2)
+        elsif pattern1.is_a?(Regexp) && pattern2.is_a?(String)
+          check_literal_regex_pair(token2, token1)
+        else
+          check_regex_pair(token1, token2)
+        end
+      end
+
+      def check_literal_pair(token1, token2)
+        lit1 = token1.pattern
+        lit2 = token2.pattern
+
+        if lit1 == lit2
+          @conflicts << Conflict.new(
+            type: :identical,
+            token1: token1.name,
+            token2: token2.name,
+            description: "identical patterns '#{lit1}' (#{token1.name} wins)"
+          )
+        elsif lit1.start_with?(lit2)
+          # lit2 is a prefix of lit1 - this is usually OK due to longest-match
+          # But we should warn if they have the same first byte
+          @conflicts << Conflict.new(
+            type: :prefix,
+            token1: token2.name,
+            token2: token1.name,
+            description: "'#{lit2}' is a prefix of '#{lit1}' (longest match: #{token1.name} preferred for '#{lit1}')"
+          )
+        elsif lit2.start_with?(lit1)
+          @conflicts << Conflict.new(
+            type: :prefix,
+            token1: token1.name,
+            token2: token2.name,
+            description: "'#{lit1}' is a prefix of '#{lit2}' (longest match: #{token2.name} preferred for '#{lit2}')"
+          )
+        end
+      end
+
+      def check_literal_regex_pair(literal_token, regex_token)
+        literal = literal_token.pattern
+        regex = regex_token.pattern
+
+        # Check if the literal matches the regex
+        return unless regex.match?(literal)
+
+        @conflicts << Conflict.new(
+          type: :literal_matches_regex,
+          token1: literal_token.name,
+          token2: regex_token.name,
+          description: "'#{literal}' matches regex /#{regex.source}/ " \
+                       "(definition order determines winner: #{literal_token.name} if defined first)"
+        )
+      end
+
+      def check_regex_pair(token1, token2)
+        # For regex pairs, we need to check if they can match the same string
+        # We do this by building a combined DFA and checking for states
+        # that accept both patterns
+
+        pattern1 = token1.pattern
+        pattern2 = token2.pattern
+
+        # Check first byte overlap
+        regex1 = DFA::RegexAST::Regex.parse(pattern1)
+        regex2 = DFA::RegexAST::Regex.parse(pattern2)
+        first_bytes1 = DFA.first_byte_set(regex1)
+        first_bytes2 = DFA.first_byte_set(regex2)
+        common_bytes = first_bytes1 & first_bytes2
+
+        return if common_bytes.empty?
+
+        # Build combined DFA and check for multi-accept states
+        begin
+          overlapping_examples = find_overlapping_strings(pattern1, pattern2)
+
+          if overlapping_examples.any?
+            examples_str = overlapping_examples.first(3).map(&:inspect).join(", ")
+            @conflicts << Conflict.new(
+              type: :regex_overlap,
+              token1: token1.name,
+              token2: token2.name,
+              description: "both patterns can match: #{examples_str} " \
+                           "(definition order determines winner)"
+            )
+          end
+        rescue StandardError
+          # If DFA construction fails, just report potential overlap based on first bytes
+          if common_bytes.size <= 10
+            bytes_str = common_bytes.to_a.map { |b| b.chr.inspect }.join(", ")
+            @conflicts << Conflict.new(
+              type: :first_byte_overlap,
+              token1: token1.name,
+              token2: token2.name,
+              description: "patterns share first bytes: #{bytes_str} (potential conflict)"
+            )
+          else
+            @conflicts << Conflict.new(
+              type: :first_byte_overlap,
+              token1: token1.name,
+              token2: token2.name,
+              description: "patterns share #{common_bytes.size} first bytes (potential conflict)"
+            )
+          end
+        end
+      end
+
+      # Find example strings that both patterns can match
+      def find_overlapping_strings(pattern1, pattern2)
+        examples = []
+
+        # Build DFA for each pattern
+        regex1 = DFA::RegexAST::Regex.parse(pattern1)
+        regex2 = DFA::RegexAST::Regex.parse(pattern2)
+        dfa1 = DFA.compile_regex(regex1, 1)
+        dfa2 = DFA.compile_regex(regex2, 2)
+
+        # BFS to find accepting strings
+        # State: [dfa1_state, dfa2_state, accumulated_string]
+        queue = [[1, 1, ""]]
+        visited = Set.new
+        max_depth = 10
+
+        while queue.any? && examples.size < 5
+          dfa1_state, dfa2_state, str = queue.shift
+
+          next if str.length > max_depth
+          next if visited.include?([dfa1_state, dfa2_state])
+
+          visited << [dfa1_state, dfa2_state]
+
+          # Check if both DFAs accept at this point
+          if dfa1.accept(dfa1_state) && dfa2.accept(dfa2_state) && !str.empty?
+            examples << str
+            next
+          end
+
+          # Explore transitions
+          (0..127).each do |byte|
+            next_state1 = dfa1.transition(dfa1_state, byte)
+            next_state2 = dfa2.transition(dfa2_state, byte)
+
+            # Both must have valid transitions
+            next if next_state1.zero? || next_state2.zero?
+
+            char = begin
+              byte.chr
+            rescue StandardError
+              next
+            end
+            next unless char.valid_encoding?
+
+            queue << [next_state1, next_state2, str + char]
+          end
+        end
+
+        examples
+      end
+    end
+  end
+end

data/lib/lexer_kit/builder/mode_def.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module LexerKit
+  class Builder
+    # ModeDef represents a lexer mode definition.
+    class ModeDef
+      attr_reader :name, :tokens, :location
+      attr_accessor :delimited
+
+      def initialize(name, location: nil)
+        @name = name
+        @tokens = []
+        @delimited = nil
+        @location = location
+      end
+
+      def add_token(token_def)
+        @tokens << token_def
+      end
+
+      # Get all literal tokens (for SWITCH_BYTE optimization)
+      def literal_tokens
+        @tokens.select(&:literal?)
+      end
+
+      # Get all regex tokens
+      def regex_tokens
+        @tokens.select(&:regex?)
+      end
+
+      def inspect
+        "#<ModeDef :#{@name} tokens=#{@tokens.size}>"
+      end
+    end
+  end
+end

data/lib/lexer_kit/builder/token_def.rb

@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module LexerKit
+  class Builder
+    # TokenDef represents a token definition from the DSL.
+    class TokenDef
+      attr_reader :name, :pattern, :skip, :push, :pop, :location, :meta, :delimited, :delimiter, :escape
+      attr_accessor :inner_mode, :token_id
+
+      def initialize(
+        name:,
+        pattern:,
+        skip: false,
+        push: nil,
+        pop: false,
+        delimited: false,
+        delimiter: nil,
+        escape: nil,
+        meta: nil,
+        location: nil
+      )
+        @name = name
+        @pattern = pattern
+        @skip = skip
+        @push = push
+        @pop = pop
+        @delimited = delimited
+        @delimiter = delimiter
+        @escape = escape
+        @meta = meta
+        @inner_mode = nil
+        @token_id = nil
+        @location = location
+      end
+
+      # Check if pattern is a literal string
+      def literal?
+        @pattern.is_a?(String)
+      end
+
+      # Check if pattern is a regex
+      def regex?
+        @pattern.is_a?(Regexp) || @pattern.is_a?(LexerKit::RegexAstProvider)
+      end
+
+      # Check if this is a delimited token (like TEXT in templates)
+      def delimited?
+        @delimited
+      end
+
+      def inspect
+        parts = ["#<TokenDef :#{@name}"]
+        parts << @pattern.inspect if @pattern
+        parts << "skip" if @skip
+        parts << "push=#{@push}" if @push
+        parts << "pop" if @pop
+        parts << "delimited=#{@delimiter.inspect}" if @delimited
+        parts << "escape=#{@escape.inspect}" if @escape
+        parts << "meta=#{@meta.inspect}" if @meta
+        parts << ">"
+        parts.join(" ")
+      end
+    end
+  end
+end

data/lib/lexer_kit/builder/validator.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+module LexerKit
+  class Builder
+    # Validator checks builder definitions for errors before compilation.
+    class Validator
+      def initialize(builder)
+        @builder = builder
+      end
+
+      # Validate all definitions
+      # @raise [LexerKit::BuildError] if validation fails
+      def validate!
+        validate_mode_references!
+        validate_reserved_token_names!
+      end
+
+      private
+
+      def validate_reserved_token_names!
+        @builder.token_defs.each do |token_def|
+          next unless token_def.name == :INVALID
+
+          message = ":INVALID is reserved for error tokens"
+          raise LexerKit::BuildError.from_location(token_def.location, message)
+        end
+      end
+
+      def validate_mode_references!
+        defined_modes = @builder.mode_defs.keys.to_set
+
+        @builder.token_defs.each do |token_def|
+          next unless token_def.push
+          next if defined_modes.include?(token_def.push)
+
+          raise_undefined_mode_error(token_def, defined_modes)
+        end
+      end
+
+      def raise_undefined_mode_error(token_def, defined_modes)
+        mode_name = token_def.push
+        suggestion = find_similar_mode(mode_name, defined_modes)
+
+        message = "undefined mode :#{mode_name}"
+        notes = suggestion ? ["did you mean :#{suggestion}?"] : nil
+
+        raise LexerKit::BuildError.from_location(token_def.location, message, notes: notes)
+      end
+
+      def find_similar_mode(target, candidates)
+        target_s = target.to_s
+        threshold = [target_s.length / 2, 2].max
+
+        candidates
+          .map { |c| [c, levenshtein(target_s, c.to_s)] }
+          .select { |_, dist| dist <= threshold }
+          .min_by { |_, dist| dist }
+          &.first
+      end
+
+      def levenshtein(str1, str2)
+        return str2.length if str1.empty?
+        return str1.length if str2.empty?
+
+        matrix = Array.new(str1.length + 1) { Array.new(str2.length + 1, 0) }
+        (0..str1.length).each { |i| matrix[i][0] = i }
+        (0..str2.length).each { |j| matrix[0][j] = j }
+
+        (1..str1.length).each do |i|
+          (1..str2.length).each do |j|
+            cost = str1[i - 1] == str2[j - 1] ? 0 : 1
+            matrix[i][j] = [
+              matrix[i - 1][j] + 1,
+              matrix[i][j - 1] + 1,
+              matrix[i - 1][j - 1] + cost
+            ].min
+          end
+        end
+
+        matrix[str1.length][str2.length]
+      end
+    end
+  end
+end

data/lib/lexer_kit/builder.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+require_relative "builder/token_def"
+require_relative "builder/mode_def"
+require_relative "builder/compiler"
+require_relative "builder/conflict_detector"
+require_relative "builder/validator"
+
+module LexerKit
+  # Builder provides the DSL for defining lexers.
+  class Builder
+    # DSL methods for defining lexers.
+    # These methods are available within the `LexerKit.build` block.
+    module DSL
+      # Define a token
+      # @param name [Symbol] token name
+      # @param pattern [String, Regexp] pattern to match
+      # @param skip [Boolean] skip this token (don't emit)
+      # @param push [Symbol] push mode after match
+      # @param pop [Boolean] pop mode after match
+      # @param meta [Hash] optional metadata for this token
+      def token(name, pattern, skip: false, push: nil, pop: false, meta: nil)
+        location = caller_locations(1, 1).first
+        token_def = TokenDef.new(
+          name: name,
+          pattern: pattern,
+          skip: skip,
+          push: push,
+          pop: pop,
+          meta: meta,
+          location: location
+        )
+        current_mode_def.add_token(token_def)
+        @token_defs << token_def
+      end
+
+      # Define a keyword (matched after identifier, checked by lookup)
+      # @param name [Symbol] keyword token name
+      # @param value [String] keyword string
+      def keyword(name, value)
+        @keywords[value.freeze] = name
+      end
+
+      # Define multiple keywords at once
+      # @param names [Array<Symbol>] keyword values (upcased for token names)
+      # @example
+      #   define_keywords :if, :else, :while
+      #   # equivalent to:
+      #   # keyword :IF, "if"
+      #   # keyword :ELSE, "else"
+      #   # keyword :WHILE, "while"
+      def define_keywords(*names)
+        names.each do |name|
+          token_name = name.to_s.upcase.to_sym
+          value = name.to_s.downcase
+          keyword(token_name, value)
+        end
+      end
+
+      # Define a mode
+      # @param name [Symbol] mode name
+      # @yield mode definition block
+      def mode(name, &block)
+        location = caller_locations(1, 1).first
+        mode_def = ModeDef.new(name, location: location)
+        @mode_defs[name] = mode_def
+
+        old_mode = @current_mode
+        @current_mode = name
+        instance_eval(&block) if block
+        @current_mode = old_mode
+      end
+
+      # Define a delimited section (for templates)
+      # Scans text until the delimiter is found, then switches to inner mode.
+      # The closing delimiter should be handled by a token in the inner mode with `pop: true`.
+      #
+      # @param name [Symbol] text token name
+      # @param delimiter [String] delimiter that marks the end of text
+      # @param escape [String, nil] escape sequence that prevents delimiter match
+      #
+      # @example
+      #   delimited :TEXT, delimiter: "{["
+      #   mode :tag do
+      #     token :CLOSE, "]}", pop: true  # closing delimiter handled here
+      #   end
+      #
+      # @example with escape
+      #   delimited :TEXT, delimiter: "{[", escape: "{[{]}" do
+      #     # {[{]} is treated as escaped delimiter and does not close TEXT
+      #   end
+      def delimited(name, delimiter: nil, escape: nil, pop: false, skip: false, &block)
+        unless delimiter
+          location = caller_locations(1, 1).first
+          message = "delimited requires `delimiter:` parameter"
+          raise LexerKit::BuildError.from_location(location, message)
+        end
+        location = caller_locations(1, 1).first
+
+        token_def = add_delimited_token(
+          name: name,
+          delimiter: delimiter,
+          escape: escape,
+          pop: pop,
+          skip: skip,
+          location: location
+        )
+
+        # If block given, define inner mode
+        return unless block
+
+        inner_mode = :"#{name}_inner"
+        mode(inner_mode, &block)
+        token_def.inner_mode = inner_mode
+      end
+
+      # Define a scan-until section with open/close delimiters.
+      #
+      # @param name [Symbol] token name for the scanned content
+      # @param open [String] opening delimiter
+      # @param close [String] closing delimiter
+      # @param escape [String, nil] escape sequence that prevents close match
+      # @param skip [Boolean] skip emitting the content token
+      def scan_until(name, open: nil, close: nil, escape: nil, skip: false)
+        location = caller_locations(1, 1).first
+        unless open
+          message = "scan_until requires `open:` parameter"
+          raise LexerKit::BuildError.from_location(location, message)
+        end
+        unless close
+          message = "scan_until requires `close:` parameter"
+          raise LexerKit::BuildError.from_location(location, message)
+        end
+
+        mode_name = next_internal_mode_name(name)
+        token :"#{name}_OPEN_#{@internal_mode_counter}", open, skip: true, push: mode_name
+
+        mode(mode_name) do
+          delimited(name, delimiter: close, escape: escape, pop: true, skip: skip)
+        end
+      end
+
+      # Get or set the version
+      # @param v [Integer, nil] version to set, or nil to get current
+      # @return [Integer] current version
+      def version(v = nil)
+        if v.nil?
+          @version
+        else
+          @version = v
+        end
+      end
+    end
+
+    include DSL
+
+    attr_reader :token_defs, :mode_defs, :keywords
+
+    # Error token is always :INVALID
+    # @return [Symbol] :INVALID
+    def error_token
+      :INVALID
+    end
+
+    def initialize
+      @token_defs = []
+      @mode_defs = { default: ModeDef.new(:default, location: nil) }
+      @current_mode = :default
+      @keywords = {}
+      @version = 1
+      @internal_mode_counter = 0
+    end
+
+    # Compile to IR
+    # @return [IR::CompiledProgram]
+    def compile
+      Validator.new(self).validate!
+      program = Compiler.new(self).compile
+      program.load_native! if LexerKit.native?
+      program
+    end
+
+    # Check for pattern conflicts
+    # @return [Array<ConflictDetector::Conflict>]
+    def check_conflicts
+      ConflictDetector.new(self).detect
+    end
+
+    # Check for pattern conflicts and print warnings
+    # @param io [IO] output stream (default: $stderr)
+    # @return [Array<ConflictDetector::Conflict>]
+    def check_conflicts!(io: $stderr)
+      conflicts = check_conflicts
+      conflicts.each do |conflict|
+        io.puts "warning: #{conflict.token1} vs #{conflict.token2}: #{conflict.description}"
+      end
+      conflicts
+    end
+
+    private
+
+    def current_mode_def
+      @mode_defs[@current_mode]
+    end
+
+    # Generate unique internal mode name for scan_until etc.
+    def next_internal_mode_name(base_name)
+      @internal_mode_counter += 1
+      :"__#{base_name}_#{@internal_mode_counter}"
+    end
+
+    # Internal helper for creating delimited token definitions.
+    def add_delimited_token(name:, delimiter:, location:, escape: nil, pop: false, skip: false)
+      token_def = TokenDef.new(
+        name: name,
+        pattern: nil,
+        delimited: true,
+        delimiter: delimiter,
+        escape: escape,
+        pop: pop,
+        skip: skip,
+        location: location
+      )
+      current_mode_def.add_token(token_def)
+      current_mode_def.delimited = token_def
+      @token_defs << token_def
+      token_def
+    end
+  end
+end