synthra 0.1.0

data/lib/synthra/parser/lexer.rb

@@ -0,0 +1,1152 @@
+# frozen_string_literal: true
+
+# =============================================================================
+
+# Synthra Lexer
+# =============================================================================
+#
+# The Lexer (also called Scanner or Tokenizer) is the first stage of DSL
+# parsing. It converts raw source text into a stream of Token objects.
+#
+# The lexer handles:
+# - Indentation tracking (DSL uses indentation-based structure)
+# - Schema and field name recognition
+# - Type name recognition
+# - Literal values (strings, numbers, ranges)
+# - Behavior annotations (@latency, @failure, etc.)
+# - Keywords (if, true, false, Ref)
+# - Comments (# ...)
+#
+# @example Basic usage
+#   lexer = Lexer.new("User:\n  name: text")
+#   tokens = lexer.tokenize
+#   # => [Token(:schema_name, "User"), Token(:colon, ":"), ...]
+#
+# @example Token stream for a simple schema
+#   # Input:
+#   # User:
+#   #   name: text
+#   #   age: number(18..80)
+#   
+#   # Output tokens:
+#   # SCHEMA_NAME("User"), COLON, NEWLINE,
+#   # INDENT(2), FIELD_NAME("name"), COLON, TYPE("text"), NEWLINE,
+#   # FIELD_NAME("age"), COLON, TYPE("number"), LPAREN, RANGE(18..80), RPAREN, NEWLINE,
+#   # DEDENT(0), EOF
+#
+# =============================================================================
+
+
+module Synthra
+  module Parser
+
+    # Lexer for the Synthra language
+    #
+    # Converts DSL source text into a stream of tokens that the parser
+    # can process. The lexer is indentation-aware and tracks nesting
+    # depth to emit INDENT and DEDENT tokens.
+    #
+    # @example Tokenize DSL source
+    #   lexer = Lexer.new(source)
+    #   tokens = lexer.tokenize
+    #
+    # @example Handle syntax errors
+    #   begin
+    #     tokens = Lexer.new(source).tokenize
+    #   rescue Synthra::SyntaxError => e
+    #     puts "Error at line #{e.line}, column #{e.column}: #{e.message}"
+    #   end
+    #
+
+    class Lexer
+
+      # Include token type constants for convenience
+      include Tokens
+
+      # Reserved keywords and their token types
+      #
+      # These words have special meaning in the DSL and cannot be used
+      # as identifiers.
+      #
+      # @return [Hash<String, Symbol>] keyword => token type mapping
+      #
+
+      KEYWORDS = {
+        "if" => IF,           # Conditional field syntax
+        "true" => TRUE_KW,    # Boolean true value
+        "false" => FALSE_KW,  # Boolean false value
+        "Ref" => REF,         # Cross-schema reference
+        "ref" => REF          # Cross-schema reference (lowercase alias)
+      }.freeze
+
+      # Simulyra DSL keywords for layered architecture
+      #
+      # These keywords define the three layers of Simulyra DSL:
+      # - schema: Data structure definitions
+      # - api: HTTP endpoint definitions
+      # - scenario: Response scenario definitions
+      #
+      # @return [Hash<String, Symbol>] keyword => token type mapping
+      #
+
+      SIMULYRA_KEYWORDS = {
+        "schema" => SCHEMA_KEYWORD,
+        "api" => API_KEYWORD,
+        "scenario" => SCENARIO_KEYWORD
+      }.freeze
+
+      # HTTP methods recognized by the lexer
+      #
+      # These are the standard HTTP methods that can be used
+      # in API definitions.
+      #
+      # @return [Array<String>] list of valid HTTP method names
+      #
+
+      HTTP_METHODS = %w[GET POST PUT PATCH DELETE OPTIONS HEAD].freeze
+
+
+      # Known built-in type names
+      #
+      # These are recognized as TYPE tokens when not inside parentheses.
+      # Custom types starting with uppercase are also recognized.
+      #
+      # @return [Array<String>] list of built-in type names
+      #
+
+      KNOWN_TYPES = %w[
+        uuid ulid id_sequence
+        name email text
+        number boolean enum
+        date past_date future_date now timestamp
+        array object custom
+        phone url ip ipv6
+        country_code currency postal_code money
+      ].freeze
+
+      # Create a new Lexer
+      #
+      # @param source [String] the DSL source code to tokenize
+      #
+      # @example
+      #   lexer = Lexer.new("User:\n  name: text")
+      #
+
+      def initialize(source)
+        @source = source                    # Full source text
+        @lines = source.lines               # Source split into lines (for errors)
+        @tokens = []                        # Accumulated tokens
+        @line = 1                           # Current line number (1-based)
+        @column = 1                         # Current column number (1-based)
+        @pos = 0                            # Current position in source
+        @indent_stack = [0]                 # Stack of indentation levels
+        @at_line_start = true               # Are we at the start of a line?
+        @paren_depth = 0                    # Parenthesis nesting depth
+      end
+
+      # Tokenize the entire source
+      #
+      # Processes the source text and returns an array of Token objects.
+      # The token stream will always end with an EOF token.
+      #
+      # @return [Array<Token>] array of tokens representing the source
+      # @raise [SyntaxError] if the source contains invalid syntax
+      #
+      # @example
+      #   tokens = lexer.tokenize
+      #   tokens.last.type  # => :eof
+      #
+
+      def tokenize
+
+        # Process source character by character
+        while @pos < @source.length
+          tokenize_line
+        end
+
+        # Emit DEDENT tokens for any remaining indentation
+        emit_remaining_dedents
+
+        # Always end with EOF token
+        emit(EOF, nil)
+
+        @tokens
+      end
+
+      private
+
+      # ==========================================================================
+
+      # Character Access Methods
+      # ==========================================================================
+
+      # Get the current character
+      #
+      # @return [String, nil] current character or nil if at end
+      #
+
+      def current_char
+        @source[@pos]
+      end
+
+      # Peek at a character ahead of current position
+      #
+      # @param offset [Integer] how far ahead to look (default: 1)
+      # @return [String, nil] character at offset or nil
+      #
+
+      def peek_char(offset = 1)
+        @source[@pos + offset]
+      end
+
+      # Advance to the next character
+      #
+      # Updates line/column tracking when crossing newlines.
+      #
+      # @return [void]
+      #
+
+      def advance
+        if current_char == "\n"
+          @line += 1
+          @column = 1
+          @at_line_start = true
+        else
+          @column += 1
+        end
+        @pos += 1
+      end
+
+      # ==========================================================================
+
+      # Token Emission
+      # ==========================================================================
+
+      # Emit a token with current position
+      #
+      # @param type [Symbol] token type (from Tokens module)
+      # @param value [Object] token value
+      # @return [void]
+      #
+
+      def emit(type, value)
+        @tokens << Token.new(type, value, line: @line, column: @column)
+      end
+
+      # ==========================================================================
+
+      # Error Handling
+      # ==========================================================================
+
+      # Get the content of the current source line
+      #
+      # Used for error messages to show context.
+      #
+      # @return [String] current line content (without trailing newline)
+      #
+
+      def current_line_content
+        # :nocov: errors only fire while @pos is within the source (the tokenize loop guards
+        # `@pos < @source.length`), so @line always indexes a real line — the `&.` nil-receiver
+        # else-arm (line out of range) is unreachable.
+        @lines[@line - 1]&.chomp || ""
+        # :nocov:
+      end
+
+      # Raise a syntax error at current position
+      #
+      # @param message [String] error description
+      # @raise [SyntaxError] always raises
+      #
+
+      def error(message)
+        raise SyntaxError.new(
+          message,
+          line: @line,
+          column: @column,
+          source_line: current_line_content
+        )
+      end
+
+      # ==========================================================================
+
+      # Main Tokenization Logic
+      # ==========================================================================
+
+      # Process tokens from current position
+      #
+      # Handles line-start vs content differently for indentation tracking.
+      #
+      # @return [void]
+      #
+
+      def tokenize_line
+        if @at_line_start
+          handle_line_start
+        else
+          tokenize_content
+        end
+      end
+
+      # Handle the start of a line
+      #
+      # At line start, we need to:
+      # 1. Skip blank lines and comments
+      # 2. Count indentation and emit INDENT/DEDENT
+      # 3. Check for Simulyra DSL keywords (schema, api, scenario)
+      # 4. Check for legacy schema definitions
+      #
+      # @return [void]
+      #
+
+      def handle_line_start
+
+        # Skip blank lines
+        if current_char == "\n"
+          advance
+          return
+        end
+
+        # Skip comment lines
+        if current_char == "#"
+          skip_comment
+          return
+        end
+
+        # Count and process indentation
+        indent = count_indentation
+        @at_line_start = false
+
+        # Check for Simulyra DSL keywords at any indentation level
+        if looking_at_simulyra_keyword?
+          process_indent(indent)
+          tokenize_simulyra_keyword
+          return
+        end
+
+        # Check if this is a legacy schema definition (no indent, identifier followed by :)
+        if indent.zero? && looking_at_schema?
+          process_indent(0)
+          tokenize_schema_name
+          return
+        end
+
+        # Process indentation and then tokenize line content
+        process_indent(indent)
+        tokenize_content
+      end
+
+      # Count indentation at current position
+      #
+      # Counts spaces and tabs (tabs = 2 spaces) at current position.
+      # Advances position past the indentation.
+      # Rejects mixed tabs and spaces to prevent parsing confusion.
+      #
+      # @return [Integer] indentation level in spaces
+      # @raise [SyntaxError] if mixed tabs and spaces are detected
+      #
+
+      def count_indentation
+        indent = 0
+        start_pos = @pos
+        # :nocov: count_indentation runs only inside the `while @pos < @source.length` tokenize
+        # loop, so @pos is always in bounds here — the nil (else) branch is unreachable.
+        start_char = @pos < @source.length ? @source[@pos] : nil
+        # :nocov:
+
+        # Determine indentation type (spaces or tabs)
+        if start_char == " "
+
+          # Only spaces allowed - count spaces
+          while @pos < @source.length && @source[@pos] == " "
+            indent += 1
+            @pos += 1
+            @column += 1
+          end
+
+          # Reject if tabs found after spaces
+          if @pos < @source.length && @source[@pos] == "\t"
+            error("Mixed tabs and spaces in indentation (use either tabs or spaces, not both)")
+          end
+
+        elsif start_char == "\t"
+
+          # Only tabs allowed - count tabs (treat as 2 spaces each)
+          while @pos < @source.length && @source[@pos] == "\t"
+            indent += 2
+            @pos += 1
+            @column += 1
+          end
+
+          # Reject if spaces found after tabs
+          if @pos < @source.length && @source[@pos] == " "
+            error("Mixed tabs and spaces in indentation (use either tabs or spaces, not both)")
+          end
+        end
+
+        indent
+      end
+
+      # Process indentation changes
+      #
+      # Compares current indentation to stack and emits INDENT or DEDENT tokens.
+      #
+      # @param indent [Integer] current indentation level
+      # @return [void]
+      #
+
+      def process_indent(indent)
+        current_indent = @indent_stack.last
+
+        if indent > current_indent
+
+          # Increased indentation - push new level
+          @indent_stack.push(indent)
+          emit(INDENT, indent)
+        elsif indent < current_indent
+
+          # Decreased indentation - pop levels until we match
+          while @indent_stack.last > indent
+            @indent_stack.pop
+            emit(DEDENT, @indent_stack.last)
+          end
+        end
+
+        # Equal indentation - no change needed
+      end
+
+      # Emit DEDENT tokens for remaining indentation at end of file
+      #
+      # @return [void]
+      #
+
+      def emit_remaining_dedents
+        while @indent_stack.length > 1
+          @indent_stack.pop
+          emit(DEDENT, @indent_stack.last)
+        end
+      end
+
+      # ==========================================================================
+
+      # Schema Recognition
+      # ==========================================================================
+
+      # Check if we're looking at a schema definition
+      #
+      # A schema definition is an identifier followed by a colon at the start
+      # of a line with no indentation.
+      #
+      # @return [Boolean] true if this looks like a schema definition
+      #
+
+      def looking_at_schema?
+
+        # Look ahead to see if line ends with : (schema definition)
+        pos = @pos
+        while pos < @source.length && @source[pos] =~ /[A-Za-z0-9_]/
+          pos += 1
+        end
+        @source[pos] == ":"
+      end
+
+      # Tokenize a schema name (SchemaName:)
+      #
+      # @return [void]
+      #
+
+      def tokenize_schema_name
+        name = read_identifier
+        emit(SCHEMA_NAME, name)
+        skip_whitespace
+        expect_char(":")
+        emit(COLON, ":")
+        advance
+        skip_to_end_of_line
+      end
+
+      # ==========================================================================
+
+      # Simulyra DSL Recognition
+      # ==========================================================================
+
+      # Check if we're looking at a Simulyra DSL keyword (schema, api, scenario)
+      #
+      # @return [Boolean] true if this looks like a Simulyra keyword
+      #
+
+      def looking_at_simulyra_keyword?
+        # Look ahead to see the word
+        pos = @pos
+        word = ""
+        while pos < @source.length && @source[pos] =~ /[a-z]/
+          word += @source[pos]
+          pos += 1
+        end
+        SIMULYRA_KEYWORDS.key?(word)
+      end
+
+      # Tokenize a Simulyra DSL keyword and its arguments
+      #
+      # Handles:
+      # - schema Name:
+      # - api METHOD /path:
+      # - scenario Name:
+      #
+      # @return [void]
+      #
+
+      def tokenize_simulyra_keyword
+        keyword = read_identifier
+
+        case keyword
+        when "schema"
+          tokenize_schema_definition
+        when "api"
+          tokenize_api_definition
+        when "scenario"
+          tokenize_scenario_definition
+        end
+      end
+
+      # Tokenize a schema definition (schema Name:)
+      #
+      # @return [void]
+      #
+
+      def tokenize_schema_definition
+        emit(SCHEMA_KEYWORD, "schema")
+        skip_whitespace
+
+        # Read schema name
+        name = read_identifier
+        error("Expected schema name after 'schema'") if name.empty?
+        emit(SCHEMA_NAME, name)
+
+        skip_whitespace
+        expect_char(":")
+        emit(COLON, ":")
+        advance
+        skip_to_end_of_line
+      end
+
+      # Tokenize an API definition (api METHOD /path:)
+      #
+      # @return [void]
+      #
+
+      def tokenize_api_definition
+        emit(API_KEYWORD, "api")
+        skip_whitespace
+
+        # Read HTTP method
+        method = read_identifier.upcase
+        unless HTTP_METHODS.include?(method)
+          error("Expected HTTP method (GET, POST, PUT, PATCH, DELETE, OPTIONS, HEAD), got '#{method}'")
+        end
+        emit(HTTP_METHOD, method.downcase.to_sym)
+
+        skip_whitespace
+
+        # Read path
+        path = read_path
+        error("Expected path starting with '/' after HTTP method") if path.empty?
+        emit(PATH, path)
+
+        skip_whitespace
+        expect_char(":")
+        emit(COLON, ":")
+        advance
+        skip_to_end_of_line
+      end
+
+      # Read a URL path from current position
+      #
+      # Paths can include:
+      # - Forward slashes: /
+      # - Identifiers: users, orders
+      # - Path parameters: :id, :user_id
+      # - Hyphens and underscores: my-api, my_api
+      #
+      # @return [String] the path
+      #
+
+      def read_path
+        return "" unless current_char == "/"
+
+        path = ""
+        while @pos < @source.length
+          char = current_char
+
+          # Path parameters start with :
+          if char == ":"
+            # Look ahead to see if this is a path parameter or the end colon
+            next_char = peek_char
+            if next_char =~ /[A-Za-z_]/
+              # This is a path parameter like :id
+              path += char
+              advance
+            else
+              # This is the terminating colon
+              break
+            end
+          elsif char =~ %r{[A-Za-z0-9_\-/]}
+            path += char
+            advance
+          else
+            break
+          end
+        end
+        path
+      end
+
+      # Tokenize a scenario definition (scenario Name:)
+      #
+      # @return [void]
+      #
+
+      def tokenize_scenario_definition
+        emit(SCENARIO_KEYWORD, "scenario")
+        skip_whitespace
+
+        # Read scenario name
+        name = read_identifier
+        error("Expected scenario name after 'scenario'") if name.empty?
+        emit(IDENTIFIER, name)
+
+        skip_whitespace
+        expect_char(":")
+        emit(COLON, ":")
+        advance
+        skip_to_end_of_line
+      end
+
+      # Check if we're looking at a schema reference: schema(:Name)
+      #
+      # @return [Boolean] true if this looks like a schema reference
+      #
+
+      def looking_at_schema_reference?
+        # Check if we're at "schema(" pattern
+        return false unless current_char == "s"
+
+        pos = @pos
+        word = ""
+
+        # Read potential "schema" word
+        while pos < @source.length && @source[pos] =~ /[a-z]/
+          word += @source[pos]
+          pos += 1
+        end
+
+        return false unless word == "schema"
+
+        # Skip whitespace
+        while pos < @source.length && @source[pos] =~ /[ \t]/
+          pos += 1
+        end
+
+        # Check for opening paren
+        @source[pos] == "("
+      end
+
+      # Tokenize a schema reference: schema(:Name) or schema(:Name).array
+      #
+      # @return [void]
+      #
+
+      def tokenize_schema_reference
+        # Read "schema" keyword (read_identifier will consume it)
+        keyword = read_identifier
+        emit(SCHEMA_KEYWORD, keyword)
+
+        skip_whitespace
+        expect_char("(")
+        emit(LPAREN, "(")
+        advance
+
+        expect_char(":")
+        advance  # Skip the colon in schema(:Name)
+
+        # Read the schema name
+        name = read_identifier
+        error("Expected schema name after 'schema(:'") if name.empty?
+        emit(SCHEMA_REF, name)
+
+        expect_char(")")
+        emit(RPAREN, ")")
+        advance
+
+        # Check for .array or .optional modifier
+        if current_char == "."
+          emit(DOT, ".")
+          advance
+          modifier = read_identifier
+          if modifier == "array"
+            emit(IDENTIFIER, "array")
+            # Check for range like .array(1..10)
+            if current_char == "("
+              emit(LPAREN, "(")
+              @paren_depth += 1
+              advance
+            end
+          elsif modifier == "optional"
+            emit(IDENTIFIER, "optional")
+          else
+            emit(IDENTIFIER, modifier)
+          end
+        end
+      end
+
+      # ==========================================================================
+
+      # Whitespace and Comment Handling
+      # ==========================================================================
+
+      # Skip a comment (from # to end of line)
+      #
+      # @return [void]
+      #
+
+      def skip_comment
+        while @pos < @source.length && current_char != "\n"
+          advance
+        end
+        advance if current_char == "\n"
+      end
+
+      # Skip horizontal whitespace (spaces and tabs)
+      #
+      # @return [void]
+      #
+
+      def skip_whitespace
+        while @pos < @source.length && current_char =~ /[ \t]/
+          advance
+        end
+      end
+
+      # Skip to end of current line
+      #
+      # @return [void]
+      #
+
+      def skip_to_end_of_line
+        while @pos < @source.length && current_char != "\n"
+          advance
+        end
+        advance if current_char == "\n"
+      end
+
+      # ==========================================================================
+
+      # Content Tokenization
+      # ==========================================================================
+
+      # Tokenize content (not at line start)
+      #
+      # Dispatches to specific tokenizers based on current character.
+      # Recursively processes until end of line.
+      #
+      # @return [void]
+      #
+
+      def tokenize_content
+        skip_whitespace
+
+        return if @pos >= @source.length
+
+        case current_char
+        when "\n"
+          emit(NEWLINE, "\n")
+          advance
+        when "#"
+          skip_comment
+        when "@"
+          tokenize_behavior
+        when ":"
+
+          # Check if this is a symbol literal (:ar, :en, etc.) or a colon separator
+          if peek_char =~ /[A-Za-z_]/
+            tokenize_symbol
+          else
+            emit(COLON, ":")
+            advance
+
+            # Process type after colon for field definitions (not inside parens)
+            tokenize_after_colon if @paren_depth.zero?
+          end
+        when "("
+          emit(LPAREN, "(")
+          @paren_depth += 1
+          advance
+        when ")"
+          emit(RPAREN, ")")
+          @paren_depth -= 1
+          advance
+        when ","
+          emit(COMMA, ",")
+          advance
+        when "-"
+
+          # Handle negative numbers (e.g., -10, -10..10)
+          if peek_char =~ /[0-9]/
+            tokenize_number
+          else
+            emit(MINUS, "-")
+            advance
+          end
+
+        when "*"
+          # Arithmetic operator, e.g. inside formula(a * b)
+          emit(STAR, "*")
+          advance
+
+        when "+"
+          # Arithmetic operator, e.g. inside formula(a + b)
+          emit(PLUS, "+")
+          advance
+
+        when "/"
+          # Arithmetic operator, e.g. inside formula(a / b). (Route paths are
+          # tokenized separately via tokenize_behavior, so a bare "/" here was
+          # previously an "Unexpected character" error.)
+          emit(SLASH, "/")
+          advance
+
+        when "."
+          if peek_char =~ /\d/
+            # .123 style decimal
+            tokenize_number
+          else
+            emit(DOT, ".")
+            advance
+          end
+        when "?"
+          emit(QUESTION, "?")
+          advance
+        when '"', "'"
+          tokenize_string
+        when /[0-9]/
+          tokenize_number
+        when /[A-Za-z_]/
+          # Check for schema reference: schema(:Name)
+          if looking_at_schema_reference?
+            tokenize_schema_reference
+          else
+            tokenize_identifier_or_keyword
+          end
+        else
+          error("Unexpected character: '#{current_char}'")
+        end
+
+        # Continue tokenizing until end of line
+        # Important: Check @at_line_start because after advancing past newline,
+        # we've set @at_line_start = true and should return to let handle_line_start
+        # process the next line's indentation
+        tokenize_content unless @pos >= @source.length || current_char == "\n" || @at_line_start
+      end
+
+      # Tokenize content after a colon (field type)
+      #
+      # After a field name and colon, the next identifier is usually a type.
+      #
+      # @return [void]
+      #
+
+      def tokenize_after_colon
+        skip_whitespace
+        return if @pos >= @source.length || current_char == "\n"
+
+        # Handle negative numbers (e.g., number(-10..10))
+        if current_char == "-"
+          tokenize_number
+          return
+        end
+
+        # Check for schema reference: schema(:Name)
+        if looking_at_schema_reference?
+          tokenize_schema_reference
+          return
+        end
+
+        # Read identifier (type name)
+        if current_char =~ /[A-Za-z_]/
+          name = read_identifier
+
+          # Check for ? suffix (nullable type)
+          has_question = current_char == "?"
+          if has_question
+            advance
+          end
+
+          if KEYWORDS.key?(name)
+            emit(KEYWORDS[name], name)
+            emit(QUESTION, "?") if has_question
+          elsif KNOWN_TYPES.include?(name)
+            emit(TYPE, has_question ? "#{name}?" : name)
+          elsif name[0] =~ /[A-Z]/
+
+            # Starts with uppercase - schema reference type
+            emit(TYPE, has_question ? "#{name}?" : name)
+          else
+            emit(IDENTIFIER, name)
+            emit(QUESTION, "?") if has_question
+          end
+        end
+      end
+
+      # ==========================================================================
+
+      # Specific Token Types
+      # ==========================================================================
+
+      # Tokenize a behavior annotation (@name)
+      #
+      # @return [void]
+      #
+
+      def tokenize_behavior
+        advance  # Skip @
+        name = read_identifier
+        emit(AT_BEHAVIOR, name)
+      end
+
+      # Tokenize a string literal ("..." or '...')
+      #
+      # Supports escape sequences: \n, \t, \r, \", \'
+      #
+      # @return [void]
+      # @raise [SyntaxError] if string is not terminated
+      #
+
+      def tokenize_string
+        quote = current_char
+        advance  # Skip opening quote
+
+        value = ""
+        while @pos < @source.length && current_char != quote
+          if current_char == "\\"
+            advance
+            value += escape_char(current_char)
+          else
+            value += current_char
+          end
+          advance
+        end
+
+        error("Unterminated string") if @pos >= @source.length
+        advance  # Skip closing quote
+        emit(STRING, value)
+      end
+
+      # Process an escape sequence character
+      #
+      # @param char [String] character after backslash
+      # @return [String] the escaped character
+      #
+
+      def escape_char(char)
+        case char
+        when "n" then "\n"
+        when "t" then "\t"
+        when "r" then "\r"
+        else char
+        end
+      end
+
+      # Tokenize a number, range, or percentage
+      #
+      # Handles:
+      # - Integers: 123
+      # - Floats: 3.14
+      # - Negative: -10
+      # - Ranges: 18..80
+      # - Percentages: 50%
+      # - Duration suffixes: 100ms, 5s, 2h
+      #
+      # @return [void]
+      #
+
+      def tokenize_number
+        start_col = @column
+        value = ""
+
+        # Handle negative numbers
+        if current_char == "-"
+          value += current_char
+          advance
+        end
+
+        # Integer part
+        while @pos < @source.length && current_char =~ /[0-9]/
+          value += current_char
+          advance
+        end
+
+        # Check for range (..)
+        if current_char == "." && peek_char == "."
+          # :nocov: `value` only holds the integer part here — the decimal part is parsed
+          # after this range check — so value.include?(".") is always false (then-arm dead).
+          left = value.include?(".") ? value.to_f : value.to_i
+          # :nocov:
+          advance  # First .
+          advance  # Second .
+
+          # Parse right side of range (may be negative)
+          right_value = ""
+          right_value += current_char and advance if current_char == "-"
+          while @pos < @source.length && current_char =~ /[0-9.]/
+            right_value += current_char
+            advance
+          end
+
+          # Handle duration suffix (ms, s, m, h, d, y)
+          suffix = ""
+          while @pos < @source.length && current_char =~ /[mshdyMHDY]/
+            suffix += current_char
+            advance
+          end
+
+          right = right_value.include?(".") ? right_value.to_f : right_value.to_i
+          emit(RANGE, { min: left, max: right, suffix: suffix.empty? ? nil : suffix.downcase })
+          return
+        end
+
+        # Decimal part
+        if current_char == "." && peek_char =~ /[0-9]/
+          value += current_char
+          advance
+          while @pos < @source.length && current_char =~ /[0-9]/
+            value += current_char
+            advance
+          end
+        end
+
+        # Check for percent
+        if current_char == "%"
+          emit(NUMBER, value.to_i)
+          emit(PERCENT, "%")
+          advance
+          return
+        end
+
+        # Check for duration suffix
+        suffix = ""
+        while @pos < @source.length && current_char =~ /[mshdyMHDY]/
+          suffix += current_char
+          advance
+        end
+
+        num = value.include?(".") ? value.to_f : value.to_i
+        if suffix.empty?
+          emit(NUMBER, num)
+        else
+          emit(NUMBER, { value: num, suffix: suffix.downcase })
+        end
+      end
+
+      # Tokenize an identifier or keyword
+      #
+      # Determines if the identifier is a:
+      # - Keyword (if, true, false, Ref)
+      # - Known type (uuid, text, etc.)
+      # - Schema reference (starts with uppercase)
+      # - Field name (followed by :)
+      # - Regular identifier
+      #
+      # @return [void]
+      #
+
+      def tokenize_identifier_or_keyword
+        name = read_identifier
+
+        # Check for ? suffix (optional/nullable)
+        has_question = current_char == "?"
+        if has_question
+          advance
+        end
+
+        # Check if followed by : (field definition)
+        # But NOT if we're inside parentheses (named arguments)
+        skip_whitespace
+        if current_char == ":" && @paren_depth.zero?
+          emit(FIELD_NAME, has_question ? "#{name}?" : name)
+          return
+        end
+
+        # Handle keywords (these apply everywhere)
+        if KEYWORDS.key?(name)
+          emit(KEYWORDS[name], name)
+          emit(QUESTION, "?") if has_question
+          return
+        end
+
+        # Inside parentheses, treat as identifiers (for named args/enum values)
+        if @paren_depth > 0
+          emit(IDENTIFIER, name)
+          emit(QUESTION, "?") if has_question
+          return
+        end
+
+        # Handle known types (only at field-definition level)
+        if KNOWN_TYPES.include?(name)
+          emit(TYPE, has_question ? "#{name}?" : name)
+          return
+        end
+
+        # Check if starts with uppercase (schema reference)
+        if name[0] =~ /[A-Z]/
+          emit(TYPE, has_question ? "#{name}?" : name)
+          return
+        end
+
+        # Regular identifier
+        emit(IDENTIFIER, name)
+        emit(QUESTION, "?") if has_question
+      end
+
+      # Tokenize a symbol literal (:identifier)
+      #
+      # Symbols are colon-prefixed identifiers like :ar, :en, :locale
+      #
+      # @return [void]
+      #
+
+      def tokenize_symbol
+        advance  # Skip the colon
+        name = read_identifier
+        emit(SYMBOL, name.to_sym)
+      end
+
+      # Read an identifier from current position
+      #
+      # Identifiers consist of letters, digits, and underscores.
+      #
+      # @return [String] the identifier
+      #
+
+      def read_identifier
+        value = ""
+        while @pos < @source.length && current_char =~ /[A-Za-z0-9_]/
+          value += current_char
+          advance
+        end
+        value
+      end
+
+      # Expect a specific character at current position
+      #
+      # @param expected [String] the expected character
+      # @raise [SyntaxError] if character doesn't match
+      # @return [void]
+      #
+
+      def expect_char(expected)
+        return if current_char == expected
+
+        error("Expected '#{expected}', got '#{current_char}'")
+      end
+    end
+  end
+end