synthra 0.1.0

data/lib/synthra/parser/tokens.rb

@@ -0,0 +1,461 @@
+# frozen_string_literal: true
+
+# =============================================================================
+
+# Synthra Token Definitions
+# =============================================================================
+#
+# This file defines all token types recognized by the DSL lexer and the
+# Token class used to represent individual tokens.
+#
+# The lexer converts DSL source text into a stream of Token objects,
+# which the parser then uses to build an AST.
+#
+# @example Token stream for "User:"
+#   Token(:schema_name, "User", line: 1, column: 1)
+#   Token(:colon, ":", line: 1, column: 5)
+#
+# @example Token stream for "name: text"
+#   Token(:field_name, "name", line: 2, column: 3)
+#   Token(:colon, ":", line: 2, column: 7)
+#   Token(:type, "text", line: 2, column: 9)
+#
+# =============================================================================
+
+
+module Synthra
+  module Parser
+
+    # Token type constants for the DSL lexer
+    #
+    # This module defines all the token types that the lexer can produce.
+    # Each token type is a symbol that identifies the kind of syntax element.
+    #
+    # Token types are organized into categories:
+    # - **Structural**: Schema and field names, colons, question marks
+    # - **Types**: Type names, Ref keyword
+    # - **Delimiters**: Parentheses, commas, dots
+    # - **Values**: Numbers, strings, identifiers, ranges, percentages
+    # - **Keywords**: if, true, false
+    # - **Behaviors**: @ prefixed behavior declarations
+    # - **Whitespace/Structure**: Newlines, indentation changes
+    # - **Special**: EOF, symbols
+    #
+    # @example Check if a token is a type
+    #   token.type == Tokens::TYPE
+    #
+    # @example Check if token type is valid
+    #   Tokens::ALL.include?(:type)  # => true
+    #
+
+    module Tokens
+      # ==========================================================================
+
+      # Structural Tokens
+      # ==========================================================================
+      # These tokens define the structure of schemas and fields
+
+      # Schema name at the top level (e.g., "User:")
+      # Value: String containing the schema name without the colon
+      # @example "User:" => Token(:schema_name, "User")
+
+      SCHEMA_NAME    = :schema_name
+
+      # Field name within a schema (e.g., "name:", "email?:")
+      # Value: String containing the field name (may include ? for optional)
+      # @example "email?:" => Token(:field_name, "email?")
+
+      FIELD_NAME     = :field_name
+
+      # Colon separator
+      # Value: ":" literal
+      # @example ":" => Token(:colon, ":")
+
+      COLON          = :colon
+
+      # Question mark for optional/nullable types
+      # Value: "?" literal
+      # @example "?" => Token(:question, "?")
+
+      QUESTION       = :question
+
+      # ==========================================================================
+
+      # Type Tokens
+      # ==========================================================================
+      # These tokens represent type declarations
+
+      # Built-in or custom type name (uuid, text, number, etc.)
+      # Value: String containing the type name
+      # @example "uuid" => Token(:type, "uuid")
+
+      TYPE           = :type
+
+      # Reference keyword for cross-schema references
+      # Value: "Ref" literal
+      # @example "Ref" => Token(:ref, "Ref")
+
+      REF            = :ref
+
+      # ==========================================================================
+
+      # Delimiter Tokens
+      # ==========================================================================
+      # These tokens separate or group elements
+
+      # Left parenthesis for type arguments
+      # Value: "(" literal
+      # @example "(" => Token(:lparen, "(")
+
+      LPAREN         = :lparen
+
+      # Right parenthesis for type arguments
+      # Value: ")" literal
+      # @example ")" => Token(:rparen, ")")
+
+      RPAREN         = :rparen
+
+      # Comma separator in argument lists
+      # Value: "," literal
+      # @example "," => Token(:comma, ",")
+
+      COMMA          = :comma
+
+      # Dot separator in references (Schema.field)
+      # Value: "." literal
+      # @example "." => Token(:dot, ".")
+
+      DOT            = :dot
+
+      # Minus sign for negative numbers
+      # Value: "-" literal
+      # @example "-" => Token(:minus, "-")
+
+      MINUS          = :minus
+
+      # Arithmetic operators (used inside formula(...) expressions)
+      # @example "*" => Token(:star, "*")
+      STAR           = :star
+
+      # @example "/" => Token(:slash, "/")
+      SLASH          = :slash
+
+      # @example "+" => Token(:plus, "+")
+      PLUS           = :plus
+
+      # ==========================================================================
+
+      # Value Tokens
+      # ==========================================================================
+      # These tokens represent literal values
+
+      # Numeric literal (integer or float)
+      # Value: Integer or Float
+      # @example "123" => Token(:number, 123)
+      # @example "3.14" => Token(:number, 3.14)
+
+      NUMBER         = :number
+
+      # String literal (single or double quoted)
+      # Value: String (quotes stripped)
+      # @example '"hello"' => Token(:string, "hello")
+
+      STRING         = :string
+
+      # Identifier (unquoted name, used in enums, etc.)
+      # Value: String
+      # @example "pending" => Token(:identifier, "pending")
+
+      IDENTIFIER     = :identifier
+
+      # Range literal (min..max)
+      # Value: Range object
+      # @example "18..80" => Token(:range, 18..80)
+
+      RANGE          = :range
+
+      # Percentage literal
+      # Value: Integer (percent without %)
+      # @example "50%" => Token(:percent, 50)
+
+      PERCENT        = :percent
+
+      # ==========================================================================
+
+      # Keyword Tokens
+      # ==========================================================================
+      # These tokens represent reserved keywords
+
+      # Conditional keyword for field conditions
+      # Value: "if" literal
+      # @example "if" => Token(:if, "if")
+
+      IF             = :if
+
+      # Boolean true keyword
+      # Value: true
+      # @example "true" => Token(:true, true)
+
+      TRUE_KW        = :true
+
+      # Boolean false keyword
+      # Value: false
+      # @example "false" => Token(:false, false)
+
+      FALSE_KW       = :false
+
+      # ==========================================================================
+
+      # Simulyra DSL Keywords
+      # ==========================================================================
+      # These tokens are specific to the Simulyra layered DSL structure
+
+      # Schema definition keyword
+      # Value: "schema" literal
+      # @example "schema User:" => Token(:schema_keyword, "schema")
+
+      SCHEMA_KEYWORD = :schema_keyword
+
+      # API definition keyword
+      # Value: "api" literal
+      # @example "api GET /users:" => Token(:api_keyword, "api")
+
+      API_KEYWORD    = :api_keyword
+
+      # Scenario definition keyword
+      # Value: "scenario" literal
+      # @example "scenario Success:" => Token(:scenario_keyword, "scenario")
+
+      SCENARIO_KEYWORD = :scenario_keyword
+
+      # HTTP method token
+      # Value: Symbol (:get, :post, :put, :patch, :delete, :options, :head)
+      # @example "GET" => Token(:http_method, :get)
+
+      HTTP_METHOD    = :http_method
+
+      # URL path token
+      # Value: String containing the path
+      # @example "/users/:id" => Token(:path, "/users/:id")
+
+      PATH           = :path
+
+      # Schema reference token for explicit schema(:Name) syntax
+      # Value: String containing the schema name
+      # @example "schema(:User)" => Token(:schema_ref, "User")
+
+      SCHEMA_REF     = :schema_ref
+
+      # ==========================================================================
+
+      # Behavior Tokens
+      # ==========================================================================
+      # These tokens represent behavior declarations
+
+      # Behavior annotation (@latency, @failure, etc.)
+      # Value: String containing behavior name without @
+      # @example "@latency" => Token(:at_behavior, "latency")
+
+      AT_BEHAVIOR    = :at_behavior
+
+      # ==========================================================================
+
+      # Whitespace/Structure Tokens
+      # ==========================================================================
+      # These tokens represent structural elements based on whitespace
+
+      # Newline character
+      # Value: "\n" literal
+      # @example "\n" => Token(:newline, "\n")
+
+      NEWLINE        = :newline
+
+      # Increase in indentation level
+      # Value: Integer (new indentation level)
+      # @example Indent from 0 to 2 => Token(:indent, 2)
+
+      INDENT         = :indent
+
+      # Decrease in indentation level
+      # Value: Integer (new indentation level)
+      # @example Dedent from 4 to 2 => Token(:dedent, 2)
+
+      DEDENT         = :dedent
+
+      # ==========================================================================
+
+      # Special Tokens
+      # ==========================================================================
+      # These tokens have special meaning
+
+      # End of file marker
+      # Value: nil
+      # @example End of input => Token(:eof, nil)
+
+      EOF            = :eof
+
+      # Ruby symbol literal (:function_name)
+      # Value: Symbol
+      # @example ":compute" => Token(:symbol, :compute)
+
+      SYMBOL         = :symbol
+
+      # ==========================================================================
+
+      # Token Type Collection
+      # ==========================================================================
+
+      # Array of all valid token types
+      #
+      # @return [Array<Symbol>] all token type symbols
+      #
+      # @example Validate token type
+      #   Tokens::ALL.include?(token.type)  # => true/false
+      #
+
+      ALL = [
+        SCHEMA_NAME, FIELD_NAME, COLON, QUESTION,
+        TYPE, REF,
+        LPAREN, RPAREN, COMMA, DOT, MINUS,
+        NUMBER, STRING, IDENTIFIER, RANGE, PERCENT,
+        IF, TRUE_KW, FALSE_KW,
+        SCHEMA_KEYWORD, API_KEYWORD, SCENARIO_KEYWORD, HTTP_METHOD, PATH, SCHEMA_REF,
+        AT_BEHAVIOR,
+        NEWLINE, INDENT, DEDENT,
+        EOF, SYMBOL
+      ].freeze
+    end
+
+    # Represents a single token from the lexer
+    #
+    # A Token captures the type, value, and source location of a syntax
+    # element in the DSL. Tokens are produced by the Lexer and consumed
+    # by the Parser.
+    #
+    # @example Create a token
+    #   token = Token.new(:type, "uuid", line: 5, column: 10)
+    #   token.type    # => :type
+    #   token.value   # => "uuid"
+    #   token.line    # => 5
+    #   token.column  # => 10
+    #
+    # @example Check token type
+    #   token.type?(:type)  # => true
+    #   token.type?(:number)  # => false
+    #
+
+    class Token
+
+      # @!attribute [r] type
+      #   The token type (one of Tokens::ALL)
+      #   @return [Symbol] the token type
+
+      attr_reader :type
+
+      # @!attribute [r] value
+      #   The token value (varies by type)
+      #   @return [Object] the token value
+
+      attr_reader :value
+
+      # @!attribute [r] line
+      #   Source line number (1-based)
+      #   @return [Integer] line number
+
+      attr_reader :line
+
+      # @!attribute [r] column
+      #   Source column number (1-based)
+      #   @return [Integer] column number
+
+      attr_reader :column
+
+      # Create a new Token
+      #
+      # @param type [Symbol] the token type (should be in Tokens::ALL)
+      # @param value [Object] the token value
+      # @param line [Integer] line number where token appears (1-based)
+      # @param column [Integer] column where token starts (1-based)
+      #
+      # @example
+      #   Token.new(:number, 42, line: 3, column: 15)
+      #
+
+      def initialize(type, value, line:, column:)
+        @type = type
+        @value = value
+        @line = line
+        @column = column
+      end
+
+      # String representation for debugging
+      #
+      # @return [String] human-readable token description
+      #
+      # @example
+      #   token.to_s  # => "Token(type, \"uuid\", line: 5, col: 10)"
+      #
+
+      def to_s
+        "Token(#{type}, #{value.inspect}, line: #{line}, col: #{column})"
+      end
+
+      # Inspect output (same as to_s)
+      #
+      # @return [String] inspect representation
+      #
+
+      def inspect
+        to_s
+      end
+
+      # Check equality with another token
+      #
+      # Two tokens are equal if they have the same type and value.
+      # Line and column are not considered for equality.
+      #
+      # @param other [Token] token to compare
+      # @return [Boolean] true if type and value match
+      #
+      # @example
+      #   token1 = Token.new(:type, "uuid", line: 1, column: 1)
+      #   token2 = Token.new(:type, "uuid", line: 2, column: 5)
+      #   token1 == token2  # => true (same type and value)
+      #
+
+      def ==(other)
+        return false unless other.is_a?(Token)
+
+        type == other.type && value == other.value
+      end
+
+      # Alias for == to support Hash key comparison
+      alias eql? ==
+
+      # Hash code for use in Hash keys and Sets
+      #
+      # Based on type and value only (not line/column).
+      #
+      # @return [Integer] hash code
+      #
+
+      def hash
+        [type, value].hash
+      end
+
+      # Check if token is of a specific type
+      #
+      # @param expected_type [Symbol] type to check against
+      # @return [Boolean] true if token type matches
+      #
+      # @example
+      #   token.type?(:type)    # => true
+      #   token.type?(:number)  # => false
+      #
+
+      def type?(expected_type)
+        type == expected_type
+      end
+    end
+  end
+end