ibex 0.1.0 → 0.2.0

data/sig/ibex/frontend/lexer.rbs

@@ -3,11 +3,24 @@
 module Ibex
   module Frontend
     # Tokenizes a racc-compatible grammar while preserving source locations.
+    # rubocop:disable Metrics/ClassLength -- token scanners stay together to preserve cursor invariants.
     class Lexer
       PUNCTUATION: Hash[String, Symbol]
 
       USER_CODE: Regexp
 
+      NODE_DIRECTIVE: Regexp
+
+      PERCENT_DIRECTIVES: Hash[String, [ Symbol, String ]]
+
+      @active_token_start: SourcePosition?
+
+      @eof_token: Token?
+
+      @emitted_tokens: Array[Token]
+
+      @segments: Array[Segment]
+
       @cursor: SourceCursor
 
       # @rbs (String source, ?file: String) -> void
@@ -16,17 +29,35 @@ module Ibex
       # @rbs () -> Array[Token]
       def tokenize: () -> Array[Token]
 
+      # @rbs () -> SourceDocument
+      def tokenize_document: () -> SourceDocument
+
       # @rbs () -> Token
       def next_token: () -> Token
 
       private
 
+      # @rbs () -> Token
+      def scan_from_cursor: () -> Token
+
       # @rbs (String character) -> Token
       def scan_regular_token: (String character) -> Token
 
       # @rbs () -> void
       def skip_ignored: () -> void
 
+      # @rbs () -> void
+      def scan_whitespace: () -> void
+
+      # @rbs () -> bool
+      def newline?: () -> bool
+
+      # @rbs () -> void
+      def scan_newline: () -> void
+
+      # @rbs () -> void
+      def scan_line_comment: () -> void
+
       # @rbs () -> void
       def skip_block_comment: () -> void
 
@@ -39,6 +70,12 @@ module Ibex
       # @rbs () -> Token
       def scan_scope: () -> Token
 
+      # @rbs () -> Token
+      def scan_percent_directive: () -> Token
+
+      # @rbs () -> Token
+      def scan_node_directive: () -> Token
+
       # @rbs () -> Token
       def scan_identifier: () -> Token
 
@@ -48,14 +85,24 @@ module Ibex
       # @rbs () -> Token
       def scan_literal: () -> Token
 
+      # @rbs () -> Token
+      # rubocop:disable Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity
+      def scan_regexp: () -> Token
+
       # @rbs () -> Token
       def scan_punctuation: () -> Token
 
       # @rbs (Symbol type, Regexp pattern) ?{ (String) -> token_value } -> Token
       def scan_match: (Symbol type, Regexp pattern) ?{ (String) -> token_value } -> Token
 
-      # @rbs (Symbol type, token_value value, ?Location location) -> Token
-      def token: (Symbol type, token_value value, ?Location location) -> Token
+      # @rbs (Symbol type, token_value value, ?Location location, ?SourceSpan? span) -> Token
+      def token: (Symbol type, token_value value, ?Location location, ?SourceSpan? span) -> Token
+
+      # @rbs (Token token, SourcePosition start) -> void
+      def emit_token: (Token token, SourcePosition start) -> void
+
+      # @rbs (Symbol kind, SourcePosition start, ?token_type: Symbol?, ?token_index: Integer?) -> void
+      def emit_segment: (Symbol kind, SourcePosition start, ?token_type: Symbol?, ?token_index: Integer?) -> void
     end
   end
 end

data/sig/ibex/frontend/lexer_recovery.rbs

@@ -0,0 +1,25 @@
+# Generated from lib/ibex/frontend/lexer_recovery.rb with RBS::Inline
+
+module Ibex
+  module Frontend
+    class Lexer
+      # Recover lexical failures while retaining every source byte.
+      # @rbs (?max_diagnostics: Integer) -> [SourceDocument, Array[Diagnostic]]
+      def tokenize_document_recovering: (?max_diagnostics: Integer) -> [ SourceDocument, Array[Diagnostic] ]
+
+      private
+
+      # @rbs (Exception error, capture: bool) -> Diagnostic?
+      def recover_lexical_error: (Exception error, capture: bool) -> Diagnostic?
+
+      # @rbs (Exception error, SourcePosition fallback) -> [Location, String]
+      def lexical_error_details: (Exception error, SourcePosition fallback) -> [ Location, String ]
+
+      # @rbs (SourcePosition start) -> void
+      def emit_invalid_segment: (SourcePosition start) -> void
+
+      # @rbs (Exception error, SourcePosition start) -> void
+      def advance_after_lexical_error: (Exception error, SourcePosition start) -> void
+    end
+  end
+end

data/sig/ibex/frontend/parser/declarations.rbs

@@ -3,6 +3,8 @@
 module Ibex
   module Frontend
     # Parses the declaration section of a grammar.
+    # rubocop:disable Metrics/ModuleLength, Metrics/CyclomaticComplexity
+    # Bootstrap declarations intentionally mirror the generated frontend's complete declaration vocabulary.
     module BootstrapParserDeclarations
       DECLARATIONS: Array[String]
 
@@ -19,6 +21,9 @@ module Ibex
       # @rbs () -> AST::declaration
       def parse_declaration: () -> AST::declaration
 
+      # @rbs () -> AST::Include
+      def parse_import: () -> AST::Include
+
       # @rbs () -> AST::Tokens
       def parse_tokens: () -> AST::Tokens
 
@@ -34,18 +39,67 @@ module Ibex
       # @rbs () -> AST::Expect
       def parse_expect: () -> AST::Expect
 
+      # @rbs () -> AST::ExpectRR
+      def parse_expect_rr: () -> AST::ExpectRR
+
+      # @rbs () -> AST::Parameter
+      def parse_parameter: () -> AST::Parameter
+
+      # @rbs () -> AST::Printer
+      def parse_printer: () -> AST::Printer
+
       # @rbs () -> AST::Start
       def parse_start: () -> AST::Start
 
+      # @rbs () -> AST::Recovery
+      def parse_recovery: () -> AST::Recovery
+
+      # @rbs () -> AST::OnErrorReduce
+      def parse_on_error_reduce: () -> AST::OnErrorReduce
+
+      # @rbs () -> AST::GrammarTest
+      def parse_grammar_test: () -> AST::GrammarTest
+
+      # @rbs () -> AST::Lexer
+      def parse_lexer: () -> AST::Lexer
+
+      # @rbs () -> Array[AST::lexer_entry]
+      def parse_lexer_entries: () -> Array[AST::lexer_entry]
+
+      # @rbs () -> AST::lexer_entry
+      def parse_lexer_entry: () -> AST::lexer_entry
+
+      # @rbs () -> AST::LexerState
+      def parse_lexer_state: () -> AST::LexerState
+
+      # @rbs () -> Token
+      def expect_lexer_pattern: () -> Token
+
+      # @rbs (Symbol kind, Token marker, Token pattern, Token? action) -> AST::LexerRule
+      def build_bootstrap_lexer_rule: (Symbol kind, Token marker, Token pattern, Token? action) -> AST::LexerRule
+
       # @rbs () -> AST::Convert
       def parse_convert: () -> AST::Convert
 
+      # @rbs (singleton(AST::DisplayName) | singleton(AST::SemanticType) node_class, String feature) ->
+      #   (AST::DisplayName | AST::SemanticType)
+      def parse_symbol_metadata: (singleton(AST::DisplayName) | singleton(AST::SemanticType) node_class, String feature) -> (AST::DisplayName | AST::SemanticType)
+
+      # @rbs () -> Token
+      def expect_metadata_value: () -> Token
+
       # @rbs (Integer line) -> Array[Token]
       def tokens_on_line: (Integer line) -> Array[Token]
 
       # @rbs (Array[Token] tokens, Location location) -> String
       def decode_conversion: (Array[Token] tokens, Location location) -> String
 
+      # @rbs (Token keyword, Token name, Token value, String feature) -> void
+      def validate_metadata_line: (Token keyword, Token name, Token value, String feature) -> void
+
+      # @rbs (Token token, String feature) -> String
+      def decode_quoted_literal: (Token token, String feature) -> String
+
       # @rbs () -> bool
       def declaration_start?: () -> bool
 

data/sig/ibex/frontend/parser/parameters.rbs

@@ -0,0 +1,28 @@
+# Generated from lib/ibex/frontend/parser/parameters.rb with RBS::Inline
+
+module Ibex
+  module Frontend
+    # Parses extended-mode parameter lists and parameterized references during bootstrap.
+    module BootstrapParserParameters
+      private
+
+      # @rbs (Token lhs) -> Array[String]
+      def parse_rule_parameters: (Token lhs) -> Array[String]
+
+      # @rbs () -> AST::item
+      def parse_parameterized_reference: () -> AST::item
+
+      # @rbs () -> bool
+      def parameterized_call?: () -> bool
+
+      # @rbs () -> bool
+      def parameterized_rule_start?: () -> bool
+
+      # @rbs (Array[Token] tail) -> bool
+      def parameter_definition_tail?: (Array[Token] tail) -> bool
+
+      # @rbs (Token left, Token right) -> bool
+      def adjacent_tokens?: (Token left, Token right) -> bool
+    end
+  end
+end

data/sig/ibex/frontend/parser/rules.rbs

@@ -17,6 +17,9 @@ module Ibex
       # @rbs (Token lhs) -> AST::Alternative
       def parse_alternative: (Token lhs) -> AST::Alternative
 
+      # @rbs () -> AST::NodeAnnotation?
+      def parse_node_annotation: () -> AST::NodeAnnotation?
+
       # @rbs () -> AST::item
       def parse_item: () -> AST::item
 

data/sig/ibex/frontend/parser.rbs

@@ -4,13 +4,79 @@ module Ibex
   module Frontend
     # Public grammar parser backed by Ibex's generated LR frontend.
     class Parser
+      DEFAULT_MAX_DIAGNOSTICS: Integer
+
+      STRICT_LEXICAL_DIAGNOSTIC_LIMIT: Integer
+
       attr_reader implementation: GeneratedParser
 
+      @lexical_diagnostics: Array[Diagnostic]
+
+      @mode: Symbol
+
+      @tokens: Array[Token]
+
+      @source_document_error_message: String?
+
+      @parse_error_message: String?
+
+      @parsed_node: AST::Root | AST::Fragment | nil
+
+      @parsed_document: SourceDocument?
+
+      @source_document: SourceDocument?
+
       # @rbs (String | Array[Token] source, ?file: String, ?mode: Symbol) -> void
       def initialize: (String | Array[Token] source, ?file: String, ?mode: Symbol) -> void
 
       # @rbs () -> AST::Root
       def parse: () -> AST::Root
+
+      # Parse an explicit fragment without resolving its includes.
+      # @rbs () -> AST::Fragment
+      def parse_fragment: () -> AST::Fragment
+
+      # Parse the grammar and return its lossless source model.
+      # @rbs () -> SourceDocument
+      def parse_document: () -> SourceDocument
+
+      # Parse either a root grammar or an explicit fragment and return its lossless source model.
+      # @rbs () -> SourceDocument
+      def parse_source_document: () -> SourceDocument
+
+      # Parse with conservative boundary recovery and collect multiple errors.
+      # @rbs (?max_diagnostics: Integer) -> ParseResult
+      def parse_with_diagnostics: (?max_diagnostics: Integer) -> ParseResult
+
+      private
+
+      # @rbs () -> SourceDocument
+      def source_document!: () -> SourceDocument
+
+      # @rbs () -> (AST::Root | AST::Fragment)
+      def parse_node: () -> (AST::Root | AST::Fragment)
+
+      # @rbs (AST::Root | AST::Fragment node) -> (AST::Root | AST::Fragment)
+      def enrich_rule_documentation: (AST::Root | AST::Fragment node) -> (AST::Root | AST::Fragment)
+
+      # @rbs (AST::Root | AST::Fragment | nil node, Array[Diagnostic] diagnostics) ->
+      #   [AST::Root?, Array[Diagnostic]]
+      def root_diagnostic_result: (AST::Root | AST::Fragment | nil node, Array[Diagnostic] diagnostics) -> [ AST::Root?, Array[Diagnostic] ]
+
+      # @rbs (String source, String file) -> Array[Token]
+      def tokenize_source: (String source, String file) -> Array[Token]
+
+      # @rbs (Integer max_diagnostics) -> Array[Diagnostic]
+      def lexical_diagnostics_for: (Integer max_diagnostics) -> Array[Diagnostic]
+
+      # @rbs (Array[Diagnostic] lexical, Array[Diagnostic] syntax, Integer limit) -> Array[Diagnostic]
+      def merge_diagnostics: (Array[Diagnostic] lexical, Array[Diagnostic] syntax, Integer limit) -> Array[Diagnostic]
+
+      # @rbs (Diagnostic diagnostic) -> [String, String, Integer, Integer, String]
+      def diagnostic_key: (Diagnostic diagnostic) -> [ String, String, Integer, Integer, String ]
+
+      # @rbs (Diagnostic diagnostic) -> [String, Integer, Integer, String, String]
+      def diagnostic_sort_key: (Diagnostic diagnostic) -> [ String, Integer, Integer, String, String ]
     end
   end
 end

data/sig/ibex/frontend/regenerator.rbs

@@ -6,11 +6,22 @@ module Ibex
     module Regenerator
       GRAMMAR_PATH: String
 
+      SHADOW_GRAMMAR_PATH: String
+
       # @rbs () -> String
       def self?.generate: () -> String
 
+      # Build the non-published parameterized/inline grammar used to verify
+      # that preview composition features can describe the production
+      # frontend without changing its observable AST.
+      # @rbs () -> String
+      def self?.generate_shadow: () -> String
+
       # @rbs () -> String
       def self?.relative_grammar_path: () -> String
+
+      # @rbs () -> String
+      def self?.relative_shadow_grammar_path: () -> String
     end
   end
 end

data/sig/ibex/frontend/resolution.rbs

@@ -0,0 +1,33 @@
+# Generated from lib/ibex/frontend/resolution.rb with RBS::Inline
+
+module Ibex
+  module Frontend
+    # Immutable result of resolving one root grammar and its fragment closure.
+    class Resolution
+      EMPTY_CHAIN: Array[IR::source_provenance]
+
+      attr_reader root: AST::Root
+
+      attr_reader root_path: String
+
+      attr_reader root_directory: String
+
+      attr_reader files: Array[String]
+
+      # @rbs (root: AST::Root, root_path: String, root_directory: String, files: Array[String],
+      #   include_chains: Hash[AST::Rule, Array[IR::source_provenance]]) -> void
+      def initialize: (root: AST::Root, root_path: String, root_directory: String, files: Array[String], include_chains: Hash[AST::Rule, Array[IR::source_provenance]]) -> void
+
+      # @rbs (AST::Rule rule) -> Array[IR::source_provenance]
+      def include_chain_for: (AST::Rule rule) -> Array[IR::source_provenance]
+
+      private
+
+      # @rbs (untyped value) -> untyped
+      def deep_freeze_ast: (untyped value) -> untyped
+
+      # @rbs (IR::source_provenance entry) -> IR::source_provenance
+      def copy_provenance: (IR::source_provenance entry) -> IR::source_provenance
+    end
+  end
+end

data/sig/ibex/frontend/resolver.rbs

@@ -0,0 +1,91 @@
+# Generated from lib/ibex/frontend/resolver.rb with RBS::Inline
+
+module Ibex
+  module Frontend
+    # An actual filesystem read failure while resolving a grammar closure.
+    class ResolutionIOError < Ibex::Error
+    end
+
+    # Loads an explicit root grammar and resolves its extended-mode fragment graph.
+    class Resolver
+      GLOB_CHARACTERS: Regexp
+
+      WINDOWS_ABSOLUTE: Regexp
+
+      @attempted_paths: Array[String]
+
+      @include_chains: Hash[AST::Rule, Array[IR::source_provenance]]
+
+      @loaded: Hash[String, bool]
+
+      @visiting: Array[String]
+
+      @files: Array[String]
+
+      @root_directory: String
+
+      @root_path: String
+
+      @loader: SourceLoader
+
+      @mode: Symbol
+
+      @input_path: String
+
+      attr_reader attempted_paths: Array[String]
+
+      # @rbs (String path, ?mode: Symbol, ?loader: SourceLoader) -> void
+      def initialize: (String path, ?mode: Symbol, ?loader: SourceLoader) -> void
+
+      # @rbs () -> Resolution
+      def resolve: () -> Resolution
+
+      # @rbs () -> Array[String]
+      def dependencies: () -> Array[String]
+
+      # Source bytes actually consumed by the parser, in resolution order.
+      # @rbs () -> Array[GenerationInput]
+      def source_records: () -> Array[GenerationInput]
+
+      private
+
+      # @rbs () -> void
+      def prepare_resolution: () -> void
+
+      # @rbs () -> String
+      def canonical_root: () -> String
+
+      # @rbs (String path) -> AST::Root
+      def parse_root: (String path) -> AST::Root
+
+      # @rbs (String path) -> AST::Fragment
+      def parse_fragment: (String path) -> AST::Fragment
+
+      # @rbs (AST::Root | AST::Fragment node, Array[IR::source_provenance] chain) ->
+      #   [Array[AST::declaration], Array[AST::Rule]]
+      def expand_node: (AST::Root | AST::Fragment node, Array[IR::source_provenance] chain) -> [ Array[AST::declaration], Array[AST::Rule] ]
+
+      # @rbs (AST::Include include_node, Array[IR::source_provenance] chain) ->
+      #   [Array[AST::declaration], Array[AST::Rule]]
+      def expand_include: (AST::Include include_node, Array[IR::source_provenance] chain) -> [ Array[AST::declaration], Array[AST::Rule] ]
+
+      # @rbs (AST::Include include_node) -> String
+      def canonical_include: (AST::Include include_node) -> String
+
+      # @rbs (AST::Include include_node) -> void
+      def validate_include_path: (AST::Include include_node) -> void
+
+      # @rbs (String path) -> bool
+      def inside_root?: (String path) -> bool
+
+      # @rbs (AST::Include include_node, String target) -> void
+      def reject_cycle: (AST::Include include_node, String target) -> void
+
+      # @rbs (String file) -> IR::source_provenance
+      def source_provenance: (String file) -> IR::source_provenance
+
+      # @rbs (AST::Include include_node, String message) -> bot
+      def fail_include: (AST::Include include_node, String message) -> bot
+    end
+  end
+end

data/sig/ibex/frontend/rule_documentation.rbs

@@ -0,0 +1,42 @@
+# Generated from lib/ibex/frontend/rule_documentation.rb with RBS::Inline
+
+module Ibex
+  module Frontend
+    # Attaches immediately preceding lossless line-comment blocks to parsed rules.
+    class RuleDocumentation
+      EMPTY_SEGMENTS: Array[Segment]
+
+      # @rbs (AST::Root | AST::Fragment node, SourceDocument document) -> (AST::Root | AST::Fragment)
+      def self.enrich: (AST::Root | AST::Fragment node, SourceDocument document) -> (AST::Root | AST::Fragment)
+
+      # @rbs (SourceDocument document) -> void
+      def initialize: (SourceDocument document) -> void
+
+      # @rbs (AST::Root | AST::Fragment node) -> (AST::Root | AST::Fragment)
+      def enrich: (AST::Root | AST::Fragment node) -> (AST::Root | AST::Fragment)
+
+      private
+
+      # @rbs (AST::Rule rule) -> AST::Rule
+      def documented_rule: (AST::Rule rule) -> AST::Rule
+
+      # @rbs (AST::Rule rule) -> Integer
+      def documentation_anchor: (AST::Rule rule) -> Integer
+
+      # @rbs (Integer rule_line) -> String?
+      def documentation_before: (Integer rule_line) -> String?
+
+      # @rbs (Integer line) -> String?
+      def documentation_line: (Integer line) -> String?
+
+      # @rbs (Segment segment, Segment comment) -> bool
+      def documentation_line_segment?: (Segment segment, Segment comment) -> bool
+
+      # @rbs (SourceDocument document) -> Hash[Integer, Array[Segment]]
+      def index_segments: (SourceDocument document) -> Hash[Integer, Array[Segment]]
+
+      # @rbs (Segment segment) -> Array[Integer]
+      def covered_lines: (Segment segment) -> Array[Integer]
+    end
+  end
+end

data/sig/ibex/frontend/source_cursor.rbs

@@ -23,15 +23,17 @@ module Ibex
     end
 
     # A grammar token and its source coordinate.
-    class Token < Struct[Symbol | token_value | Location]
+    class Token < Struct[Symbol | token_value | Location | SourceSpan?]
       attr_accessor type(): Symbol
 
       attr_accessor value(): token_value
 
       attr_accessor location(): Location
 
-      def self.new: (?type: Symbol, ?value: token_value, ?location: Location) -> instance
-                  | ({ ?type: Symbol, ?value: token_value, ?location: Location }) -> instance
+      attr_accessor span(): SourceSpan?
+
+      def self.new: (?type: Symbol, ?value: token_value, ?location: Location, ?span: SourceSpan?) -> instance
+                  | ({ ?type: Symbol, ?value: token_value, ?location: Location, ?span: SourceSpan? }) -> instance
     end
 
     class Token
@@ -47,6 +49,8 @@ module Ibex
 
       attr_reader index: Integer
 
+      attr_reader byte_offset: Integer
+
       attr_reader line: Integer
 
       attr_reader column: Integer
@@ -66,8 +70,37 @@ module Ibex
       # @rbs () -> Location
       def location: () -> Location
 
+      # @rbs () -> SourcePosition
+      def position: () -> SourcePosition
+
+      # @rbs (SourcePosition start) -> SourceSpan
+      def span_from: (SourcePosition start) -> SourceSpan
+
       # @rbs (?Integer count) -> void
       def advance: (?Integer count) -> void
+
+      private
+
+      # @rbs (Integer count) -> void
+      def advance_ascii: (Integer count) -> void
+
+      # @rbs (Integer count) -> void
+      def advance_utf8: (Integer count) -> void
+
+      # @rbs (Integer character_index) -> Integer
+      def byte_offset_for_character: (Integer character_index) -> Integer
+
+      # @rbs (Integer byte_offset, Integer from_index, Integer to_index) -> Integer
+      def walk_to_character: (Integer byte_offset, Integer from_index, Integer to_index) -> Integer
+
+      # @rbs (Integer byte_offset) -> Integer
+      def previous_character_byte_offset: (Integer byte_offset) -> Integer
+
+      # @rbs (Integer byte_offset) -> Integer
+      def utf8_character_width: (Integer byte_offset) -> Integer
+
+      # @rbs (Integer byte) -> bool
+      def continuation_byte?: (Integer byte) -> bool
     end
   end
 end