ibex 0.1.0 → 0.2.0

data/lib/ibex/frontend/parser.rb

@@ -4,19 +4,197 @@ module Ibex
   module Frontend
     # Public grammar parser backed by Ibex's generated LR frontend.
     class Parser
+      DEFAULT_MAX_DIAGNOSTICS = 20 #: Integer
+      STRICT_LEXICAL_DIAGNOSTIC_LIMIT = 1 #: Integer
+      private_constant :STRICT_LEXICAL_DIAGNOSTIC_LIMIT
+
       attr_reader :implementation #: GeneratedParser
 
+      # @rbs @source_document: SourceDocument?
+      # @rbs @parsed_document: SourceDocument?
+      # @rbs @parsed_node: AST::Root | AST::Fragment | nil
+      # @rbs @parse_error_message: String?
+      # @rbs @source_document_error_message: String?
+      # @rbs @tokens: Array[Token]
+      # @rbs @mode: Symbol
+      # @rbs @lexical_diagnostics: Array[Diagnostic]
+
       # @rbs (String | Array[Token] source, ?file: String, ?mode: Symbol) -> void
-      def initialize(source, file: "(grammar)", mode: :racc)
-        raise ArgumentError, "mode must be :racc or :extended" unless %i[racc extended].include?(mode)
+      def initialize(source, file: "(grammar)", mode: :default)
+        raise ArgumentError, "mode must be :default or :extended" unless %i[default extended].include?(mode)
 
-        tokens = source.is_a?(Array) ? source : Lexer.new(source, file: file).tokenize
+        @mode = mode
+        @lexical_diagnostics = [] #: Array[Diagnostic]
+        @parse_error_message = nil
+        @source_document_error_message = nil
+        tokens = source.is_a?(Array) ? source : tokenize_source(source, file)
+        @tokens = tokens
         @implementation = GeneratedParser.new(tokens, mode: mode)
       end
 
       # @rbs () -> AST::Root
       def parse
-        @implementation.parse
+        node = parse_node
+        return node if node.is_a?(AST::Root)
+
+        raise Ibex::Error, "#{node.loc}: fragment input requires Parser#parse_fragment"
+      end
+
+      # Parse an explicit fragment without resolving its includes.
+      # @rbs () -> AST::Fragment
+      def parse_fragment
+        node = parse_node
+        return node if node.is_a?(AST::Fragment)
+
+        raise Ibex::Error, "#{node.loc}: root grammar input cannot be parsed as a fragment"
+      end
+
+      # Parse the grammar and return its lossless source model.
+      # @rbs () -> SourceDocument
+      def parse_document
+        parsed_document = @parsed_document
+        return parsed_document if parsed_document && parsed_document.ast.is_a?(AST::Root)
+
+        @parsed_document = source_document!.with_ast(parse)
+      end
+
+      # Parse either a root grammar or an explicit fragment and return its lossless source model.
+      # @rbs () -> SourceDocument
+      def parse_source_document
+        parsed_document = @parsed_document
+        return parsed_document if parsed_document
+
+        @parsed_document = source_document!.with_ast(parse_node)
+      end
+
+      # Parse with conservative boundary recovery and collect multiple errors.
+      # @rbs (?max_diagnostics: Integer) -> ParseResult
+      def parse_with_diagnostics(max_diagnostics: DEFAULT_MAX_DIAGNOSTICS)
+        unless max_diagnostics.is_a?(Integer) && max_diagnostics.positive?
+          raise ArgumentError, "max_diagnostics must be a positive integer"
+        end
+
+        lexical_diagnostics = lexical_diagnostics_for(max_diagnostics)
+        if @source_document.nil? && @lexical_diagnostics.any?
+          return ParseResult.new(diagnostics: lexical_diagnostics, ast: nil, document: nil)
+        end
+
+        recovery = DiagnosticRecovery.new(
+          @tokens, mode: @mode, max_diagnostics: max_diagnostics
+        )
+        node, syntax_diagnostics = recovery.parse
+        node = enrich_rule_documentation(node) if node
+        ast, syntax_diagnostics = root_diagnostic_result(node, syntax_diagnostics)
+        diagnostics = merge_diagnostics(lexical_diagnostics, syntax_diagnostics, max_diagnostics)
+        document = if diagnostics.empty? && ast
+                     @source_document&.with_ast(ast)
+                   else
+                     @source_document
+                   end
+        ParseResult.new(diagnostics: diagnostics, ast: ast, document: document)
+      end
+
+      private
+
+      # @rbs () -> SourceDocument
+      def source_document!
+        source_document_error_message = @source_document_error_message
+        raise Ibex::Error, source_document_error_message if source_document_error_message
+
+        source_document = @source_document
+        raise ArgumentError, "source document parsing requires String source" unless source_document
+
+        source_document
+      end
+
+      # @rbs () -> (AST::Root | AST::Fragment)
+      def parse_node
+        lexical = @lexical_diagnostics.first
+        raise Ibex::Error, lexical.to_s if lexical
+
+        parse_error_message = @parse_error_message
+        raise Ibex::Error, parse_error_message if parse_error_message
+
+        node = @parsed_node
+        return node if node
+
+        begin
+          @parsed_node = enrich_rule_documentation(@implementation.parse)
+        rescue Ibex::Error => e
+          @parse_error_message = e.message.dup.freeze
+          raise
+        end
+      end
+
+      # @rbs (AST::Root | AST::Fragment node) -> (AST::Root | AST::Fragment)
+      def enrich_rule_documentation(node)
+        source_document = @source_document
+        return node unless source_document
+
+        RuleDocumentation.enrich(node, source_document)
+      end
+
+      # @rbs (AST::Root | AST::Fragment | nil node, Array[Diagnostic] diagnostics) ->
+      #   [AST::Root?, Array[Diagnostic]]
+      def root_diagnostic_result(node, diagnostics)
+        return [node, diagnostics] unless node.is_a?(AST::Fragment)
+
+        diagnostic = Diagnostic.new(
+          code: "frontend.syntax_error", phase: :syntax,
+          message: "fragment input requires Parser#parse_fragment", location: node.loc
+        )
+        [nil, diagnostics + [diagnostic]]
+      end
+
+      # @rbs (String source, String file) -> Array[Token]
+      def tokenize_source(source, file)
+        document, diagnostics = Lexer.new(source, file: file).tokenize_document_recovering(
+          max_diagnostics: STRICT_LEXICAL_DIAGNOSTIC_LIMIT
+        )
+        @source_document = document
+        @lexical_diagnostics = diagnostics
+        document.tokens
+      rescue Ibex::Error => e
+        location = Location.new(file: file, line: 1, column: 1)
+        @lexical_diagnostics = [
+          Diagnostic.new(code: "frontend.lexical_error", phase: :lexical, message: e.message,
+                         location: location, rendered: e.message)
+        ]
+        @source_document_error_message = e.message.dup.freeze
+        [Token.new(type: :eof, value: nil, location: location, span: nil)]
+      end
+
+      # @rbs (Integer max_diagnostics) -> Array[Diagnostic]
+      def lexical_diagnostics_for(max_diagnostics)
+        return @lexical_diagnostics if @lexical_diagnostics.empty? || max_diagnostics == 1
+
+        source_document = @source_document
+        return @lexical_diagnostics.first(max_diagnostics) unless source_document
+
+        _, diagnostics = Lexer.new(source_document.source, file: source_document.file)
+                              .tokenize_document_recovering(max_diagnostics: max_diagnostics)
+        diagnostics
+      end
+
+      # @rbs (Array[Diagnostic] lexical, Array[Diagnostic] syntax, Integer limit) -> Array[Diagnostic]
+      def merge_diagnostics(lexical, syntax, limit)
+        (lexical + syntax)
+          .uniq { |diagnostic| diagnostic_key(diagnostic) }
+          .sort_by { |diagnostic| diagnostic_sort_key(diagnostic) }
+          .first(limit)
+          .freeze
+      end
+
+      # @rbs (Diagnostic diagnostic) -> [String, String, Integer, Integer, String]
+      def diagnostic_key(diagnostic)
+        location = diagnostic.location
+        [diagnostic.phase.to_s, location.file, location.line, location.column, diagnostic.message]
+      end
+
+      # @rbs (Diagnostic diagnostic) -> [String, Integer, Integer, String, String]
+      def diagnostic_sort_key(diagnostic)
+        location = diagnostic.location
+        [location.file, location.line, location.column, diagnostic.phase.to_s, diagnostic.code]
       end
     end
   end

data/lib/ibex/frontend/regenerator.rb

@@ -2,12 +2,15 @@
 
 require_relative "../error"
 require_relative "../tables"
+require_relative "source_span"
 require_relative "source_cursor"
+require_relative "source_document"
 require_relative "action_scanner"
 require_relative "lexer"
 require_relative "ast"
 require_relative "parser/declarations"
 require_relative "parser/rules"
+require_relative "parser/parameters"
 require_relative "bootstrap_parser"
 require_relative "../ir"
 require_relative "../normalize"
@@ -20,6 +23,7 @@ module Ibex
     # Builds the committed frontend parser from its canonical Ibex grammar.
     module Regenerator
       GRAMMAR_PATH = File.expand_path("grammar.y", File.dirname(__FILE__)) #: String
+      SHADOW_GRAMMAR_PATH = File.expand_path("shadow_grammar.y", File.dirname(__FILE__)) #: String
 
       module_function
 
@@ -29,13 +33,34 @@ module Ibex
         ast = BootstrapParser.new(source, file: relative_grammar_path).parse
         grammar = Normalizer.new(ast).normalize
         automaton = LALR::Builder.new(grammar).build
-        Codegen::Ruby.new(automaton, table: :compact, line_convert: false).generate
+        Codegen::Ruby.new(
+          automaton, table: :compact, line_convert: false, runtime_require: nil
+        ).generate
+      end
+
+      # Build the non-published parameterized/inline grammar used to verify
+      # that preview composition features can describe the production
+      # frontend without changing its observable AST.
+      # @rbs () -> String
+      def generate_shadow
+        source = File.read(SHADOW_GRAMMAR_PATH)
+        ast = BootstrapParser.new(source, file: relative_shadow_grammar_path, mode: :extended).parse
+        grammar = Normalizer.new(ast, mode: :extended).normalize
+        automaton = LALR::Builder.new(grammar).build
+        Codegen::Ruby.new(
+          automaton, table: :compact, line_convert: false, runtime_require: nil
+        ).generate
       end
 
       # @rbs () -> String
       def relative_grammar_path
         "lib/ibex/frontend/grammar.y"
       end
+
+      # @rbs () -> String
+      def relative_shadow_grammar_path
+        "lib/ibex/frontend/shadow_grammar.y"
+      end
     end
   end
 end

data/lib/ibex/frontend/resolution.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Immutable result of resolving one root grammar and its fragment closure.
+    class Resolution
+      EMPTY_CHAIN = Array.new(0).freeze #: Array[IR::source_provenance]
+      private_constant :EMPTY_CHAIN
+
+      attr_reader :root #: AST::Root
+      attr_reader :root_path #: String
+      attr_reader :root_directory #: String
+      attr_reader :files #: Array[String]
+
+      # @rbs (root: AST::Root, root_path: String, root_directory: String, files: Array[String],
+      #   include_chains: Hash[AST::Rule, Array[IR::source_provenance]]) -> void
+      def initialize(root:, root_path:, root_directory:, files:, include_chains:)
+        @root = deep_freeze_ast(root)
+        @root_path = root_path.dup.freeze
+        @root_directory = root_directory.dup.freeze
+        @files = files.map { |file| file.dup.freeze }.freeze
+        chains = {} #: Hash[AST::Rule, Array[IR::source_provenance]]
+        @include_chains = chains.compare_by_identity
+        include_chains.each do |rule, chain|
+          frozen_chain = chain.map { |entry| copy_provenance(entry) }.freeze
+          @include_chains[rule] = frozen_chain
+        end
+        @include_chains.freeze
+        freeze
+      end
+
+      # @rbs (AST::Rule rule) -> Array[IR::source_provenance]
+      def include_chain_for(rule)
+        @include_chains.fetch(rule, EMPTY_CHAIN)
+      end
+
+      private
+
+      # @rbs (untyped value) -> untyped
+      def deep_freeze_ast(value)
+        case value
+        when Struct
+          value.each_pair { |_name, child| deep_freeze_ast(child) }
+        when Array
+          value.each { |child| deep_freeze_ast(child) }
+        when Hash
+          value.each do |key, child|
+            deep_freeze_ast(key)
+            deep_freeze_ast(child)
+          end
+        end
+        value.freeze
+      end
+
+      # @rbs (IR::source_provenance entry) -> IR::source_provenance
+      def copy_provenance(entry)
+        span = entry[:byte_span]
+        frozen_span = span && { start: span[:start], end: span[:end] } #: IR::byte_span?
+        frozen_span&.freeze
+        provenance = {
+          file: entry[:file]&.dup&.freeze,
+          root: entry[:root]&.dup&.freeze,
+          byte_span: frozen_span
+        } #: IR::source_provenance
+        provenance.freeze
+      end
+    end
+  end
+end

data/lib/ibex/frontend/resolver.rb

@@ -0,0 +1,217 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # An actual filesystem read failure while resolving a grammar closure.
+    class ResolutionIOError < Ibex::Error; end
+
+    # Loads an explicit root grammar and resolves its extended-mode fragment graph.
+    class Resolver
+      GLOB_CHARACTERS = /[*?\[\]{}]/ #: Regexp
+      WINDOWS_ABSOLUTE = %r{\A(?:[A-Za-z]:[\\/]|\\\\)} #: Regexp
+
+      # @rbs @input_path: String
+      # @rbs @mode: Symbol
+      # @rbs @loader: SourceLoader
+      # @rbs @root_path: String
+      # @rbs @root_directory: String
+      # @rbs @files: Array[String]
+      # @rbs @visiting: Array[String]
+      # @rbs @loaded: Hash[String, bool]
+      # @rbs @include_chains: Hash[AST::Rule, Array[IR::source_provenance]]
+      # @rbs @attempted_paths: Array[String]
+
+      attr_reader :attempted_paths #: Array[String]
+
+      # @rbs (String path, ?mode: Symbol, ?loader: SourceLoader) -> void
+      def initialize(path, mode: :default, loader: SourceLoader.new)
+        raise ArgumentError, "mode must be :default or :extended" unless %i[default extended].include?(mode)
+
+        @input_path = path
+        @mode = mode
+        @loader = loader
+        @attempted_paths = [File.expand_path(path)]
+      end
+
+      # @rbs () -> Resolution
+      def resolve
+        cached = @resolution
+        return cached if cached
+
+        prepare_resolution
+        parsed_root = parse_root(@root_path)
+        declarations, rules = expand_node(parsed_root, [])
+        root = AST::Root.new(
+          class_name: parsed_root.class_name, superclass: parsed_root.superclass,
+          declarations: declarations, rules: rules, user_code: parsed_root.user_code, loc: parsed_root.loc,
+          extended: parsed_root.extended, cst: parsed_root.cst
+        )
+        @loaded[@root_path] = true
+        @visiting.pop
+        @resolution = Resolution.new(
+          root: root, root_path: @root_path, root_directory: @root_directory,
+          files: @files, include_chains: @include_chains
+        )
+      end
+
+      # @rbs () -> Array[String]
+      def dependencies
+        resolve.files
+      end
+
+      # Source bytes actually consumed by the parser, in resolution order.
+      # @rbs () -> Array[GenerationInput]
+      def source_records
+        @loader.read_records
+      end
+
+      private
+
+      # @rbs () -> void
+      def prepare_resolution
+        @root_path = canonical_root
+        @root_directory = File.dirname(@root_path)
+        @files = [@root_path] #: Array[String]
+        @visiting = [@root_path] #: Array[String]
+        @loaded = {} #: Hash[String, bool]
+        chains = {} #: Hash[AST::Rule, Array[IR::source_provenance]]
+        @include_chains = chains.compare_by_identity
+        @attempted_paths = [File.expand_path(@input_path)]
+      end
+
+      # @rbs () -> String
+      def canonical_root
+        path = @loader.canonical_path(@input_path)
+        raise Ibex::Error, "#{@input_path}:1:1: root grammar must be a file" unless @loader.file?(path)
+
+        @loader.record_access(path, @input_path)
+        path
+      rescue SystemCallError => e
+        raise ResolutionIOError, "#{@input_path}:1:1: cannot read root grammar: #{e.message}"
+      end
+
+      # @rbs (String path) -> AST::Root
+      def parse_root(path)
+        Parser.new(@loader.read(path), file: path, mode: @mode).parse
+      rescue SystemCallError => e
+        raise ResolutionIOError, "#{path}:1:1: cannot read grammar: #{e.message}"
+      end
+
+      # @rbs (String path) -> AST::Fragment
+      def parse_fragment(path)
+        Parser.new(@loader.read(path), file: path, mode: :extended).parse_fragment
+      rescue SystemCallError => e
+        raise ResolutionIOError, "#{path}:1:1: cannot read fragment: #{e.message}"
+      end
+
+      # @rbs (AST::Root | AST::Fragment node, Array[IR::source_provenance] chain) ->
+      #   [Array[AST::declaration], Array[AST::Rule]]
+      def expand_node(node, chain)
+        declarations = [] #: Array[AST::declaration]
+        rules = [] #: Array[AST::Rule]
+        node.declarations.each do |declaration|
+          if declaration.is_a?(AST::Include)
+            included_declarations, included_rules = expand_include(declaration, chain)
+            declarations.concat(included_declarations)
+            rules.concat(included_rules)
+          else
+            declarations << declaration
+          end
+        end
+        node.rules.each do |rule|
+          @include_chains[rule] = chain
+          rules << rule
+        end
+        [declarations, rules]
+      end
+
+      # @rbs (AST::Include include_node, Array[IR::source_provenance] chain) ->
+      #   [Array[AST::declaration], Array[AST::Rule]]
+      def expand_include(include_node, chain)
+        target = canonical_include(include_node)
+        reject_cycle(include_node, target)
+        return [[], []] if @loaded[target]
+
+        @files << target
+        @visiting << target
+        fragment = parse_fragment(target)
+        next_chain = chain + [source_provenance(target)]
+        declarations, rules = expand_node(fragment, next_chain)
+        @visiting.pop
+        @loaded[target] = true
+        [declarations, rules]
+      end
+
+      # @rbs (AST::Include include_node) -> String
+      def canonical_include(include_node)
+        validate_include_path(include_node)
+        candidate = File.expand_path(include_node.path, File.dirname(include_node.loc.file))
+        @attempted_paths << candidate unless @attempted_paths.include?(candidate)
+        canonical = @loader.canonical_path(candidate)
+        unless @loader.file?(canonical)
+          fail_include(include_node, "include path is not a file: #{include_node.path.inspect}")
+        end
+
+        unless inside_root?(canonical)
+          message = "include resolves outside the root grammar directory: #{include_node.path.inspect}"
+          fail_include(include_node, message)
+        end
+        @loader.record_access(canonical, candidate)
+        canonical
+      rescue Errno::ENOENT, Errno::ENOTDIR
+        fail_include(include_node, "include file does not exist: #{include_node.path.inspect}")
+      rescue SystemCallError => e
+        message = "#{include_node.loc}: cannot read include #{include_node.path.inspect}: #{e.message}"
+        raise ResolutionIOError, message
+      end
+
+      # @rbs (AST::Include include_node) -> void
+      def validate_include_path(include_node)
+        path = include_node.path
+        fail_include(include_node, "include path must not be empty") if path.empty?
+        fail_include(include_node, "include path must not contain NUL") if path.include?("\0")
+        if path.start_with?("/") || path.match?(WINDOWS_ABSOLUTE)
+          fail_include(include_node, "include path must be relative")
+        end
+        if path.split(%r{[\\/]}).include?("..")
+          fail_include(include_node, "include path must not contain parent traversal")
+        end
+        return unless path.match?(GLOB_CHARACTERS)
+
+        fail_include(include_node, "include path must not contain glob metacharacters")
+      end
+
+      # @rbs (String path) -> bool
+      def inside_root?(path)
+        directory = File.dirname(path)
+        loop do
+          return true if directory == @root_directory
+
+          parent = File.dirname(directory)
+          return false if parent == directory
+
+          directory = parent
+        end
+      end
+
+      # @rbs (AST::Include include_node, String target) -> void
+      def reject_cycle(include_node, target)
+        start = @visiting.index(target)
+        return unless start
+
+        cycle = @visiting.drop(start) + [target]
+        fail_include(include_node, "include cycle: #{cycle.join(' -> ')}")
+      end
+
+      # @rbs (String file) -> IR::source_provenance
+      def source_provenance(file)
+        { file: file, root: @root_directory, byte_span: nil }
+      end
+
+      # @rbs (AST::Include include_node, String message) -> bot
+      def fail_include(include_node, message)
+        raise Ibex::Error, "#{include_node.loc}: #{message}"
+      end
+    end
+  end
+end

data/lib/ibex/frontend/rule_documentation.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Ibex
+  module Frontend
+    # Attaches immediately preceding lossless line-comment blocks to parsed rules.
+    class RuleDocumentation
+      EMPTY_SEGMENTS = Array.new(0).freeze #: Array[Segment]
+      private_constant :EMPTY_SEGMENTS
+
+      # @rbs (AST::Root | AST::Fragment node, SourceDocument document) -> (AST::Root | AST::Fragment)
+      def self.enrich(node, document)
+        new(document).enrich(node)
+      end
+
+      # @rbs (SourceDocument document) -> void
+      def initialize(document)
+        @segments_by_line = index_segments(document) #: Hash[Integer, Array[Segment]]
+        @inline_lines = document.tokens.filter_map do |token|
+          token.location.line if token.type == :inline
+        end #: Array[Integer]
+      end
+
+      # @rbs (AST::Root | AST::Fragment node) -> (AST::Root | AST::Fragment)
+      def enrich(node)
+        rules = node.rules.map { |rule| documented_rule(rule) }
+        if node.is_a?(AST::Root)
+          AST::Root.new(
+            class_name: node.class_name, superclass: node.superclass, declarations: node.declarations,
+            rules: rules, user_code: node.user_code, loc: node.loc, extended: node.extended, cst: node.cst
+          )
+        else
+          AST::Fragment.new(declarations: node.declarations, rules: rules, loc: node.loc)
+        end
+      end
+
+      private
+
+      # @rbs (AST::Rule rule) -> AST::Rule
+      def documented_rule(rule)
+        AST::Rule.new(
+          lhs: rule.lhs, parameters: rule.parameters, alternatives: rule.alternatives, loc: rule.loc,
+          documentation: documentation_before(documentation_anchor(rule)), inline: rule.inline
+        )
+      end
+
+      # @rbs (AST::Rule rule) -> Integer
+      def documentation_anchor(rule)
+        return rule.loc.line unless rule.inline
+
+        @inline_lines.reverse_each.find { |line| line <= rule.loc.line } || rule.loc.line
+      end
+
+      # @rbs (Integer rule_line) -> String?
+      def documentation_before(rule_line)
+        lines = [] #: Array[String]
+        line = rule_line - 1
+        while line.positive?
+          content = documentation_line(line)
+          break unless content
+
+          lines.unshift(content)
+          line -= 1
+        end
+        lines.empty? ? nil : lines.join("\n").freeze
+      end
+
+      # @rbs (Integer line) -> String?
+      def documentation_line(line)
+        segments = @segments_by_line.fetch(line, EMPTY_SEGMENTS)
+        comments = segments.select { |segment| segment.kind == :line_comment }
+        return unless comments.length == 1
+
+        comment = comments.fetch(0)
+        return unless comment.text.start_with?("##")
+        return unless segments.all? { |segment| documentation_line_segment?(segment, comment) }
+
+        comment.text.delete_prefix("##").delete_prefix(" ")
+      end
+
+      # @rbs (Segment segment, Segment comment) -> bool
+      def documentation_line_segment?(segment, comment)
+        segment.equal?(comment) || %i[whitespace newline].include?(segment.kind)
+      end
+
+      # @rbs (SourceDocument document) -> Hash[Integer, Array[Segment]]
+      def index_segments(document)
+        indexed = Hash.new { |hash, line| hash[line] = [] } #: Hash[Integer, Array[Segment]]
+        document.cst.each do |segment|
+          covered_lines(segment).each { |line| indexed[line] << segment }
+        end
+        indexed
+      end
+
+      # @rbs (Segment segment) -> Array[Integer]
+      def covered_lines(segment)
+        first = segment.span.start.line
+        return [first] if segment.kind == :newline
+
+        (first..segment.span.finish.line).to_a
+      end
+    end
+  end
+end