ibex 0.1.0 → 0.3.0

data/lib/ibex/codegen/rbs.rb

@@ -1,18 +1,29 @@
 # frozen_string_literal: true
 
+require_relative "generated_action_abi"
+require_relative "cst_metadata"
+
 module Ibex
   module Codegen
     # Generates an RBS declaration for the public surface of a generated parser.
+    # rubocop:disable Metrics/ClassLength -- one generator owns the complete parser signature.
     class RBS
+      # @rbs!
+      #   type syntax_definition = { name: String, kind: Integer, fields: Hash[String, CSTMetadata::field_slot] }
+
       # @rbs @automaton: IR::Automaton
       # @rbs @grammar: IR::Grammar
       # @rbs @superclass: String
+      # @rbs @omit_action_call: bool
+      # @rbs @generated_action_abi: GeneratedActionABI::Cache
 
-      # @rbs (IR::Automaton automaton, ?superclass: String?) -> void
-      def initialize(automaton, superclass: nil)
+      # @rbs (IR::Automaton automaton, ?superclass: String?, ?omit_action_call: bool?) -> void
+      def initialize(automaton, superclass: nil, omit_action_call: nil)
         @automaton = automaton
         @grammar = automaton.grammar
         @superclass = superclass || @grammar.superclass || "Ibex::Runtime::Parser"
+        @omit_action_call = omit_action_call.nil? ? @grammar.options[:omit_action_call] : omit_action_call
+        @generated_action_abi = GeneratedActionABI::Cache.new
       end
 
       # @rbs () -> String
@@ -21,7 +32,12 @@ module Ibex
         modules, class_name = class_parts
         modules.each { |name| lines << "module #{name}" }
         lines << "class #{class_name} < #{@superclass}"
+        append_ast_contract(lines)
+        append_syntax_contract(lines)
         append_contract(lines)
+        append_lexer_contract(lines)
+        append_value_printer_signatures(lines)
+        append_actions(lines)
         lines << "end"
         modules.reverse_each { lines << "end" }
         "#{lines.join("\n")}\n"
@@ -29,14 +45,326 @@ module Ibex
 
       private
 
+      # @rbs (Array[String] lines) -> void
+      def append_ast_contract(lines)
+        definitions = ast_node_definitions
+        return if definitions.empty?
+
+        lines << "  module AST"
+        definitions.each_value { |node| append_ast_node_contract(lines, node) }
+        append_visitor_contract(lines, definitions)
+        append_listener_contract(lines, definitions)
+        lines.push("  end", "")
+      end
+
+      # @rbs (Array[String] lines) -> void
+      def append_syntax_contract(lines)
+        definitions = syntax_node_definitions
+        return if definitions.empty? || @grammar.options[:cst] != true
+
+        lines << "  module Syntax"
+        definitions.each_value { |definition| append_syntax_node_contract(lines, definition) }
+        lines.push("  end", "")
+      end
+
+      # @rbs () -> Hash[String, syntax_definition]
+      def syntax_node_definitions
+        metadata = CSTMetadata.new(@grammar).build
+        slots = metadata.fetch(:slots)
+        definitions = {} #: Hash[String, syntax_definition]
+        @grammar.productions.each do |production|
+          node = production.node
+          next unless node
+
+          slot = slots.fetch(production.id)
+          name = node.fetch(:name)
+          previous = definitions[name]
+          fields = previous ? merge_syntax_fields(previous.fetch(:fields), slot.fetch(:fields)) : slot.fetch(:fields)
+          definition = { name: name, kind: slot.fetch(:node_kind), fields: fields }.freeze #: syntax_definition
+          definitions[name] = definition
+        end
+        definitions
+      end
+
+      # @rbs (Hash[String, CSTMetadata::field_slot] left, Hash[String, CSTMetadata::field_slot] right) -> Hash[String, CSTMetadata::field_slot]
+      def merge_syntax_fields(left, right)
+        left.to_h do |name, left_slot|
+          right_slot = right.fetch(name)
+          left_index = left_slot.is_a?(Hash) ? left_slot.fetch(:index) : left_slot
+          right_index = right_slot.is_a?(Hash) ? right_slot.fetch(:index) : right_slot
+          raise Ibex::Error, "inconsistent CST slot for #{name}" unless left_index == right_index
+
+          [name, left_slot == right_slot ? left_slot : left_index]
+        end.freeze
+      end
+
+      # @rbs (Array[String] lines, syntax_definition definition) -> void
+      def append_syntax_node_contract(lines, definition)
+        name = definition.fetch(:name)
+        fields = definition.fetch(:fields)
+        lines.push(
+          "    class #{name} < Ibex::Runtime::CST::TypedNode",
+          "      KIND: Integer",
+          "      def self.cast: (Ibex::Runtime::CST::SyntaxNode node) -> #{name}?"
+        )
+        fields.each_with_index do |(field, _slot), index|
+          lines << "      def #{field}: () -> #{syntax_field_type(name, index)}"
+        end
+        append_repetition_contracts(lines, fields)
+        lines << "    end"
+      end
+
+      # @rbs (String node_name, Integer index) -> String
+      def syntax_field_type(node_name, index)
+        symbol_ids = @grammar.productions.filter_map do |production|
+          production.rhs.fetch(index) if production.node&.fetch(:name) == node_name
+        end
+        types = symbol_ids.map do |symbol_id|
+          symbol = @grammar.symbols.fetch(symbol_id)
+          symbol.terminal? ? "Ibex::Runtime::CST::SyntaxToken" : "Ibex::Runtime::CST::SyntaxNode"
+        end.uniq
+        types.one? ? types.fetch(0) : "(#{types.join(' | ')})"
+      end
+
+      # @rbs (Array[String] lines, Hash[String, CSTMetadata::field_slot] fields) -> void
+      def append_repetition_contracts(lines, fields)
+        repeated = fields.select { |_field, slot| slot.is_a?(Hash) && slot[:extraction] }
+        element = "Enumerator[Ibex::Runtime::CST::SyntaxNode | Ibex::Runtime::CST::SyntaxToken, void]"
+        repeated.each do |field, slot|
+          next unless slot.is_a?(Hash)
+
+          lines << "      def each_#{field}_element: () -> #{element}"
+          lines << "      def each_#{field}_separator: () -> #{element}" if slot.fetch(:extraction) == :separated_list
+        end
+        return unless repeated.one?
+
+        _field, slot = repeated.first
+        return unless slot.is_a?(Hash)
+
+        lines << "      def each_element: () -> #{element}"
+        lines << "      def each_separator: () -> #{element}" if slot.fetch(:extraction) == :separated_list
+      end
+
+      # @rbs () -> Hash[String, IR::node_annotation]
+      def ast_node_definitions
+        @grammar.productions.filter_map(&:node).to_h { |node| [node.fetch(:name), node] }
+      end
+
+      # @rbs (Array[String] lines, IR::node_annotation node) -> void
+      def append_ast_node_contract(lines, node)
+        name = node.fetch(:name)
+        fields = node.fetch(:fields)
+        types = ast_node_field_types(name, fields.length)
+        lines << "    class #{name} < Data"
+        fields.each_with_index { |field, index| lines << "      attr_reader #{field}: #{types.fetch(index)}" }
+        keywords = fields.each_with_index.map { |field, index| "#{field}: #{types.fetch(index)}" }
+        lines << "      def self.new: (#{keywords.join(', ')}) -> instance"
+        lines << "      def initialize: (#{keywords.join(', ')}) -> void"
+        lines << "    end"
+      end
+
+      # @rbs (String name, Integer length) -> Array[String]
+      def ast_node_field_types(name, length)
+        productions = @grammar.productions.select { |production| production.node&.fetch(:name) == name }
+        Array.new(length) do |index|
+          types = productions.map { |production| semantic_type(production.rhs.fetch(index)) }.uniq
+          types.length == 1 ? types.fetch(0) : "(#{types.join(' | ')})"
+        end
+      end
+
+      # @rbs (Array[String] lines, Hash[String, IR::node_annotation] definitions) -> void
+      def append_visitor_contract(lines, definitions)
+        lines.push("    class Visitor", "      def visit: (untyped node) -> untyped")
+        definitions.each_value do |node|
+          name = node.fetch(:name)
+          lines << "      def visit_#{ast_method_name(name)}: (#{name} node) -> untyped"
+        end
+        lines.push("      def visit_children: (untyped node) -> untyped", "    end")
+      end
+
+      # @rbs (Array[String] lines, Hash[String, IR::node_annotation] definitions) -> void
+      def append_listener_contract(lines, definitions)
+        lines.push("    class Listener", "      def walk: (untyped node) -> untyped",
+                   "      def enter: (untyped node) -> void", "      def exit: (untyped node) -> void")
+        definitions.each_value do |node|
+          name = node.fetch(:name)
+          method = ast_method_name(name)
+          lines << "      def enter_#{method}: (#{name} node) -> void"
+          lines << "      def exit_#{method}: (#{name} node) -> void"
+        end
+        lines.push("      def listener_children: (untyped node) -> Array[untyped]", "    end")
+      end
+
+      # @rbs (String name) -> String
+      def ast_method_name(name)
+        name.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2')
+            .gsub(/([a-z\d])([A-Z])/, '\1_\2')
+            .downcase
+      end
+
       # @rbs (Array[String] lines) -> void
       def append_contract(lines)
         lines.push("  PARSER_TABLE_FORMAT_VERSION: Integer",
+                   "  GRAMMAR_DIGEST: String", "  STATE_COUNT: Integer", "  PRODUCTION_COUNT: Integer",
                    "  TOKEN_IDS: Hash[untyped, Integer]", "  TOKEN_NAMES: Hash[Integer, String]",
                    "  ACTIONS: untyped", "  GOTOS: untyped", "  DEFAULT_ACTIONS: Array[untyped]",
+                   "  EAGER_REDUCTIONS: Hash[Integer, [:reduce, Integer]]",
                    "  PRODUCTIONS: Array[Hash[Symbol, untyped]]",
+                   "  ERROR_MESSAGES: Hash[Integer, String | { id: String, message: String }]",
                    "  PARSER_TABLES: Hash[Symbol, untyped]", "  DEBUG_ENABLED: bool", "",
                    "  def self.parser_tables: () -> Hash[Symbol, untyped]")
+        lines << "  SYMBOL_NAMES: Hash[Integer, String]" if @grammar.options[:cst] == true
+        lines << "  CST_METADATA: Hash[Symbol, untyped]" if @grammar.options[:cst] == true
+        lines << "  CST_KIND_MODEL: Ibex::Runtime::CST::Kind" if @grammar.options[:cst] == true
+        append_entry_contract(lines)
+        append_parameter_contract(lines)
+      end
+
+      # @rbs (Array[String] lines) -> void
+      def append_entry_contract(lines)
+        return unless @grammar.starts.length > 1
+
+        lines << "  ENTRY_STATES: Hash[Symbol, Integer]"
+        @grammar.starts.each { |name| lines << "  def parse_#{name}: () -> untyped" }
+      end
+
+      # @rbs (Array[String] lines) -> void
+      def append_parameter_contract(lines)
+        parameters = @grammar.parser_parameters
+        return if parameters.empty?
+
+        parameters.each do |parameter|
+          lines << "  @#{parameter[:name]}: #{parameter[:semantic_type] || 'untyped'}"
+        end
+        keywords = parameters.map do |parameter|
+          "#{parameter[:name]}: #{parameter[:semantic_type] || 'untyped'}"
+        end
+        lines << ""
+        lines << "  def initialize: (#{keywords.join(', ')}, **untyped) -> void"
+      end
+
+      # @rbs (Array[String] lines) -> void
+      def append_lexer_contract(lines)
+        lexer = @grammar.lexer
+        return unless lexer
+
+        lines.push(
+          "  LEXER_STATES: Array[String]",
+          "  LEXER_RULES_BY_STATE: Hash[Symbol, Array[Hash[Symbol, untyped]]]",
+          "",
+          "  def lex: (String | IO | Fiber source, ?file: String) -> self",
+          "  def parse: (String | IO | Fiber source, ?file: String) -> untyped",
+          "  def lexer_state: () -> Symbol",
+          "  def lexer_state=: (Symbol | String state) -> Symbol",
+          "  def next_token: () -> [untyped, untyped, Hash[Symbol, untyped]]"
+        )
+        if @grammar.options[:cst] == true
+          lines << "  def parse_with_syntax: (String | IO | Fiber source, ?file: String) -> " \
+                   "Ibex::Runtime::CST::ParseResult"
+        end
+        lexer.rules.each do |rule|
+          lines << "  private def _ibex_lexer_action_#{rule.id}: (String lexeme) -> untyped" if rule.action
+        end
+      end
+
+      # @rbs (Array[String] lines) -> void
+      def append_value_printer_signatures(lines)
+        @grammar.value_printers.each do |printer|
+          symbol = @grammar.symbol(printer[:symbol]) || raise(Ibex::Error, "missing printer symbol #{printer[:symbol]}")
+          lines << ""
+          lines << "  private def _ibex_value_printer_#{symbol.id}: " \
+                   "(#{symbol.semantic_type || 'untyped'} value) -> untyped"
+        end
+      end
+
+      # @rbs (Array[String] lines) -> void
+      def append_actions(lines)
+        actions = @grammar.productions.select { |production| action_method?(production) }
+        return if actions.empty?
+
+        lines << ""
+        actions.each do |production|
+          append_action_signature(lines, production)
+          append_composed_fragment_signatures(lines, production) if composed_action?(production)
+        end
+      end
+
+      # @rbs (Array[String] lines, IR::Production production) -> void
+      def append_action_signature(lines, production)
+        parameters = production.rhs.map { |symbol_id| semantic_type(symbol_id) }.join(", ")
+        result = production.node ? "AST::#{production.node.fetch(:name)}" : semantic_type(production.lhs)
+        if @generated_action_abi.positional_values?(production)
+          lines << "  private def _ibex_action_#{production.id}: (#{parameters}) -> #{result}"
+          return
+        end
+        if @generated_action_abi.values_only?(production)
+          lines << "  private def _ibex_action_#{production.id}: ([#{parameters}]) -> #{result}"
+          return
+        end
+
+        locations = Array.new(production.rhs.length, "untyped").join(", ")
+        lookahead = composed_action?(production) ? ", untyped" : ""
+        lines << "  private def _ibex_action_#{production.id}: " \
+                 "([#{parameters}], Array[untyped], [#{locations}], Array[untyped], " \
+                 "Ibex::Runtime::LocationSpan?#{lookahead}) -> #{result}"
+      end
+
+      # @rbs (Array[String] lines, IR::Production production) -> void
+      def append_composed_fragment_signatures(lines, production)
+        plan = production.action&.composition&.dig(:plan)
+        raise Ibex::Error, "missing action composition plan" unless plan
+
+        plan.fetch(:steps).each_with_index do |step, index|
+          next unless step[:code]
+
+          inputs = step.fetch(:inputs).map do |slot|
+            composition_slot_type(production, plan, slot, index)
+          end
+          locations = Array.new(inputs.length, "untyped").join(", ")
+          result = step[:result_type] || "untyped"
+          lines << "  private def _ibex_inline_fragment_#{production.id}_#{index}: " \
+                   "([#{inputs.join(', ')}], Array[untyped], [#{locations}], Array[untyped], " \
+                   "Ibex::Runtime::LocationSpan?) -> #{result}"
+        end
+      end
+
+      # @rbs (IR::Production production, IR::action_composition_plan plan, Integer slot, Integer step_index) -> String
+      def composition_slot_type(production, plan, slot, step_index)
+        physical = plan.fetch(:physical)
+        return semantic_type(production.rhs.fetch(slot)) if slot < physical
+
+        prior_index = slot - physical
+        raise Ibex::Error, "invalid composed action slot #{slot}" if prior_index >= step_index
+
+        plan.fetch(:steps).fetch(prior_index)[:result_type] || "untyped"
+      end
+
+      # @rbs (IR::Production production) -> bool
+      def action_method?(production)
+        !!(production.node || production.action || !@omit_action_call)
+      end
+
+      # @rbs (IR::Production production) -> bool
+      def composed_action?(production)
+        production.action&.composition&.dig(:plan, :version) == 1
+      end
+
+      # @rbs (Integer symbol_id) -> String
+      def semantic_type(symbol_id)
+        symbol = @grammar.symbol_by_id(symbol_id) || raise(Ibex::Error, "missing grammar symbol id #{symbol_id}")
+        symbol.semantic_type || inferred_ast_type(symbol_id) || "untyped"
+      end
+
+      # @rbs (Integer symbol_id) -> String?
+      def inferred_ast_type(symbol_id)
+        productions = @grammar.productions.select { |production| production.lhs == symbol_id }
+        return if productions.empty? || productions.any? { |production| production.node.nil? }
+
+        names = productions.map { |production| production.node&.fetch(:name) }.compact.uniq
+        return if names.empty?
+
+        types = names.map { |name| "AST::#{name}" }
+        types.length == 1 ? types.fetch(0) : "(#{types.join(' | ')})"
       end
 
       # @rbs () -> [Array[String], String]
@@ -50,5 +378,6 @@ module Ibex
         @automaton.grammar_digest.delete_prefix("sha256:").chars.first(12).join
       end
     end
+    # rubocop:enable Metrics/ClassLength
   end
 end

data/lib/ibex/codegen/report.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../lalr"
 require_relative "symbol_labels"
 
 module Ibex
@@ -11,23 +12,25 @@ module Ibex
       #     Array[String] lines,
       #     IR::AutomatonState state,
       #     IR::Grammar grammar,
-      #     Array[IR::counterexample] examples,
+      #     Array[LALR::search_counterexample] examples,
       #     Hash[Integer, String] labels
       #   ) -> void
       #   private def self.append_state: (
       #     Array[String] lines,
       #     IR::AutomatonState state,
       #     IR::Grammar grammar,
-      #     Array[IR::counterexample] examples,
+      #     Array[LALR::search_counterexample] examples,
       #     Hash[Integer, String] labels
       #   ) -> void
-      #   private def append_counterexample: (Array[String] lines, IR::counterexample example, IR::Grammar grammar,
+      #   private def append_counterexample: (Array[String] lines, LALR::search_counterexample example,
+      #     IR::Grammar grammar,
       #     Hash[Integer, String] labels) -> void
-      #   private def self.append_counterexample: (Array[String] lines, IR::counterexample example, IR::Grammar grammar,
+      #   private def self.append_counterexample: (Array[String] lines, LALR::search_counterexample example,
+      #     IR::Grammar grammar,
       #     Hash[Integer, String] labels) -> void
-      #   private def append_tree: (Array[String] lines, untyped tree, String indentation, IR::Grammar grammar,
+      #   private def append_tree: (Array[String] lines, Object? tree, String indentation, IR::Grammar grammar,
       #     Hash[Integer, String] labels) -> void
-      #   private def self.append_tree: (Array[String] lines, untyped tree, String indentation, IR::Grammar grammar,
+      #   private def self.append_tree: (Array[String] lines, Object? tree, String indentation, IR::Grammar grammar,
       #     Hash[Integer, String] labels) -> void
       #   private def format_item: (IR::AutomatonItem item, IR::Grammar grammar, Hash[Integer, String] labels) -> String
       #   private def self.format_item: (IR::AutomatonItem item, IR::Grammar grammar,
@@ -36,15 +39,24 @@ module Ibex
       #   private def self.format_action: (IR::parser_action action) -> String
       #   private def symbol_name: (Hash[Integer, String] labels, Integer id) -> String
       #   private def self.symbol_name: (Hash[Integer, String] labels, Integer id) -> String
-      #   private def tree_label: (IR::Grammar grammar, Hash[Integer, String] labels, untyped value) -> untyped
-      #   private def self.tree_label: (IR::Grammar grammar, Hash[Integer, String] labels, untyped value) -> untyped
+      #   private def tree_label: (IR::Grammar grammar, Hash[Integer, String] labels, Object? value) -> Object?
+      #   private def self.tree_label: (IR::Grammar grammar, Hash[Integer, String] labels, Object? value) -> Object?
+      #   private def display_conflict: (IR::conflict conflict, IR::Grammar grammar,
+      #     Hash[Integer, String] labels) -> Hash[Symbol, Object?]
+      #   private def self.display_conflict: (IR::conflict conflict, IR::Grammar grammar,
+      #     Hash[Integer, String] labels) -> Hash[Symbol, Object?]
+      #   private def format_value: (Object? value) -> String
+      #   private def self.format_value: (Object? value) -> String
 
       # @rbs (IR::Automaton automaton, ?max_tokens: Integer, ?max_configurations: Integer) -> String
       def render(automaton, max_tokens: LALR::Counterexample::DEFAULT_MAX_TOKENS,
                  max_configurations: LALR::Counterexample::DEFAULT_MAX_CONFIGURATIONS)
         grammar = automaton.grammar
         labels = SymbolLabels.build(grammar)
-        lines = ["Algorithm: #{automaton.algorithm}", "States: #{automaton.states.length}", ""]
+        entries = automaton.entry_states.map { |name, state| "#{name}=#{state}" }.join(", ")
+        lines = ["Algorithm: #{automaton.algorithm}", "States: #{automaton.states.length}"]
+        lines << "Entries: #{entries}" if automaton.entry_states.length > 1
+        lines << ""
         examples = LALR::Counterexample.new(
           automaton, max_tokens: max_tokens, max_configurations: max_configurations
         ).all.group_by { |example| example[:state] }
@@ -69,16 +81,29 @@ module Ibex
         end
         lines << "  default: #{format_action(state.default_action)}" if state.default_action
         state.gotos.each { |symbol_id, target| lines << "  goto #{symbol_name(labels, symbol_id)}: #{target}" }
-        state.conflicts.each { |conflict| lines << "  conflict: #{conflict.inspect}" }
+        state.conflicts.each do |conflict|
+          lines << "  conflict: #{format_value(display_conflict(conflict, grammar, labels))}"
+        end
         examples.each { |example| append_counterexample(lines, example, grammar, labels) }
         lines << ""
       end
 
       # @rbs skip
       def append_counterexample(lines, example, grammar, labels)
-        label = example[:unifying] ? "unifying counterexample" : "nonunifying witness"
-        sentence = example[:sentence].dup.insert(example[:lookahead_index], "•").join(" ")
+        label = if example[:unifying]
+                  "unifying counterexample"
+                elsif example[:inconclusive]
+                  "inconclusive reachability witness"
+                else
+                  "nonunifying witness"
+                end
+        sentence = example[:sentence].map { |name| tree_label(grammar, labels, name) }
+        sentence = sentence.dup.insert(example[:lookahead_index], "•").join(" ")
         lines << "  #{label}: #{sentence}"
+        if example[:inconclusive]
+          search = example.fetch(:search)
+          lines << "    search exhausted after #{search.fetch(:explored)} configurations; no classification was made"
+        end
         example[:interpretations].each do |interpretation|
           lines << "    #{interpretation[:kind]} derivation:"
           append_tree(lines, interpretation[:tree], "      ", grammar, labels)
@@ -106,7 +131,8 @@ module Ibex
       # @rbs skip
       def format_item(item, grammar, labels)
         if item.production == LALR::Builder::AUGMENTED_PRODUCTION
-          rhs = [grammar.start]
+          start = grammar.symbol(grammar.start)
+          rhs = [grammar.starts.one? && start ? symbol_name(labels, start.id) : "<entry>"]
           lhs = "$accept"
         else
           production = grammar.productions.fetch(item.production)
@@ -137,12 +163,38 @@ module Ibex
         symbol = grammar.symbol(value.to_s)
         symbol ? symbol_name(labels, symbol.id) : value
       end
+
+      # @rbs skip
+      def display_conflict(conflict, grammar, labels)
+        symbol = grammar.symbol(conflict[:symbol])
+        return conflict unless symbol
+
+        conflict.merge(symbol: symbol_name(labels, symbol.id))
+      end
+
+      # Keep reports byte-identical across Ruby versions whose Hash#inspect
+      # formatting differs for symbol keys.
+      # @rbs skip
+      def format_value(value)
+        case value
+        when Hash
+          entries = value.map do |key, item|
+            label = key.is_a?(Symbol) ? "#{key}:" : "#{format_value(key)} =>"
+            "#{label} #{format_value(item)}"
+          end
+          "{#{entries.join(', ')}}"
+        when Array
+          "[#{value.map { |item| format_value(item) }.join(', ')}]"
+        else
+          value.inspect
+        end
+      end
       module_function :append_state, :append_counterexample, :append_tree, :format_item, :format_action, :symbol_name,
-                      :tree_label
+                      :tree_label, :display_conflict, :format_value
 
       class << self
         private :append_state, :append_counterexample, :append_tree, :format_item, :format_action, :symbol_name,
-                :tree_label
+                :tree_label, :display_conflict, :format_value
       end
     end
   end