ibex 0.1.0 → 0.2.0

data/lib/ibex/codegen/action_locations.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module Codegen
+    # Rewrites semantic-location expressions without touching Ruby literals,
+    # comments, heredoc bodies, or ordinary instance/class variables.
+    class ActionLocations
+      SEMANTIC_HELPERS = %w[loc result_loc].freeze #: Array[String]
+      private_constant :SEMANTIC_HELPERS
+
+      # @rbs @source: String
+      # @rbs @maximum: Integer
+      # @rbs @location: IR::location
+
+      # @rbs (String source, maximum: Integer, location: IR::location) -> void
+      def initialize(source, maximum:, location:)
+        @source = source
+        @maximum = maximum
+        @location = location
+      end
+
+      # @rbs () -> String
+      def rewrite
+        rewritten = mutable_binary_source_copy
+        return rewritten.force_encoding(@source.encoding) unless @source.match?(/@(?:\$|\d+)/)
+
+        replacements = semantic_references.map do |offset, spelling|
+          [offset, spelling.bytesize, replacement_for(spelling)]
+        end #: Array[[Integer, Integer, String]]
+        replacements.reverse_each { |offset, length, replacement| rewritten[offset, length] = replacement }
+        rewritten.force_encoding(@source.encoding)
+      end
+
+      # @rbs () -> bool
+      def references?
+        return true if @source.match?(/@(?:\$|\d+)/) && !semantic_references.empty?
+        return false unless @source.match?(/\b(?:loc|result_loc)\b/)
+
+        require "ripper"
+        tokens = Object.const_get(:Ripper).__send__(:lex, lexable_source)
+        tokens.any? do |_position, type, token, _state|
+          type == :on_ident && SEMANTIC_HELPERS.include?(token)
+        end
+      end
+
+      private
+
+      # @rbs () -> String
+      def mutable_binary_source_copy
+        String.new(encoding: Encoding::BINARY).replace(@source.b)
+      end
+
+      # @rbs () -> Array[[Integer, String]]
+      def semantic_references
+        require "ripper"
+
+        offsets = line_offsets
+        tokens = Object.const_get(:Ripper).__send__(:lex, lexable_source)
+        # @type var tokens: Array[[[Integer, Integer], Symbol, String, untyped]]
+        tokens.filter_map do |position, type, _token, _state|
+          next unless type == :on_ident
+
+          line, column = position
+          offset = offsets.fetch(line - 1) + column
+          spelling = @source.b.byteslice(offset..)&.match(/\A@(?:\$|\d+)/)&.[](0)
+          [offset, spelling] if spelling
+        end
+      end
+
+      # Give every Ripper implementation syntactically valid input while
+      # preserving byte offsets and lexical string/comment boundaries.
+      # @rbs () -> String
+      def lexable_source
+        @source.gsub(/@(?:\$|\d+)/) { |spelling| "_" * spelling.bytesize }
+      end
+
+      # @rbs () -> Array[Integer]
+      def line_offsets
+        offset = 0
+        @source.lines.map do |line|
+          current = offset
+          offset += line.bytesize
+          current
+        end
+      end
+
+      # @rbs (String spelling) -> String
+      def replacement_for(spelling)
+        return "_ibex_location" if spelling == "@$"
+
+        index = spelling.delete_prefix("@").to_i
+        if index.zero? || index > @maximum
+          raise Ibex::Error,
+                "#{@location[:file]}:#{@location[:line]}:#{@location[:column]}: " \
+                "semantic location #{spelling} is outside 1..#{@maximum}"
+        end
+
+        "_ibex_locations[#{index - 1}]"
+      end
+    end
+  end
+end

data/lib/ibex/codegen/action_method_source.rb

@@ -0,0 +1,210 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "action_locations"
+require_relative "generated_action_abi"
+
+module Ibex
+  module Codegen
+    # Builds semantic method source shared by runtime and static shadow output.
+    class ActionMethodSource
+      # @rbs @grammar: IR::Grammar
+      # @rbs @generated_action_abi: GeneratedActionABI::Cache
+
+      # @rbs (IR::Grammar grammar, ?generated_action_abi: GeneratedActionABI::Cache?) -> void
+      def initialize(grammar, generated_action_abi: nil)
+        @grammar = grammar
+        @generated_action_abi = generated_action_abi || GeneratedActionABI::Cache.new
+      end
+
+      # @rbs (IR::Production production, Hash[Symbol, untyped] step, Integer index) -> String
+      def composed_fragment_method_source(production, step, index)
+        source = "private def #{composed_fragment_name(production, index)}" \
+                 "(val, _values, _ibex_locations, _ibex_location_stack, _ibex_location); "
+        append_parameter_values(source)
+        context_length = step.fetch(:context_length)
+        if context_length.positive?
+          source << "val = _values.last(#{context_length}); "
+          source << "_ibex_locations = _ibex_location_stack.last(#{context_length}); "
+        end
+        named_refs = step.fetch(:named_refs)
+        named_refs.each { |reference| source << "#{reference[:name]} = val[#{reference[:index]}]; " }
+        append_named_values(source, named_refs)
+        source << "result = val[0]; " if step.fetch(:result_var)
+        source << composed_semantic_code(step)
+        source << "\nresult" if step.fetch(:result_var)
+        source << "\nend"
+      end
+
+      # @rbs (IR::Production production, Integer index) -> String
+      def composed_fragment_name(production, index)
+        "_ibex_inline_fragment_#{production.id}_#{index}"
+      end
+
+      # @rbs (IR::Production production) -> String
+      def compiled_action_method_source(production)
+        action = production.action
+        source = "private def _ibex_action_#{production.id}#{action_parameters(production)}; "
+        append_parameter_values(source, production: production)
+        return "#{source}#{default_value_expression(production)}\nend" unless action
+
+        if action.context_length.positive?
+          source << "val = _values.last(#{action.context_length}); "
+          source << "_ibex_locations = _ibex_location_stack.last(#{action.context_length}); "
+        end
+        action.named_refs.each do |reference|
+          source << "#{reference[:name]} = #{value_expression(production, reference[:index])}; "
+        end
+        append_named_values(source, action.named_refs)
+        append_action_body(source, production, action)
+        source << "\nend"
+      end
+
+      # @rbs (IR::Production production) -> String
+      def direct_action_method_source(production)
+        action = production.action
+        lines = [
+          "private def _ibex_action_#{production.id}#{action_parameters(production)}"
+        ]
+        append_direct_parameter_values(lines, production: production)
+        if action&.context_length&.positive?
+          lines << "  val = _values.last(#{action.context_length})"
+          lines << "  _ibex_locations = _ibex_location_stack.last(#{action.context_length})"
+        end
+        if action&.named_refs&.any?
+          action.named_refs.each do |reference|
+            lines << "  #{reference[:name]} = #{value_expression(production, reference[:index])}"
+          end
+          names = action.named_refs.map { |reference| reference[:name] }
+          lines << "  _ibex_named_values = [#{names.join(', ')}]"
+        end
+        append_direct_action_body(lines, production, action)
+        lines << "end"
+        lines.join("\n")
+      end
+
+      # @rbs (Integer symbol_id, IR::value_printer printer) -> String
+      def value_printer_method_source(symbol_id, printer)
+        source = "private def _ibex_value_printer_#{symbol_id}(value); "
+        append_parameter_values(source)
+        source << printer[:code]
+        source << "\nend"
+      end
+
+      # @rbs (String source) -> bool
+      def column_sensitive?(source)
+        return false unless source.include?("<<")
+
+        require "ripper"
+        tokens = Object.const_get(:Ripper).__send__(:lex, source)
+        # @type var tokens: Array[[[Integer, Integer], Symbol, String, untyped]]
+        tokens.any? { |_position, event, _token, _state| event == :on_heredoc_beg }
+      end
+
+      private
+
+      # @rbs (IR::Production production) -> String
+      def action_parameters(production)
+        if @generated_action_abi.positional_values?(production)
+          parameters = Array.new(production.rhs.length) { |index| "v#{index}" }
+          return "(#{parameters.join(', ')})"
+        end
+        return "(val)" if @generated_action_abi.values_only?(production)
+
+        "(val, _values, _ibex_locations, _ibex_location_stack, _ibex_location)"
+      end
+
+      # @rbs (IR::Production production, Integer index) -> String
+      def value_expression(production, index)
+        return "v#{index}" if @generated_action_abi.positional_values?(production)
+
+        "val[#{index}]"
+      end
+
+      # @rbs (String source, ?production: IR::Production?) -> void
+      def append_parameter_values(source, production: nil)
+        excluded = if production && @generated_action_abi.positional_values?(production)
+                     Array.new(production.rhs.length) { |index| "v#{index}" }
+                   else
+                     [] #: Array[String]
+                   end
+        @grammar.parser_parameters.each do |parameter|
+          next if excluded.include?(parameter[:name])
+
+          source << "#{parameter[:name]} = @#{parameter[:name]}; "
+        end
+      end
+
+      # @rbs (Array[String] lines, production: IR::Production) -> void
+      def append_direct_parameter_values(lines, production:)
+        excluded = if @generated_action_abi.positional_values?(production)
+                     Array.new(production.rhs.length) { |index| "v#{index}" }
+                   else
+                     [] #: Array[String]
+                   end
+        @grammar.parser_parameters.each do |parameter|
+          next if excluded.include?(parameter[:name])
+
+          lines << "  #{parameter[:name]} = @#{parameter[:name]}"
+        end
+      end
+
+      # @rbs (Hash[Symbol, untyped] step) -> String
+      def composed_semantic_code(step)
+        maximum = [step.fetch(:inputs).length, step.fetch(:context_length)].max
+        ActionLocations.new(
+          step.fetch(:code), maximum: maximum, location: step.fetch(:loc)
+        ).rewrite
+      end
+
+      # @rbs (String source, Array[IR::named_ref] named_refs) -> void
+      def append_named_values(source, named_refs)
+        return if named_refs.empty?
+
+        names = named_refs.map { |reference| reference[:name] }
+        source << "_ibex_named_values = [#{names.join(', ')}]; "
+      end
+
+      # @rbs (String source, IR::Production production, IR::Action action) -> void
+      def append_action_body(source, production, action)
+        source << "result = #{default_value_expression(production)}; " if @grammar.options[:result_var]
+        source << semantic_action_code(production, action)
+        source << "\nresult" if @grammar.options[:result_var]
+      end
+
+      # @rbs (Array[String] lines, IR::Production production, IR::Action? action) -> void
+      def append_direct_action_body(lines, production, action)
+        unless action
+          lines << "  #{default_value_expression(production)}"
+          return
+        end
+
+        lines << "  result = #{default_value_expression(production)}" if @grammar.options[:result_var]
+        semantic_code = semantic_action_code(production, action)
+        if column_sensitive?(semantic_code)
+          lines << semantic_code
+        else
+          semantic_code.lines.each { |line| lines << "  #{line.rstrip}" }
+        end
+        lines << "  result" if @grammar.options[:result_var]
+      end
+
+      # @rbs (IR::Production production, IR::Action action) -> String
+      def semantic_action_code(production, action)
+        positional = @generated_action_abi.positional_action_source(production)
+        return positional unless positional.nil?
+
+        maximum = action.context_length.positive? ? action.context_length : production.rhs.length
+        ActionLocations.new(action.code, maximum: maximum, location: action.location).rewrite
+      end
+
+      # @rbs (IR::Production production) -> String
+      def default_value_expression(production)
+        return "nil" if production.rhs.empty?
+        return "v0" if @generated_action_abi.positional_values?(production)
+
+        "val[0]"
+      end
+    end
+  end
+end

data/lib/ibex/codegen/action_source.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "action_method_source"
+require_relative "ruby_ast"
+
+module Ibex
+  module Codegen
+    # Generates static-check-only Ruby containing semantic action methods.
+    class ActionSource
+      include RubyAST
+
+      # @rbs @automaton: IR::Automaton
+      # @rbs @grammar: IR::Grammar
+      # @rbs @omit_action_call: bool
+      # @rbs @method_source: ActionMethodSource
+
+      # @rbs (IR::Automaton automaton, ?omit_action_call: bool?) -> void
+      def initialize(automaton, omit_action_call: nil)
+        @automaton = automaton
+        @grammar = automaton.grammar
+        @omit_action_call = omit_action_call.nil? ? @grammar.options[:omit_action_call] : omit_action_call
+        @method_source = ActionMethodSource.new(@grammar)
+      end
+
+      # @rbs () -> String
+      def generate
+        lines = [
+          "# frozen_string_literal: true",
+          "# Generated by Ibex #{grammar_digest_comment}.",
+          "# Static-check-only semantic action shadow source.",
+          "# DO NOT LOAD OR EXECUTE: runtime wiring and user code are intentionally absent.",
+          ""
+        ]
+        modules, class_name = class_parts
+        modules.each { |name| lines << "module #{name}" }
+        lines << "class #{class_name}"
+        append_ast(lines)
+        append_value_printers(lines)
+        append_actions(lines)
+        lines << "end"
+        modules.reverse_each { lines << "end" }
+        "#{lines.join("\n")}\n"
+      end
+
+      private
+
+      # @rbs (Array[String] lines) -> void
+      def append_value_printers(lines)
+        @grammar.value_printers.each do |printer|
+          symbol = @grammar.symbol(printer[:symbol]) || raise(Ibex::Error, "missing printer symbol #{printer[:symbol]}")
+          append_method(lines, printer[:loc], @method_source.value_printer_method_source(symbol.id, printer))
+        end
+      end
+
+      # @rbs (Array[String] lines) -> void
+      def append_actions(lines)
+        @grammar.productions.each do |production|
+          if production.node
+            append_method(lines, production.node.fetch(:loc), ast_node_action_source(production))
+          elsif composed_action?(production)
+            append_composed_actions(lines, production)
+          elsif production.action || !@omit_action_call
+            append_action(lines, production)
+          end
+        end
+      end
+
+      # @rbs (Array[String] lines, IR::Production production) -> void
+      def append_composed_actions(lines, production)
+        plan = production.action&.composition&.dig(:plan)
+        raise Ibex::Error, "missing action composition plan" unless plan
+
+        plan.fetch(:steps).each_with_index do |step, index|
+          next unless step[:code]
+
+          append_method(
+            lines, step.fetch(:loc),
+            @method_source.composed_fragment_method_source(production, step, index)
+          )
+        end
+      end
+
+      # @rbs (Array[String] lines, IR::Production production) -> void
+      def append_action(lines, production)
+        location = production.action&.location || production.origin.fetch(:loc)
+        append_method(lines, location, @method_source.compiled_action_method_source(production))
+      end
+
+      # @rbs (Array[String] lines, IR::location location, String source) -> void
+      def append_method(lines, location, source)
+        lines << ""
+        lines << "  # Grammar action: #{location.fetch(:file).inspect}:" \
+                 "#{location.fetch(:line)}:#{location.fetch(:column)}"
+        # Keep action bytes at column zero. Ruby lexical nesting does not require
+        # indentation, while changing it can invalidate <<HEREDOC terminators or
+        # alter <<~HEREDOC dedentation.
+        lines << source
+      end
+
+      # @rbs (IR::Production production) -> bool
+      def composed_action?(production)
+        production.action&.composition&.dig(:plan, :version) == 1
+      end
+
+      # @rbs () -> [Array[String], String]
+      def class_parts
+        parts = @grammar.class_name.split("::")
+        [parts.take(parts.length - 1), parts.fetch(-1)]
+      end
+
+      # @rbs () -> String
+      def grammar_digest_comment
+        @automaton.grammar_digest.delete_prefix("sha256:").chars.first(12).join
+      end
+    end
+  end
+end

data/lib/ibex/codegen/ambiguity.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "../lalr"
+
+module Ibex
+  module Codegen
+    # @rbs!
+    #   type ambiguity_search = {
+    #     max_tokens: Integer,
+    #     max_configurations: Integer
+    #   }
+    #   type ambiguity_summary = {
+    #     conflicts: Integer,
+    #     ambiguous: Integer,
+    #     not_found: Integer,
+    #     configuration_budget_exhausted: Integer
+    #   }
+    #   type ambiguity_check = {
+    #     state: Integer,
+    #     type: String,
+    #     token: String,
+    #     result: String,
+    #     sentence: Array[String],
+    #     lookahead_index: Integer?,
+    #     interpretations: Array[IR::interpretation],
+    #     explored_configurations: Integer,
+    #     configuration_budget_exhausted: bool
+    #   }
+    #   type ambiguity_document = {
+    #     ibex_check: String,
+    #     schema_version: Integer,
+    #     algorithm: String,
+    #     search: ambiguity_search,
+    #     status: String,
+    #     summary: ambiguity_summary,
+    #     conflicts: Array[ambiguity_check]
+    #   }
+
+    # Renders a bounded ambiguity search over every parser conflict.
+    class Ambiguity
+      SCHEMA_VERSION = 1 #: Integer
+
+      # @rbs @automaton: IR::Automaton
+      # @rbs @grammar: IR::Grammar
+      # @rbs @max_tokens: Integer
+      # @rbs @max_configurations: Integer
+      # @rbs @checks: Array[ambiguity_check]
+
+      # @rbs (IR::Automaton automaton, ?max_tokens: Integer, ?max_configurations: Integer) -> void
+      def initialize(
+        automaton,
+        max_tokens: LALR::Counterexample::DEFAULT_MAX_TOKENS,
+        max_configurations: LALR::Counterexample::DEFAULT_MAX_CONFIGURATIONS
+      )
+        LALR::ConflictSearchLimits.validate!(
+          max_tokens: max_tokens, max_configurations: max_configurations
+        )
+        @automaton = automaton
+        @grammar = automaton.grammar
+        @max_tokens = max_tokens
+        @max_configurations = max_configurations
+        @checks = check_conflicts
+      end
+
+      # @rbs () -> Integer
+      def exit_status
+        return 1 if status == "ambiguous"
+        return 2 if status == "inconclusive"
+
+        0
+      end
+
+      # @rbs () -> ambiguity_document
+      def to_h
+        {
+          ibex_check: "ambiguity",
+          schema_version: SCHEMA_VERSION,
+          algorithm: @automaton.algorithm,
+          search: { max_tokens: @max_tokens, max_configurations: @max_configurations },
+          status: status,
+          summary: summary,
+          conflicts: @checks
+        }
+      end
+
+      # @rbs () -> String
+      def render_text
+        counts = summary
+        lines = [
+          "Ibex ambiguity check v#{SCHEMA_VERSION}",
+          "Algorithm: #{@automaton.algorithm}",
+          "Search budget: #{@max_tokens} tokens, #{@max_configurations} configurations per conflict",
+          "Result: #{status}",
+          "Conflicts: #{counts.fetch(:conflicts)}, ambiguous: #{counts.fetch(:ambiguous)}, " \
+          "exhausted: #{counts.fetch(:configuration_budget_exhausted)}"
+        ]
+        @checks.each_with_index { |check, index| append_check(lines, check, index + 1) }
+        "#{lines.join("\n")}\n"
+      end
+
+      private
+
+      # @rbs () -> String
+      def status
+        return "ambiguous" if @checks.any? { |check| check.fetch(:result) == "ambiguous" }
+        return "inconclusive" if @checks.any? { |check| check.fetch(:configuration_budget_exhausted) }
+
+        "no_ambiguity_found_within_bounds"
+      end
+
+      # @rbs () -> ambiguity_summary
+      def summary
+        {
+          conflicts: @checks.length,
+          ambiguous: @checks.count { |check| check.fetch(:result) == "ambiguous" },
+          not_found: @checks.count { |check| check.fetch(:result) == "not_found" },
+          configuration_budget_exhausted: @checks.count do |check|
+            check.fetch(:configuration_budget_exhausted)
+          end
+        }
+      end
+
+      # @rbs () -> Array[ambiguity_check]
+      def check_conflicts
+        @automaton.states.flat_map do |state|
+          state.conflicts.map { |conflict| check_conflict(state, conflict) }
+        end
+      end
+
+      # @rbs (IR::AutomatonState state, IR::conflict conflict) -> ambiguity_check
+      def check_conflict(state, conflict)
+        search = LALR::ConflictSearch.new(
+          @automaton, state, conflict,
+          max_tokens: @max_tokens, max_configurations: @max_configurations
+        )
+        result = search.call
+        {
+          state: state.id,
+          type: conflict.fetch(:type).to_s,
+          token: conflict.fetch(:symbol),
+          result: result ? "ambiguous" : "not_found",
+          sentence: result ? symbol_names(result.fetch(:sentence_ids)) : [],
+          lookahead_index: result&.fetch(:lookahead_index),
+          interpretations: result ? result.fetch(:interpretations) : [],
+          explored_configurations: search.explored,
+          configuration_budget_exhausted: search.exhausted?
+        }
+      end
+
+      # @rbs (Array[Integer] ids) -> Array[String]
+      def symbol_names(ids)
+        ids.map do |id|
+          symbol = @grammar.symbol_by_id(id) || raise(Ibex::Error, "missing grammar symbol id #{id}")
+          symbol.display_name || symbol.name
+        end
+      end
+
+      # @rbs (Array[String] lines, ambiguity_check check, Integer number) -> void
+      def append_check(lines, check, number)
+        lines << ""
+        lines << "#{number}. State #{check.fetch(:state)}, #{check.fetch(:type)}, token #{check.fetch(:token)}"
+        lines << "   Result: #{check.fetch(:result)}"
+        sentence = check.fetch(:sentence)
+        lines << "   Sentence: #{sentence.join(' ')}" unless sentence.empty?
+        lines << "   Explored configurations: #{check.fetch(:explored_configurations)}"
+        return unless check.fetch(:configuration_budget_exhausted)
+
+        lines << "   Configuration budget exhausted before a counterexample was found."
+      end
+    end
+  end
+end