ibex 0.1.0 → 0.3.0

data/lib/ibex/table_simulation/simulator.rb

@@ -0,0 +1,253 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "../error"
+
+module Ibex
+  module TableSimulation
+    # Deterministic, semantic-action-free table interpreter.
+    class Simulator
+      DEFAULT_MAX_STEPS = 100_000 #: Integer
+      DEFAULT_MAX_STACK = 10_000 #: Integer
+      EOF_NAME = "$eof" #: String
+      ERROR_NAME = "error" #: String
+      IMPLICIT_ERROR = { type: :error }.freeze #: IR::error_action
+
+      attr_reader :automaton #: IR::Automaton
+      attr_reader :max_steps #: Integer
+      attr_reader :max_stack #: Integer
+      attr_reader :eof #: IR::GrammarSymbol
+
+      # @rbs (IR::Automaton automaton, ?max_steps: Integer, ?max_stack: Integer) -> void
+      def initialize(automaton, max_steps: DEFAULT_MAX_STEPS, max_stack: DEFAULT_MAX_STACK)
+        raise ArgumentError, "max_steps must be positive" unless max_steps.positive?
+        raise ArgumentError, "max_stack must be positive" unless max_stack.positive?
+
+        @automaton = automaton
+        @max_steps = max_steps
+        @max_stack = max_stack
+        @terminals = automaton.grammar.terminals.to_h { |symbol| [symbol.name, symbol] }.freeze
+        @terminal_aliases = terminal_aliases(automaton.grammar.terminals).freeze
+        @eof = @terminals[EOF_NAME] || raise(Ibex::Error, "(debug):1:1: automaton has no $eof terminal")
+      end
+
+      # @rbs () -> Session
+      def start
+        Session.new(self)
+      end
+
+      # @rbs (Array[String] tokens) -> Result
+      def simulate(tokens)
+        session = start
+        tokens.each do |token|
+          session.push(token)
+          break if session.status
+        end
+        session.finish
+      end
+
+      # @rbs (String spelling) -> IR::GrammarSymbol
+      def terminal(spelling)
+        if [EOF_NAME, ERROR_NAME].include?(spelling)
+          raise Ibex::Error, "(debug):1:1: reserved terminal #{spelling.inspect} cannot be supplied"
+        end
+
+        terminal = @terminals[spelling]
+        return terminal if terminal
+
+        empty = [] #: Array[IR::GrammarSymbol]
+        aliases = @terminal_aliases.fetch(spelling, empty)
+        return aliases.first if aliases.length == 1
+
+        if aliases.length > 1
+          names = aliases.map(&:name).sort.join(", ")
+          raise Ibex::Error, "(debug):1:1: terminal display name #{spelling.inspect} is ambiguous: #{names}"
+        end
+
+        raise Ibex::Error, "(debug):1:1: unknown terminal #{spelling.inspect}"
+      end
+
+      private
+
+      # @rbs (Array[IR::GrammarSymbol] terminals) -> Hash[String, Array[IR::GrammarSymbol]]
+      def terminal_aliases(terminals)
+        aliases = Hash.new { |hash, key| hash[key] = [] } #: Hash[String, Array[IR::GrammarSymbol]]
+        terminals.each do |terminal|
+          aliases[terminal.display_name] << terminal if terminal.display_name
+        end
+        aliases.each_value(&:freeze)
+        aliases
+      end
+    end
+
+    # Stateful token-at-a-time table simulation session.
+    class Session
+      # @rbs @steps: Array[Step]
+      # @rbs @tokens: Array[String]
+
+      attr_reader :status #: Symbol?
+
+      # @rbs (Simulator simulator) -> void
+      def initialize(simulator)
+        @simulator = simulator
+        @state_stack = [0] #: Array[Integer]
+        @status = nil
+        @steps = []
+        @tokens = []
+        @finished = false
+      end
+
+      # @rbs (String spelling) -> Array[Step]
+      def push(spelling)
+        ensure_active!
+        terminal = @simulator.terminal(spelling)
+        @tokens << terminal.name
+        process(terminal, eof: false)
+      end
+
+      # @rbs () -> Array[Step]
+      def steps = @steps.dup.freeze
+
+      # @rbs () -> Array[String]
+      def tokens = @tokens.dup.freeze
+
+      # @rbs () -> Result
+      def finish
+        raise Ibex::Error, "(debug):1:1: simulation session is already finished" if @finished
+
+        @finished = true
+        process(@simulator.eof, eof: true) until @status
+        Result.new(
+          grammar_digest: @simulator.automaton.grammar_digest,
+          algorithm: @simulator.automaton.algorithm,
+          tokens: @tokens,
+          status: @status || raise(Ibex::Error, "(debug):1:1: simulation did not terminate"),
+          steps: @steps
+        )
+      end
+
+      private
+
+      # @rbs (IR::GrammarSymbol terminal, eof: bool) -> Array[Step]
+      def process(terminal, eof:)
+        first = @steps.length
+        loop do
+          state = current_state
+          action, source = action_for(state, terminal.id)
+          case action.fetch(:type)
+          when :shift
+            record_shift(state, terminal, action, source)
+            next if eof
+
+            break
+          when :reduce then record_reduce(state, terminal, action, source)
+          when :accept
+            record_terminal(state, terminal, "accept", source)
+            @status = :accepted
+            break
+          when :error
+            record_terminal(state, terminal, "error", source)
+            @status = :error
+            break
+          else raise Ibex::Error, "(debug):1:1: unknown table action #{action.inspect}"
+          end
+        end
+        @steps.drop(first).freeze
+      end
+
+      # @rbs (IR::AutomatonState state, Integer token_id) -> [IR::parser_action, String]
+      def action_for(state, token_id)
+        return [state.actions.fetch(token_id), "explicit"] if state.actions.key?(token_id)
+        return [state.default_action, "default"] if state.default_action
+
+        [Simulator::IMPLICIT_ERROR, "implicit"]
+      end
+
+      # @rbs (IR::AutomatonState state, IR::GrammarSymbol terminal, IR::parser_action action, String source) -> void
+      def record_shift(state, terminal, action, source)
+        shift = action #: IR::shift_action
+        before = @state_stack.length
+        target = shift.fetch(:state)
+        @state_stack << target
+        enforce_stack_budget!
+        append_step(state, terminal, "shift", source, target_state: target, before: before)
+      end
+
+      # @rbs (IR::AutomatonState state, IR::GrammarSymbol terminal, IR::parser_action action, String source) -> void
+      def record_reduce(state, terminal, action, source)
+        reduction = action #: IR::reduce_action
+        production_id = reduction.fetch(:production)
+        production = @simulator.automaton.grammar.productions.fetch(production_id)
+        length = production.rhs.length
+        before = @state_stack.length
+        if length >= before
+          raise Ibex::Error, "(debug):1:1: production #{production_id} underflows the simulated state stack"
+        end
+
+        @state_stack.pop(length)
+        goto = current_state.gotos[production.lhs]
+        unless goto
+          raise Ibex::Error, "(debug):1:1: missing goto for production #{production_id} from state #{current_state.id}"
+        end
+
+        @state_stack << goto
+        enforce_stack_budget!
+        lhs = @simulator.automaton.grammar.symbol_by_id(production.lhs)
+        raise Ibex::Error, "(debug):1:1: missing lhs symbol #{production.lhs}" unless lhs
+
+        append_step(
+          state, terminal, "reduce", source,
+          production_id: production_id, lhs: lhs.name, rhs_length: length,
+          target_state: goto, before: before
+        )
+      end
+
+      # @rbs (IR::AutomatonState state, IR::GrammarSymbol terminal, String action, String source) -> void
+      def record_terminal(state, terminal, action, source)
+        append_step(state, terminal, action, source, before: @state_stack.length)
+      end
+
+      # @rbs (IR::AutomatonState state, IR::GrammarSymbol terminal, String action, String source,
+      #   before: Integer, ?production_id: Integer?, ?lhs: String?, ?rhs_length: Integer?,
+      #   ?target_state: Integer?) -> void
+      def append_step(state, terminal, action, source, before:, production_id: nil, lhs: nil, rhs_length: nil,
+                      target_state: nil)
+        raise Ibex::Error, "(debug):1:1: simulation exceeded #{@simulator.max_steps} actions" \
+          if @steps.length >= @simulator.max_steps
+
+        @steps << Step.new(
+          sequence: @steps.length + 1,
+          state: state.id,
+          token_id: terminal.id,
+          token: terminal.display_name || terminal.name,
+          action: action,
+          action_source: source,
+          production_id: production_id,
+          lhs: lhs,
+          rhs_length: rhs_length,
+          target_state: target_state,
+          stack_depth_before: before,
+          stack_depth_after: @state_stack.length
+        )
+      end
+
+      # @rbs () -> IR::AutomatonState
+      def current_state
+        id = @state_stack.last || raise(Ibex::Error, "(debug):1:1: simulated state stack is empty")
+        @simulator.automaton.states.fetch(id)
+      end
+
+      # @rbs () -> void
+      def enforce_stack_budget!
+        return if @state_stack.length <= @simulator.max_stack
+
+        raise Ibex::Error, "(debug):1:1: simulation exceeded stack depth #{@simulator.max_stack}"
+      end
+
+      # @rbs () -> void
+      def ensure_active!
+        raise Ibex::Error, "(debug):1:1: simulation session is already finished" if @finished || @status
+      end
+    end
+  end
+end

data/lib/ibex/table_simulation/step.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module TableSimulation
+    # One immutable parser-table action.
+    class Step
+      # @rbs! type document_value = String | Integer | nil
+
+      attr_reader :sequence #: Integer
+      attr_reader :state #: Integer
+      attr_reader :token_id #: Integer
+      attr_reader :token #: String
+      attr_reader :action #: String
+      attr_reader :action_source #: String
+      attr_reader :production_id #: Integer?
+      attr_reader :lhs #: String?
+      attr_reader :rhs_length #: Integer?
+      attr_reader :target_state #: Integer?
+      attr_reader :stack_depth_before #: Integer
+      attr_reader :stack_depth_after #: Integer
+
+      # @rbs (sequence: Integer, state: Integer, token_id: Integer, token: String, action: String,
+      #   action_source: String, production_id: Integer?, lhs: String?, rhs_length: Integer?,
+      #   target_state: Integer?, stack_depth_before: Integer, stack_depth_after: Integer) -> void
+      def initialize(sequence:, state:, token_id:, token:, action:, action_source:, production_id:, lhs:, rhs_length:,
+                     target_state:, stack_depth_before:, stack_depth_after:)
+        @sequence = sequence
+        @state = state
+        @token_id = token_id
+        @token = token.dup.freeze
+        @action = action.dup.freeze
+        @action_source = action_source.dup.freeze
+        @production_id = production_id
+        @lhs = lhs&.dup&.freeze
+        @rhs_length = rhs_length
+        @target_state = target_state
+        @stack_depth_before = stack_depth_before
+        @stack_depth_after = stack_depth_after
+        freeze
+      end
+
+      # @rbs () -> Hash[String, document_value]
+      def to_h
+        {
+          "sequence" => @sequence,
+          "state" => @state,
+          "token_id" => @token_id,
+          "token" => @token,
+          "action" => @action,
+          "action_source" => @action_source,
+          "production_id" => @production_id,
+          "lhs" => @lhs,
+          "rhs_length" => @rhs_length,
+          "target_state" => @target_state,
+          "stack_depth_before" => @stack_depth_before,
+          "stack_depth_after" => @stack_depth_after
+        }.freeze
+      end
+    end
+  end
+end

data/lib/ibex/table_simulation/text.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+module Ibex
+  module TableSimulation
+    # Stable one-line renderer for simulation steps.
+    module Text
+      class << self
+        # @rbs (Step step) -> String
+        def render_step(step)
+          detail = case step.action
+                   when "shift" then " target=#{step.target_state}"
+                   when "reduce"
+                     " production=#{step.production_id} lhs=#{step.lhs} rhs=#{step.rhs_length} " \
+                     "goto=#{step.target_state}"
+                   else ""
+                   end
+          "#{step.sequence}: state=#{step.state} token=#{step.token.inspect} action=#{step.action}" \
+            "#{detail} source=#{step.action_source} stack=#{step.stack_depth_before}->#{step.stack_depth_after}"
+        end
+
+        # @rbs (Result result) -> String
+        def render(result)
+          lines = result.steps.map { |step| render_step(step) }
+          lines << "status=#{result.status}"
+          "#{lines.join("\n")}\n"
+        end
+      end
+    end
+  end
+end

data/lib/ibex/table_simulation.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "table_simulation/step"
+require_relative "table_simulation/result"
+require_relative "table_simulation/simulator"
+require_relative "table_simulation/text"
+
+module Ibex
+  # Safe LR table simulation over validated Automaton IR.
+  module TableSimulation
+  end
+end

data/lib/ibex/tables.rb

@@ -1,82 +1,31 @@
 # frozen_string_literal: true
 
+require "ibex/tables/compact"
+require "ibex/tables/compact_actions"
+require "ibex/tables/compact_productions"
+
 module Ibex
   # Parser table construction and row-displacement compression.
   module Tables
     # @rbs!
+    #   type action_table = Array[Hash[Integer, IR::runtime_action]] | CompactActions
+    #   type goto_table = Array[Hash[Integer, Integer]] | Compact
     #   private def runtime_action: (IR::parser_action action) -> IR::runtime_action
     #   private def self.runtime_action: (IR::parser_action action) -> IR::runtime_action
 
+    # @rbs skip
     TableSet = Struct.new(:actions, :gotos, :default_actions, keyword_init: true)
 
-    # Sparse table represented by per-row offsets and ownership checks.
-    class Compact
-      attr_reader :offsets #: Array[Integer]
-      attr_reader :values #: Array[untyped]
-      attr_reader :checks #: Array[Integer?]
-      attr_reader :row_count #: Integer
-
-      class << self
-        # @rbs (Array[Hash[Integer, untyped]] rows) -> Compact
-        def build(rows)
-          offsets = Array.new(rows.length, 0)
-          values = [] #: Array[untyped]
-          checks = [] #: Array[Integer?]
-          rows.each_index.sort_by { |row| [-rows[row].length, row] }.each do |row|
-            offset = find_offset(rows[row].keys, checks)
-            offsets[row] = offset
-            rows[row].each do |column, value|
-              index = offset + column
-              values[index] = value
-              checks[index] = row
-            end
-          end
-          new(offsets: offsets, values: values, checks: checks, row_count: rows.length)
-        end
-
-        private
-
-        # @rbs (Array[Integer] columns, Array[Integer?] checks) -> Integer
-        def find_offset(columns, checks)
-          offset = 0
-          offset += 1 while columns.any? { |column| checks[offset + column] }
-          offset
-        end
-      end
-
-      # @rbs (offsets: Array[Integer], values: Array[untyped], checks: Array[Integer?], row_count: Integer) -> void
-      def initialize(offsets:, values:, checks:, row_count:)
-        @offsets = offsets.freeze
-        @values = values.freeze
-        @checks = checks.freeze
-        @row_count = row_count
-        freeze
-      end
-
-      # @rbs (Integer row, Integer column) -> untyped
-      def lookup(row, column)
-        return nil unless row.between?(0, @row_count - 1)
-        return nil if column.negative?
-
-        index = @offsets.fetch(row) + column
-        return nil unless index.between?(0, @checks.length - 1)
-
-        @checks[index] == row ? @values[index] : nil
-      end
-
-      # @rbs (Integer row) -> Hash[Integer, untyped]
-      def row(row)
-        return {} unless row.between?(0, @row_count - 1)
-
-        result = {} #: Hash[Integer, untyped]
-        @checks.each_index do |index|
-          next unless @checks[index] == row
-
-          result[index - @offsets[row]] = @values[index]
-        end
-        result
-      end
-    end
+    # @rbs!
+    #   class TableSet < Struct[Object?]
+    #     attr_accessor actions: action_table
+    #     attr_accessor gotos: goto_table
+    #     attr_accessor default_actions: Array[IR::runtime_action?]
+    #     def self.new: (?actions: action_table, ?gotos: goto_table,
+    #       ?default_actions: Array[IR::runtime_action?]) -> instance
+    #                    | ({ ?actions: action_table, ?gotos: goto_table,
+    #       ?default_actions: Array[IR::runtime_action?] }) -> instance
+    #   end
 
     # @rbs (IR::Automaton automaton, ?format: Symbol | String) -> TableSet
     def build(automaton, format: :compact)
@@ -95,7 +44,11 @@ module Ibex
       end
       raise ArgumentError, "unknown table format #{format.inspect}" unless format.to_sym == :compact
 
-      TableSet.new(actions: Compact.build(action_rows), gotos: Compact.build(goto_rows), default_actions: defaults)
+      TableSet.new(
+        actions: CompactActions.build(action_rows),
+        gotos: Compact.build(goto_rows),
+        default_actions: defaults
+      )
     end
     module_function :build
 

data/lib/ibex/verifiable_generation_bundle.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+# rbs_inline: enabled
+
+require_relative "artifact_set"
+require_relative "generation_manifest"
+require_relative "generation_transaction"
+require_relative "table_artifact"
+require_relative "verification_report"
+
+module Ibex
+  # Explicitly renders and publishes one manifest-bound verification bundle.
+  class VerifiableGenerationBundle
+    # rubocop:disable Metrics/ParameterLists -- persisted artifact paths and verification bounds are explicit inputs.
+    # @rbs (IR::Automaton automaton, wrapper_path: String, wrapper_source: String,
+    #   table_path: String, report_path: String, manifest_path: String,
+    #   source_records: Array[GenerationInput], ?manifest_options: Hash[String, Object?],
+    #   ?wrapper_mode: Integer?, ?representation: Symbol | String, ?cst_trivia: Symbol | String?,
+    #   ?omit_action_call: bool?, ?strict: bool, ?max_states: Integer, ?max_items: Integer) -> void
+    def initialize(automaton, wrapper_path:, wrapper_source:, table_path:, report_path:, manifest_path:,
+                   source_records:, manifest_options: {}, wrapper_mode: nil, representation: :compact,
+                   cst_trivia: nil, omit_action_call: nil, strict: false,
+                   max_states: VerificationReport::DEFAULT_MAX_STATES,
+                   max_items: VerificationReport::DEFAULT_MAX_ITEMS)
+      @automaton = automaton
+      @wrapper_path = wrapper_path
+      @wrapper_source = wrapper_source
+      @table_path = table_path
+      VerificationReport::LogicalPath.table(table_path)
+      if source_records.length > VerificationReport::LogicalPath::MAX_INPUT_FILES
+        raise ArgumentError,
+              "verification reports support at most #{VerificationReport::LogicalPath::MAX_INPUT_FILES} input files"
+      end
+      @report_path = report_path
+      @manifest_path = manifest_path
+      @source_records = source_records
+      @manifest_options = manifest_options
+      @wrapper_mode = wrapper_mode
+      @representation = representation
+      @cst_trivia = cst_trivia
+      @omit_action_call = omit_action_call
+      @strict = strict
+      @max_states = max_states
+      @max_items = max_items
+    end
+    # rubocop:enable Metrics/ParameterLists
+
+    # Render table, wrapper, and report before the non-cyclic manifest marker.
+    # @rbs () -> ArtifactSet
+    def render
+      canonical_automaton = VerificationReport.canonical_automaton(
+        @automaton, source_records: @source_records
+      )
+      table = TableArtifact.build(
+        canonical_automaton, representation: @representation, cst_trivia: @cst_trivia,
+                             omit_action_call: @omit_action_call
+      )
+      artifacts = ArtifactSet.new
+      artifacts.add(kind: :parser_table, path: @table_path, content: table.dump)
+      artifacts.add(kind: :parser, path: @wrapper_path, content: @wrapper_source, mode: @wrapper_mode)
+      report = VerificationReport.render(
+        canonical_automaton, table: table, source_records: @source_records, table_path: @table_path,
+                             strict: @strict, max_states: @max_states, max_items: @max_items
+      )
+      artifacts.add(kind: :verification_report, path: @report_path, content: report)
+      manifest = GenerationManifest.render(
+        artifacts, source_records: @source_records, options: @manifest_options
+      )
+      artifacts.add(kind: :manifest, path: @manifest_path, content: manifest)
+      artifacts
+    end
+
+    # @rbs (?warning: ^(String) -> void, ?stability_check: ^() -> bool) -> ArtifactSet
+    def publish(warning: ->(_message) {}, stability_check: -> { @source_records.all?(&:current?) })
+      artifacts = render
+      GenerationTransaction.new(
+        artifacts, warning: warning, stability_check: stability_check, source_records: @source_records
+      ).commit
+      artifacts
+    end
+
+    # @rbs (String manifest_path) -> Hash[String, VerificationReport::Validator::json_value]
+    def self.validate_file(manifest_path)
+      VerificationReport.validate_bundle_file(manifest_path)
+    end
+  end
+end