phronomy 0.23.0 → 0.24.0

data/lib/phronomy/agent/recovery_coordinator/continuation.rb

@@ -3,231 +3,135 @@
 module Phronomy
   module Agent
     class RecoveryCoordinator
-      # Continuation portion of durable Agent Recovery.
+      # Both resolution apply and restart use this interpretation of saved facts.
+      # Runtime tasks and sessions are projections, never continuation authority.
       # @api private
       module Continuation
-        # simplecov:disable
-
         private
 
-        def continue_provider_completed_after_resolution(
-          event_loop,
-          old_state,
-          execution,
-          completion
-        )
-          manifest_ref = execution.metadata.fetch(
-            "manifest_ref"
-          )
-          _manifest, projection =
-            RecoverySupport.materialize_projection(
-              agent,
-              manifest_ref
-            )
-          main = agent.send(:execution_coordinator)
-          invocation =
-            RecoverySupport.build_chat_for_recovery(
-              agent,
-              execution,
-              projection,
-              main,
-              agent.send(:_phronomy_event_listener)
-            )
-          output, usage =
-            RecoverySupport.provider_output_and_usage(
-              agent,
-              execution
-            )
-          invocation.output = output
-          invocation.usage = usage
-          AgentInvocationSessionBuilder.send(
-            :output_filtering_action,
-            agent,
-            invocation
-          )
+        def recovery_action(execution)
+          return :framework_tools if framework_batch?(execution)
 
-          event_loop.replace_agent_execution(
-            execution.execution_id,
-            execution: execution,
-            runtime_projection: projection,
-            invocation: invocation,
-            fsm_session_id: nil
-          )
-          event_loop.mark_agent_execution_admission(
-            agent.agent_id,
-            execution_id: execution.execution_id,
-            state: :executing
-          )
-          start_output_completion_session(
-            event_loop,
-            main,
-            execution,
-            invocation,
-            completion
-          )
+          case execution.phase.to_sym
+          when :recovery_provider_completed
+            execution.metadata["framework_calls_pending"] ? :framework_calls : :output
+          when :recovery_tools_completed
+            execution.metadata["framework_calls_pending"] ? :framework_calls : :followup
+          when :recovery_resolved_failed
+            :failed_terminal
+          else
+            :resolution_required
+          end
         end
 
-        def continue_tools_completed_after_resolution(
-          event_loop,
-          old_state,
-          execution,
-          completion
-        )
-          manifest_ref = execution.metadata.fetch(
-            "manifest_ref"
-          )
-          _manifest, projection =
-            RecoverySupport.materialize_projection(
-              agent,
-              manifest_ref
-            )
-          main = agent.send(:execution_coordinator)
-          invocation =
-            RecoverySupport.build_chat_for_recovery(
-              agent,
-              execution,
-              projection,
-              main,
-              agent.send(:_phronomy_event_listener)
-            )
-          event_loop.replace_agent_execution(
-            execution.execution_id,
-            execution: execution,
-            runtime_projection: projection,
-            invocation: invocation,
-            fsm_session_id: nil
-          )
-          event_loop.mark_agent_execution_admission(
-            agent.agent_id,
-            execution_id: execution.execution_id,
-            state: :executing
-          )
-          start_followup_session(
-            event_loop,
-            main,
-            execution,
-            invocation,
-            completion
-          )
+        # Read durable inputs before entering EventLoop. The returned messages are
+        # operation-local projections transferred to the new invocation, never a
+        # shared cache or a second source of continuation decisions.
+        def prepare_recovery_material(execution, projection: nil)
+          action = recovery_action(execution)
+          suspended = execution.status == :suspended || execution.phase.to_sym == :resuming
+          return if !suspended && %i[resolution_required failed_terminal].include?(action)
+
+          unless projection
+            _manifest, projection = RecoverySupport.materialize_projection(agent, execution.metadata.fetch("manifest_ref"))
+          end
+          records = execution.working_records.select { |record| %i[assistant_message tool_message].include?(record.kind.to_sym) }
+          materializer = RubyLLMMaterializer.new(agent: agent, persistence: agent.persistence)
+          messages = records.map { |record| materializer.materialize_journal_record(record) }.freeze
+          assistant = messages.reverse.find { |message| message.role.to_sym == :assistant }
+          output, usage = RecoverySupport.provider_output_and_usage(agent, execution) if execution.phase.to_sym == :recovery_provider_completed
+          RecoveryMaterial.new(projection: projection, messages: messages,
+            assistant_message: assistant, output: output, usage: usage)
         end
 
-        def continue_failed_after_resolution(
-          event_loop,
-          old_state,
-          execution,
-          completion,
-          failure
-        )
-          main = agent.send(:execution_coordinator)
-          invocation = Phronomy::Agent::AgentInvocation.new(
-            agent: agent,
-            input: nil,
-            config: {
-              execution_id: execution.execution_id,
-              phronomy_execution_coordinator: main
-            },
-            event_listener:
-              agent.send(:_phronomy_event_listener),
-            mode: (
-              execution.metadata[
-                RecoverySupport::INVOCATION_MODE_KEY
-              ] || "invoke"
-            ).to_sym,
-            execution_id: execution.execution_id
-          )
-          event_loop.replace_agent_execution(
-            execution.execution_id,
-            execution: execution,
-            invocation: invocation,
-            fsm_session_id: nil
-          )
-          state = event_loop.agent_execution_state(
-            execution.execution_id
-          )
-          main.send(
-            :begin_terminal_commit_on_event_loop,
-            state,
-            completion,
-            invocation,
-            RecoverySupport.error_from_failure(failure),
-            fsm_session_id: nil
-          )
+        def continue_recovery_on_event_loop(execution, completion, material:)
+          event_loop = @runtime.event_loop
+          action = recovery_action(execution)
+          if action == :resolution_required
+            event_loop.mark_agent_execution_admission(agent.agent_id,
+              execution_id: execution.execution_id, state: :recovery_required)
+            deliver_resolution_required(execution, coordination_recovery_descriptor(execution))
+            completion.complete({execution_id: execution.execution_id,
+                                 execution_revision: execution.execution_revision,
+                                 recovery: :resolution_required}.freeze)
+            return
+          end
+
+          observe_recovery_execution(completion, execution)
+          main = agent.send(:execution_coordinator_for, agent.__coordination_config)
+          projection = event_loop.agent_execution_state(execution.execution_id).runtime_projection
+          if action == :failed_terminal
+            invocation = build_failed_recovery_invocation(execution, main)
+          else
+            projection = material.projection
+            invocation = if action == :framework_tools
+              RecoverySupport.build_invocation_for_suspended(agent, execution, projection, main,
+                agent.send(:_phronomy_event_listener), assistant_message: material.assistant_message)
+            else
+              RecoverySupport.build_chat_for_recovery(agent, execution, projection, main,
+                agent.send(:_phronomy_event_listener), messages: material.messages)
+            end
+            if execution.phase.to_sym == :recovery_provider_completed
+              invocation.output, invocation.usage = material.output, material.usage
+            end
+            prepare_saved_provider_calls(execution, invocation, material.assistant_message) if action == :framework_calls
+          end
+
+          event_loop.replace_agent_execution(execution.execution_id, execution: execution,
+            runtime_projection: projection, invocation: invocation, fsm_session_id: nil)
+          if action == :failed_terminal
+            failure = execution.metadata.dig(RecoverySupport::RECOVERY_METADATA_KEY, "failure") ||
+              {"class" => "Phronomy::Error", "message" => "Recovery-resolved failure"}
+            main.send(:begin_terminal_commit_on_event_loop,
+              event_loop.agent_execution_state(execution.execution_id), completion, invocation,
+              RecoverySupport.error_from_failure(failure), fsm_session_id: nil)
+            return
+          end
+
+          event_loop.mark_agent_execution_admission(agent.agent_id,
+            execution_id: execution.execution_id, state: :executing)
+          case action
+          when :framework_tools
+            event_loop.register_agent_completion_waiter(execution.execution_id, completion)
+            main.send(:start_framework_tools_on_event_loop, execution.execution_id, completion)
+          when :framework_calls, :output
+            # Re-enter the ordinary FSM before output filtering so its failures
+            # settle through the same durable terminal barrier as a live call.
+            start_recovery_session(event_loop, main, execution, invocation, completion,
+              resume_event: :llm_completed, resume_phase: :calling_llm)
+          when :followup
+            start_recovery_session(event_loop, main, execution, invocation, completion,
+              resume_event: :state_completed, resume_phase: :recording_tool_results)
+          end
         end
 
-        def start_output_completion_session(
-          event_loop,
-          main,
-          execution,
-          invocation,
-          completion
-        )
-          session =
-            AgentInvocationSessionBuilder.build_for_resume(
-              agent_invocation: invocation,
-              resume_event: :state_completed,
-              resume_phase: :output_filtering,
-              runtime: Phronomy::Runtime.instance
-            )
-          event_loop.replace_agent_execution(
-            execution.execution_id,
-            invocation: invocation,
-            fsm_session_id: session.id
-          )
-          event_loop.register_agent_completion_waiter(
-            execution.execution_id,
-            completion
-          )
-          source = Phronomy::Task.deferred(
-            name: "#{completion.name}-source"
-          )
-          source.on_complete do |completed, error|
-            main.send(
-              :finish_on_event_loop,
-              execution.execution_id,
-              completion,
-              completed || session.context,
-              error,
-              fsm_session_id: session.id
-            )
+        def prepare_saved_provider_calls(execution, invocation, message)
+          record = RecoverySupport.latest_assistant_record(execution)
+          calls = message.tool_calls.respond_to?(:values) ? message.tool_calls.values : Array(message.tool_calls)
+          framework_calls = calls.select { |call| agent.__framework_call?(call.name) }
+          if framework_calls.empty?
+            raise Phronomy::ExecutionRehydrationRequiredError, "Saved framework call wiring is missing"
           end
-          event_loop.register(session, completion: source)
+          invocation.accept_tool_calls!(framework_calls, llm_call_id: record.llm_call_id)
         end
 
-        def start_followup_session(
-          event_loop,
-          main,
-          execution,
-          invocation,
-          completion
-        )
-          session =
-            AgentInvocationSessionBuilder.build_for_resume(
-              agent_invocation: invocation,
-              resume_event: :state_completed,
-              resume_phase: :recording_tool_results,
-              runtime: Phronomy::Runtime.instance
-            )
-          event_loop.replace_agent_execution(
-            execution.execution_id,
-            invocation: invocation,
-            fsm_session_id: session.id
-          )
-          event_loop.register_agent_completion_waiter(
-            execution.execution_id,
-            completion
-          )
-          source = Phronomy::Task.deferred(
-            name: "#{completion.name}-source"
-          )
+        def build_failed_recovery_invocation(execution, main)
+          Phronomy::Agent::AgentInvocation.new(agent: agent, input: nil,
+            config: {execution_id: execution.execution_id, phronomy_execution_coordinator: main},
+            event_listener: agent.send(:_phronomy_event_listener),
+            mode: (execution.metadata[RecoverySupport::INVOCATION_MODE_KEY] || "invoke").to_sym,
+            execution_id: execution.execution_id)
+        end
+
+        def start_recovery_session(event_loop, main, execution, invocation, completion, resume_event:, resume_phase:)
+          session = AgentInvocationSessionBuilder.build_for_resume(agent_invocation: invocation,
+            resume_event: resume_event, resume_phase: resume_phase, runtime: @runtime)
+          event_loop.replace_agent_execution(execution.execution_id, invocation: invocation, fsm_session_id: session.id)
+          event_loop.register_agent_completion_waiter(execution.execution_id, completion)
+          source = Phronomy::Task.deferred(name: "#{completion.name}-source")
           source.on_complete do |completed, error|
-            main.send(
-              :finish_on_event_loop,
-              execution.execution_id,
-              completion,
-              completed || session.context,
-              error,
-              fsm_session_id: session.id
-            )
+            main.send(:finish_on_event_loop, execution.execution_id, completion,
+              completed || session.context, error, fsm_session_id: session.id)
           end
           event_loop.register(session, completion: source)
         end
@@ -264,7 +168,6 @@ module Phronomy
           end
           nil
         end
-        # simplecov:enable
       end
     end
   end

data/lib/phronomy/agent/recovery_coordinator/installation.rb

@@ -8,6 +8,35 @@ module Phronomy
       module Installation
         private
 
+        def framework_batch?(execution)
+          return false unless %i[dispatching_tools resuming recovery_tools].include?(execution.phase.to_sym)
+          return false if execution.status == :suspended
+          entries = Array(execution.metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY])
+          pending = entries.reject { |entry| %w[completed rejected failed cancelled].include?(entry.fetch("status")) }
+          !pending.empty? && pending.all? do |entry|
+            entry.fetch("status") == "authorized" && agent.__framework_tool_replayable?(entry.fetch("tool_name"))
+          end
+        end
+
+        # Framework operations never become Application factual questions. In a
+        # mixed batch, external facts are resolved first; authorized framework
+        # slots then resume through their normal Tool sessions.
+        def coordination_recovery_descriptor(execution)
+          metadata = execution.metadata.dup
+          if metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY]
+            metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY] = metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY].sort_by do |entry|
+              agent.__framework_tool_replayable?(entry.fetch("tool_name")) ? 1 : 0
+            end
+          end
+          if (recovery = metadata[RecoverySupport::RECOVERY_METADATA_KEY]) && recovery["subjects"]
+            metadata[RecoverySupport::RECOVERY_METADATA_KEY] = recovery.merge("subjects" => recovery["subjects"].sort_by do |entry|
+              agent.__framework_tool_replayable?(entry.fetch("tool_name")) ? 1 : 0
+            end)
+          end
+          RecoverySupport.recovery_descriptor(execution.with(execution_revision: execution.execution_revision,
+            updated_at: execution.updated_at, metadata: metadata))
+        end
+
         def prepare_plan(execution)
           root = agent.agent_root
           manifest_ref = execution.metadata["manifest_ref"]
@@ -34,11 +63,28 @@ module Phronomy
             manifest: manifest,
             base_manifest: base_manifest,
             projection: projection,
-            classification: classification
+            classification: classification,
+            material: prepare_recovery_material(execution, projection: projection)
           )
         end
 
         def classify(execution)
+          if framework_batch?(execution)
+            return Phronomy::Recovery::Classification.new(disposition: Phronomy::Recovery::RESUMABLE, reason: :framework_tools)
+          end
+          if execution.status == :preparing &&
+              execution.phase.to_sym == :preparing
+            unless execution.metadata["preparation_replayable"] == true
+              raise Phronomy::ExecutionRehydrationRequiredError,
+                "execution #{execution.execution_id} is :preparing but replay safety " \
+                  "was not durably established"
+            end
+            return Phronomy::Recovery::Classification.new(
+              disposition: Phronomy::Recovery::RESUMABLE,
+              reason: :initial_preparation
+            )
+          end
+
           if execution.status == :suspended &&
               execution.phase.to_sym == :approval
             return Phronomy::Recovery::Classification.new(
@@ -77,7 +123,7 @@ module Phronomy
             end
           end
 
-          descriptor = RecoverySupport.recovery_descriptor(
+          descriptor = coordination_recovery_descriptor(
             execution
           )
           if descriptor
@@ -97,10 +143,10 @@ module Phronomy
         end
 
         def install_on_event_loop(command)
-          event_loop = Phronomy::Runtime.instance.event_loop
+          event_loop = @runtime.event_loop
           plan = command.plan
           execution = plan.execution
-          main = agent.send(:execution_coordinator)
+          main = agent.send(:execution_coordinator_for, agent.__coordination_config)
           admitted = false
           bound = false
           installed = false
@@ -118,60 +164,10 @@ module Phronomy
           bound = true
 
           invocation = nil
-          if execution.status == :suspended ||
-              execution.phase.to_sym == :resuming
-            invocation =
-              RecoverySupport.build_invocation_for_suspended(
-                agent,
-                execution,
-                plan.projection,
-                main,
-                agent.send(:_phronomy_event_listener)
-              )
-          elsif execution.phase.to_sym ==
-              :recovery_tools_completed
-            invocation =
-              RecoverySupport.build_chat_for_recovery(
-                agent,
-                execution,
-                plan.projection,
-                main,
-                agent.send(:_phronomy_event_listener)
-              )
-          elsif execution.phase.to_sym ==
-              :recovery_provider_completed
-            invocation =
-              RecoverySupport.build_chat_for_recovery(
-                agent,
-                execution,
-                plan.projection,
-                main,
-                agent.send(:_phronomy_event_listener)
-              )
-            output, usage =
-              RecoverySupport.provider_output_and_usage(
-                agent,
-                execution
-              )
-            invocation.output = output
-            invocation.usage = usage
-          elsif execution.phase.to_sym ==
-              :recovery_resolved_failed
-            invocation = Phronomy::Agent::AgentInvocation.new(
-              agent: agent,
-              input: nil,
-              config: {
-                execution_id: execution.execution_id,
-                phronomy_execution_coordinator: main
-              },
-              event_listener:
-                agent.send(:_phronomy_event_listener),
-              mode: (
-                execution.metadata[
-                  RecoverySupport::INVOCATION_MODE_KEY
-                ] || "invoke"
-              ).to_sym,
-              execution_id: execution.execution_id
+          if execution.status == :suspended || execution.phase.to_sym == :resuming
+            invocation = RecoverySupport.build_invocation_for_suspended(
+              agent, execution, plan.projection, main, agent.send(:_phronomy_event_listener),
+              assistant_message: plan.material.assistant_message
             )
           end
 
@@ -254,8 +250,7 @@ module Phronomy
           when Phronomy::Recovery::RESUMABLE
             continue_resumable_on_event_loop(
               execution,
-              invocation,
-              completion: command.completion
+              completion: command.completion, material: plan.material
             )
           else
             raise Phronomy::Error,
@@ -313,14 +308,36 @@ module Phronomy
 
         def continue_resumable_on_event_loop(
           execution,
-          invocation,
-          completion:
+          completion:, material:
         )
           # simplecov:disable
-          event_loop = Phronomy::Runtime.instance.event_loop
-          main = agent.send(:execution_coordinator)
+          event_loop = @runtime.event_loop
+          main = agent.send(:execution_coordinator_for, agent.__coordination_config)
+
+          if framework_batch?(execution)
+            internal = Phronomy::Task.deferred(name: "framework-tool-recovery:#{execution.execution_id}")
+            continue_recovery_on_event_loop(execution, internal, material: material)
+            completion.complete(agent)
+            return
+          end
 
           case execution.phase.to_sym
+          when :preparing
+            internal_task = Phronomy::Task.deferred(
+              name: "agent-recovery-auto:#{execution.execution_id}"
+            )
+            observe_recovery_execution(internal_task, execution)
+            event_loop.mark_agent_execution_admission(
+              agent.agent_id,
+              execution_id: execution.execution_id,
+              state: :executing
+            )
+            main.send(
+              :start_initial_preparation_recovery_on_event_loop,
+              execution,
+              internal_task,
+              load_completion: completion
+            )
           when :resuming
             approved = execution.approval_request &&
               (
@@ -348,72 +365,9 @@ module Phronomy
               config: {}
             )
             completion.complete(agent)
-          when :recovery_tools_completed
-            internal_task = Phronomy::Task.deferred(
-              name: "agent-recovery-auto:#{execution.execution_id}"
-            )
-            observe_recovery_execution(internal_task, execution)
-            event_loop.mark_agent_execution_admission(
-              agent.agent_id,
-              execution_id: execution.execution_id,
-              state: :executing
-            )
-            start_followup_session(
-              event_loop,
-              main,
-              execution,
-              invocation,
-              internal_task
-            )
-            completion.complete(agent)
-          when :recovery_provider_completed
-            AgentInvocationSessionBuilder.send(
-              :output_filtering_action,
-              agent,
-              invocation
-            )
-            internal_task = Phronomy::Task.deferred(
-              name: "agent-recovery-auto:#{execution.execution_id}"
-            )
-            observe_recovery_execution(internal_task, execution)
-            event_loop.mark_agent_execution_admission(
-              agent.agent_id,
-              execution_id: execution.execution_id,
-              state: :executing
-            )
-            start_output_completion_session(
-              event_loop,
-              main,
-              execution,
-              invocation,
-              internal_task
-            )
-            completion.complete(agent)
-          when :recovery_resolved_failed
-            failure = (
-              execution.metadata[
-                RecoverySupport::RECOVERY_METADATA_KEY
-              ] || {}
-            )["failure"] || {
-              "class" => "Phronomy::Error",
-              "message" => "Recovery-resolved failure"
-            }
-            error = RecoverySupport.error_from_failure(failure)
-            internal_task = Phronomy::Task.deferred(
-              name: "agent-recovery-auto:#{execution.execution_id}"
-            )
-            observe_recovery_execution(internal_task, execution)
-            state = event_loop.agent_execution_state(
-              execution.execution_id
-            )
-            main.send(
-              :begin_terminal_commit_on_event_loop,
-              state,
-              internal_task,
-              invocation,
-              error,
-              fsm_session_id: nil
-            )
+          when :recovery_tools_completed, :recovery_provider_completed, :recovery_resolved_failed
+            internal_task = Phronomy::Task.deferred(name: "agent-recovery-auto:#{execution.execution_id}")
+            continue_recovery_on_event_loop(execution, internal_task, material: material)
             completion.complete(agent)
           else
             raise Phronomy::ExecutionRehydrationRequiredError,