phronomy 0.22.0 → 0.24.0

data/lib/phronomy/agent/recovery_coordinator/installation.rb

@@ -0,0 +1,381 @@
+# frozen_string_literal: true
+
+module Phronomy
+  module Agent
+    class RecoveryCoordinator
+      # Installation portion of durable Agent Recovery.
+      # @api private
+      module Installation
+        private
+
+        def framework_batch?(execution)
+          return false unless %i[dispatching_tools resuming recovery_tools].include?(execution.phase.to_sym)
+          return false if execution.status == :suspended
+          entries = Array(execution.metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY])
+          pending = entries.reject { |entry| %w[completed rejected failed cancelled].include?(entry.fetch("status")) }
+          !pending.empty? && pending.all? do |entry|
+            entry.fetch("status") == "authorized" && agent.__framework_tool_replayable?(entry.fetch("tool_name"))
+          end
+        end
+
+        # Framework operations never become Application factual questions. In a
+        # mixed batch, external facts are resolved first; authorized framework
+        # slots then resume through their normal Tool sessions.
+        def coordination_recovery_descriptor(execution)
+          metadata = execution.metadata.dup
+          if metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY]
+            metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY] = metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY].sort_by do |entry|
+              agent.__framework_tool_replayable?(entry.fetch("tool_name")) ? 1 : 0
+            end
+          end
+          if (recovery = metadata[RecoverySupport::RECOVERY_METADATA_KEY]) && recovery["subjects"]
+            metadata[RecoverySupport::RECOVERY_METADATA_KEY] = recovery.merge("subjects" => recovery["subjects"].sort_by do |entry|
+              agent.__framework_tool_replayable?(entry.fetch("tool_name")) ? 1 : 0
+            end)
+          end
+          RecoverySupport.recovery_descriptor(execution.with(execution_revision: execution.execution_revision,
+            updated_at: execution.updated_at, metadata: metadata))
+        end
+
+        def prepare_plan(execution)
+          root = agent.agent_root
+          manifest_ref = execution.metadata["manifest_ref"]
+          base_ref = execution.metadata["base_manifest_ref"] ||
+            manifest_ref
+          manifest = base_manifest = projection = nil
+          if manifest_ref
+            manifest, projection =
+              RecoverySupport.materialize_projection(
+                agent,
+                manifest_ref
+              )
+            base_manifest = if base_ref.to_s == manifest_ref.to_s
+              manifest
+            else
+              RecoverySupport.manifest_from_ref(agent, base_ref)
+            end
+          end
+
+          classification = classify(execution)
+          RecoveryPlan.new(
+            execution: execution,
+            root: root,
+            manifest: manifest,
+            base_manifest: base_manifest,
+            projection: projection,
+            classification: classification,
+            material: prepare_recovery_material(execution, projection: projection)
+          )
+        end
+
+        def classify(execution)
+          if framework_batch?(execution)
+            return Phronomy::Recovery::Classification.new(disposition: Phronomy::Recovery::RESUMABLE, reason: :framework_tools)
+          end
+          if execution.status == :preparing &&
+              execution.phase.to_sym == :preparing
+            unless execution.metadata["preparation_replayable"] == true
+              raise Phronomy::ExecutionRehydrationRequiredError,
+                "execution #{execution.execution_id} is :preparing but replay safety " \
+                  "was not durably established"
+            end
+            return Phronomy::Recovery::Classification.new(
+              disposition: Phronomy::Recovery::RESUMABLE,
+              reason: :initial_preparation
+            )
+          end
+
+          if execution.status == :suspended &&
+              execution.phase.to_sym == :approval
+            return Phronomy::Recovery::Classification.new(
+              disposition: Phronomy::Recovery::RESUMABLE,
+              reason: :approval_wait,
+              facts: {
+                approval_request_id:
+                  execution.approval_request &&
+                    (
+                      execution.approval_request["id"] ||
+                      execution.approval_request[:id]
+                    )
+              }.compact
+            )
+          end
+
+          case execution.phase.to_sym
+          when :recovery_provider_completed,
+            :recovery_tools_completed,
+            :recovery_resolved_failed
+            return Phronomy::Recovery::Classification.new(
+              disposition: Phronomy::Recovery::RESUMABLE,
+              reason: execution.phase
+            )
+          when :resuming
+            approved = execution.approval_request &&
+              (
+                execution.approval_request["approved"] ||
+                execution.approval_request[:approved]
+              )
+            unless approved
+              return Phronomy::Recovery::Classification.new(
+                disposition: Phronomy::Recovery::RESUMABLE,
+                reason: :approval_rejection_committed
+              )
+            end
+          end
+
+          descriptor = coordination_recovery_descriptor(
+            execution
+          )
+          if descriptor
+            return Phronomy::Recovery::Classification.new(
+              disposition: Phronomy::Recovery::RESOLUTION_REQUIRED,
+              reason: descriptor.fetch(:reason),
+              subject: descriptor.fetch(:subject),
+              allowed_outcomes:
+                descriptor.fetch(:allowed_outcomes),
+              facts: descriptor.fetch(:facts)
+            )
+          end
+
+          raise Phronomy::ExecutionRehydrationRequiredError,
+            "execution #{execution.execution_id} is not at an restart-safe durable continuation point " \
+            "(status=#{execution.status.inspect}, phase=#{execution.phase.inspect})"
+        end
+
+        def install_on_event_loop(command)
+          event_loop = @runtime.event_loop
+          plan = command.plan
+          execution = plan.execution
+          main = agent.send(:execution_coordinator_for, agent.__coordination_config)
+          admitted = false
+          bound = false
+          installed = false
+
+          event_loop.admit_agent_execution(
+            agent.agent_id,
+            owner_token: command.owner_token
+          )
+          admitted = true
+          event_loop.bind_agent_execution_admission(
+            agent.agent_id,
+            owner_token: command.owner_token,
+            execution_id: execution.execution_id
+          )
+          bound = true
+
+          invocation = nil
+          if execution.status == :suspended || execution.phase.to_sym == :resuming
+            invocation = RecoverySupport.build_invocation_for_suspended(
+              agent, execution, plan.projection, main, agent.send(:_phronomy_event_listener),
+              assistant_message: plan.material.assistant_message
+            )
+          end
+
+          event_loop.install_agent_execution(
+            execution_id: execution.execution_id,
+            agent: agent,
+            coordinator: main,
+            execution: execution,
+            runtime_projection: plan.projection,
+            base_manifest: plan.base_manifest,
+            invocation: invocation,
+            fsm_session_id: nil
+          )
+          installed = true
+
+          if execution.status == :suspended
+            event_loop.mark_agent_execution_admission(
+              agent.agent_id,
+              execution_id: execution.execution_id,
+              state: :suspended
+            )
+            request = invocation.approval_request
+            callback_error = agent.send(
+              :_deliver_stream_event,
+              agent.send(:_phronomy_event_listener),
+              StreamEvent.new(
+                type: :approval_required,
+                payload: {request: request}.freeze
+              )
+            )
+            if callback_error
+              raise agent.send(
+                :_build_stream_callback_error,
+                event_type: :approval_required,
+                callback_error: callback_error,
+                result: {
+                  execution_id: execution.execution_id,
+                  suspended: true
+                }
+              )
+            end
+            command.completion.complete(agent)
+            return
+          end
+
+          case plan.classification.disposition
+          when Phronomy::Recovery::RESOLUTION_REQUIRED
+            event_loop.mark_agent_execution_admission(
+              agent.agent_id,
+              execution_id: execution.execution_id,
+              state: :recovery_required
+            )
+            payload = RecoverySupport.event_payload(
+              execution,
+              {
+                reason: plan.classification.reason,
+                subject: plan.classification.subject,
+                allowed_outcomes:
+                  plan.classification.allowed_outcomes,
+                facts: plan.classification.facts
+              }
+            )
+            callback_error = agent.send(
+              :_deliver_stream_event,
+              agent.send(:_phronomy_event_listener),
+              StreamEvent.new(
+                type: :recovery_resolution_required,
+                payload: payload
+              )
+            )
+            if callback_error
+              raise agent.send(
+                :_build_stream_callback_error,
+                event_type: :recovery_resolution_required,
+                callback_error: callback_error,
+                result: payload
+              )
+            end
+            command.completion.complete(agent)
+          when Phronomy::Recovery::RESUMABLE
+            continue_resumable_on_event_loop(
+              execution,
+              completion: command.completion, material: plan.material
+            )
+          else
+            raise Phronomy::Error,
+              "unsupported Recovery installation disposition: #{plan.classification.disposition.inspect}"
+          end
+        rescue => caught
+          # simplecov:disable
+          if installed
+            begin
+              event_loop.release_agent_execution(
+              execution.execution_id
+            )
+            rescue
+              nil
+            end
+          end
+          if admitted
+            if bound
+              begin
+                event_loop.release_agent_execution_admission(
+                  agent.agent_id,
+                  execution_id: execution&.execution_id
+                )
+              rescue
+                nil
+              end
+            else
+              begin
+                event_loop.release_agent_execution_admission(
+                  agent.agent_id,
+                  owner_token: command.owner_token
+                )
+              rescue
+                nil
+              end
+            end
+          end
+          command.completion.fail(caught)
+          # simplecov:enable
+        end
+
+        def observe_recovery_execution(task, execution)
+          Phronomy::Tracing::Automatic.observe_task(
+            task,
+            "agent.execution",
+            agent_id: agent.agent_id,
+            execution_id: execution.execution_id,
+            mode: (
+              execution.metadata[
+                RecoverySupport::INVOCATION_MODE_KEY
+              ] || "invoke"
+            ).to_sym
+          )
+        end
+
+        def continue_resumable_on_event_loop(
+          execution,
+          completion:, material:
+        )
+          # simplecov:disable
+          event_loop = @runtime.event_loop
+          main = agent.send(:execution_coordinator_for, agent.__coordination_config)
+
+          if framework_batch?(execution)
+            internal = Phronomy::Task.deferred(name: "framework-tool-recovery:#{execution.execution_id}")
+            continue_recovery_on_event_loop(execution, internal, material: material)
+            completion.complete(agent)
+            return
+          end
+
+          case execution.phase.to_sym
+          when :preparing
+            internal_task = Phronomy::Task.deferred(
+              name: "agent-recovery-auto:#{execution.execution_id}"
+            )
+            observe_recovery_execution(internal_task, execution)
+            event_loop.mark_agent_execution_admission(
+              agent.agent_id,
+              execution_id: execution.execution_id,
+              state: :executing
+            )
+            main.send(
+              :start_initial_preparation_recovery_on_event_loop,
+              execution,
+              internal_task,
+              load_completion: completion
+            )
+          when :resuming
+            approved = execution.approval_request &&
+              (
+                execution.approval_request["approved"] ||
+                execution.approval_request[:approved]
+              )
+            if approved
+              raise Phronomy::ExecutionRehydrationRequiredError,
+                "approved resuming execution requires Tool outcome resolution"
+            end
+            internal_task = Phronomy::Task.deferred(
+              name: "agent-recovery-auto:#{execution.execution_id}"
+            )
+            observe_recovery_execution(internal_task, execution)
+            event_loop.mark_agent_execution_admission(
+              agent.agent_id,
+              execution_id: execution.execution_id,
+              state: :executing
+            )
+            main.send(
+              :start_resume_on_event_loop,
+              execution.execution_id,
+              internal_task,
+              approved: false,
+              config: {}
+            )
+            completion.complete(agent)
+          when :recovery_tools_completed, :recovery_provider_completed, :recovery_resolved_failed
+            internal_task = Phronomy::Task.deferred(name: "agent-recovery-auto:#{execution.execution_id}")
+            continue_recovery_on_event_loop(execution, internal_task, material: material)
+            completion.complete(agent)
+          else
+            raise Phronomy::ExecutionRehydrationRequiredError,
+              "no automatic continuation for #{execution.phase.inspect}"
+          end
+          # simplecov:enable
+        end
+      end
+    end
+  end
+end