phronomy 0.22.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. checksums.yaml +4 -4
  2. data/.mutant.yml +3 -4
  3. data/CHANGELOG.md +215 -10
  4. data/CONTRIBUTING.md +81 -9
  5. data/README.md +15 -6
  6. data/VERIFY.sh +587 -0
  7. data/benchmark/bench_agent_invoke.rb +2 -2
  8. data/benchmark/bench_context_assembler.rb +39 -68
  9. data/benchmark/bench_regression.rb +2 -2
  10. data/docs/architecture/agent-context.md +174 -0
  11. data/docs/architecture/before-llm-input.md +78 -0
  12. data/docs/architecture/context-management.md +232 -0
  13. data/docs/architecture/knowledge-and-rag.md +130 -0
  14. data/docs/architecture/multi-agent-handoff.md +147 -0
  15. data/docs/architecture/persistence.md +186 -0
  16. data/docs/architecture/removed/agent-context.md +72 -0
  17. data/docs/architecture/security-boundaries.md +173 -0
  18. data/docs/architecture/tracing.md +194 -0
  19. data/docs/architecture.md +89 -0
  20. data/docs/archive/design/archived/04_api_design.md +507 -0
  21. data/docs/archive/design/archived/09_guardrails.md +186 -0
  22. data/docs/archive/design/archived/17_rails_integration.md +175 -0
  23. data/docs/archive/design/historical/00_design_philosophy.md +122 -0
  24. data/docs/archive/design/historical/01_rubyllm_evaluation.md +178 -0
  25. data/docs/archive/design/historical/06_design_decisions.md +143 -0
  26. data/docs/changelog/0.14-and-earlier.md +1 -1
  27. data/docs/decisions/001-rubyllm-as-provider-layer.md +6 -1
  28. data/docs/decisions/002-workflow-context-immutability.md +26 -1
  29. data/docs/decisions/006-no-built-in-guardrails.md +2 -1
  30. data/docs/decisions/012-canonical-execution-log-and-context-policy.md +120 -38
  31. data/docs/decisions/014-unified-persistence-durable-state.md +9 -2
  32. data/docs/decisions/016-semantic-multi-agent-handoff.md +114 -0
  33. data/docs/decisions/017-design-authority-and-adr-governance.md +200 -0
  34. data/docs/decisions/018-durability-guarantees-and-failure-model.md +488 -0
  35. data/docs/decisions/019-filter-contract-and-security-boundaries.md +229 -0
  36. data/docs/decisions/020-canonical-workflow-instance-identity.md +177 -0
  37. data/docs/decisions/021-generic-agent-invocation-identity-removal.md +119 -0
  38. data/docs/decisions/022-agent-execution-parent-identity-and-runtime-routing-boundary.md +193 -0
  39. data/docs/decisions/023-fsm-session-incarnation-identity-and-routing.md +139 -0
  40. data/docs/decisions/024-event-loop-single-writer-agent-runtime.md +188 -0
  41. data/docs/decisions/025-process-local-agent-ownership-and-runtime-admission.md +249 -0
  42. data/docs/decisions/026-workflow-runtime-admission-and-durable-terminal-barrier.md +257 -0
  43. data/docs/decisions/027-llm-adapter-provider-boundary.md +93 -0
  44. data/docs/decisions/028-preparing-recovery-replay-contract.md +106 -0
  45. data/docs/decisions/029-semantic-completion-and-application-effect-boundary.md +220 -0
  46. data/docs/decisions/030-agent-handoff-domain-and-durable-responsibility.md +235 -0
  47. data/docs/decisions/031-durable-multi-agent-coordination.md +301 -0
  48. data/docs/decisions/README.md +176 -0
  49. data/docs/design/durable-semantic-coordination/CHANGELOG_V2_REVISION_2.md +33 -0
  50. data/docs/design/durable-semantic-coordination/CONTINUATION_DECISION_REFACTOR.md +191 -0
  51. data/docs/design/durable-semantic-coordination/IMPLEMENTATION_DESIGN_V2.md +862 -0
  52. data/docs/design/durable-semantic-coordination/IMPLEMENTATION_REPORT.md +106 -0
  53. data/docs/design/durable-semantic-coordination/RECOVERY_CONTRACT_CLARIFICATIONS.md +179 -0
  54. data/docs/design/durable-semantic-coordination/RESPONSIBILITY_BOUNDARY_REVIEW.md +302 -0
  55. data/docs/features.md +65 -11
  56. data/docs/getting-started.md +77 -45
  57. data/docs/migrations/0.19.md +14 -7
  58. data/docs/migrations/0.22.md +390 -0
  59. data/docs/migrations/durable-semantic-coordination-v2.md +65 -0
  60. data/docs/persistence-backends.md +129 -40
  61. data/docs/runtime-and-concurrency.md +227 -33
  62. data/examples/README.md +13 -0
  63. data/lib/phronomy/agent/agent_execution.rb +21 -17
  64. data/lib/phronomy/agent/agent_invocation.rb +288 -93
  65. data/lib/phronomy/agent/agent_invocation_session_builder.rb +236 -202
  66. data/lib/phronomy/agent/agent_root.rb +3 -3
  67. data/lib/phronomy/agent/approval_evaluation_request.rb +37 -19
  68. data/lib/phronomy/agent/async_event_api.rb +162 -72
  69. data/lib/phronomy/agent/base.rb +416 -181
  70. data/lib/phronomy/agent/concerns/before_llm_input.rb +1 -1
  71. data/lib/phronomy/agent/context_assembler.rb +437 -178
  72. data/lib/phronomy/agent/context_candidate_resolver.rb +2 -2
  73. data/lib/phronomy/agent/context_plan.rb +18 -13
  74. data/lib/phronomy/agent/context_plan_validator.rb +246 -88
  75. data/lib/phronomy/agent/context_policies/default.rb +123 -34
  76. data/lib/phronomy/agent/context_policy.rb +109 -3
  77. data/lib/phronomy/agent/context_policy_input.rb +244 -0
  78. data/lib/phronomy/agent/context_policy_input_builder.rb +241 -0
  79. data/lib/phronomy/agent/exact_execution.rb +153 -0
  80. data/lib/phronomy/agent/execution_cancellation.rb +25 -0
  81. data/lib/phronomy/agent/execution_coordinator.rb +2417 -572
  82. data/lib/phronomy/agent/handoff.rb +43 -0
  83. data/lib/phronomy/agent/handoff_capability_factory.rb +45 -0
  84. data/lib/phronomy/agent/handoff_context.rb +120 -0
  85. data/lib/phronomy/agent/handoff_execution_coordinator.rb +143 -0
  86. data/lib/phronomy/agent/handoff_policy.rb +143 -0
  87. data/lib/phronomy/agent/handoff_projection.rb +207 -0
  88. data/lib/phronomy/agent/handoff_request.rb +45 -0
  89. data/lib/phronomy/agent/handoff_runner.rb +178 -0
  90. data/lib/phronomy/agent/handoff_state.rb +43 -0
  91. data/lib/phronomy/agent/journal_record.rb +17 -4
  92. data/lib/phronomy/agent/llm_input_build_context.rb +1 -1
  93. data/lib/phronomy/agent/llm_input_manifest.rb +277 -2
  94. data/lib/phronomy/agent/llm_operation_result.rb +12 -7
  95. data/lib/phronomy/agent/phase_machine_builder.rb +19 -7
  96. data/lib/phronomy/agent/provider_call_outcome.rb +23 -7
  97. data/lib/phronomy/agent/recovery_coordinator/continuation.rb +174 -0
  98. data/lib/phronomy/agent/recovery_coordinator/installation.rb +381 -0
  99. data/lib/phronomy/agent/recovery_coordinator/resolution.rb +511 -0
  100. data/lib/phronomy/agent/recovery_coordinator.rb +217 -0
  101. data/lib/phronomy/agent/recovery_support.rb +504 -0
  102. data/lib/phronomy/agent/ruby_llm_materializer.rb +16 -13
  103. data/lib/phronomy/agent/selection/candidate.rb +53 -0
  104. data/lib/phronomy/agent/selection/constraint.rb +49 -0
  105. data/lib/phronomy/agent/shared_state.rb +38 -1
  106. data/lib/phronomy/agent/tool_approval_request.rb +33 -5
  107. data/lib/phronomy/agent/tool_definition_set.rb +49 -3
  108. data/lib/phronomy/agent/tool_invocation.rb +339 -103
  109. data/lib/phronomy/agent/tool_invocation_session_builder.rb +49 -45
  110. data/lib/phronomy/agent.rb +20 -2
  111. data/lib/phronomy/agent_already_exists_error.rb +5 -0
  112. data/lib/phronomy/agent_purged_error.rb +5 -0
  113. data/lib/phronomy/engine/concurrency/offload_pool.rb +17 -3
  114. data/lib/phronomy/engine/concurrency/physical_completion_task.rb +135 -0
  115. data/lib/phronomy/engine/event_loop.rb +622 -63
  116. data/lib/phronomy/engine/fsm_session.rb +194 -21
  117. data/lib/phronomy/engine/runtime/agent_ownership_registry.rb +352 -0
  118. data/lib/phronomy/engine/runtime/team_ownership_registry.rb +77 -0
  119. data/lib/phronomy/engine/runtime.rb +92 -20
  120. data/lib/phronomy/generator_verifier.rb +12 -14
  121. data/lib/phronomy/invocation_context.rb +9 -29
  122. data/lib/phronomy/multi_agent/admission_registry.rb +51 -0
  123. data/lib/phronomy/multi_agent/durable_subagent_coordinator.rb +134 -0
  124. data/lib/phronomy/multi_agent/fan_out_invocation.rb +24 -33
  125. data/lib/phronomy/multi_agent/fan_out_session_builder.rb +12 -19
  126. data/lib/phronomy/multi_agent/orchestrator.rb +71 -26
  127. data/lib/phronomy/multi_agent/team_coordinator.rb +473 -125
  128. data/lib/phronomy/multi_agent/team_execution.rb +44 -0
  129. data/lib/phronomy/multi_agent/team_root.rb +41 -0
  130. data/lib/phronomy/persistence/durable_codec.rb +706 -0
  131. data/lib/phronomy/persistence/durable_record.rb +117 -0
  132. data/lib/phronomy/persistence/in_memory.rb +471 -133
  133. data/lib/phronomy/persistence/migration/initial_format_migration.rb +226 -0
  134. data/lib/phronomy/persistence/repository_facades.rb +535 -0
  135. data/lib/phronomy/persistence.rb +174 -40
  136. data/lib/phronomy/recovery.rb +186 -0
  137. data/lib/phronomy/testing/persistence_contract/a_journal_repository.rb +2 -2
  138. data/lib/phronomy/testing/persistence_contract/a_persistence_backend.rb +2 -1
  139. data/lib/phronomy/testing/persistence_contract/a_workflow_state_repository.rb +19 -19
  140. data/lib/phronomy/testing/persistence_contract/an_agent_repository.rb +3 -3
  141. data/lib/phronomy/testing/persistence_contract/an_execution_repository.rb +5 -5
  142. data/lib/phronomy/testing/persistence_contract/coordination_repositories.rb +137 -0
  143. data/lib/phronomy/testing/persistence_contract.rb +5 -0
  144. data/lib/phronomy/tools/agent.rb +1 -1
  145. data/lib/phronomy/tracing/automatic.rb +176 -0
  146. data/lib/phronomy/tracing/base.rb +11 -2
  147. data/lib/phronomy/tracing/langfuse_tracer.rb +20 -12
  148. data/lib/phronomy/version.rb +1 -1
  149. data/lib/phronomy/workflow.rb +3 -6
  150. data/lib/phronomy/workflow_context.rb +14 -5
  151. data/lib/phronomy/workflow_recovery.rb +123 -0
  152. data/lib/phronomy/workflow_runner.rb +468 -256
  153. data/lib/phronomy.rb +6 -0
  154. data/scripts/api_snapshot.rb +12 -0
  155. data/sig/phronomy/agent.rbs +209 -7
  156. data/sig/phronomy/handoff.rbs +41 -0
  157. data/sig/phronomy/multi_agent.rbs +35 -0
  158. data/sig/phronomy/persistence.rbs +123 -4
  159. data/sig/phronomy/runtime.rbs +1 -4
  160. data/sig/phronomy/workflow.rbs +2 -2
  161. data/sig/phronomy.rbs +10 -0
  162. metadata +84 -18
  163. data/examples/workflows/agent_event_mapping.rb +0 -101
  164. data/examples/workflows/generic_task_event_mapping.rb +0 -66
  165. data/lib/phronomy/agent/activation_registry.rb +0 -28
  166. data/lib/phronomy/agent/agent_execution_activation.rb +0 -172
  167. data/lib/phronomy/agent/context_candidate.rb +0 -47
  168. data/lib/phronomy/agent/context_parts/budget/token_budget_packer.rb +0 -53
  169. data/lib/phronomy/agent/context_parts/requirements/required_context_resolver.rb +0 -56
  170. data/lib/phronomy/agent/context_parts/selectors/recent_first_selector.rb +0 -30
  171. data/lib/phronomy/agent/context_parts/unit_builders/dependency_aware_unit_builder.rb +0 -118
  172. data/lib/phronomy/agent/context_policy_descriptor.rb +0 -49
  173. data/lib/phronomy/agent/context_policy_registry.rb +0 -46
  174. data/lib/phronomy/agent/context_request.rb +0 -35
  175. data/lib/phronomy/agent/context_selection_unit.rb +0 -38
  176. data/lib/phronomy/agent/derived_content_spec.rb +0 -34
  177. data/lib/phronomy/agent/runner.rb +0 -97
  178. data/lib/phronomy/multi_agent/handoff.rb +0 -64
@@ -1,48 +1,158 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require "time"
4
+ require "securerandom"
4
5
 
5
6
  module Phronomy
6
7
  module Agent
7
8
  class ExecutionCoordinator
8
- Prepared = Data.define(:execution, :runtime_projection, :filtered_input, :config)
9
- AgentTerminalCommand = Struct.new(:coordinator, :activation, :result_task, :outcome, :commit_error)
10
- PrepFailureCommand = Struct.new(:coordinator, :on_event_listener, :result_task, :error)
9
+ # External/API -> EventLoop control messages.
10
+ StartCommand = Data.define(
11
+ :coordinator, :input, :config, :mode,
12
+ :approval_policy, :approval_listener, :on_event, :result_task,
13
+ :admission_token
14
+ )
15
+ ResumeCommand = Data.define(
16
+ :coordinator, :execution_id, :approval_request_id,
17
+ :approved, :config, :result_task
18
+ )
19
+
20
+ # EventLoop -> Offload operation-specific immutable snapshots.
21
+ InitialPreparationCommand = Data.define(
22
+ :root, :journal_records, :input, :config, :preparation_replayable
23
+ )
24
+ ProviderDispatchPreparationCommand = Data.define(
25
+ :execution_id, :fsm_session_id, :expected_execution_revision,
26
+ :root, :journal_records, :execution, :base_manifest,
27
+ :invocation_config, :runtime_snapshot, :streaming,
28
+ :pending_llm_call_id, :pending_llm_started_at
29
+ )
30
+ ToolDispatchPreparationCommand = Data.define(
31
+ :execution_id, :fsm_session_id, :expected_execution_revision,
32
+ :root, :execution, :runtime_snapshot, :tool_batch_snapshot
33
+ )
34
+ ProviderDispatchPreparationReconciliationCommand = Data.define(
35
+ :operation, :intended_result, :original_error
36
+ )
37
+ ToolDispatchPreparationReconciliationCommand = Data.define(
38
+ :operation, :intended_result, :original_error
39
+ )
40
+ ResumeCommitCommand = Data.define(
41
+ :execution_id, :expected_execution_revision,
42
+ :root, :execution, :approval_request_id, :approved
43
+ )
44
+ HandoffTerminalView = Data.define(
45
+ :target_agent_id, :responsibility, :selection_intent,
46
+ :llm_call_id, :tool_call_id, :policy
47
+ )
48
+ TerminalView = Data.define(
49
+ :phase, :output, :usage, :approval_request, :rejected,
50
+ :input_blocked, :output_blocked, :block_error,
51
+ :invocation_error, :handoff, :callback_failure, :source_error, :cancel_requested
52
+ )
53
+ TerminalCommitCommand = Data.define(
54
+ :execution_id, :fsm_session_id, :expected_execution_revision,
55
+ :root, :journal_records, :execution, :runtime_snapshot,
56
+ :terminal_view, :state_required
57
+ )
58
+ TerminalDelivery = Data.define(
59
+ :result_task, :application_listener, :approval_listener,
60
+ :handoff_request, :handoff_manifest
61
+ )
62
+
63
+ # Offload -> EventLoop operation results. No live Agent/Invocation object is
64
+ # mutated by a worker; these values are validated and applied on EventLoop.
65
+ InitialPreparationResult = Data.define(
66
+ :execution, :root, :runtime_projection, :filtered_input,
67
+ :config, :appended_records, :error, :admission_outcome
68
+ )
69
+ ProviderDispatchPreparationResult = Data.define(:execution, :runtime_projection, :error)
70
+ ToolDispatchPreparationResult = Data.define(:execution)
71
+ ProviderDispatchPreparationReconciliationResult = Data.define(
72
+ :disposition, :preparation_result
73
+ )
74
+ ToolDispatchPreparationReconciliationResult = Data.define(
75
+ :disposition, :preparation_result
76
+ )
77
+ ResumeCommitResult = Data.define(:execution, :root)
78
+ TerminalOutcome = Data.define(
79
+ :type, :execution, :root, :appended_records,
80
+ :result, :error, :approval_request
81
+ )
82
+
83
+ InitialPreparationReady = Data.define(
84
+ :coordinator, :request, :result, :error
85
+ )
86
+ InitialPreparationRecoveryReady = Data.define(
87
+ :coordinator, :execution_id, :expected_execution_revision,
88
+ :result_task, :load_completion, :result, :error
89
+ )
90
+ ProviderDispatchPreparationReady = Data.define(
91
+ :coordinator, :operation, :result, :error
92
+ )
93
+ ToolDispatchPreparationReady = Data.define(
94
+ :coordinator, :operation, :result, :error
95
+ )
96
+ ProviderDispatchPreparationReconciliationReady = Data.define(
97
+ :coordinator, :command, :result, :error
98
+ )
99
+ ToolDispatchPreparationReconciliationReady = Data.define(
100
+ :coordinator, :command, :result, :error
101
+ )
102
+ ResumeCommitReady = Data.define(
103
+ :coordinator, :request, :operation, :result, :error
104
+ )
105
+ TerminalCommitReady = Data.define(
106
+ :coordinator, :operation, :delivery, :outcome, :error
107
+ )
108
+ DeferredTerminalCommand = Data.define(
109
+ :coordinator, :execution_id, :result_task, :invocation,
110
+ :source_error, :fsm_session_id
111
+ )
112
+
113
+ class PreparationOutcomeUnknownError < Phronomy::Error
114
+ attr_reader :original_error, :intended_result
115
+
116
+ def initialize(original_error, intended_result)
117
+ @original_error = original_error
118
+ @intended_result = intended_result
119
+ super(
120
+ "durable dispatch preparation outcome is uncertain: " \
121
+ "#{original_error.class}: #{original_error.message}"
122
+ )
123
+ set_backtrace(original_error.backtrace)
124
+ end
125
+ end
126
+ private_constant :PreparationOutcomeUnknownError
11
127
 
12
128
  def initialize(agent)
13
129
  @agent = agent
14
130
  end
15
131
 
16
132
  def start(
17
- input, thread_id: nil, config: {}, mode: :invoke,
133
+ input, config: {}, mode: :invoke,
18
134
  approval_policy: nil, approval_listener: nil, on_event: nil
19
135
  )
136
+ @agent.send(:__assert_live_agent!)
137
+ @agent.send(:_reject_removed_generic_identity_keys!, config)
20
138
  result_task = Phronomy::Task.deferred(name: "agent-#{@agent.agent_id}-#{mode}")
21
- runtime = Phronomy::Runtime.instance
22
- preparation = runtime.offload.submit(on_full: :raise) do
23
- prepare(input, thread_id: thread_id, config: config)
24
- end
25
- preparation.on_complete do |prepared, error|
26
- if error
27
- post_prep_failure(runtime, on_event, result_task, translated(error))
28
- else
29
- register(
30
- prepared,
31
- result_task,
32
- mode: mode,
33
- approval_policy: approval_policy,
34
- approval_listener: approval_listener,
35
- on_event: on_event
36
- )
37
- end
139
+ command = StartCommand.new(
140
+ coordinator: self,
141
+ input: input,
142
+ config: @agent.__invocation_config(config).dup.freeze,
143
+ mode: mode.to_sym,
144
+ approval_policy: approval_policy,
145
+ approval_listener: approval_listener,
146
+ on_event: on_event,
147
+ result_task: result_task,
148
+ admission_token: Object.new.freeze
149
+ )
150
+ unless post_control(Phronomy::Runtime.instance, command)
151
+ fail_task(result_task, runtime_rejected_error(:start))
38
152
  end
39
153
  result_task
40
154
  rescue => error
41
- if defined?(runtime) && runtime
42
- post_prep_failure(runtime, on_event, result_task, translated(error))
43
- else
44
- fail_task(result_task, translated(error))
45
- end
155
+ fail_task(result_task, translated(error)) if defined?(result_task) && result_task
46
156
  result_task
47
157
  end
48
158
 
@@ -52,156 +162,393 @@ module Phronomy
52
162
  approved:,
53
163
  config: {}
54
164
  )
165
+ @agent.send(:__assert_live_agent!)
166
+ @agent.send(:_reject_removed_generic_identity_keys!, config)
55
167
  result_task = Phronomy::Task.deferred(
56
168
  name: "agent-approval-resume:#{execution_id}"
57
169
  )
170
+ command = ResumeCommand.new(
171
+ coordinator: self,
172
+ execution_id: execution_id.to_s.freeze,
173
+ approval_request_id: approval_request_id.to_s.freeze,
174
+ approved: !!approved,
175
+ config: config.dup.freeze,
176
+ result_task: result_task
177
+ )
178
+ unless post_control(Phronomy::Runtime.instance, command)
179
+ fail_task(result_task, runtime_rejected_error(:resume))
180
+ end
181
+ result_task
182
+ rescue => error
183
+ fail_task(result_task, translated(error)) if defined?(result_task) && result_task
184
+ result_task
185
+ end
186
+
187
+ # Called only from an Agent FSMSession external-operation dispatch state on
188
+ # EventLoop. Each method captures one operation-specific immutable snapshot
189
+ # and sends only that snapshot to OffloadPool.
190
+ # @api private
191
+ def prepare_provider_dispatch(invocation, event_sink:, streaming:)
192
+ runtime = Phronomy::Runtime.instance
193
+ event_loop = runtime.event_loop
194
+ assert_event_loop!(event_loop)
195
+ state = event_loop.agent_execution_state(invocation.execution_id)
196
+ validate_live_session!(state, invocation, event_sink.fsm_session_id)
197
+ unless event_loop.fsm_session_state(event_sink.fsm_session_id) == :calling_llm
198
+ raise Phronomy::Error,
199
+ "Provider dispatch preparation requires the owning FSMSession to be in :calling_llm"
200
+ end
201
+
202
+ operation = ProviderDispatchPreparationCommand.new(
203
+ execution_id: state.execution_id,
204
+ fsm_session_id: event_sink.fsm_session_id.to_s.freeze,
205
+ expected_execution_revision: state.execution.execution_revision,
206
+ root: @agent.agent_root,
207
+ journal_records: @agent.send(:_journal_records_snapshot),
208
+ execution: state.execution,
209
+ base_manifest: state.base_manifest,
210
+ invocation_config: invocation.config.dup.freeze,
211
+ runtime_snapshot: invocation.runtime_snapshot,
212
+ streaming: !!streaming,
213
+ pending_llm_call_id: SecureRandom.uuid.to_s.freeze,
214
+ pending_llm_started_at: Time.now.utc.iso8601(6).freeze
215
+ )
216
+ submit_provider_dispatch_preparation(operation)
217
+ nil
218
+ rescue => error
219
+ event_sink.post(:llm_setup_failed, translated(error))
220
+ nil
221
+ end
222
+
223
+ # @api private
224
+ def prepare_tool_dispatch(invocation, event_sink:)
58
225
  runtime = Phronomy::Runtime.instance
59
- activation = runtime.__agent_activations.fetch(execution_id)
60
- unless activation
61
- fail_task(
62
- result_task,
63
- Phronomy::ExecutionRehydrationRequiredError.new(
64
- "no live activation for #{execution_id}; durable rehydration is required"
226
+ event_loop = runtime.event_loop
227
+ assert_event_loop!(event_loop)
228
+ state = event_loop.agent_execution_state(invocation.execution_id)
229
+ validate_live_session!(state, invocation, event_sink.fsm_session_id)
230
+ unless event_loop.fsm_session_state(event_sink.fsm_session_id) == :dispatching_tools
231
+ raise Phronomy::Error,
232
+ "Tool dispatch preparation requires the owning FSMSession to be in :dispatching_tools"
233
+ end
234
+
235
+ tool_batch_snapshot = RecoverySupport.build_tool_batch_snapshot(invocation)
236
+ if tool_batch_snapshot.empty?
237
+ raise Phronomy::Error, "Tool dispatch preparation requires a non-empty Tool batch"
238
+ end
239
+
240
+ operation = ToolDispatchPreparationCommand.new(
241
+ execution_id: state.execution_id,
242
+ fsm_session_id: event_sink.fsm_session_id.to_s.freeze,
243
+ expected_execution_revision: state.execution.execution_revision,
244
+ root: @agent.agent_root,
245
+ execution: state.execution,
246
+ runtime_snapshot: invocation.runtime_snapshot,
247
+ tool_batch_snapshot: tool_batch_snapshot
248
+ )
249
+ submit_tool_dispatch_preparation(operation)
250
+ nil
251
+ rescue => error
252
+ event_sink.post(:tool_setup_failed, translated(error))
253
+ nil
254
+ end
255
+
256
+ def submit_provider_dispatch_preparation(operation)
257
+ runtime = Phronomy::Runtime.instance
258
+ task = runtime.offload.submit(on_full: :raise) do
259
+ perform_provider_dispatch_preparation(operation)
260
+ end
261
+ task.on_complete do |result, error|
262
+ ready = ProviderDispatchPreparationReady.new(
263
+ coordinator: self,
264
+ operation: operation,
265
+ result: result,
266
+ error: error
267
+ )
268
+ unless post_control(runtime, ready)
269
+ Phronomy.configuration.logger&.warn(
270
+ "[Phronomy] EventLoop rejected Provider dispatch preparation result for " \
271
+ "#{operation.execution_id}"
65
272
  )
66
- )
67
- return result_task
273
+ end
68
274
  end
275
+ end
69
276
 
70
- unless activation.agent.equal?(@agent) &&
71
- activation.execution.status == :suspended
72
- fail_task(
73
- result_task,
74
- ArgumentError.new(
75
- "execution is not a suspended execution of this agent: #{execution_id}"
277
+ def submit_tool_dispatch_preparation(operation)
278
+ runtime = Phronomy::Runtime.instance
279
+ task = runtime.offload.submit(on_full: :raise) do
280
+ perform_tool_dispatch_preparation(operation)
281
+ end
282
+ task.on_complete do |result, error|
283
+ ready = ToolDispatchPreparationReady.new(
284
+ coordinator: self,
285
+ operation: operation,
286
+ result: result,
287
+ error: error
288
+ )
289
+ unless post_control(runtime, ready)
290
+ Phronomy.configuration.logger&.warn(
291
+ "[Phronomy] EventLoop rejected Tool dispatch preparation result for " \
292
+ "#{operation.execution_id}"
76
293
  )
77
- )
78
- return result_task
294
+ end
79
295
  end
296
+ end
80
297
 
81
- preparation = runtime.offload.submit(on_full: :raise) do
82
- mark_resuming!(
83
- activation,
84
- approval_request_id: approval_request_id,
85
- approved: approved
86
- )
87
- activation
298
+ def submit_provider_dispatch_preparation_reconciliation(operation, uncertainty)
299
+ # simplecov:disable
300
+ runtime = Phronomy::Runtime.instance
301
+ command = ProviderDispatchPreparationReconciliationCommand.new(
302
+ operation: operation,
303
+ intended_result: uncertainty.intended_result,
304
+ original_error: uncertainty.original_error
305
+ )
306
+ task = runtime.offload.submit(on_full: :raise) do
307
+ perform_provider_dispatch_preparation_reconciliation(command)
88
308
  end
89
- preparation.on_complete do |prepared_activation, error|
90
- if error
91
- fail_task(result_task, translated(error))
92
- next
309
+ task.on_complete do |result, error|
310
+ ready = ProviderDispatchPreparationReconciliationReady.new(
311
+ coordinator: self,
312
+ command: command,
313
+ result: result,
314
+ error: error
315
+ )
316
+ unless post_control(runtime, ready)
317
+ Phronomy.configuration.logger&.warn(
318
+ "[Phronomy] EventLoop rejected Provider dispatch reconciliation result for #{operation.execution_id}"
319
+ )
93
320
  end
321
+ end
322
+ rescue => error
323
+ mark_barrier_recovery_required(operation, error)
324
+ # simplecov:enable
325
+ end
94
326
 
95
- begin
96
- start_resume(
97
- prepared_activation,
98
- result_task,
99
- approved: approved,
100
- config: config
327
+ def submit_tool_dispatch_preparation_reconciliation(operation, uncertainty)
328
+ # simplecov:disable
329
+ runtime = Phronomy::Runtime.instance
330
+ command = ToolDispatchPreparationReconciliationCommand.new(
331
+ operation: operation,
332
+ intended_result: uncertainty.intended_result,
333
+ original_error: uncertainty.original_error
334
+ )
335
+ task = runtime.offload.submit(on_full: :raise) do
336
+ perform_tool_dispatch_preparation_reconciliation(command)
337
+ end
338
+ task.on_complete do |result, error|
339
+ ready = ToolDispatchPreparationReconciliationReady.new(
340
+ coordinator: self,
341
+ command: command,
342
+ result: result,
343
+ error: error
344
+ )
345
+ unless post_control(runtime, ready)
346
+ Phronomy.configuration.logger&.warn(
347
+ "[Phronomy] EventLoop rejected Tool dispatch reconciliation result for #{operation.execution_id}"
101
348
  )
102
- rescue => start_error
103
- commit = runtime.offload.submit(on_full: :raise) do
104
- commit_failed(
105
- prepared_activation,
106
- prepared_activation.invocation,
107
- start_error
108
- )
109
- end
110
- commit.on_complete do |terminal, commit_error|
111
- if commit_error
112
- fail_task(result_task, commit_error)
113
- else
114
- runtime.__agent_activations.delete(execution_id)
115
- fail_task(result_task, terminal.fetch(:error))
116
- end
117
- end
118
349
  end
119
350
  end
120
- result_task
121
351
  rescue => error
122
- fail_task(result_task, translated(error)) if result_task
123
- result_task
352
+ mark_barrier_recovery_required(operation, error)
353
+ # simplecov:enable
124
354
  end
125
355
 
126
- # Persists facts from the previous Provider/Tool cycle and fixes the next
127
- # Manifest from the current local Agent state. This runs on OffloadPool.
128
- def prepare_next_llm_call(activation)
129
- snapshot = activation.runtime_snapshot
130
- root = @agent.agent_root
131
- current = activation.execution
132
- journal_records = @agent.send(:_journal_records_snapshot)
133
- manifest = manifest_ref = updated = nil
356
+ # Every control message is delivered by EventLoop. This method is the only
357
+ # coordinator entry point allowed to advance Phronomy-managed live state.
358
+ # @api private
359
+ def deliver_on_event_loop(command)
360
+ event_loop = Phronomy::Runtime.instance.event_loop
361
+ assert_event_loop!(event_loop)
134
362
 
135
- @agent.persistence.transaction do |tx|
136
- assert_local_durable_base!(tx, root)
137
- encoded_records, call_records = encode_runtime_records(
138
- activation,
139
- tx: tx,
140
- snapshot: snapshot,
141
- context_candidate: true,
142
- agent_root: root
143
- )
144
- staged = current.with(
145
- execution_revision: current.execution_revision,
146
- phase: :preparing_llm_call,
147
- working_records: current.working_records + encoded_records,
148
- llm_calls: current.llm_calls + call_records
363
+ case command
364
+ when StartCommand
365
+ begin_start_on_event_loop(command)
366
+ when InitialPreparationReady
367
+ apply_initial_preparation_on_event_loop(command)
368
+ when InitialPreparationRecoveryReady
369
+ apply_initial_preparation_recovery_on_event_loop(command)
370
+ when ProviderDispatchPreparationReady
371
+ apply_provider_dispatch_preparation_on_event_loop(command)
372
+ when ToolDispatchPreparationReady
373
+ apply_tool_dispatch_preparation_on_event_loop(command)
374
+ when ProviderDispatchPreparationReconciliationReady
375
+ apply_provider_dispatch_preparation_reconciliation_on_event_loop(command)
376
+ when ToolDispatchPreparationReconciliationReady
377
+ apply_tool_dispatch_preparation_reconciliation_on_event_loop(command)
378
+ when ResumeCommand
379
+ begin_resume_on_event_loop(command)
380
+ when ResumeCommitReady
381
+ apply_resume_commit_on_event_loop(command)
382
+ when TerminalCommitReady
383
+ apply_terminal_commit_on_event_loop(command)
384
+ when DeferredTerminalCommand
385
+ resume_deferred_terminal_on_event_loop(command)
386
+ else
387
+ raise Phronomy::Error, "unknown Agent control command: #{command.class}"
388
+ end
389
+ end
390
+
391
+ private
392
+
393
+ # ----------------------------------------------------------------------
394
+ # Initial preparation
395
+ # ----------------------------------------------------------------------
396
+
397
+ def begin_start_on_event_loop(request)
398
+ runtime = Phronomy::Runtime.instance
399
+ event_loop = runtime.event_loop
400
+ root = @agent.agent_root
401
+ if root.lifecycle_status == :closed
402
+ deliver_start_failure_on_event_loop(
403
+ request,
404
+ Phronomy::Error.new("agent is closed: #{@agent.agent_id}")
149
405
  )
150
- invocation_config = activation.invocation&.config || {}
151
- patch = @agent.send(
152
- :run_before_llm_input_hooks,
153
- call_sequence: staged.llm_calls.length + 1,
154
- config: invocation_config
406
+ return
407
+ end
408
+
409
+ @agent.send(:__assert_live_agent!)
410
+ admitted = false
411
+ submitted = false
412
+ event_loop.admit_agent_execution(
413
+ @agent.agent_id,
414
+ owner_token: request.admission_token
415
+ )
416
+ admitted = true
417
+
418
+ # Ownership purge may begin on an application thread between the first
419
+ # live-owner check and EventLoop admission. Re-check after the slot is
420
+ # installed: once this check succeeds, purge observes the admission and
421
+ # must abort instead of deleting the Agent underneath this start.
422
+ @agent.send(:__assert_live_agent!)
423
+
424
+ operation = InitialPreparationCommand.new(
425
+ root: root,
426
+ journal_records: @agent.send(:_journal_records_snapshot),
427
+ input: request.input,
428
+ config: request.config,
429
+ preparation_replayable: initial_preparation_replayable?(
430
+ request.input,
431
+ request.config,
432
+ request.approval_policy
155
433
  )
156
- manifest, manifest_ref = ContextAssembler.new(
157
- agent: @agent,
158
- persistence: tx,
159
- policy: context_policy_for(staged),
160
- journal_records: journal_records
161
- ).build_followup(
162
- base_manifest: activation.base_manifest,
163
- agent_root: root,
164
- execution: staged,
165
- config: invocation_config,
166
- patch: patch
167
- )
168
- refs = Array(staged.metadata["manifest_refs"]) + [manifest_ref]
169
- updated = staged.with(
170
- phase: :calling_llm,
171
- metadata: staged.metadata.merge(
172
- "manifest_ref" => manifest_ref,
173
- "manifest_refs" => refs
434
+ )
435
+ task = runtime.offload.submit(on_full: :raise) do
436
+ perform_initial_preparation(operation)
437
+ end
438
+ submitted = true
439
+ task.on_complete do |result, error|
440
+ ready = InitialPreparationReady.new(
441
+ coordinator: self,
442
+ request: request,
443
+ result: result,
444
+ error: error
445
+ )
446
+ fail_task(request.result_task, runtime_rejected_error(:initial_preparation)) unless
447
+ post_control(runtime, ready)
448
+ end
449
+ rescue => error
450
+ if admitted
451
+ if submitted
452
+ event_loop.mark_agent_admission_recovery_required(
453
+ @agent.agent_id,
454
+ owner_token: request.admission_token
174
455
  )
175
- )
176
- tx.executions.save(
177
- current.execution_id,
178
- expected_revision: current.execution_revision,
179
- execution: updated
456
+ else
457
+ event_loop.release_agent_execution_admission(
458
+ @agent.agent_id,
459
+ owner_token: request.admission_token
460
+ )
461
+ end
462
+ end
463
+ deliver_start_failure_on_event_loop(request, translated(error))
464
+ end
465
+
466
+ def perform_initial_preparation(operation)
467
+ begin
468
+ raw_message = @agent.send(:extract_message, operation.input)
469
+ rescue => error
470
+ return InitialPreparationResult.new(
471
+ execution: nil,
472
+ root: operation.root,
473
+ runtime_projection: nil,
474
+ filtered_input: nil,
475
+ config: operation.config,
476
+ appended_records: [].freeze,
477
+ error: translated(error),
478
+ admission_outcome: :not_established
180
479
  )
181
480
  end
182
481
 
183
- activation.replace_execution(updated)
184
- activation.acknowledge_runtime_snapshot(snapshot)
482
+ begin
483
+ execution, active_root = admit_execution(
484
+ raw_message,
485
+ root: operation.root,
486
+ mode: (operation.config[:phronomy_recovery_mode] || :invoke).to_sym,
487
+ config: operation.config,
488
+ preparation_replayable: operation.preparation_replayable
489
+ )
490
+ rescue Phronomy::AgentBusyError => error
491
+ # A durable busy conflict proves that another nonterminal logical
492
+ # Execution already exists for this agent_id. Process loss does not
493
+ # make that execution terminal; keep Runtime admission fail-closed
494
+ # until ACS-15 recovery/reconciliation can establish current lineage.
495
+ return InitialPreparationResult.new(
496
+ execution: nil,
497
+ root: operation.root,
498
+ runtime_projection: nil,
499
+ filtered_input: nil,
500
+ config: operation.config,
501
+ appended_records: [].freeze,
502
+ error: translated(error),
503
+ admission_outcome: :recovery_required
504
+ )
505
+ rescue Phronomy::Persistence::ConflictError,
506
+ Phronomy::Persistence::NotFoundError,
507
+ Phronomy::Persistence::SerializationError,
508
+ ArgumentError,
509
+ Phronomy::ConfigurationError => error
510
+ return InitialPreparationResult.new(
511
+ execution: nil,
512
+ root: operation.root,
513
+ runtime_projection: nil,
514
+ filtered_input: nil,
515
+ config: operation.config,
516
+ appended_records: [].freeze,
517
+ error: translated(error),
518
+ admission_outcome: :not_established
519
+ )
520
+ rescue => error
521
+ return InitialPreparationResult.new(
522
+ execution: nil,
523
+ root: operation.root,
524
+ runtime_projection: nil,
525
+ filtered_input: nil,
526
+ config: operation.config,
527
+ appended_records: [].freeze,
528
+ error: translated(error),
529
+ admission_outcome: :outcome_unknown
530
+ )
531
+ end
185
532
 
186
- projection = RubyLLMMaterializer.new(
187
- agent: @agent,
188
- persistence: @agent.persistence
189
- ).materialize(manifest: manifest, manifest_ref: manifest_ref)
190
- activation.replace_runtime_projection(projection)
191
- projection
533
+ perform_admitted_initial_preparation(
534
+ input: operation.input,
535
+ config: operation.config,
536
+ execution: execution,
537
+ active_root: active_root,
538
+ journal_records: operation.journal_records
539
+ )
192
540
  end
193
541
 
194
- def prepare(input, thread_id:, config:)
195
- raw_message = @agent.send(:extract_message, input)
196
- execution, active_root = admit_execution(
197
- raw_message,
198
- thread_id: thread_id
199
- )
200
- @agent.__replace_root(active_root)
542
+ def perform_admitted_initial_preparation(
543
+ input:,
544
+ config:,
545
+ execution:,
546
+ active_root:,
547
+ journal_records:
548
+ )
201
549
  current_execution = execution
202
-
203
550
  begin
204
- effective_config = thread_id ? config.merge(thread_id: thread_id) : config
551
+ effective_config = config
205
552
  @agent.send(
206
553
  :check_cancellation!,
207
554
  effective_config,
@@ -215,46 +562,56 @@ module Phronomy
215
562
  )
216
563
  filtered_message = @agent.send(:extract_message, filtered_input)
217
564
  manifest = manifest_ref = active_execution = nil
218
- local_journal = @agent.send(:_journal_records_snapshot)
219
565
 
220
- @agent.persistence.transaction do |tx|
221
- assert_local_durable_base!(tx, active_root)
222
- filtered_ref = tx.contents.put_text(filtered_message)
223
- input_record = JournalRecord.new(
224
- agent_id: @agent.agent_id,
225
- execution_id: current_execution.execution_id,
226
- kind: :external_message,
227
- channel: :external,
228
- role: :user,
229
- content_ref: filtered_ref,
230
- correlation_id: thread_id,
231
- context_generation: active_root.transcript_generation,
232
- context_candidate: true
233
- )
234
- staged = current_execution.with(
235
- execution_revision: current_execution.execution_revision,
236
- working_records: current_execution.working_records + [input_record],
237
- metadata: current_execution.metadata.merge(
238
- "current_input_ref" => filtered_ref,
239
- "current_input_record_id" => input_record.record_id
240
- )
566
+ # Content-addressed source material may be written before the Policy call.
567
+ # No mutable Agent/Execution repository state is committed here.
568
+ filtered_ref = @agent.persistence.contents.put_text(filtered_message)
569
+ input_record = JournalRecord.new(
570
+ agent_id: @agent.agent_id,
571
+ execution_id: current_execution.execution_id,
572
+ kind: :external_message,
573
+ channel: :external,
574
+ role: :user,
575
+ content_ref: filtered_ref,
576
+ context_generation: active_root.transcript_generation,
577
+ context_candidate: true,
578
+ metadata: {"handoff_policy_category" => "current_request"}
579
+ )
580
+ staged = current_execution.with(
581
+ execution_revision: current_execution.execution_revision,
582
+ working_records: current_execution.working_records + [input_record],
583
+ metadata: current_execution.metadata.merge(
584
+ "current_input_ref" => filtered_ref,
585
+ "current_input_record_id" => input_record.record_id
241
586
  )
242
- manifest, manifest_ref = ContextAssembler.new(
243
- agent: @agent,
244
- persistence: tx,
245
- policy: context_policy_for(staged),
246
- journal_records: local_journal
247
- ).build_initial(
248
- input: filtered_input,
249
- agent_root: active_root,
250
- execution: staged,
251
- config: effective_config,
252
- patch: @agent.send(
253
- :run_before_llm_input_hooks,
254
- call_sequence: 1,
255
- config: effective_config
256
- )
587
+ )
588
+ assembler = ContextAssembler.new(
589
+ agent: @agent,
590
+ persistence: @agent.persistence,
591
+ journal_records: journal_records
592
+ )
593
+ prepared = assembler.prepare_initial(
594
+ input: filtered_input,
595
+ agent_root: active_root,
596
+ execution: staged,
597
+ config: effective_config,
598
+ patch: @agent.send(
599
+ :run_before_llm_input_hooks,
600
+ call_sequence: 1,
601
+ config: effective_config
257
602
  )
603
+ )
604
+ @agent.send(
605
+ :check_cancellation!,
606
+ effective_config,
607
+ "invocation cancelled after context policy"
608
+ )
609
+
610
+ # Revalidate the durable base only after Application Policy returns.
611
+ # ContextPolicy never runs while this transaction is open.
612
+ @agent.persistence.transaction do |tx|
613
+ assert_local_durable_base!(tx, active_root)
614
+ manifest, manifest_ref = assembler.finalize(prepared, persistence: tx)
258
615
  active_execution = staged.with(
259
616
  status: :active,
260
617
  phase: :calling_llm,
@@ -276,52 +633,94 @@ module Phronomy
276
633
  agent: @agent,
277
634
  persistence: @agent.persistence
278
635
  ).materialize(manifest: manifest, manifest_ref: manifest_ref)
279
- Phronomy::Agent::ExecutionCoordinator::Prepared.new(
636
+ InitialPreparationResult.new(
280
637
  execution: active_execution,
638
+ root: active_root,
281
639
  runtime_projection: projection,
282
640
  filtered_input: filtered_input,
283
- config: effective_config
641
+ config: effective_config,
642
+ appended_records: [].freeze,
643
+ error: nil,
644
+ admission_outcome: :active
284
645
  )
285
646
  rescue => error
286
- commit_preparation_failure(current_execution, error)
287
- raise
647
+ failure = commit_preparation_failure(
648
+ current_execution,
649
+ active_root,
650
+ error
651
+ )
652
+ InitialPreparationResult.new(
653
+ execution: failure.fetch(:execution),
654
+ root: failure.fetch(:root),
655
+ runtime_projection: nil,
656
+ filtered_input: nil,
657
+ config: config,
658
+ appended_records: failure.fetch(:appended_records),
659
+ error: failure.fetch(:error),
660
+ admission_outcome: :terminal
661
+ )
288
662
  end
289
663
  end
290
664
 
291
- def admit_execution(raw_message, thread_id:)
292
- root = @agent.agent_root
293
- raise Phronomy::Error, "agent is closed: #{@agent.agent_id}" if root.lifecycle_status == :closed
665
+ def admit_execution(
666
+ raw_message,
667
+ root:,
668
+ mode: :invoke,
669
+ config: {},
670
+ preparation_replayable: false
671
+ )
672
+ if root.lifecycle_status == :closed
673
+ raise Phronomy::Error,
674
+ "agent is closed: #{@agent.agent_id}"
675
+ end
294
676
 
295
677
  execution = next_root = nil
678
+ mode = mode.to_sym
679
+ pending_llm_call_id = SecureRandom.uuid.to_s.freeze
680
+ pending_started_at = Time.now.utc.iso8601(6).freeze
681
+
296
682
  @agent.persistence.transaction do |tx|
297
683
  input_ref = tx.contents.put_text(raw_message)
684
+ durable_context_ref = if config.key?(:durable_context)
685
+ tx.contents.put_json(config.fetch(:durable_context))
686
+ end
298
687
  input_record = JournalRecord.new(
299
688
  agent_id: @agent.agent_id,
300
689
  kind: :input_received,
301
690
  channel: :external,
302
691
  role: :user,
303
692
  content_ref: input_ref,
304
- correlation_id: thread_id,
305
693
  context_generation: root.transcript_generation,
306
694
  context_candidate: false
307
695
  )
308
- policy_descriptor = ContextPolicies::Default.new.descriptor
309
696
  execution = AgentExecution.start(
310
697
  agent_root: root,
311
698
  input_record: input_record,
699
+ execution_id: config[:phronomy_reserved_execution_id] || SecureRandom.uuid,
312
700
  metadata: {
313
- "thread_id" => thread_id,
701
+ "coordination" => config[:phronomy_coordination],
314
702
  "current_input_ref" => input_ref,
315
- "context_policy" => policy_descriptor.to_h
703
+ "durable_context_ref" => durable_context_ref,
704
+ "preparation_replayable" => !!preparation_replayable,
705
+ RecoverySupport::CONTRACT_VERSION_KEY =>
706
+ RecoverySupport::CONTRACT_VERSION,
707
+ RecoverySupport::INVOCATION_MODE_KEY => mode.to_s,
708
+ RecoverySupport::PENDING_LLM_ID_KEY =>
709
+ pending_llm_call_id,
710
+ RecoverySupport::PENDING_LLM_STARTED_AT_KEY =>
711
+ pending_started_at
316
712
  }.compact
317
713
  )
318
714
  input_record = JournalRecord.from_h(
319
- input_record.to_h.merge("execution_id" => execution.execution_id)
715
+ input_record.to_h.merge(
716
+ "execution_id" => execution.execution_id
717
+ )
320
718
  )
321
719
  execution = execution.with(
322
720
  execution_revision: 0,
323
721
  working_records: [input_record]
324
722
  )
723
+ validate_coordination_admission!(tx, execution)
325
724
  tx.executions.create_active(execution)
326
725
  next_root = root.with(
327
726
  agent_revision: root.agent_revision + 1,
@@ -336,9 +735,70 @@ module Phronomy
336
735
  [execution, next_root]
337
736
  end
338
737
 
339
- def commit_preparation_failure(execution, error)
738
+ def validate_coordination_admission!(tx, execution)
739
+ owner = execution.metadata["coordination"]
740
+ return unless owner
741
+ case owner.fetch("kind")
742
+ when "team"
743
+ team = tx.team_executions.load(owner.fetch("team_execution_id"))
744
+ unless team.team_id == owner.fetch("team_id") && team.active? && !team.metadata["cancel_requested"]
745
+ raise Phronomy::CancellationError, "Team run is no longer dispatchable"
746
+ end
747
+ slot = if owner.fetch("slot") == "coordinator"
748
+ team.coordinator
749
+ else
750
+ assignment = team.assignments.find { |entry| entry.fetch("task_id") == owner.fetch("slot") }
751
+ worker = assignment && team.workers.fetch(assignment.fetch("worker"))
752
+ worker&.merge("execution_id" => assignment.fetch("execution_id"))
753
+ end
754
+ unless slot && slot.fetch("execution_id") == execution.execution_id && slot.fetch("agent_id") == execution.agent_id
755
+ raise Phronomy::Persistence::ConflictError, "Team reserved child identity mismatch"
756
+ end
757
+ when "subagent"
758
+ parent = tx.executions.load(owner.fetch("parent_execution_id"))
759
+ unless parent.agent_id == owner.fetch("parent_agent_id") && parent.active? && !parent.metadata["coordination_cancel_requested"]
760
+ raise Phronomy::CancellationError, "Parent run is no longer dispatchable"
761
+ end
762
+ snapshot = tx.contents.fetch_json(parent.metadata.fetch("multi_agent_coordination_ref"))
763
+ slot = snapshot.fetch("children").find { |entry| entry.fetch("slot") == owner.fetch("slot") }
764
+ unless slot && slot.fetch("agent_id") == execution.agent_id && slot.fetch("execution_id") == execution.execution_id
765
+ raise Phronomy::Persistence::ConflictError, "Parent reserved child identity mismatch"
766
+ end
767
+ when "handoff"
768
+ routing = tx.handoff_states.load(owner.fetch("main_agent_id"))
769
+ unless routing && routing.active_agent_id == execution.agent_id
770
+ raise Phronomy::Persistence::ConflictError, "Handoff responsibility changed before admission"
771
+ end
772
+ if Array(routing.metadata["cancelled_execution_ids"]).include?(execution.execution_id)
773
+ raise Phronomy::CancellationError, "Handoff reservation was cancelled"
774
+ end
775
+ if routing.phase != "stable" && routing.pending_target_execution_id != execution.execution_id
776
+ raise Phronomy::Persistence::ConflictError, "Handoff reserved Target identity mismatch"
777
+ end
778
+ else
779
+ raise Phronomy::ConfigurationError, "Unknown coordination owner kind"
780
+ end
781
+ end
782
+
783
+ def initial_preparation_replayable?(input, config, approval_policy)
784
+ return false unless input.is_a?(String)
785
+ return false unless approval_policy.nil?
786
+ if config.key?(:phronomy_handoff_bindings) ||
787
+ config.key?(:phronomy_handoff_context)
788
+ return false unless config[:phronomy_coordination]&.fetch("kind", nil) == "handoff"
789
+ end
790
+
791
+ invocation_context = config[:invocation_context]
792
+ return true unless invocation_context
793
+
794
+ %i[approval_policy redaction_policy token_budget].none? do |name|
795
+ invocation_context.respond_to?(name) &&
796
+ !invocation_context.public_send(name).nil?
797
+ end
798
+ end
799
+
800
+ def commit_preparation_failure(execution, root, error)
340
801
  translated_error = translated(error)
341
- root = @agent.agent_root
342
802
  failed = next_root = appended = nil
343
803
 
344
804
  @agent.persistence.transaction do |tx|
@@ -389,24 +849,95 @@ module Phronomy
389
849
  root: next_root
390
850
  )
391
851
  end
392
- @agent.send(:_append_journal_records, appended)
393
- @agent.__replace_root(next_root)
394
- failed
852
+ {
853
+ error: translated_error,
854
+ execution: failed,
855
+ root: next_root,
856
+ appended_records: Array(appended).freeze
857
+ }.freeze
395
858
  end
396
859
 
397
- def register(prepared, result_task, mode:, approval_policy:, approval_listener:, on_event:)
398
- activation = AgentExecutionActivation.new(
399
- execution: prepared.execution,
400
- agent: @agent,
401
- runtime_projection: prepared.runtime_projection,
402
- coordinator: self,
403
- application_listener: on_event
860
+ def apply_initial_preparation_on_event_loop(ready)
861
+ request = ready.request
862
+ event_loop = Phronomy::Runtime.instance.event_loop
863
+ if ready.error
864
+ event_loop.mark_agent_admission_recovery_required(
865
+ @agent.agent_id,
866
+ owner_token: request.admission_token
867
+ )
868
+ deliver_start_failure_on_event_loop(request, translated(ready.error))
869
+ return
870
+ end
871
+
872
+ result = ready.result
873
+ case result.admission_outcome
874
+ when :not_established
875
+ event_loop.release_agent_execution_admission(
876
+ @agent.agent_id,
877
+ owner_token: request.admission_token
878
+ )
879
+ deliver_start_failure_on_event_loop(request, result.error)
880
+ return
881
+ when :outcome_unknown, :recovery_required
882
+ event_loop.mark_agent_admission_recovery_required(
883
+ @agent.agent_id,
884
+ owner_token: request.admission_token
885
+ )
886
+ deliver_start_failure_on_event_loop(request, result.error)
887
+ return
888
+ when :active, :terminal
889
+ event_loop.bind_agent_execution_admission(
890
+ @agent.agent_id,
891
+ owner_token: request.admission_token,
892
+ execution_id: result.execution.execution_id
893
+ )
894
+ else
895
+ event_loop.mark_agent_admission_recovery_required(
896
+ @agent.agent_id,
897
+ owner_token: request.admission_token
898
+ )
899
+ raise Phronomy::Error,
900
+ "unknown initial admission outcome: #{result.admission_outcome.inspect}"
901
+ end
902
+
903
+ apply_agent_live_state(
904
+ root: result.root,
905
+ appended_records: result.appended_records
906
+ )
907
+ if result.admission_outcome == :terminal
908
+ event_loop.release_agent_execution_admission(
909
+ @agent.agent_id,
910
+ execution_id: result.execution.execution_id
911
+ )
912
+ deliver_start_failure_on_event_loop(request, result.error)
913
+ return
914
+ end
915
+
916
+ register_initial_session_on_event_loop(request, result)
917
+ rescue => error
918
+ # Preparation may already be durably active. Terminalize through another
919
+ # offloaded durable operation rather than releasing the admission blindly.
920
+ begin_terminal_without_session_on_event_loop(
921
+ request: request,
922
+ prepared: ready.result,
923
+ error: error
404
924
  )
925
+ end
926
+
927
+ def register_initial_session_on_event_loop(request, prepared)
405
928
  runtime = Phronomy::Runtime.instance
406
- runtime.__agent_activations.register(activation)
407
929
  event_loop = runtime.event_loop
930
+ Phronomy::Tracing::Automatic.observe_task(
931
+ request.result_task,
932
+ "agent.execution",
933
+ input: request.input,
934
+ agent_id: @agent.agent_id,
935
+ execution_id: prepared.execution.execution_id,
936
+ mode: request.mode,
937
+ **@agent.send(:_build_caller_meta, prepared.config)
938
+ )
408
939
  effective_config = prepared.config.merge(
409
- phronomy_activation: activation,
940
+ phronomy_execution_coordinator: self,
410
941
  phronomy_runtime_projection: prepared.runtime_projection,
411
942
  phronomy_filtered_input: prepared.filtered_input,
412
943
  execution_id: prepared.execution.execution_id
@@ -415,128 +946,1384 @@ module Phronomy
415
946
  agent: @agent,
416
947
  input: prepared.filtered_input,
417
948
  config: effective_config,
418
- approval_policy: approval_policy,
419
- approval_listener: approval_listener,
420
- mode: mode,
421
- on_event: ->(event) { activation.record_event(event) },
949
+ approval_policy: request.approval_policy,
950
+ approval_listener: request.approval_listener,
951
+ mode: request.mode,
952
+ on_event: request.on_event,
422
953
  runtime: runtime
423
954
  )
424
- activation.invocation = session.context
425
- activation.session = session
426
- source_task = Phronomy::Task.deferred(name: "#{result_task.name}-source")
955
+ event_loop.install_agent_execution(
956
+ execution_id: prepared.execution.execution_id,
957
+ agent: @agent,
958
+ coordinator: self,
959
+ execution: prepared.execution,
960
+ runtime_projection: prepared.runtime_projection,
961
+ base_manifest: prepared.runtime_projection.manifest,
962
+ invocation: session.context,
963
+ fsm_session_id: session.id
964
+ )
965
+ event_loop.register_agent_completion_waiter(
966
+ prepared.execution.execution_id,
967
+ request.result_task
968
+ )
969
+ source_task = Phronomy::Task.deferred(name: "#{request.result_task.name}-source")
427
970
  source_task.on_complete do |invocation, error|
428
- finish(
429
- activation,
430
- result_task,
971
+ finish_on_event_loop(
972
+ prepared.execution.execution_id,
973
+ request.result_task,
431
974
  invocation || session.context,
432
- error
975
+ error,
976
+ fsm_session_id: session.id
433
977
  )
434
978
  end
435
979
  event_loop.register(session, completion: source_task)
436
- rescue => error
437
- activation ||= AgentExecutionActivation.new(
438
- execution: prepared.execution,
439
- agent: @agent,
440
- runtime_projection: prepared.runtime_projection,
441
- coordinator: self,
442
- application_listener: on_event
443
- )
444
- finish(activation, result_task, activation.invocation, error)
980
+ # simplecov:disable
981
+ rescue => _error
982
+ event_loop&.release_agent_execution(prepared.execution.execution_id) if
983
+ event_loop&.current? && event_loop.agent_execution_state(prepared.execution.execution_id)
984
+ raise
445
985
  end
986
+ # simplecov:enable
446
987
 
447
- def finish(activation, result_task, invocation, error)
988
+ def deliver_start_failure_on_event_loop(request, error)
989
+ callback_error = @agent.send(
990
+ :_deliver_stream_event,
991
+ request.on_event,
992
+ StreamEvent.new(
993
+ type: terminal_event_type(error),
994
+ payload: {error: error}
995
+ )
996
+ )
997
+ settle_after_terminal(
998
+ request.result_task,
999
+ callback_error,
1000
+ terminal_event_type(error),
1001
+ nil,
1002
+ error
1003
+ )
1004
+ end
1005
+
1006
+ # Restarts only the replay-safe initial preparation region of an already
1007
+ # durably admitted logical Execution. Runtime-only caller state is not
1008
+ # reconstructed here; the durable preparation inputs are authoritative.
1009
+ # @api private
1010
+ def start_initial_preparation_recovery_on_event_loop(
1011
+ execution,
1012
+ result_task,
1013
+ load_completion:
1014
+ )
448
1015
  runtime = Phronomy::Runtime.instance
449
- operation = runtime.offload.submit(on_full: :raise) do
450
- compute_terminal(activation, invocation, error)
451
- end
452
- operation.on_complete do |outcome, commit_error|
453
- command = AgentTerminalCommand.new(
454
- self, activation, result_task, outcome, commit_error
455
- )
456
- posted = runtime.event_loop.post(
457
- Phronomy::Event.new(
458
- type: :agent_terminal_ready,
459
- target_id: Phronomy::EventLoop::SYSTEM_CHANNEL_ID,
460
- payload: {command: command}
461
- )
1016
+ event_loop = runtime.event_loop
1017
+ assert_event_loop!(event_loop)
1018
+ state = event_loop.agent_execution_state(execution.execution_id)
1019
+ unless state && state.agent.equal?(@agent) &&
1020
+ state.execution.execution_revision == execution.execution_revision &&
1021
+ state.execution.status == :preparing &&
1022
+ state.execution.phase.to_sym == :preparing
1023
+ error = Phronomy::ExecutionRehydrationRequiredError.new(
1024
+ "stale :preparing Recovery installation for #{execution.execution_id}"
1025
+ )
1026
+ fail_task(result_task, error)
1027
+ load_completion.fail(error)
1028
+ return
1029
+ end
1030
+
1031
+ root = @agent.agent_root
1032
+ journal_records = @agent.send(:_journal_records_snapshot)
1033
+ task = runtime.offload.submit(on_full: :raise) do
1034
+ perform_initial_preparation_recovery(
1035
+ execution,
1036
+ root: root,
1037
+ journal_records: journal_records
462
1038
  )
463
- unless posted
464
- settle_without_listener(
465
- activation, result_task, outcome, commit_error
1039
+ end
1040
+ task.on_complete do |result, error|
1041
+ ready = InitialPreparationRecoveryReady.new(
1042
+ coordinator: self,
1043
+ execution_id: execution.execution_id.to_s.freeze,
1044
+ expected_execution_revision: execution.execution_revision,
1045
+ result_task: result_task,
1046
+ load_completion: load_completion,
1047
+ result: result,
1048
+ error: error
1049
+ )
1050
+ unless post_control(runtime, ready)
1051
+ rejected = runtime_rejected_error(:initial_preparation_recovery)
1052
+ fail_task(result_task, rejected)
1053
+ load_completion.fail(rejected)
1054
+ end
1055
+ end
1056
+ rescue => error
1057
+ # simplecov:disable
1058
+ begin
1059
+ if defined?(event_loop) && event_loop.current?
1060
+ release_initial_preparation_recovery_runtime_state(
1061
+ event_loop,
1062
+ execution.execution_id
466
1063
  )
467
1064
  end
1065
+ rescue
1066
+ nil
468
1067
  end
469
- rescue => caught
470
- fail_task(result_task, translated(caught))
1068
+ translated_error = translated(error)
1069
+ fail_task(result_task, translated_error)
1070
+ load_completion.fail(translated_error)
1071
+ # simplecov:enable
471
1072
  end
472
1073
 
473
- public
1074
+ def perform_initial_preparation_recovery(
1075
+ execution,
1076
+ root:,
1077
+ journal_records:
1078
+ )
1079
+ unless execution.metadata["preparation_replayable"] == true
1080
+ raise Phronomy::ExecutionRehydrationRequiredError,
1081
+ "execution #{execution.execution_id} has no replay-safe initial preparation contract"
1082
+ end
474
1083
 
475
- def deliver_on_event_loop(command)
476
- case command
477
- when PrepFailureCommand
478
- callback_error = @agent.send(
479
- :_deliver_stream_event,
480
- command.on_event_listener,
481
- StreamEvent.new(
482
- type: terminal_event_type(command.error),
483
- payload: {error: command.error}
484
- )
1084
+ input_ref = execution.metadata.fetch("current_input_ref")
1085
+ input = @agent.persistence.contents.fetch_text(input_ref)
1086
+ mode = (
1087
+ execution.metadata[RecoverySupport::INVOCATION_MODE_KEY] || "invoke"
1088
+ ).to_sym
1089
+ config = {phronomy_recovery_mode: mode}.merge(@agent.__coordination_config)
1090
+
1091
+ if execution.metadata.key?("durable_context_ref")
1092
+ durable_context_ref = execution.metadata.fetch("durable_context_ref")
1093
+ durable_context = @agent.persistence.contents.fetch_json(
1094
+ durable_context_ref
1095
+ )
1096
+ unless durable_context.is_a?(Hash)
1097
+ raise Phronomy::ExecutionRehydrationRequiredError,
1098
+ "execution #{execution.execution_id} has a non-Hash durable_context"
1099
+ end
1100
+ Phronomy::Agent::Immutable.validate_canonical_json!(
1101
+ durable_context,
1102
+ label: "Recovered durable_context"
1103
+ )
1104
+ config[:durable_context] =
1105
+ Phronomy::Agent::Immutable.copy(durable_context)
1106
+ end
1107
+
1108
+ perform_admitted_initial_preparation(
1109
+ input: input,
1110
+ config: config.freeze,
1111
+ execution: execution,
1112
+ active_root: root,
1113
+ journal_records: journal_records
1114
+ )
1115
+ end
1116
+
1117
+ def apply_initial_preparation_recovery_on_event_loop(ready)
1118
+ event_loop = Phronomy::Runtime.instance.event_loop
1119
+ state = event_loop.agent_execution_state(ready.execution_id)
1120
+ unless state && state.agent.equal?(@agent) &&
1121
+ state.execution.execution_revision == ready.expected_execution_revision &&
1122
+ state.execution.status == :preparing &&
1123
+ state.execution.phase.to_sym == :preparing
1124
+ error = Phronomy::ExecutionRehydrationRequiredError.new(
1125
+ "stale initial preparation Recovery result for #{ready.execution_id}"
1126
+ )
1127
+ fail_task(ready.result_task, error)
1128
+ ready.load_completion.fail(error)
1129
+ return
1130
+ end
1131
+
1132
+ if ready.error
1133
+ release_initial_preparation_recovery_runtime_state(
1134
+ event_loop,
1135
+ ready.execution_id
1136
+ )
1137
+ error = translated(ready.error)
1138
+ fail_task(ready.result_task, error)
1139
+ ready.load_completion.fail(error)
1140
+ return
1141
+ end
1142
+
1143
+ result = ready.result
1144
+ case result.admission_outcome
1145
+ when :terminal
1146
+ apply_agent_live_state(
1147
+ root: result.root,
1148
+ appended_records: result.appended_records
1149
+ )
1150
+ event_loop.replace_agent_execution(
1151
+ ready.execution_id,
1152
+ execution: result.execution
1153
+ )
1154
+ event_loop.release_agent_execution(ready.execution_id)
1155
+ event_loop.release_agent_execution_admission(
1156
+ @agent.agent_id,
1157
+ execution_id: ready.execution_id
1158
+ )
1159
+ event_type = terminal_event_type(result.error)
1160
+ callback_error = deliver_terminal(
1161
+ @agent.send(:_phronomy_event_listener),
1162
+ event_type,
1163
+ error: result.error
485
1164
  )
486
1165
  settle_after_terminal(
487
- command.result_task,
1166
+ ready.result_task,
488
1167
  callback_error,
489
- terminal_event_type(command.error),
1168
+ event_type,
490
1169
  nil,
491
- command.error
1170
+ result.error
1171
+ )
1172
+ ready.load_completion.complete(@agent)
1173
+ when :active
1174
+ apply_agent_live_state(
1175
+ root: result.root,
1176
+ appended_records: result.appended_records
1177
+ )
1178
+ event_loop.release_agent_execution(ready.execution_id)
1179
+ mode = (
1180
+ result.execution.metadata[RecoverySupport::INVOCATION_MODE_KEY] ||
1181
+ "invoke"
1182
+ ).to_sym
1183
+ request = StartCommand.new(
1184
+ coordinator: self,
1185
+ input: result.filtered_input,
1186
+ config: result.config,
1187
+ mode: mode,
1188
+ approval_policy: nil,
1189
+ approval_listener: nil,
1190
+ on_event: @agent.send(:_phronomy_event_listener),
1191
+ result_task: ready.result_task,
1192
+ admission_token: Object.new.freeze
492
1193
  )
493
- when AgentTerminalCommand
494
- deliver_execution_terminal(command)
1194
+ begin
1195
+ register_initial_session_on_event_loop(request, result)
1196
+ rescue => error
1197
+ begin_terminal_without_session_on_event_loop(
1198
+ request: request,
1199
+ prepared: result,
1200
+ error: error
1201
+ )
1202
+ end
1203
+ ready.load_completion.complete(@agent)
495
1204
  else
496
- raise Phronomy::Error, "unknown terminal command: #{command.class}"
1205
+ error = Phronomy::ExecutionRehydrationRequiredError.new(
1206
+ "unexpected initial preparation Recovery outcome: #{result.admission_outcome.inspect}"
1207
+ )
1208
+ release_initial_preparation_recovery_runtime_state(
1209
+ event_loop,
1210
+ ready.execution_id
1211
+ )
1212
+ fail_task(ready.result_task, error)
1213
+ ready.load_completion.fail(error)
1214
+ end
1215
+ rescue => error
1216
+ # simplecov:disable
1217
+ begin
1218
+ release_initial_preparation_recovery_runtime_state(
1219
+ event_loop,
1220
+ ready.execution_id
1221
+ )
1222
+ rescue
1223
+ nil
1224
+ end
1225
+ translated_error = translated(error)
1226
+ fail_task(ready.result_task, translated_error)
1227
+ ready.load_completion.fail(translated_error)
1228
+ # simplecov:enable
1229
+ end
1230
+
1231
+ def release_initial_preparation_recovery_runtime_state(
1232
+ event_loop,
1233
+ execution_id
1234
+ )
1235
+ state = event_loop.agent_execution_state(execution_id)
1236
+ event_loop.release_agent_execution(execution_id) if state
1237
+ event_loop.release_agent_execution_admission(
1238
+ @agent.agent_id,
1239
+ execution_id: execution_id
1240
+ )
1241
+ end
1242
+
1243
+ # ----------------------------------------------------------------------
1244
+ # External-operation causal durable barriers
1245
+ # ----------------------------------------------------------------------
1246
+
1247
+ def perform_provider_dispatch_preparation(operation)
1248
+ metadata = operation.execution.metadata.dup
1249
+ metadata.delete(RecoverySupport::TOOL_BATCH_METADATA_KEY)
1250
+ metadata.delete(RecoverySupport::RECOVERY_METADATA_KEY)
1251
+ metadata[RecoverySupport::PENDING_LLM_ID_KEY] =
1252
+ operation.pending_llm_call_id
1253
+ metadata[RecoverySupport::PENDING_LLM_STARTED_AT_KEY] =
1254
+ operation.pending_llm_started_at
1255
+ metadata[RecoverySupport::CONTRACT_VERSION_KEY] =
1256
+ RecoverySupport::CONTRACT_VERSION
1257
+ staged_execution = operation.execution.with(
1258
+ execution_revision: operation.execution.execution_revision,
1259
+ metadata: metadata
1260
+ )
1261
+
1262
+ manifest = manifest_ref = updated = intended = nil
1263
+
1264
+ # Encode the immutable runtime snapshot first. This transaction may add
1265
+ # content-addressed blobs, but it does not advance Execution state.
1266
+ encoded_records = call_records = nil
1267
+ @agent.persistence.transaction do |tx|
1268
+ assert_local_durable_base!(tx, operation.root)
1269
+ encoded_records, call_records = encode_runtime_records(
1270
+ staged_execution,
1271
+ tx: tx,
1272
+ snapshot: operation.runtime_snapshot,
1273
+ context_candidate: true,
1274
+ agent_root: operation.root
1275
+ )
1276
+ end
1277
+ staged = staged_execution.with(
1278
+ execution_revision: staged_execution.execution_revision,
1279
+ phase: :preparing_llm_call,
1280
+ working_records: staged_execution.working_records + encoded_records,
1281
+ llm_calls: staged_execution.llm_calls + call_records
1282
+ )
1283
+ patch = @agent.send(
1284
+ :run_before_llm_input_hooks,
1285
+ call_sequence: staged.llm_calls.length + 1,
1286
+ config: operation.invocation_config
1287
+ )
1288
+ assembler = ContextAssembler.new(
1289
+ agent: @agent,
1290
+ persistence: @agent.persistence,
1291
+ journal_records: operation.journal_records
1292
+ )
1293
+ prepared = assembler.prepare_followup(
1294
+ base_manifest: operation.base_manifest,
1295
+ agent_root: operation.root,
1296
+ execution: staged,
1297
+ config: operation.invocation_config,
1298
+ patch: patch
1299
+ )
1300
+ @agent.send(
1301
+ :check_cancellation!,
1302
+ operation.invocation_config,
1303
+ "invocation cancelled after context policy"
1304
+ )
1305
+
1306
+ begin
1307
+ @agent.persistence.transaction do |tx|
1308
+ assert_local_durable_base!(tx, operation.root)
1309
+ manifest, manifest_ref = assembler.finalize(prepared, persistence: tx)
1310
+ refs = Array(staged.metadata["manifest_refs"]) + [manifest_ref]
1311
+ updated = staged.with(
1312
+ phase: :calling_llm,
1313
+ metadata: staged.metadata.merge(
1314
+ "manifest_ref" => manifest_ref,
1315
+ "manifest_refs" => refs
1316
+ )
1317
+ )
1318
+ updated = @agent.__prepare_coordination_record(updated, tx: tx)
1319
+ tx.executions.save(
1320
+ operation.execution.execution_id,
1321
+ expected_revision: operation.execution.execution_revision,
1322
+ execution: updated
1323
+ )
1324
+ intended = ProviderDispatchPreparationResult.new(
1325
+ execution: updated,
1326
+ runtime_projection: nil,
1327
+ error: nil
1328
+ )
1329
+ end
1330
+ rescue => caught
1331
+ raise if known_durable_preparation_failure?(caught) || intended.nil?
1332
+
1333
+ raise PreparationOutcomeUnknownError.new(caught, intended)
1334
+ end
1335
+
1336
+ projection = materialization_error = nil
1337
+ begin
1338
+ projection = RubyLLMMaterializer.new(
1339
+ agent: @agent,
1340
+ persistence: @agent.persistence
1341
+ ).materialize(manifest: manifest, manifest_ref: manifest_ref)
1342
+ rescue => error
1343
+ materialization_error = error
497
1344
  end
1345
+ ProviderDispatchPreparationResult.new(
1346
+ execution: updated,
1347
+ runtime_projection: projection,
1348
+ error: materialization_error
1349
+ )
498
1350
  end
499
1351
 
500
- private
1352
+ def perform_tool_dispatch_preparation(operation)
1353
+ intended = nil
1354
+ begin
1355
+ @agent.persistence.transaction do |tx|
1356
+ assert_local_durable_base!(tx, operation.root)
1357
+ encoded_records, call_records = encode_runtime_records(
1358
+ operation.execution,
1359
+ tx: tx,
1360
+ snapshot: operation.runtime_snapshot,
1361
+ context_candidate: true,
1362
+ agent_root: operation.root
1363
+ )
1364
+ metadata = operation.execution.metadata.dup
1365
+ metadata.delete(RecoverySupport::PENDING_LLM_ID_KEY)
1366
+ metadata.delete(RecoverySupport::PENDING_LLM_STARTED_AT_KEY)
1367
+ metadata.delete(RecoverySupport::RECOVERY_METADATA_KEY)
1368
+ metadata.delete("framework_calls_pending")
1369
+ metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY] =
1370
+ RecoverySupport.canonical_copy(operation.tool_batch_snapshot)
1371
+ metadata[RecoverySupport::CONTRACT_VERSION_KEY] =
1372
+ RecoverySupport::CONTRACT_VERSION
1373
+ updated = operation.execution.with(
1374
+ phase: :dispatching_tools,
1375
+ working_records: operation.execution.working_records + encoded_records,
1376
+ llm_calls: operation.execution.llm_calls + call_records,
1377
+ metadata: metadata
1378
+ )
1379
+ updated = @agent.__prepare_coordination_record(updated, tx: tx)
1380
+ tx.executions.save(
1381
+ operation.execution.execution_id,
1382
+ expected_revision: operation.execution.execution_revision,
1383
+ execution: updated
1384
+ )
1385
+ intended = ToolDispatchPreparationResult.new(execution: updated)
1386
+ end
1387
+ rescue => caught
1388
+ raise if known_durable_preparation_failure?(caught) || intended.nil?
1389
+
1390
+ raise PreparationOutcomeUnknownError.new(caught, intended)
1391
+ end
1392
+ intended
1393
+ end
1394
+
1395
+ def apply_provider_dispatch_preparation_on_event_loop(ready)
1396
+ operation = ready.operation
1397
+ state = authoritative_state_for_operation(
1398
+ execution_id: operation.execution_id,
1399
+ fsm_session_id: operation.fsm_session_id,
1400
+ expected_execution_revision: operation.expected_execution_revision,
1401
+ expected_fsm_state: :calling_llm
1402
+ )
1403
+ return unless state
1404
+
1405
+ if ready.error
1406
+ if ready.error.is_a?(PreparationOutcomeUnknownError)
1407
+ submit_provider_dispatch_preparation_reconciliation(
1408
+ operation,
1409
+ ready.error
1410
+ )
1411
+ else
1412
+ state.invocation.event_sink.post(:llm_setup_failed, translated(ready.error))
1413
+ end
1414
+ return
1415
+ end
1416
+
1417
+ apply_confirmed_provider_dispatch_preparation_on_event_loop(
1418
+ operation,
1419
+ ready.result,
1420
+ state
1421
+ )
1422
+ rescue => error
1423
+ state&.invocation&.event_sink&.post(:llm_setup_failed, translated(error))
1424
+ end
1425
+
1426
+ def apply_tool_dispatch_preparation_on_event_loop(ready)
1427
+ operation = ready.operation
1428
+ state = authoritative_state_for_operation(
1429
+ execution_id: operation.execution_id,
1430
+ fsm_session_id: operation.fsm_session_id,
1431
+ expected_execution_revision: operation.expected_execution_revision,
1432
+ expected_fsm_state: :dispatching_tools
1433
+ )
1434
+ return unless state
1435
+
1436
+ if ready.error
1437
+ if ready.error.is_a?(PreparationOutcomeUnknownError)
1438
+ submit_tool_dispatch_preparation_reconciliation(
1439
+ operation,
1440
+ ready.error
1441
+ )
1442
+ else
1443
+ state.invocation.event_sink.post(:tool_setup_failed, translated(ready.error))
1444
+ end
1445
+ return
1446
+ end
1447
+
1448
+ apply_confirmed_tool_dispatch_preparation_on_event_loop(
1449
+ operation,
1450
+ ready.result,
1451
+ state
1452
+ )
1453
+ rescue => error
1454
+ state&.invocation&.event_sink&.post(:tool_setup_failed, translated(error))
1455
+ end
1456
+
1457
+ def apply_confirmed_provider_dispatch_preparation_on_event_loop(
1458
+ operation,
1459
+ result,
1460
+ state
1461
+ )
1462
+ event_loop = Phronomy::Runtime.instance.event_loop
1463
+ if result.runtime_projection
1464
+ event_loop.replace_agent_execution(
1465
+ operation.execution_id,
1466
+ execution: result.execution,
1467
+ runtime_projection: result.runtime_projection
1468
+ )
1469
+ else
1470
+ event_loop.replace_agent_execution(
1471
+ operation.execution_id,
1472
+ execution: result.execution
1473
+ )
1474
+ end
1475
+ state.invocation.acknowledge_runtime_snapshot(operation.runtime_snapshot)
1476
+
1477
+ if result.error
1478
+ state.invocation.event_sink.post(:llm_setup_failed, translated(result.error))
1479
+ return
1480
+ end
1481
+
1482
+ Agent::AgentInvocationSessionBuilder.start_prepared_provider_call(
1483
+ agent: @agent,
1484
+ runtime: Phronomy::Runtime.instance,
1485
+ event_sink: state.invocation.event_sink,
1486
+ invocation: state.invocation,
1487
+ projection: result.runtime_projection,
1488
+ streaming: operation.streaming
1489
+ )
1490
+ end
1491
+
1492
+ def apply_confirmed_tool_dispatch_preparation_on_event_loop(
1493
+ operation,
1494
+ result,
1495
+ state
1496
+ )
1497
+ event_loop = Phronomy::Runtime.instance.event_loop
1498
+ event_loop.replace_agent_execution(
1499
+ operation.execution_id,
1500
+ execution: result.execution
1501
+ )
1502
+ state.invocation.acknowledge_runtime_snapshot(operation.runtime_snapshot)
1503
+ Agent::AgentInvocationSessionBuilder.start_prepared_tool_dispatch(
1504
+ runtime: Phronomy::Runtime.instance,
1505
+ event_sink: state.invocation.event_sink,
1506
+ invocation: state.invocation
1507
+ )
1508
+ end
1509
+
1510
+ def perform_provider_dispatch_preparation_reconciliation(command)
1511
+ # simplecov:disable
1512
+ disposition, current = preparation_reconciliation_state(
1513
+ command.operation,
1514
+ command.intended_result.execution
1515
+ )
1516
+ preparation_result = nil
1517
+ if disposition == :committed
1518
+ projection = materialization_error = nil
1519
+ begin
1520
+ _manifest, projection = RecoverySupport.materialize_projection(
1521
+ @agent,
1522
+ current.metadata.fetch("manifest_ref")
1523
+ )
1524
+ rescue => error
1525
+ materialization_error = error
1526
+ end
1527
+ preparation_result = ProviderDispatchPreparationResult.new(
1528
+ execution: current,
1529
+ runtime_projection: projection,
1530
+ error: materialization_error
1531
+ )
1532
+ end
1533
+ ProviderDispatchPreparationReconciliationResult.new(
1534
+ disposition: disposition,
1535
+ preparation_result: preparation_result
1536
+ )
1537
+ end
1538
+
1539
+ def perform_tool_dispatch_preparation_reconciliation(command)
1540
+ disposition, current = preparation_reconciliation_state(
1541
+ command.operation,
1542
+ command.intended_result.execution
1543
+ )
1544
+ preparation_result = if disposition == :committed
1545
+ ToolDispatchPreparationResult.new(execution: current)
1546
+ end
1547
+ ToolDispatchPreparationReconciliationResult.new(
1548
+ disposition: disposition,
1549
+ preparation_result: preparation_result
1550
+ )
1551
+ end
1552
+
1553
+ def apply_provider_dispatch_preparation_reconciliation_on_event_loop(ready)
1554
+ command = ready.command
1555
+ operation = command.operation
1556
+ state = authoritative_state_for_operation(
1557
+ execution_id: operation.execution_id,
1558
+ fsm_session_id: operation.fsm_session_id,
1559
+ expected_execution_revision: operation.expected_execution_revision,
1560
+ expected_fsm_state: :calling_llm
1561
+ )
1562
+ return unless state
1563
+
1564
+ if ready.error
1565
+ mark_barrier_recovery_required(operation, ready.error)
1566
+ return
1567
+ end
1568
+
1569
+ case ready.result.disposition
1570
+ when :committed
1571
+ apply_confirmed_provider_dispatch_preparation_on_event_loop(
1572
+ operation,
1573
+ ready.result.preparation_result,
1574
+ state
1575
+ )
1576
+ when :not_committed
1577
+ state.invocation.event_sink.post(
1578
+ :llm_setup_failed,
1579
+ translated(command.original_error)
1580
+ )
1581
+ when :conflict
1582
+ mark_barrier_recovery_required(operation, command.original_error)
1583
+ else
1584
+ raise Phronomy::Error,
1585
+ "unknown Provider dispatch reconciliation disposition: " \
1586
+ "#{ready.result.disposition.inspect}"
1587
+ end
1588
+ rescue => error
1589
+ mark_barrier_recovery_required(operation, error)
1590
+ end
1591
+
1592
+ def apply_tool_dispatch_preparation_reconciliation_on_event_loop(ready)
1593
+ command = ready.command
1594
+ operation = command.operation
1595
+ state = authoritative_state_for_operation(
1596
+ execution_id: operation.execution_id,
1597
+ fsm_session_id: operation.fsm_session_id,
1598
+ expected_execution_revision: operation.expected_execution_revision,
1599
+ expected_fsm_state: :dispatching_tools
1600
+ )
1601
+ return unless state
1602
+
1603
+ if ready.error
1604
+ mark_barrier_recovery_required(operation, ready.error)
1605
+ return
1606
+ end
1607
+
1608
+ case ready.result.disposition
1609
+ when :committed
1610
+ apply_confirmed_tool_dispatch_preparation_on_event_loop(
1611
+ operation,
1612
+ ready.result.preparation_result,
1613
+ state
1614
+ )
1615
+ when :not_committed
1616
+ state.invocation.event_sink.post(
1617
+ :tool_setup_failed,
1618
+ translated(command.original_error)
1619
+ )
1620
+ when :conflict
1621
+ mark_barrier_recovery_required(operation, command.original_error)
1622
+ else
1623
+ raise Phronomy::Error,
1624
+ "unknown Tool dispatch reconciliation disposition: " \
1625
+ "#{ready.result.disposition.inspect}"
1626
+ end
1627
+ rescue => error
1628
+ mark_barrier_recovery_required(operation, error)
1629
+ # simplecov:enable
1630
+ end
1631
+
1632
+ # Worker-only Persistence readback for causal-barrier F1 reconciliation.
1633
+ # EventLoop submits one operation-specific ReconciliationCommand and only
1634
+ # applies this result after it returns through the normal control path.
1635
+ def preparation_reconciliation_state(operation, intended_execution)
1636
+ current = @agent.persistence.executions.load(operation.execution_id)
1637
+ if current.execution_revision == intended_execution.execution_revision &&
1638
+ current.to_h == intended_execution.to_h
1639
+ return [:committed, current]
1640
+ end
1641
+ if current.execution_revision == operation.execution.execution_revision &&
1642
+ current.to_h == operation.execution.to_h
1643
+ return [:not_committed, current]
1644
+ end
1645
+
1646
+ [:conflict, current]
1647
+ end
1648
+
1649
+ def known_durable_preparation_failure?(error)
1650
+ error.is_a?(Phronomy::Persistence::ConflictError) ||
1651
+ error.is_a?(Phronomy::Persistence::NotFoundError) ||
1652
+ error.is_a?(Phronomy::Persistence::SerializationError) ||
1653
+ error.is_a?(Phronomy::Persistence::UnsupportedBackendError) ||
1654
+ error.is_a?(ArgumentError) ||
1655
+ error.is_a?(Phronomy::ConfigurationError)
1656
+ end
1657
+
1658
+ def mark_barrier_recovery_required(operation, error)
1659
+ event_loop = Phronomy::Runtime.instance.event_loop
1660
+ event_loop.mark_agent_execution_admission(
1661
+ @agent.agent_id,
1662
+ execution_id: operation.execution_id,
1663
+ state: :recovery_required
1664
+ )
1665
+ Phronomy.configuration.logger&.warn(
1666
+ "[Phronomy] causal durable barrier requires recovery for " \
1667
+ "#{operation.execution_id}: #{error.class}: #{error.message}"
1668
+ )
1669
+ end
1670
+
1671
+ # ----------------------------------------------------------------------
1672
+ # Approval resume
1673
+ # ----------------------------------------------------------------------
1674
+
1675
+ def begin_resume_on_event_loop(request)
1676
+ event_loop = Phronomy::Runtime.instance.event_loop
1677
+ state = event_loop.agent_execution_state(request.execution_id)
1678
+ if state&.agent&.equal?(@agent) && state&.invocation
1679
+ @recovery_resume_snapshot_mutex ||= Mutex.new
1680
+ snapshot = RecoverySupport.build_tool_batch_snapshot(
1681
+ state.invocation
1682
+ )
1683
+ @recovery_resume_snapshot_mutex.synchronize do
1684
+ @recovery_resume_snapshots ||= {}
1685
+ @recovery_resume_snapshots[request.execution_id.to_s] =
1686
+ snapshot
1687
+ end
1688
+ end
1689
+
1690
+ runtime = Phronomy::Runtime.instance
1691
+ event_loop = runtime.event_loop
1692
+ state = event_loop.agent_execution_state(request.execution_id)
1693
+ unless state
1694
+ fail_task(
1695
+ request.result_task,
1696
+ Phronomy::ExecutionRehydrationRequiredError.new(
1697
+ "no live execution owner for #{request.execution_id}; durable rehydration is required"
1698
+ )
1699
+ )
1700
+ return
1701
+ end
1702
+
1703
+ unless state.agent.equal?(@agent) && state.execution.status == :suspended
1704
+ fail_task(
1705
+ request.result_task,
1706
+ ArgumentError.new(
1707
+ "execution is not a suspended execution of this agent: #{request.execution_id}"
1708
+ )
1709
+ )
1710
+ return
1711
+ end
1712
+
1713
+ approval = state.execution.approval_request || {}
1714
+ current_request_id = approval["id"] || approval[:id]
1715
+ unless current_request_id.to_s == request.approval_request_id
1716
+ fail_task(
1717
+ request.result_task,
1718
+ ArgumentError.new(
1719
+ "approval request does not match execution #{request.execution_id}"
1720
+ )
1721
+ )
1722
+ return
1723
+ end
1724
+
1725
+ operation = ResumeCommitCommand.new(
1726
+ execution_id: state.execution_id,
1727
+ expected_execution_revision: state.execution.execution_revision,
1728
+ root: @agent.agent_root,
1729
+ execution: state.execution,
1730
+ approval_request_id: request.approval_request_id,
1731
+ approved: request.approved
1732
+ )
1733
+ resume_transition_started = false
1734
+ submitted = false
1735
+ event_loop.mark_agent_execution_admission(
1736
+ @agent.agent_id,
1737
+ execution_id: state.execution_id,
1738
+ state: :resuming
1739
+ )
1740
+ resume_transition_started = true
1741
+ task = runtime.offload.submit(on_full: :raise) do
1742
+ perform_resume_commit(operation)
1743
+ end
1744
+ submitted = true
1745
+ task.on_complete do |result, error|
1746
+ ready = ResumeCommitReady.new(
1747
+ coordinator: self,
1748
+ request: request,
1749
+ operation: operation,
1750
+ result: result,
1751
+ error: error
1752
+ )
1753
+ fail_task(request.result_task, runtime_rejected_error(:resume_commit)) unless
1754
+ post_control(runtime, ready)
1755
+ end
1756
+ rescue => error
1757
+ if resume_transition_started
1758
+ event_loop.mark_agent_execution_admission(
1759
+ @agent.agent_id,
1760
+ execution_id: state.execution_id,
1761
+ state: submitted ? :recovery_required : :suspended
1762
+ )
1763
+ end
1764
+ fail_task(request.result_task, translated(error))
1765
+ raise unless resume_transition_started
1766
+ end
1767
+
1768
+ def perform_resume_commit(operation)
1769
+ snapshot = nil
1770
+ @recovery_resume_snapshot_mutex&.synchronize do
1771
+ snapshot = @recovery_resume_snapshots&.delete(
1772
+ operation.execution_id.to_s
1773
+ )
1774
+ end
1775
+
1776
+ if snapshot
1777
+ staged_execution = RecoverySupport.with_recovery_metadata(
1778
+ operation.execution,
1779
+ RecoverySupport::TOOL_BATCH_METADATA_KEY => snapshot
1780
+ )
1781
+ operation = operation.class.new(
1782
+ **operation.to_h.merge(execution: staged_execution)
1783
+ )
1784
+ end
1785
+
1786
+ current = operation.execution
1787
+ current_root = operation.root
1788
+ request = current.approval_request || {}
1789
+ request_id = request["id"] || request[:id]
1790
+ unless request_id.to_s == operation.approval_request_id
1791
+ raise ArgumentError,
1792
+ "approval request does not match execution #{current.execution_id}"
1793
+ end
1794
+
1795
+ updated = next_root = nil
1796
+ @agent.persistence.transaction do |tx|
1797
+ decision_ref = tx.contents.put_json(
1798
+ "approval_request_id" => request_id.to_s,
1799
+ "approved" => operation.approved
1800
+ )
1801
+ decision_record = JournalRecord.new(
1802
+ agent_id: @agent.agent_id,
1803
+ execution_id: current.execution_id,
1804
+ kind: :approval_decided,
1805
+ channel: :approval,
1806
+ content_ref: decision_ref,
1807
+ context_generation: current_root.transcript_generation,
1808
+ context_candidate: false
1809
+ )
1810
+ updated = current.with(
1811
+ status: :active,
1812
+ phase: :resuming,
1813
+ working_records: current.working_records + [decision_record],
1814
+ approval_request: request.merge("approved" => operation.approved)
1815
+ )
1816
+ tx.executions.save(
1817
+ current.execution_id,
1818
+ expected_revision: current.execution_revision,
1819
+ execution: updated
1820
+ )
1821
+ next_root = current_root.with(
1822
+ agent_revision: current_root.agent_revision + 1,
1823
+ lifecycle_status: :active
1824
+ )
1825
+ tx.agents.save(
1826
+ current_root.agent_id,
1827
+ expected_revision: current_root.agent_revision,
1828
+ root: next_root
1829
+ )
1830
+ end
1831
+ ResumeCommitResult.new(execution: updated, root: next_root)
1832
+ end
1833
+
1834
+ def apply_resume_commit_on_event_loop(ready)
1835
+ request = ready.request
1836
+ operation = ready.operation
1837
+ event_loop = Phronomy::Runtime.instance.event_loop
1838
+ state = event_loop.agent_execution_state(operation.execution_id)
1839
+ unless state && state.agent.equal?(@agent) &&
1840
+ state.execution.execution_revision == operation.expected_execution_revision &&
1841
+ state.execution.status == :suspended
1842
+ fail_task(
1843
+ request.result_task,
1844
+ Phronomy::Error.new(
1845
+ "stale approval resume result for #{operation.execution_id}"
1846
+ )
1847
+ )
1848
+ return
1849
+ end
1850
+
1851
+ if ready.error
1852
+ event_loop.mark_agent_execution_admission(
1853
+ @agent.agent_id,
1854
+ execution_id: operation.execution_id,
1855
+ state: :recovery_required
1856
+ )
1857
+ fail_task(request.result_task, translated(ready.error))
1858
+ return
1859
+ end
1860
+
1861
+ apply_agent_live_state(root: ready.result.root, appended_records: [])
1862
+ event_loop.mark_agent_execution_admission(
1863
+ @agent.agent_id,
1864
+ execution_id: operation.execution_id,
1865
+ state: :executing
1866
+ )
1867
+ event_loop.replace_agent_execution(
1868
+ operation.execution_id,
1869
+ execution: ready.result.execution
1870
+ )
1871
+ event_loop.register_agent_completion_waiter(
1872
+ operation.execution_id,
1873
+ request.result_task
1874
+ )
1875
+ trace_config = state.invocation ?
1876
+ state.invocation.config.merge(request.config) : request.config
1877
+ trace_mode = state.invocation&.mode ||
1878
+ (
1879
+ ready.result.execution.metadata[
1880
+ RecoverySupport::INVOCATION_MODE_KEY
1881
+ ] || "invoke"
1882
+ ).to_sym
1883
+ Phronomy::Tracing::Automatic.observe_task(
1884
+ request.result_task,
1885
+ "agent.execution",
1886
+ agent_id: @agent.agent_id,
1887
+ execution_id: operation.execution_id,
1888
+ mode: trace_mode,
1889
+ **@agent.send(:_build_caller_meta, trace_config)
1890
+ )
1891
+ start_resume_on_event_loop(
1892
+ operation.execution_id,
1893
+ request.result_task,
1894
+ approved: request.approved,
1895
+ config: request.config
1896
+ )
1897
+ rescue => error
1898
+ state = event_loop&.agent_execution_state(operation.execution_id) if event_loop&.current?
1899
+ if state
1900
+ begin_terminal_commit_on_event_loop(
1901
+ state,
1902
+ request.result_task,
1903
+ state.invocation,
1904
+ error,
1905
+ fsm_session_id: state.fsm_session_id
1906
+ )
1907
+ else
1908
+ fail_task(request.result_task, translated(error))
1909
+ end
1910
+ end
1911
+
1912
+ def start_resume_on_event_loop(execution_id, result_task, approved:, config:)
1913
+ runtime = Phronomy::Runtime.instance
1914
+ event_loop = runtime.event_loop
1915
+ state = event_loop.agent_execution_state(execution_id)
1916
+ invocation = state.invocation
1917
+ invocation.merge_config!(config)
1918
+ invocation.begin_approval_resume!(approved: approved)
1919
+ parent_session = Agent::AgentInvocationSessionBuilder.build_for_resume(
1920
+ agent_invocation: invocation,
1921
+ resume_event: :resume,
1922
+ resume_phase: :suspended,
1923
+ runtime: runtime
1924
+ )
1925
+ event_loop.replace_agent_execution(
1926
+ execution_id,
1927
+ invocation: invocation,
1928
+ fsm_session_id: parent_session.id
1929
+ )
1930
+ source_task = Phronomy::Task.deferred(name: "#{result_task.name}-source")
1931
+ source_task.on_complete do |completed, error|
1932
+ finish_on_event_loop(
1933
+ execution_id,
1934
+ result_task,
1935
+ completed || parent_session.context,
1936
+ error,
1937
+ fsm_session_id: parent_session.id
1938
+ )
1939
+ end
1940
+ event_loop.register(parent_session, completion: source_task)
1941
+ invocation.tool_invocations.each do |child|
1942
+ child_session = if child.awaiting_approval?
1943
+ Agent::ToolInvocationSessionBuilder.build_for_resume(
1944
+ tool_invocation: child,
1945
+ parent_event_sink: parent_session.event_sink,
1946
+ resume_event: approved ? :approve : :reject,
1947
+ resume_phase: :awaiting_approval,
1948
+ runtime: runtime
1949
+ )
1950
+ elsif !approved && child.authorized?
1951
+ Agent::ToolInvocationSessionBuilder.build_for_resume(
1952
+ tool_invocation: child,
1953
+ parent_event_sink: parent_session.event_sink,
1954
+ resume_event: :cancel,
1955
+ resume_phase: :authorized,
1956
+ runtime: runtime
1957
+ )
1958
+ end
1959
+ register_child(event_loop, child, child_session, parent_session.event_sink) if child_session
1960
+ end
1961
+ end
1962
+
1963
+ def start_framework_tools_on_event_loop(execution_id, result_task)
1964
+ runtime = Phronomy::Runtime.instance
1965
+ event_loop = runtime.event_loop
1966
+ state = event_loop.agent_execution_state(execution_id)
1967
+ invocation = state.invocation
1968
+ session = AgentInvocationSessionBuilder.build_for_resume(agent_invocation: invocation,
1969
+ resume_event: :resume, resume_phase: :suspended, runtime: runtime)
1970
+ event_loop.replace_agent_execution(execution_id, invocation: invocation, fsm_session_id: session.id)
1971
+ event_loop.mark_agent_execution_admission(@agent.agent_id, execution_id: execution_id, state: :executing)
1972
+ source = Phronomy::Task.deferred(name: "framework-tool-recovery-source")
1973
+ source.on_complete do |result, error|
1974
+ finish_on_event_loop(execution_id, result_task, result || session.context, error, fsm_session_id: session.id)
1975
+ end
1976
+ event_loop.register(session, completion: source)
1977
+ invocation.tool_invocations.select(&:authorized?).each do |child|
1978
+ child_session = ToolInvocationSessionBuilder.build_for_resume(tool_invocation: child,
1979
+ parent_event_sink: session.event_sink, resume_event: :dispatch, resume_phase: :authorized, runtime: runtime)
1980
+ register_child(event_loop, child, child_session, session.event_sink)
1981
+ end
1982
+ end
1983
+
1984
+ def register_child(event_loop, child, session, parent_event_sink)
1985
+ completion = Phronomy::Task.deferred(name: "tool-session:#{child.id}")
1986
+ completion.on_complete do |_result, error|
1987
+ next unless error
1988
+
1989
+ child.mark_framework_failed!(error)
1990
+ parent_event_sink.post(:tool_failed, {tool_invocation_id: child.id})
1991
+ end
1992
+ event_loop.register(session, completion: completion)
1993
+ end
1994
+
1995
+ # ----------------------------------------------------------------------
1996
+ # Terminal durable barrier
1997
+ # ----------------------------------------------------------------------
501
1998
 
502
- def compute_terminal(activation, invocation, error)
503
- if (failure = activation.callback_failure)
504
- terminal = commit_failed(
505
- activation,
506
- invocation,
507
- failure.to_stream_callback_error
1999
+ def finish_on_event_loop(execution_id, result_task, invocation, error, fsm_session_id:)
2000
+ event_loop = Phronomy::Runtime.instance.event_loop
2001
+ assert_event_loop!(event_loop)
2002
+ state = event_loop.agent_execution_state(execution_id)
2003
+ return fail_task(result_task, runtime_rejected_error(:terminal)) unless state
2004
+
2005
+ unless state.fsm_session_id.to_s == fsm_session_id.to_s &&
2006
+ state.invocation.equal?(invocation)
2007
+ return fail_task(
2008
+ result_task,
2009
+ Phronomy::Error.new("stale Agent terminal callback for #{execution_id}")
2010
+ )
2011
+ end
2012
+
2013
+ terminal_error = error || invocation&.error
2014
+ unless event_loop.agent_execution_quiescent?(execution_id)
2015
+ wait_state = quiescence_sensitive_terminal_error?(terminal_error) ?
2016
+ :cancelling : :terminalizing
2017
+ event_loop.mark_agent_execution_admission(
2018
+ @agent.agent_id,
2019
+ execution_id: execution_id,
2020
+ state: wait_state
2021
+ )
2022
+ event_loop.defer_agent_terminal_until_quiescent(
2023
+ execution_id,
2024
+ DeferredTerminalCommand.new(
2025
+ coordinator: self,
2026
+ execution_id: execution_id.to_s.freeze,
2027
+ result_task: result_task,
2028
+ invocation: invocation,
2029
+ source_error: error,
2030
+ fsm_session_id: fsm_session_id.to_s.freeze
2031
+ )
2032
+ )
2033
+ return
2034
+ end
2035
+
2036
+ begin_terminal_commit_on_event_loop(
2037
+ state,
2038
+ result_task,
2039
+ invocation,
2040
+ error,
2041
+ fsm_session_id: fsm_session_id
2042
+ )
2043
+ end
2044
+
2045
+ def resume_deferred_terminal_on_event_loop(command)
2046
+ event_loop = Phronomy::Runtime.instance.event_loop
2047
+ state = event_loop.agent_execution_state(command.execution_id)
2048
+ unless state && state.agent.equal?(@agent) &&
2049
+ state.fsm_session_id.to_s == command.fsm_session_id.to_s &&
2050
+ state.invocation.equal?(command.invocation)
2051
+ Phronomy.configuration.logger&.warn(
2052
+ "[Phronomy] Dropped stale deferred terminal for #{command.execution_id}"
508
2053
  )
509
- return {type: :failed, error: terminal.fetch(:error)}
2054
+ return
510
2055
  end
511
- if error
512
- terminal = commit_failed(activation, invocation, error)
513
- return {type: :failed, error: terminal.fetch(:error)}
2056
+
2057
+ unless event_loop.agent_execution_quiescent?(command.execution_id)
2058
+ raise Phronomy::Error,
2059
+ "deferred terminal resumed before quiescence for #{command.execution_id}"
514
2060
  end
2061
+
2062
+ begin_terminal_commit_on_event_loop(
2063
+ state,
2064
+ command.result_task,
2065
+ command.invocation,
2066
+ command.source_error,
2067
+ fsm_session_id: command.fsm_session_id
2068
+ )
2069
+ end
2070
+
2071
+ def quiescence_sensitive_terminal_error?(error)
2072
+ error.is_a?(Phronomy::CancellationError) ||
2073
+ error.is_a?(Phronomy::TimeoutError)
2074
+ end
2075
+
2076
+ def begin_terminal_commit_on_event_loop(
2077
+ state,
2078
+ result_task,
2079
+ invocation,
2080
+ source_error,
2081
+ fsm_session_id:
2082
+ )
515
2083
  if invocation&.phase == :suspended
516
- return {
517
- type: :suspended,
518
- result: commit_suspended(activation, invocation)
519
- }
2084
+ snapshot = RecoverySupport.build_tool_batch_snapshot(invocation)
2085
+ staged_execution = RecoverySupport.with_recovery_metadata(
2086
+ state.execution,
2087
+ RecoverySupport::TOOL_BATCH_METADATA_KEY => snapshot
2088
+ )
2089
+ state = state.class.new(
2090
+ **state.to_h.merge(execution: staged_execution)
2091
+ )
2092
+ end
2093
+
2094
+ operation = build_terminal_operation(
2095
+ execution: state.execution,
2096
+ root: @agent.agent_root,
2097
+ journal_records: @agent.send(:_journal_records_snapshot),
2098
+ invocation: invocation,
2099
+ source_error: source_error,
2100
+ fsm_session_id: fsm_session_id,
2101
+ state_required: true
2102
+ )
2103
+ delivery = TerminalDelivery.new(
2104
+ result_task: result_task,
2105
+ application_listener: invocation&.event_listener,
2106
+ approval_listener: invocation&.approval_listener,
2107
+ handoff_request: invocation&.handoff_request,
2108
+ handoff_manifest: state.runtime_projection&.manifest
2109
+ )
2110
+ submit_terminal_operation(operation, delivery)
2111
+ end
2112
+
2113
+ def begin_terminal_without_session_on_event_loop(request:, prepared:, error:)
2114
+ return deliver_start_failure_on_event_loop(request, translated(error)) unless prepared&.execution
2115
+
2116
+ operation = build_terminal_operation(
2117
+ execution: prepared.execution,
2118
+ root: prepared.root,
2119
+ journal_records: @agent.send(:_journal_records_snapshot),
2120
+ invocation: nil,
2121
+ source_error: error,
2122
+ fsm_session_id: nil,
2123
+ state_required: false
2124
+ )
2125
+ delivery = TerminalDelivery.new(
2126
+ result_task: request.result_task,
2127
+ application_listener: request.on_event,
2128
+ approval_listener: request.approval_listener,
2129
+ handoff_request: nil,
2130
+ handoff_manifest: nil
2131
+ )
2132
+ submit_terminal_operation(operation, delivery)
2133
+ end
2134
+
2135
+ def build_terminal_operation(
2136
+ execution:,
2137
+ root:,
2138
+ journal_records:,
2139
+ invocation:,
2140
+ source_error:,
2141
+ fsm_session_id:,
2142
+ state_required:
2143
+ )
2144
+ TerminalCommitCommand.new(
2145
+ execution_id: execution.execution_id.to_s.freeze,
2146
+ fsm_session_id: fsm_session_id&.to_s&.freeze,
2147
+ expected_execution_revision: execution.execution_revision,
2148
+ root: root,
2149
+ journal_records: journal_records,
2150
+ execution: execution,
2151
+ runtime_snapshot: invocation ? invocation.runtime_snapshot : empty_runtime_snapshot,
2152
+ terminal_view: terminal_view(invocation, source_error),
2153
+ state_required: state_required
2154
+ )
2155
+ end
2156
+
2157
+ def submit_terminal_operation(operation, delivery)
2158
+ runtime = Phronomy::Runtime.instance
2159
+ event_loop = runtime.event_loop
2160
+ terminal_transition_started = false
2161
+ event_loop.mark_agent_execution_admission(
2162
+ @agent.agent_id,
2163
+ execution_id: operation.execution_id,
2164
+ state: :terminalizing
2165
+ )
2166
+ terminal_transition_started = true
2167
+ event_loop.register_agent_completion_waiter(
2168
+ operation.execution_id,
2169
+ delivery.result_task
2170
+ )
2171
+ task = runtime.offload.submit(on_full: :raise) do
2172
+ # Only the operation-specific immutable durable snapshot crosses the
2173
+ # worker boundary. Task/listener delivery state stays outside it.
2174
+ compute_terminal(operation)
2175
+ end
2176
+ task.on_complete do |outcome, error|
2177
+ ready = TerminalCommitReady.new(
2178
+ coordinator: self,
2179
+ operation: operation,
2180
+ delivery: delivery,
2181
+ outcome: outcome,
2182
+ error: error
2183
+ )
2184
+ fail_task(delivery.result_task, runtime_rejected_error(:terminal_apply)) unless
2185
+ post_control(runtime, ready)
2186
+ end
2187
+ rescue => error
2188
+ if terminal_transition_started
2189
+ event_loop.mark_agent_execution_admission(
2190
+ @agent.agent_id,
2191
+ execution_id: operation.execution_id,
2192
+ state: :recovery_required
2193
+ )
2194
+ Phronomy.configuration.logger&.warn(
2195
+ "[Phronomy] Agent terminal transition requires recovery for " \
2196
+ "#{operation.execution_id}: #{error.class}: #{error.message}"
2197
+ )
2198
+ return
520
2199
  end
521
- raise invocation.block_error if invocation.input_blocked? || invocation.output_blocked?
522
- raise invocation.error if invocation.error
523
2200
 
524
- {type: :completed, result: commit_completed(activation, invocation)}
2201
+ fail_task(delivery.result_task, translated(error))
2202
+ raise
2203
+ end
2204
+
2205
+ def terminal_view(invocation, source_error)
2206
+ TerminalView.new(
2207
+ phase: invocation&.phase,
2208
+ output: invocation&.output,
2209
+ usage: invocation&.usage,
2210
+ approval_request: invocation&.approval_request,
2211
+ rejected: invocation ? !!invocation.rejected : false,
2212
+ input_blocked: invocation ? invocation.input_blocked? : false,
2213
+ output_blocked: invocation ? invocation.output_blocked? : false,
2214
+ block_error: invocation&.block_error,
2215
+ invocation_error: invocation&.error,
2216
+ handoff: handoff_terminal_view(invocation&.handoff_request),
2217
+ callback_failure: invocation&.callback_failure,
2218
+ source_error: source_error,
2219
+ cancel_requested: invocation&.config&.fetch(:cancellation_token, nil)&.cancelled? || false
2220
+ )
2221
+ end
2222
+
2223
+ def handoff_terminal_view(request)
2224
+ return unless request
2225
+
2226
+ HandoffTerminalView.new(
2227
+ target_agent_id: request.handoff.target_agent.agent_id.to_s.freeze,
2228
+ responsibility: request.responsibility.to_s.freeze,
2229
+ selection_intent: request.selection_intent.to_h do |category, included|
2230
+ [category.to_s.freeze, !!included]
2231
+ end.freeze,
2232
+ llm_call_id: request.llm_call_id&.to_s&.freeze,
2233
+ tool_call_id: request.tool_call_id&.to_s&.freeze,
2234
+ policy: Immutable.copy(request.handoff.policy.to_h)
2235
+ )
2236
+ end
2237
+
2238
+ def empty_runtime_snapshot
2239
+ {llm_results: [].freeze, runtime_events: [].freeze, active_call: nil}.freeze
2240
+ end
2241
+
2242
+ def compute_terminal(operation)
2243
+ view = operation.terminal_view
2244
+ error = view.callback_failure&.to_stream_callback_error || view.source_error ||
2245
+ view.block_error || view.invocation_error
2246
+ if operation.execution.metadata["multi_agent_coordination_ref"] &&
2247
+ (error.is_a?(Phronomy::ExecutionRehydrationRequiredError) || view.cancel_requested)
2248
+ waiting = commit_coordination_wait(operation, error)
2249
+ return waiting if waiting
2250
+ end
2251
+ raise error if error.is_a?(Phronomy::ExecutionRehydrationRequiredError)
2252
+ return commit_failed_outcome(operation, error) if error
2253
+ return commit_suspended(operation) if view.phase == :suspended
2254
+ commit_completed(operation)
525
2255
  rescue => caught
526
- terminal = commit_failed(activation, invocation, caught)
527
- {type: :failed, error: terminal.fetch(:error)}
2256
+ reconcile_terminal_error(operation, caught)
528
2257
  end
529
2258
 
530
- def commit_suspended(activation, invocation)
531
- current = activation.execution
532
- root = @agent.agent_root
533
- request = invocation.approval_request
534
- runtime_snapshot = activation.runtime_snapshot
2259
+ # Reuse the Agent barrier to retain unfinished owned children. This is an
2260
+ # active AgentExecution snapshot, not a second scheduler or terminal state.
2261
+ def commit_coordination_wait(operation, error)
2262
+ current = operation.execution
2263
+ waiting = nil
2264
+ @agent.persistence.transaction do |tx|
2265
+ refreshed = @agent.__prepare_coordination_record(current, tx: tx)
2266
+ snapshot = tx.contents.fetch_json(refreshed.metadata.fetch("multi_agent_coordination_ref"))
2267
+ unresolved = snapshot.fetch("children").any? do |child|
2268
+ !%w[completed failed cancelled rejected blocked handed_off].include?(child.fetch("state")) &&
2269
+ !(operation.terminal_view.cancel_requested && child.fetch("state") == "reserved")
2270
+ end
2271
+ next unless unresolved
2272
+ waiting = current.with(metadata: refreshed.metadata.merge(
2273
+ "coordination_cancel_requested" => operation.terminal_view.cancel_requested || current.metadata["coordination_cancel_requested"] == true
2274
+ ))
2275
+ tx.executions.save(current.execution_id, expected_revision: current.execution_revision, execution: waiting)
2276
+ end
2277
+ return unless waiting
2278
+ coordination_wait_outcome(operation, waiting, error)
2279
+ rescue => failure
2280
+ confirmed = @agent.persistence.executions.load(current.execution_id)
2281
+ raise failure unless waiting && confirmed.to_h == waiting.to_h
2282
+ coordination_wait_outcome(operation, confirmed, error)
2283
+ end
2284
+
2285
+ def coordination_wait_outcome(operation, execution, error)
2286
+ TerminalOutcome.new(type: :coordination_wait, execution: execution, root: operation.root,
2287
+ appended_records: [].freeze, result: nil,
2288
+ error: error.is_a?(Phronomy::ExecutionRehydrationRequiredError) ? error :
2289
+ Phronomy::ExecutionRehydrationRequiredError.new("Execution #{execution.execution_id} retains unfinished children for cancellation/recovery"),
2290
+ approval_request: nil)
2291
+ end
2292
+
2293
+ # F1: terminal atomicity alone does not establish commit outcome certainty.
2294
+ # Reuse an exact committed outcome; an absent/active/read-failed result is
2295
+ # never permission to write a second terminal transition.
2296
+ def reconcile_terminal_error(operation, error)
2297
+ confirmed = @agent.persistence.executions.load(operation.execution_id)
2298
+ raise error unless confirmed.terminal? &&
2299
+ confirmed.agent_id == @agent.agent_id &&
2300
+ confirmed.execution_revision == operation.expected_execution_revision + 1
2301
+ root = @agent.persistence.agents.load(@agent.agent_id)
2302
+ records = @agent.persistence.journals.read(@agent.agent_id,
2303
+ after: operation.root.journal_position, limit: root.journal_position - operation.root.journal_position)
2304
+ result = @agent.persistence.execution_result(confirmed.execution_id)
2305
+ failure = result[:error] && RecoverySupport.error_from_failure(result[:error])
2306
+ type = if failure
2307
+ :failed
2308
+ else
2309
+ ((confirmed.status == :handed_off) ? :handed_off : :completed)
2310
+ end
2311
+ TerminalOutcome.new(type: type, execution: confirmed, root: root,
2312
+ appended_records: records.freeze,
2313
+ result: failure ? nil : result_base(confirmed, root).merge(output: result[:result]).freeze,
2314
+ error: failure, approval_request: nil)
2315
+ end
2316
+
2317
+ def commit_suspended(operation)
2318
+ current = operation.execution
2319
+ root = operation.root
2320
+ request = operation.terminal_view.approval_request
2321
+ runtime_snapshot = operation.runtime_snapshot
535
2322
  suspended = next_root = nil
536
2323
 
537
2324
  @agent.persistence.transaction do |tx|
538
2325
  encoded_records, call_records = encode_runtime_records(
539
- activation,
2326
+ current,
540
2327
  tx: tx,
541
2328
  snapshot: runtime_snapshot,
542
2329
  context_candidate: true,
@@ -549,7 +2336,6 @@ module Phronomy
549
2336
  kind: :approval_required,
550
2337
  channel: :approval,
551
2338
  content_ref: request_ref,
552
- correlation_id: request.id.to_s,
553
2339
  context_generation: root.transcript_generation,
554
2340
  context_candidate: false
555
2341
  )
@@ -575,31 +2361,49 @@ module Phronomy
575
2361
  root: next_root
576
2362
  )
577
2363
  end
578
- activation.acknowledge_runtime_snapshot(runtime_snapshot)
579
- activation.replace_execution(suspended)
580
- @agent.__replace_root(next_root)
581
- dispatch_approval_listener(invocation, request)
582
- result_base(suspended, next_root).merge(
2364
+ result = result_base(suspended, next_root).merge(
583
2365
  suspended: true,
584
2366
  approval_request: request
585
2367
  )
2368
+ TerminalOutcome.new(
2369
+ type: :suspended,
2370
+ execution: suspended,
2371
+ root: next_root,
2372
+ appended_records: [].freeze,
2373
+ result: result.freeze,
2374
+ error: nil,
2375
+ approval_request: request
2376
+ )
586
2377
  end
587
2378
 
588
- def commit_completed(activation, invocation)
589
- current = activation.execution
590
- root = @agent.agent_root
591
- runtime_snapshot = activation.runtime_snapshot
592
- completed = next_root = appended = nil
2379
+ def synchronize_handoff_target(tx, execution)
2380
+ coordination = execution.metadata["coordination"]
2381
+ return unless coordination && coordination["kind"] == "handoff"
2382
+ routing = tx.handoff_states.load(coordination.fetch("main_agent_id"))
2383
+ return unless routing && routing.pending_target_execution_id == execution.execution_id && routing.phase != "stable"
2384
+ unless routing.active_agent_id == execution.agent_id
2385
+ raise Phronomy::Persistence::ConflictError, "Handoff Target owner mismatch"
2386
+ end
2387
+ tx.handoff_states.save(routing.main_agent_id, expected_revision: routing.handoff_revision,
2388
+ state: routing.with(phase: "stable"))
2389
+ end
2390
+
2391
+ def commit_completed(operation)
2392
+ current = operation.execution
2393
+ root = operation.root
2394
+ view = operation.terminal_view
2395
+ runtime_snapshot = operation.runtime_snapshot
2396
+ completed = next_root = appended = messages = nil
593
2397
 
594
2398
  @agent.persistence.transaction do |tx|
595
2399
  encoded_records, call_records = encode_runtime_records(
596
- activation,
2400
+ current,
597
2401
  tx: tx,
598
2402
  snapshot: runtime_snapshot,
599
2403
  context_candidate: true,
600
2404
  agent_root: root
601
2405
  )
602
- output_ref = tx.contents.put_text(invocation.output.to_s)
2406
+ output_ref = tx.contents.put_text(view.output.to_s)
603
2407
  final_output_record = JournalRecord.new(
604
2408
  agent_id: @agent.agent_id,
605
2409
  execution_id: current.execution_id,
@@ -613,7 +2417,7 @@ module Phronomy
613
2417
  completed_record = JournalRecord.new(
614
2418
  agent_id: @agent.agent_id,
615
2419
  execution_id: current.execution_id,
616
- kind: invocation.rejected ? :execution_rejected : :execution_completed,
2420
+ kind: view.rejected ? :execution_rejected : :execution_completed,
617
2421
  channel: :audit,
618
2422
  content_ref: output_ref,
619
2423
  context_generation: root.transcript_generation,
@@ -627,14 +2431,16 @@ module Phronomy
627
2431
  records: all_records
628
2432
  )
629
2433
  completed = current.with(
630
- status: invocation.rejected ? :rejected : :completed,
2434
+ status: view.rejected ? :rejected : :completed,
631
2435
  phase: :completed,
632
2436
  working_records: [],
633
2437
  llm_calls: current.llm_calls + call_records,
634
2438
  approval_request: nil,
635
2439
  result_ref: output_ref,
636
- terminal_reason: invocation.rejected ? "rejected" : "completed"
2440
+ terminal_reason: view.rejected ? "rejected" : "completed"
637
2441
  )
2442
+ completed = @agent.__prepare_coordination_record(completed, tx: tx)
2443
+ synchronize_handoff_target(tx, completed)
638
2444
  tx.executions.save(
639
2445
  current.execution_id,
640
2446
  expected_revision: current.execution_revision,
@@ -652,29 +2458,45 @@ module Phronomy
652
2458
  expected_revision: root.agent_revision,
653
2459
  root: next_root
654
2460
  )
2461
+
2462
+ # Materialize the caller-facing transcript through the transaction
2463
+ # view before commit. If materialization fails, the terminal durable
2464
+ # transition rolls back instead of committing and then attempting a
2465
+ # second terminal transition from a stale execution revision.
2466
+ records_after_commit = operation.journal_records + Array(appended)
2467
+ messages = transcript_messages(
2468
+ next_root,
2469
+ records_after_commit,
2470
+ persistence: tx
2471
+ )
655
2472
  end
656
- activation.acknowledge_runtime_snapshot(runtime_snapshot)
657
- activation.replace_execution(completed)
658
- @agent.send(:_append_journal_records, appended)
659
- @agent.__replace_root(next_root)
660
- result_base(completed, next_root).merge(
661
- output: invocation.output,
662
- rejected: invocation.rejected || nil,
663
- usage: invocation.usage,
664
- messages: transcript_messages(next_root)
2473
+ result = result_base(completed, next_root).merge(
2474
+ output: view.output,
2475
+ rejected: view.rejected || nil,
2476
+ usage: view.usage,
2477
+ messages: messages
665
2478
  ).compact
2479
+ TerminalOutcome.new(
2480
+ type: :completed,
2481
+ execution: completed,
2482
+ root: next_root,
2483
+ appended_records: Array(appended).freeze,
2484
+ result: result.freeze,
2485
+ error: nil,
2486
+ approval_request: nil
2487
+ )
666
2488
  end
667
2489
 
668
- def commit_failed(activation, _invocation, error)
2490
+ def commit_failed_outcome(operation, error)
669
2491
  translated_error = translated(error)
670
- current = activation.execution
671
- root = @agent.agent_root
672
- runtime_snapshot = activation.runtime_snapshot
2492
+ current = operation.execution
2493
+ root = operation.root
2494
+ runtime_snapshot = operation.runtime_snapshot
673
2495
  failed = next_root = appended = nil
674
2496
 
675
2497
  @agent.persistence.transaction do |tx|
676
2498
  encoded_records, call_records = encode_runtime_records(
677
- activation,
2499
+ current,
678
2500
  tx: tx,
679
2501
  snapshot: runtime_snapshot,
680
2502
  context_candidate: false,
@@ -712,6 +2534,8 @@ module Phronomy
712
2534
  error_ref: error_ref,
713
2535
  terminal_reason: translated_error.class.name
714
2536
  )
2537
+ failed = @agent.__prepare_coordination_record(failed, tx: tx)
2538
+ synchronize_handoff_target(tx, failed)
715
2539
  tx.executions.save(
716
2540
  current.execution_id,
717
2541
  expected_revision: current.execution_revision,
@@ -728,21 +2552,155 @@ module Phronomy
728
2552
  root: next_root
729
2553
  )
730
2554
  end
731
- activation.acknowledge_runtime_snapshot(runtime_snapshot)
732
- activation.replace_execution(failed)
733
- @agent.send(:_append_journal_records, appended)
734
- @agent.__replace_root(next_root)
735
- {error: translated_error, execution: failed, root: next_root}
2555
+ TerminalOutcome.new(
2556
+ type: :failed,
2557
+ execution: failed,
2558
+ root: next_root,
2559
+ appended_records: Array(appended).freeze,
2560
+ result: nil,
2561
+ error: translated_error,
2562
+ approval_request: nil
2563
+ )
2564
+ end
2565
+
2566
+ def apply_terminal_commit_on_event_loop(ready)
2567
+ operation = ready.operation
2568
+ delivery = ready.delivery
2569
+ event_loop = Phronomy::Runtime.instance.event_loop
2570
+ state = event_loop.agent_execution_state(operation.execution_id)
2571
+
2572
+ if operation.state_required
2573
+ unless state && state.agent.equal?(@agent) &&
2574
+ state.execution.execution_revision == operation.expected_execution_revision &&
2575
+ state.fsm_session_id.to_s == operation.fsm_session_id.to_s
2576
+ Phronomy.configuration.logger&.warn(
2577
+ "[Phronomy] Dropped stale Agent terminal result for #{operation.execution_id}"
2578
+ )
2579
+ return
2580
+ end
2581
+ end
2582
+
2583
+ if ready.error
2584
+ if operation.execution.metadata["coordination"] || operation.execution.metadata["multi_agent_coordination_ref"]
2585
+ event_loop.release_agent_execution(operation.execution_id) if state
2586
+ event_loop.release_agent_execution_admission(@agent.agent_id, execution_id: operation.execution_id)
2587
+ event_loop.take_agent_completion_waiters(operation.execution_id, fallback: delivery.result_task).each do |task|
2588
+ task.fail(ready.error)
2589
+ end
2590
+ return
2591
+ end
2592
+ event_loop.mark_agent_execution_admission(
2593
+ @agent.agent_id,
2594
+ execution_id: operation.execution_id,
2595
+ state: :recovery_required
2596
+ )
2597
+ Phronomy.configuration.logger&.warn(
2598
+ "[Phronomy] Agent terminal durable outcome requires recovery for " \
2599
+ "#{operation.execution_id}: #{ready.error.class}: #{ready.error.message}"
2600
+ )
2601
+ return
2602
+ end
2603
+
2604
+ outcome = ready.outcome
2605
+ apply_agent_live_state(
2606
+ root: outcome.root,
2607
+ appended_records: outcome.appended_records
2608
+ )
2609
+ if state
2610
+ state.invocation&.acknowledge_runtime_snapshot(operation.runtime_snapshot)
2611
+ event_loop.replace_agent_execution(
2612
+ operation.execution_id,
2613
+ execution: outcome.execution,
2614
+ fsm_session_id: (outcome.type == :suspended) ? nil : state.fsm_session_id
2615
+ )
2616
+ end
2617
+
2618
+ case outcome.type
2619
+ when :coordination_wait
2620
+ event_loop.release_agent_execution(operation.execution_id) if state
2621
+ event_loop.release_agent_execution_admission(@agent.agent_id, execution_id: operation.execution_id)
2622
+ event_loop.take_agent_completion_waiters(operation.execution_id, fallback: delivery.result_task).each { |task| task.fail(outcome.error) }
2623
+ when :suspended
2624
+ event_loop.mark_agent_execution_admission(
2625
+ @agent.agent_id,
2626
+ execution_id: operation.execution_id,
2627
+ state: :suspended
2628
+ )
2629
+ dispatch_approval_listener(delivery.approval_listener, outcome.approval_request)
2630
+ callback_error = deliver_terminal(
2631
+ delivery.application_listener,
2632
+ :approval_required,
2633
+ request: outcome.approval_request
2634
+ )
2635
+ report_nonterminal_callback_error(
2636
+ callback_error,
2637
+ :approval_required,
2638
+ outcome.result
2639
+ )
2640
+ Array(state&.invocation&.config&.fetch(:phronomy_exact_observers, [])).each do |task|
2641
+ task.fail(Phronomy::ExecutionRehydrationRequiredError.new("Execution #{operation.execution_id} requires approval"))
2642
+ end
2643
+ when :completed
2644
+ event_loop.release_agent_execution(operation.execution_id) if state
2645
+ event_loop.release_agent_execution_admission(
2646
+ @agent.agent_id,
2647
+ execution_id: operation.execution_id
2648
+ )
2649
+ callback_error = deliver_terminal(delivery.application_listener, :done, outcome.result)
2650
+ settle_execution_waiters(
2651
+ event_loop, operation.execution_id, delivery.result_task,
2652
+ callback_error, :done, outcome.result
2653
+ )
2654
+ when :handed_off
2655
+ event_loop.release_agent_execution(operation.execution_id) if state
2656
+ event_loop.release_agent_execution_admission(
2657
+ @agent.agent_id,
2658
+ execution_id: operation.execution_id
2659
+ )
2660
+ result = outcome.result.merge(
2661
+ handoff_request: delivery.handoff_request,
2662
+ _phronomy_handoff_manifest: delivery.handoff_manifest
2663
+ ).freeze
2664
+ callback_error = deliver_terminal(delivery.application_listener, :handoff, result)
2665
+ settle_execution_waiters(
2666
+ event_loop, operation.execution_id, delivery.result_task,
2667
+ callback_error, :handoff, result
2668
+ )
2669
+ when :failed
2670
+ event_loop.release_agent_execution(operation.execution_id) if state
2671
+ event_loop.release_agent_execution_admission(
2672
+ @agent.agent_id,
2673
+ execution_id: operation.execution_id
2674
+ )
2675
+ type = terminal_event_type(outcome.error)
2676
+ callback_error = deliver_terminal(
2677
+ delivery.application_listener,
2678
+ type,
2679
+ error: outcome.error
2680
+ )
2681
+ settle_execution_waiters(
2682
+ event_loop, operation.execution_id, delivery.result_task,
2683
+ callback_error, type, nil, outcome.error
2684
+ )
2685
+ else
2686
+ fail_task(
2687
+ delivery.result_task,
2688
+ Phronomy::Error.new("unknown terminal outcome: #{outcome.type.inspect}")
2689
+ )
2690
+ end
736
2691
  end
737
2692
 
2693
+ # ----------------------------------------------------------------------
2694
+ # Durable record encoding helpers
2695
+ # ----------------------------------------------------------------------
2696
+
738
2697
  def encode_runtime_records(
739
- activation,
2698
+ execution,
740
2699
  tx:,
741
2700
  snapshot:,
742
2701
  context_candidate:,
743
2702
  agent_root:
744
2703
  )
745
- execution = activation.execution
746
2704
  root = agent_root
747
2705
  records = []
748
2706
  calls = []
@@ -846,6 +2804,9 @@ module Phronomy
846
2804
  llm_call_id = payload[:llm_call_id] || payload["llm_call_id"]
847
2805
  tool_call_id = payload.fetch(:tool_call_id).to_s
848
2806
  tool_name = payload.fetch(:tool_name).to_s
2807
+ next if execution.working_records.any? do |record|
2808
+ record.kind == :tool_message && record.causation_id.to_s == tool_call_id && record.llm_call_id.to_s == llm_call_id.to_s
2809
+ end
849
2810
  result_ref = put_runtime_content(tx, payload.fetch(:tool_result))
850
2811
  records << JournalRecord.new(
851
2812
  agent_id: @agent.agent_id,
@@ -889,168 +2850,6 @@ module Phronomy
889
2850
  [records, calls]
890
2851
  end
891
2852
 
892
- def mark_resuming!(activation, approval_request_id:, approved:)
893
- current = activation.execution
894
- current_root = @agent.agent_root
895
- unless current.agent_id == @agent.agent_id && current.status == :suspended
896
- raise ArgumentError, "execution is not suspended: #{current.execution_id}"
897
- end
898
-
899
- request = current.approval_request || {}
900
- request_id = request["id"] || request[:id]
901
- unless request_id.to_s == approval_request_id.to_s
902
- raise ArgumentError,
903
- "approval request does not match execution #{current.execution_id}"
904
- end
905
-
906
- updated = next_root = nil
907
- @agent.persistence.transaction do |tx|
908
- decision_ref = tx.contents.put_json(
909
- "approval_request_id" => request_id.to_s,
910
- "approved" => !!approved
911
- )
912
- decision_record = JournalRecord.new(
913
- agent_id: @agent.agent_id,
914
- execution_id: current.execution_id,
915
- kind: :approval_decided,
916
- channel: :approval,
917
- content_ref: decision_ref,
918
- correlation_id: request_id.to_s,
919
- context_generation: current_root.transcript_generation,
920
- context_candidate: false
921
- )
922
- updated = current.with(
923
- status: :active,
924
- phase: :resuming,
925
- working_records: current.working_records + [decision_record],
926
- approval_request: request.merge("approved" => approved)
927
- )
928
- tx.executions.save(
929
- current.execution_id,
930
- expected_revision: current.execution_revision,
931
- execution: updated
932
- )
933
- next_root = current_root.with(
934
- agent_revision: current_root.agent_revision + 1,
935
- lifecycle_status: :active
936
- )
937
- tx.agents.save(
938
- current_root.agent_id,
939
- expected_revision: current_root.agent_revision,
940
- root: next_root
941
- )
942
- end
943
- activation.replace_execution(updated)
944
- @agent.__replace_root(next_root)
945
- updated
946
- end
947
-
948
- def assert_local_durable_base!(tx, root)
949
- tx.assert_agent_watermark!(
950
- agent_id: root.agent_id,
951
- agent_revision: root.agent_revision,
952
- journal_position: root.journal_position
953
- )
954
- end
955
-
956
- def transcript_messages(root)
957
- materializer = RubyLLMMaterializer.new(
958
- agent: @agent,
959
- persistence: @agent.persistence
960
- )
961
- materializer.materialize_journal_records(
962
- JournalProjection.new(
963
- agent_root: root,
964
- records: @agent.send(:_journal_records_snapshot)
965
- ).transcript_records
966
- )
967
- end
968
-
969
- def deliver_execution_terminal(command)
970
- activation = command.activation
971
- registry = Phronomy::Runtime.instance.__agent_activations
972
- if command.commit_error
973
- callback_error = deliver_terminal(
974
- activation,
975
- :error,
976
- error: command.commit_error
977
- )
978
- settle_after_terminal(
979
- command.result_task,
980
- callback_error,
981
- :error,
982
- nil,
983
- command.commit_error
984
- )
985
- return
986
- end
987
-
988
- outcome = command.outcome
989
- case outcome.fetch(:type)
990
- when :suspended
991
- result = outcome.fetch(:result)
992
- callback_error = deliver_terminal(
993
- activation,
994
- :approval_required,
995
- request: result.fetch(:approval_request)
996
- )
997
- settle_after_terminal(
998
- command.result_task,
999
- callback_error,
1000
- :approval_required,
1001
- result
1002
- )
1003
- when :completed
1004
- registry.delete(activation.execution_id)
1005
- result = outcome.fetch(:result)
1006
- callback_error = deliver_terminal(activation, :done, result)
1007
- settle_after_terminal(
1008
- command.result_task,
1009
- callback_error,
1010
- :done,
1011
- result
1012
- )
1013
- when :failed
1014
- registry.delete(activation.execution_id)
1015
- error = outcome.fetch(:error)
1016
- type = terminal_event_type(error)
1017
- callback_error = deliver_terminal(activation, type, error: error)
1018
- settle_after_terminal(
1019
- command.result_task,
1020
- callback_error,
1021
- type,
1022
- nil,
1023
- error
1024
- )
1025
- end
1026
- end
1027
-
1028
- def settle_without_listener(activation, result_task, outcome, commit_error)
1029
- registry = Phronomy::Runtime.instance.__agent_activations
1030
- if commit_error
1031
- return fail_task(result_task, commit_error)
1032
- end
1033
-
1034
- case outcome&.fetch(:type)
1035
- when :suspended
1036
- complete_task(result_task, outcome.fetch(:result))
1037
- when :completed
1038
- registry.delete(activation.execution_id)
1039
- complete_task(result_task, outcome.fetch(:result))
1040
- when :failed
1041
- registry.delete(activation.execution_id)
1042
- fail_task(result_task, outcome.fetch(:error))
1043
- else
1044
- registry.delete(activation.execution_id)
1045
- fail_task(
1046
- result_task,
1047
- Phronomy::RuntimeShutdownError.new(
1048
- "EventLoop is not accepting terminal delivery"
1049
- )
1050
- )
1051
- end
1052
- end
1053
-
1054
2853
  def assistant_message_ref(tx, outcome)
1055
2854
  return unless outcome
1056
2855
 
@@ -1084,64 +2883,76 @@ module Phronomy
1084
2883
  tx.contents.put_text(value) : tx.contents.put_json(json_value(value))
1085
2884
  end
1086
2885
 
1087
- def start_resume(activation, result_task, approved:, config:)
1088
- invocation = activation.invocation
1089
- invocation.merge_config!(config)
1090
- invocation.begin_approval_resume!(approved: approved)
1091
- runtime = Phronomy::Runtime.instance
1092
- event_loop = runtime.event_loop
1093
- parent_session = Agent::AgentInvocationSessionBuilder.build_for_resume(
1094
- agent_invocation: invocation,
1095
- resume_event: :resume,
1096
- resume_phase: :suspended,
1097
- runtime: runtime
2886
+ # ----------------------------------------------------------------------
2887
+ # EventLoop validation/apply helpers
2888
+ # ----------------------------------------------------------------------
2889
+
2890
+ def authoritative_state_for_operation(
2891
+ execution_id:,
2892
+ fsm_session_id:,
2893
+ expected_execution_revision:,
2894
+ expected_fsm_state:
2895
+ )
2896
+ event_loop = Phronomy::Runtime.instance.event_loop
2897
+ state = event_loop.agent_execution_state(execution_id)
2898
+ authoritative = state &&
2899
+ state.agent.equal?(@agent) &&
2900
+ state.execution.execution_revision == expected_execution_revision &&
2901
+ state.fsm_session_id.to_s == fsm_session_id.to_s &&
2902
+ event_loop.fsm_session_state(fsm_session_id) == expected_fsm_state
2903
+ return state if authoritative
2904
+
2905
+ Phronomy.configuration.logger&.warn(
2906
+ "[Phronomy] Dropped stale Agent operation result: " \
2907
+ "execution_id=#{execution_id} fsm_session_id=#{fsm_session_id}"
1098
2908
  )
1099
- activation.session = parent_session
1100
- source_task = Phronomy::Task.deferred(name: "#{result_task.name}-source")
1101
- source_task.on_complete do |completed, error|
1102
- finish(
1103
- activation,
1104
- result_task,
1105
- completed || parent_session.context,
1106
- error
1107
- )
1108
- end
1109
- event_loop.register(parent_session, completion: source_task)
1110
- invocation.tool_invocations.each do |child|
1111
- child_session = if child.awaiting_approval?
1112
- Agent::ToolInvocationSessionBuilder.build_for_resume(
1113
- tool_invocation: child,
1114
- resume_event: approved ? :approve : :reject,
1115
- resume_phase: :awaiting_approval,
1116
- runtime: runtime
1117
- )
1118
- elsif !approved && child.authorized?
1119
- Agent::ToolInvocationSessionBuilder.build_for_resume(
1120
- tool_invocation: child,
1121
- resume_event: :cancel,
1122
- resume_phase: :authorized,
1123
- runtime: runtime
1124
- )
1125
- end
1126
- register_child(event_loop, runtime, child, child_session) if child_session
2909
+ nil
2910
+ rescue
2911
+ nil
2912
+ end
2913
+
2914
+ def validate_live_session!(state, invocation, fsm_session_id)
2915
+ unless state && state.agent.equal?(@agent) &&
2916
+ state.invocation.equal?(invocation) &&
2917
+ state.fsm_session_id.to_s == fsm_session_id.to_s
2918
+ raise Phronomy::Error,
2919
+ "Agent operation does not belong to the current live FSMSession"
1127
2920
  end
1128
2921
  end
1129
2922
 
1130
- def register_child(event_loop, runtime, child, session)
1131
- completion = Phronomy::Task.deferred(name: "tool-session:#{child.id}")
1132
- completion.on_complete do |_result, error|
1133
- next unless error
2923
+ def apply_agent_live_state(root:, appended_records:)
2924
+ event_loop = Phronomy::Runtime.instance.event_loop
2925
+ assert_event_loop!(event_loop)
2926
+ @agent.send(:_append_journal_records, appended_records)
2927
+ @agent.__replace_root(root)
2928
+ end
1134
2929
 
1135
- child.mark_framework_failed!(error)
1136
- runtime.event_loop.post_to_session(
1137
- Phronomy::Event.new(
1138
- type: :tool_failed,
1139
- target_id: child.parent_agent_invocation_id,
1140
- payload: {tool_invocation_id: child.id}
1141
- )
1142
- )
1143
- end
1144
- event_loop.register(session, completion: completion)
2930
+ def assert_event_loop!(event_loop)
2931
+ return if event_loop.current?
2932
+
2933
+ raise Phronomy::Error,
2934
+ "ExecutionCoordinator live-state apply must run on EventLoop"
2935
+ end
2936
+
2937
+ def assert_local_durable_base!(tx, root)
2938
+ tx.assert_agent_watermark!(
2939
+ agent_id: root.agent_id,
2940
+ agent_revision: root.agent_revision,
2941
+ journal_position: root.journal_position
2942
+ )
2943
+ end
2944
+
2945
+ def transcript_messages(root, journal_records, persistence: @agent.persistence)
2946
+ materializer = RubyLLMMaterializer.new(
2947
+ agent: @agent,
2948
+ persistence: persistence
2949
+ )
2950
+ materializer.materialize_journal_records(
2951
+ JournalProjection.new(
2952
+ agent_root: root,
2953
+ records: journal_records
2954
+ ).transcript_records
2955
+ )
1145
2956
  end
1146
2957
 
1147
2958
  def result_base(execution, root)
@@ -1154,16 +2965,7 @@ module Phronomy
1154
2965
  }
1155
2966
  end
1156
2967
 
1157
- def context_policy_for(execution)
1158
- descriptor_hash = execution.metadata.fetch("context_policy") do
1159
- ContextPolicies::Default.new.descriptor.to_h
1160
- end
1161
- descriptor = ContextPolicyDescriptor.from_h(descriptor_hash)
1162
- ContextPolicyRegistry.default.resolve(descriptor)
1163
- end
1164
-
1165
- def deliver_terminal(activation, type, payload)
1166
- listener = activation.application_listener
2968
+ def deliver_terminal(listener, type, payload)
1167
2969
  return nil unless listener
1168
2970
 
1169
2971
  @agent.send(
@@ -1173,16 +2975,43 @@ module Phronomy
1173
2975
  )
1174
2976
  end
1175
2977
 
1176
- def post_prep_failure(runtime, listener, result_task, error)
1177
- command = PrepFailureCommand.new(self, listener, result_task, error)
1178
- posted = runtime.event_loop.post(
1179
- Phronomy::Event.new(
1180
- type: :agent_terminal_ready,
1181
- target_id: Phronomy::EventLoop::SYSTEM_CHANNEL_ID,
1182
- payload: {command: command}
2978
+ def dispatch_approval_listener(listener, request)
2979
+ return unless listener
2980
+
2981
+ Phronomy::Runtime.instance.offload.submit(on_full: :raise) do
2982
+ listener.call(request)
2983
+ end
2984
+ rescue => error
2985
+ Phronomy.configuration.logger&.warn(
2986
+ "[Phronomy] approval listener dispatch failed: #{error.class}: #{error.message}"
2987
+ )
2988
+ end
2989
+
2990
+ def settle_execution_waiters(
2991
+ event_loop, execution_id, fallback_task, callback_error, event_type,
2992
+ result, execution_error = nil
2993
+ )
2994
+ waiters = event_loop.take_agent_completion_waiters(
2995
+ execution_id,
2996
+ fallback: fallback_task
2997
+ )
2998
+ waiters.each do |task|
2999
+ settle_after_terminal(
3000
+ task, callback_error, event_type, result, execution_error
1183
3001
  )
3002
+ end
3003
+ end
3004
+
3005
+ def report_nonterminal_callback_error(callback_error, event_type, result)
3006
+ return unless callback_error
3007
+
3008
+ @agent.send(
3009
+ :_report_stream_callback_error,
3010
+ callback_error,
3011
+ event: StreamEvent.new(type: event_type, payload: result),
3012
+ execution_id: result&.fetch(:execution_id, nil),
3013
+ callback_error_policy: Phronomy.configuration.stream_callback_error_policy
1184
3014
  )
1185
- fail_task(result_task, error) unless posted
1186
3015
  end
1187
3016
 
1188
3017
  def settle_after_terminal(
@@ -1201,7 +3030,7 @@ module Phronomy
1201
3030
  type: event_type,
1202
3031
  payload: result || {error: execution_error}
1203
3032
  ),
1204
- invocation_id: nil,
3033
+ execution_id: result&.fetch(:execution_id, nil),
1205
3034
  callback_error_policy: policy
1206
3035
  )
1207
3036
  if policy == :fail_task && execution_error.nil?
@@ -1216,8 +3045,15 @@ module Phronomy
1216
3045
  )
1217
3046
  end
1218
3047
  end
1219
- execution_error ?
1220
- fail_task(result_task, execution_error) : complete_task(result_task, result)
3048
+ if execution_error
3049
+ if execution_error.is_a?(Phronomy::CancellationError)
3050
+ result_task.cancel!(execution_error)
3051
+ else
3052
+ fail_task(result_task, execution_error)
3053
+ end
3054
+ else
3055
+ complete_task(result_task, result)
3056
+ end
1221
3057
  end
1222
3058
 
1223
3059
  def terminal_event_type(error)
@@ -1227,15 +3063,6 @@ module Phronomy
1227
3063
  :error
1228
3064
  end
1229
3065
 
1230
- def dispatch_approval_listener(invocation, request)
1231
- listener = invocation.approval_listener
1232
- return unless listener
1233
-
1234
- Phronomy::Runtime.instance.offload.submit(on_full: :raise) do
1235
- listener.call(request)
1236
- end
1237
- end
1238
-
1239
3066
  def json_value(value)
1240
3067
  case value
1241
3068
  when Hash
@@ -1277,6 +3104,24 @@ module Phronomy
1277
3104
  }.fetch(status)
1278
3105
  end
1279
3106
 
3107
+ def post_control(runtime, command)
3108
+ runtime.event_loop.post(
3109
+ Phronomy::Event.new(
3110
+ type: :agent_control,
3111
+ target_id: Phronomy::EventLoop::SYSTEM_CHANNEL_ID,
3112
+ payload: {command: command}
3113
+ )
3114
+ )
3115
+ rescue Phronomy::RuntimeShutdownError
3116
+ false
3117
+ end
3118
+
3119
+ def runtime_rejected_error(operation)
3120
+ Phronomy::RuntimeShutdownError.new(
3121
+ "EventLoop is not accepting Agent #{operation} control delivery"
3122
+ )
3123
+ end
3124
+
1280
3125
  def translated(error)
1281
3126
  @agent.send(:_translated_error, error)
1282
3127
  end