phronomy 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.mutant.yml +3 -4
- data/CHANGELOG.md +215 -10
- data/CONTRIBUTING.md +81 -9
- data/README.md +15 -6
- data/VERIFY.sh +587 -0
- data/benchmark/bench_agent_invoke.rb +2 -2
- data/benchmark/bench_context_assembler.rb +39 -68
- data/benchmark/bench_regression.rb +2 -2
- data/docs/architecture/agent-context.md +174 -0
- data/docs/architecture/before-llm-input.md +78 -0
- data/docs/architecture/context-management.md +232 -0
- data/docs/architecture/knowledge-and-rag.md +130 -0
- data/docs/architecture/multi-agent-handoff.md +147 -0
- data/docs/architecture/persistence.md +186 -0
- data/docs/architecture/removed/agent-context.md +72 -0
- data/docs/architecture/security-boundaries.md +173 -0
- data/docs/architecture/tracing.md +194 -0
- data/docs/architecture.md +89 -0
- data/docs/archive/design/archived/04_api_design.md +507 -0
- data/docs/archive/design/archived/09_guardrails.md +186 -0
- data/docs/archive/design/archived/17_rails_integration.md +175 -0
- data/docs/archive/design/historical/00_design_philosophy.md +122 -0
- data/docs/archive/design/historical/01_rubyllm_evaluation.md +178 -0
- data/docs/archive/design/historical/06_design_decisions.md +143 -0
- data/docs/changelog/0.14-and-earlier.md +1 -1
- data/docs/decisions/001-rubyllm-as-provider-layer.md +6 -1
- data/docs/decisions/002-workflow-context-immutability.md +26 -1
- data/docs/decisions/006-no-built-in-guardrails.md +2 -1
- data/docs/decisions/012-canonical-execution-log-and-context-policy.md +120 -38
- data/docs/decisions/014-unified-persistence-durable-state.md +9 -2
- data/docs/decisions/016-semantic-multi-agent-handoff.md +114 -0
- data/docs/decisions/017-design-authority-and-adr-governance.md +200 -0
- data/docs/decisions/018-durability-guarantees-and-failure-model.md +488 -0
- data/docs/decisions/019-filter-contract-and-security-boundaries.md +229 -0
- data/docs/decisions/020-canonical-workflow-instance-identity.md +177 -0
- data/docs/decisions/021-generic-agent-invocation-identity-removal.md +119 -0
- data/docs/decisions/022-agent-execution-parent-identity-and-runtime-routing-boundary.md +193 -0
- data/docs/decisions/023-fsm-session-incarnation-identity-and-routing.md +139 -0
- data/docs/decisions/024-event-loop-single-writer-agent-runtime.md +188 -0
- data/docs/decisions/025-process-local-agent-ownership-and-runtime-admission.md +249 -0
- data/docs/decisions/026-workflow-runtime-admission-and-durable-terminal-barrier.md +257 -0
- data/docs/decisions/027-llm-adapter-provider-boundary.md +93 -0
- data/docs/decisions/028-preparing-recovery-replay-contract.md +106 -0
- data/docs/decisions/029-semantic-completion-and-application-effect-boundary.md +220 -0
- data/docs/decisions/030-agent-handoff-domain-and-durable-responsibility.md +235 -0
- data/docs/decisions/031-durable-multi-agent-coordination.md +301 -0
- data/docs/decisions/README.md +176 -0
- data/docs/design/durable-semantic-coordination/CHANGELOG_V2_REVISION_2.md +33 -0
- data/docs/design/durable-semantic-coordination/CONTINUATION_DECISION_REFACTOR.md +191 -0
- data/docs/design/durable-semantic-coordination/IMPLEMENTATION_DESIGN_V2.md +862 -0
- data/docs/design/durable-semantic-coordination/IMPLEMENTATION_REPORT.md +106 -0
- data/docs/design/durable-semantic-coordination/RECOVERY_CONTRACT_CLARIFICATIONS.md +179 -0
- data/docs/design/durable-semantic-coordination/RESPONSIBILITY_BOUNDARY_REVIEW.md +302 -0
- data/docs/features.md +65 -11
- data/docs/getting-started.md +77 -45
- data/docs/migrations/0.19.md +14 -7
- data/docs/migrations/0.22.md +390 -0
- data/docs/migrations/durable-semantic-coordination-v2.md +65 -0
- data/docs/persistence-backends.md +129 -40
- data/docs/runtime-and-concurrency.md +227 -33
- data/examples/README.md +13 -0
- data/lib/phronomy/agent/agent_execution.rb +21 -17
- data/lib/phronomy/agent/agent_invocation.rb +288 -93
- data/lib/phronomy/agent/agent_invocation_session_builder.rb +236 -202
- data/lib/phronomy/agent/agent_root.rb +3 -3
- data/lib/phronomy/agent/approval_evaluation_request.rb +37 -19
- data/lib/phronomy/agent/async_event_api.rb +162 -72
- data/lib/phronomy/agent/base.rb +416 -181
- data/lib/phronomy/agent/concerns/before_llm_input.rb +1 -1
- data/lib/phronomy/agent/context_assembler.rb +437 -178
- data/lib/phronomy/agent/context_candidate_resolver.rb +2 -2
- data/lib/phronomy/agent/context_plan.rb +18 -13
- data/lib/phronomy/agent/context_plan_validator.rb +246 -88
- data/lib/phronomy/agent/context_policies/default.rb +123 -34
- data/lib/phronomy/agent/context_policy.rb +109 -3
- data/lib/phronomy/agent/context_policy_input.rb +244 -0
- data/lib/phronomy/agent/context_policy_input_builder.rb +241 -0
- data/lib/phronomy/agent/exact_execution.rb +153 -0
- data/lib/phronomy/agent/execution_cancellation.rb +25 -0
- data/lib/phronomy/agent/execution_coordinator.rb +2417 -572
- data/lib/phronomy/agent/handoff.rb +43 -0
- data/lib/phronomy/agent/handoff_capability_factory.rb +45 -0
- data/lib/phronomy/agent/handoff_context.rb +120 -0
- data/lib/phronomy/agent/handoff_execution_coordinator.rb +143 -0
- data/lib/phronomy/agent/handoff_policy.rb +143 -0
- data/lib/phronomy/agent/handoff_projection.rb +207 -0
- data/lib/phronomy/agent/handoff_request.rb +45 -0
- data/lib/phronomy/agent/handoff_runner.rb +178 -0
- data/lib/phronomy/agent/handoff_state.rb +43 -0
- data/lib/phronomy/agent/journal_record.rb +17 -4
- data/lib/phronomy/agent/llm_input_build_context.rb +1 -1
- data/lib/phronomy/agent/llm_input_manifest.rb +277 -2
- data/lib/phronomy/agent/llm_operation_result.rb +12 -7
- data/lib/phronomy/agent/phase_machine_builder.rb +19 -7
- data/lib/phronomy/agent/provider_call_outcome.rb +23 -7
- data/lib/phronomy/agent/recovery_coordinator/continuation.rb +174 -0
- data/lib/phronomy/agent/recovery_coordinator/installation.rb +381 -0
- data/lib/phronomy/agent/recovery_coordinator/resolution.rb +511 -0
- data/lib/phronomy/agent/recovery_coordinator.rb +217 -0
- data/lib/phronomy/agent/recovery_support.rb +504 -0
- data/lib/phronomy/agent/ruby_llm_materializer.rb +16 -13
- data/lib/phronomy/agent/selection/candidate.rb +53 -0
- data/lib/phronomy/agent/selection/constraint.rb +49 -0
- data/lib/phronomy/agent/shared_state.rb +38 -1
- data/lib/phronomy/agent/tool_approval_request.rb +33 -5
- data/lib/phronomy/agent/tool_definition_set.rb +49 -3
- data/lib/phronomy/agent/tool_invocation.rb +339 -103
- data/lib/phronomy/agent/tool_invocation_session_builder.rb +49 -45
- data/lib/phronomy/agent.rb +20 -2
- data/lib/phronomy/agent_already_exists_error.rb +5 -0
- data/lib/phronomy/agent_purged_error.rb +5 -0
- data/lib/phronomy/engine/concurrency/offload_pool.rb +17 -3
- data/lib/phronomy/engine/concurrency/physical_completion_task.rb +135 -0
- data/lib/phronomy/engine/event_loop.rb +622 -63
- data/lib/phronomy/engine/fsm_session.rb +194 -21
- data/lib/phronomy/engine/runtime/agent_ownership_registry.rb +352 -0
- data/lib/phronomy/engine/runtime/team_ownership_registry.rb +77 -0
- data/lib/phronomy/engine/runtime.rb +92 -20
- data/lib/phronomy/generator_verifier.rb +12 -14
- data/lib/phronomy/invocation_context.rb +9 -29
- data/lib/phronomy/multi_agent/admission_registry.rb +51 -0
- data/lib/phronomy/multi_agent/durable_subagent_coordinator.rb +134 -0
- data/lib/phronomy/multi_agent/fan_out_invocation.rb +24 -33
- data/lib/phronomy/multi_agent/fan_out_session_builder.rb +12 -19
- data/lib/phronomy/multi_agent/orchestrator.rb +71 -26
- data/lib/phronomy/multi_agent/team_coordinator.rb +473 -125
- data/lib/phronomy/multi_agent/team_execution.rb +44 -0
- data/lib/phronomy/multi_agent/team_root.rb +41 -0
- data/lib/phronomy/persistence/durable_codec.rb +706 -0
- data/lib/phronomy/persistence/durable_record.rb +117 -0
- data/lib/phronomy/persistence/in_memory.rb +471 -133
- data/lib/phronomy/persistence/migration/initial_format_migration.rb +226 -0
- data/lib/phronomy/persistence/repository_facades.rb +535 -0
- data/lib/phronomy/persistence.rb +174 -40
- data/lib/phronomy/recovery.rb +186 -0
- data/lib/phronomy/testing/persistence_contract/a_journal_repository.rb +2 -2
- data/lib/phronomy/testing/persistence_contract/a_persistence_backend.rb +2 -1
- data/lib/phronomy/testing/persistence_contract/a_workflow_state_repository.rb +19 -19
- data/lib/phronomy/testing/persistence_contract/an_agent_repository.rb +3 -3
- data/lib/phronomy/testing/persistence_contract/an_execution_repository.rb +5 -5
- data/lib/phronomy/testing/persistence_contract/coordination_repositories.rb +137 -0
- data/lib/phronomy/testing/persistence_contract.rb +5 -0
- data/lib/phronomy/tools/agent.rb +1 -1
- data/lib/phronomy/tracing/automatic.rb +176 -0
- data/lib/phronomy/tracing/base.rb +11 -2
- data/lib/phronomy/tracing/langfuse_tracer.rb +20 -12
- data/lib/phronomy/version.rb +1 -1
- data/lib/phronomy/workflow.rb +3 -6
- data/lib/phronomy/workflow_context.rb +14 -5
- data/lib/phronomy/workflow_recovery.rb +123 -0
- data/lib/phronomy/workflow_runner.rb +468 -256
- data/lib/phronomy.rb +6 -0
- data/scripts/api_snapshot.rb +12 -0
- data/sig/phronomy/agent.rbs +209 -7
- data/sig/phronomy/handoff.rbs +41 -0
- data/sig/phronomy/multi_agent.rbs +35 -0
- data/sig/phronomy/persistence.rbs +123 -4
- data/sig/phronomy/runtime.rbs +1 -4
- data/sig/phronomy/workflow.rbs +2 -2
- data/sig/phronomy.rbs +10 -0
- metadata +84 -18
- data/examples/workflows/agent_event_mapping.rb +0 -101
- data/examples/workflows/generic_task_event_mapping.rb +0 -66
- data/lib/phronomy/agent/activation_registry.rb +0 -28
- data/lib/phronomy/agent/agent_execution_activation.rb +0 -172
- data/lib/phronomy/agent/context_candidate.rb +0 -47
- data/lib/phronomy/agent/context_parts/budget/token_budget_packer.rb +0 -53
- data/lib/phronomy/agent/context_parts/requirements/required_context_resolver.rb +0 -56
- data/lib/phronomy/agent/context_parts/selectors/recent_first_selector.rb +0 -30
- data/lib/phronomy/agent/context_parts/unit_builders/dependency_aware_unit_builder.rb +0 -118
- data/lib/phronomy/agent/context_policy_descriptor.rb +0 -49
- data/lib/phronomy/agent/context_policy_registry.rb +0 -46
- data/lib/phronomy/agent/context_request.rb +0 -35
- data/lib/phronomy/agent/context_selection_unit.rb +0 -38
- data/lib/phronomy/agent/derived_content_spec.rb +0 -34
- data/lib/phronomy/agent/runner.rb +0 -97
- data/lib/phronomy/multi_agent/handoff.rb +0 -64
|
@@ -1,48 +1,158 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "time"
|
|
4
|
+
require "securerandom"
|
|
4
5
|
|
|
5
6
|
module Phronomy
|
|
6
7
|
module Agent
|
|
7
8
|
class ExecutionCoordinator
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
# External/API -> EventLoop control messages.
|
|
10
|
+
StartCommand = Data.define(
|
|
11
|
+
:coordinator, :input, :config, :mode,
|
|
12
|
+
:approval_policy, :approval_listener, :on_event, :result_task,
|
|
13
|
+
:admission_token
|
|
14
|
+
)
|
|
15
|
+
ResumeCommand = Data.define(
|
|
16
|
+
:coordinator, :execution_id, :approval_request_id,
|
|
17
|
+
:approved, :config, :result_task
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
# EventLoop -> Offload operation-specific immutable snapshots.
|
|
21
|
+
InitialPreparationCommand = Data.define(
|
|
22
|
+
:root, :journal_records, :input, :config, :preparation_replayable
|
|
23
|
+
)
|
|
24
|
+
ProviderDispatchPreparationCommand = Data.define(
|
|
25
|
+
:execution_id, :fsm_session_id, :expected_execution_revision,
|
|
26
|
+
:root, :journal_records, :execution, :base_manifest,
|
|
27
|
+
:invocation_config, :runtime_snapshot, :streaming,
|
|
28
|
+
:pending_llm_call_id, :pending_llm_started_at
|
|
29
|
+
)
|
|
30
|
+
ToolDispatchPreparationCommand = Data.define(
|
|
31
|
+
:execution_id, :fsm_session_id, :expected_execution_revision,
|
|
32
|
+
:root, :execution, :runtime_snapshot, :tool_batch_snapshot
|
|
33
|
+
)
|
|
34
|
+
ProviderDispatchPreparationReconciliationCommand = Data.define(
|
|
35
|
+
:operation, :intended_result, :original_error
|
|
36
|
+
)
|
|
37
|
+
ToolDispatchPreparationReconciliationCommand = Data.define(
|
|
38
|
+
:operation, :intended_result, :original_error
|
|
39
|
+
)
|
|
40
|
+
ResumeCommitCommand = Data.define(
|
|
41
|
+
:execution_id, :expected_execution_revision,
|
|
42
|
+
:root, :execution, :approval_request_id, :approved
|
|
43
|
+
)
|
|
44
|
+
HandoffTerminalView = Data.define(
|
|
45
|
+
:target_agent_id, :responsibility, :selection_intent,
|
|
46
|
+
:llm_call_id, :tool_call_id, :policy
|
|
47
|
+
)
|
|
48
|
+
TerminalView = Data.define(
|
|
49
|
+
:phase, :output, :usage, :approval_request, :rejected,
|
|
50
|
+
:input_blocked, :output_blocked, :block_error,
|
|
51
|
+
:invocation_error, :handoff, :callback_failure, :source_error, :cancel_requested
|
|
52
|
+
)
|
|
53
|
+
TerminalCommitCommand = Data.define(
|
|
54
|
+
:execution_id, :fsm_session_id, :expected_execution_revision,
|
|
55
|
+
:root, :journal_records, :execution, :runtime_snapshot,
|
|
56
|
+
:terminal_view, :state_required
|
|
57
|
+
)
|
|
58
|
+
TerminalDelivery = Data.define(
|
|
59
|
+
:result_task, :application_listener, :approval_listener,
|
|
60
|
+
:handoff_request, :handoff_manifest
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# Offload -> EventLoop operation results. No live Agent/Invocation object is
|
|
64
|
+
# mutated by a worker; these values are validated and applied on EventLoop.
|
|
65
|
+
InitialPreparationResult = Data.define(
|
|
66
|
+
:execution, :root, :runtime_projection, :filtered_input,
|
|
67
|
+
:config, :appended_records, :error, :admission_outcome
|
|
68
|
+
)
|
|
69
|
+
ProviderDispatchPreparationResult = Data.define(:execution, :runtime_projection, :error)
|
|
70
|
+
ToolDispatchPreparationResult = Data.define(:execution)
|
|
71
|
+
ProviderDispatchPreparationReconciliationResult = Data.define(
|
|
72
|
+
:disposition, :preparation_result
|
|
73
|
+
)
|
|
74
|
+
ToolDispatchPreparationReconciliationResult = Data.define(
|
|
75
|
+
:disposition, :preparation_result
|
|
76
|
+
)
|
|
77
|
+
ResumeCommitResult = Data.define(:execution, :root)
|
|
78
|
+
TerminalOutcome = Data.define(
|
|
79
|
+
:type, :execution, :root, :appended_records,
|
|
80
|
+
:result, :error, :approval_request
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
InitialPreparationReady = Data.define(
|
|
84
|
+
:coordinator, :request, :result, :error
|
|
85
|
+
)
|
|
86
|
+
InitialPreparationRecoveryReady = Data.define(
|
|
87
|
+
:coordinator, :execution_id, :expected_execution_revision,
|
|
88
|
+
:result_task, :load_completion, :result, :error
|
|
89
|
+
)
|
|
90
|
+
ProviderDispatchPreparationReady = Data.define(
|
|
91
|
+
:coordinator, :operation, :result, :error
|
|
92
|
+
)
|
|
93
|
+
ToolDispatchPreparationReady = Data.define(
|
|
94
|
+
:coordinator, :operation, :result, :error
|
|
95
|
+
)
|
|
96
|
+
ProviderDispatchPreparationReconciliationReady = Data.define(
|
|
97
|
+
:coordinator, :command, :result, :error
|
|
98
|
+
)
|
|
99
|
+
ToolDispatchPreparationReconciliationReady = Data.define(
|
|
100
|
+
:coordinator, :command, :result, :error
|
|
101
|
+
)
|
|
102
|
+
ResumeCommitReady = Data.define(
|
|
103
|
+
:coordinator, :request, :operation, :result, :error
|
|
104
|
+
)
|
|
105
|
+
TerminalCommitReady = Data.define(
|
|
106
|
+
:coordinator, :operation, :delivery, :outcome, :error
|
|
107
|
+
)
|
|
108
|
+
DeferredTerminalCommand = Data.define(
|
|
109
|
+
:coordinator, :execution_id, :result_task, :invocation,
|
|
110
|
+
:source_error, :fsm_session_id
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
class PreparationOutcomeUnknownError < Phronomy::Error
|
|
114
|
+
attr_reader :original_error, :intended_result
|
|
115
|
+
|
|
116
|
+
def initialize(original_error, intended_result)
|
|
117
|
+
@original_error = original_error
|
|
118
|
+
@intended_result = intended_result
|
|
119
|
+
super(
|
|
120
|
+
"durable dispatch preparation outcome is uncertain: " \
|
|
121
|
+
"#{original_error.class}: #{original_error.message}"
|
|
122
|
+
)
|
|
123
|
+
set_backtrace(original_error.backtrace)
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
private_constant :PreparationOutcomeUnknownError
|
|
11
127
|
|
|
12
128
|
def initialize(agent)
|
|
13
129
|
@agent = agent
|
|
14
130
|
end
|
|
15
131
|
|
|
16
132
|
def start(
|
|
17
|
-
input,
|
|
133
|
+
input, config: {}, mode: :invoke,
|
|
18
134
|
approval_policy: nil, approval_listener: nil, on_event: nil
|
|
19
135
|
)
|
|
136
|
+
@agent.send(:__assert_live_agent!)
|
|
137
|
+
@agent.send(:_reject_removed_generic_identity_keys!, config)
|
|
20
138
|
result_task = Phronomy::Task.deferred(name: "agent-#{@agent.agent_id}-#{mode}")
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
approval_listener: approval_listener,
|
|
35
|
-
on_event: on_event
|
|
36
|
-
)
|
|
37
|
-
end
|
|
139
|
+
command = StartCommand.new(
|
|
140
|
+
coordinator: self,
|
|
141
|
+
input: input,
|
|
142
|
+
config: @agent.__invocation_config(config).dup.freeze,
|
|
143
|
+
mode: mode.to_sym,
|
|
144
|
+
approval_policy: approval_policy,
|
|
145
|
+
approval_listener: approval_listener,
|
|
146
|
+
on_event: on_event,
|
|
147
|
+
result_task: result_task,
|
|
148
|
+
admission_token: Object.new.freeze
|
|
149
|
+
)
|
|
150
|
+
unless post_control(Phronomy::Runtime.instance, command)
|
|
151
|
+
fail_task(result_task, runtime_rejected_error(:start))
|
|
38
152
|
end
|
|
39
153
|
result_task
|
|
40
154
|
rescue => error
|
|
41
|
-
if defined?(
|
|
42
|
-
post_prep_failure(runtime, on_event, result_task, translated(error))
|
|
43
|
-
else
|
|
44
|
-
fail_task(result_task, translated(error))
|
|
45
|
-
end
|
|
155
|
+
fail_task(result_task, translated(error)) if defined?(result_task) && result_task
|
|
46
156
|
result_task
|
|
47
157
|
end
|
|
48
158
|
|
|
@@ -52,156 +162,393 @@ module Phronomy
|
|
|
52
162
|
approved:,
|
|
53
163
|
config: {}
|
|
54
164
|
)
|
|
165
|
+
@agent.send(:__assert_live_agent!)
|
|
166
|
+
@agent.send(:_reject_removed_generic_identity_keys!, config)
|
|
55
167
|
result_task = Phronomy::Task.deferred(
|
|
56
168
|
name: "agent-approval-resume:#{execution_id}"
|
|
57
169
|
)
|
|
170
|
+
command = ResumeCommand.new(
|
|
171
|
+
coordinator: self,
|
|
172
|
+
execution_id: execution_id.to_s.freeze,
|
|
173
|
+
approval_request_id: approval_request_id.to_s.freeze,
|
|
174
|
+
approved: !!approved,
|
|
175
|
+
config: config.dup.freeze,
|
|
176
|
+
result_task: result_task
|
|
177
|
+
)
|
|
178
|
+
unless post_control(Phronomy::Runtime.instance, command)
|
|
179
|
+
fail_task(result_task, runtime_rejected_error(:resume))
|
|
180
|
+
end
|
|
181
|
+
result_task
|
|
182
|
+
rescue => error
|
|
183
|
+
fail_task(result_task, translated(error)) if defined?(result_task) && result_task
|
|
184
|
+
result_task
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# Called only from an Agent FSMSession external-operation dispatch state on
|
|
188
|
+
# EventLoop. Each method captures one operation-specific immutable snapshot
|
|
189
|
+
# and sends only that snapshot to OffloadPool.
|
|
190
|
+
# @api private
|
|
191
|
+
def prepare_provider_dispatch(invocation, event_sink:, streaming:)
|
|
192
|
+
runtime = Phronomy::Runtime.instance
|
|
193
|
+
event_loop = runtime.event_loop
|
|
194
|
+
assert_event_loop!(event_loop)
|
|
195
|
+
state = event_loop.agent_execution_state(invocation.execution_id)
|
|
196
|
+
validate_live_session!(state, invocation, event_sink.fsm_session_id)
|
|
197
|
+
unless event_loop.fsm_session_state(event_sink.fsm_session_id) == :calling_llm
|
|
198
|
+
raise Phronomy::Error,
|
|
199
|
+
"Provider dispatch preparation requires the owning FSMSession to be in :calling_llm"
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
operation = ProviderDispatchPreparationCommand.new(
|
|
203
|
+
execution_id: state.execution_id,
|
|
204
|
+
fsm_session_id: event_sink.fsm_session_id.to_s.freeze,
|
|
205
|
+
expected_execution_revision: state.execution.execution_revision,
|
|
206
|
+
root: @agent.agent_root,
|
|
207
|
+
journal_records: @agent.send(:_journal_records_snapshot),
|
|
208
|
+
execution: state.execution,
|
|
209
|
+
base_manifest: state.base_manifest,
|
|
210
|
+
invocation_config: invocation.config.dup.freeze,
|
|
211
|
+
runtime_snapshot: invocation.runtime_snapshot,
|
|
212
|
+
streaming: !!streaming,
|
|
213
|
+
pending_llm_call_id: SecureRandom.uuid.to_s.freeze,
|
|
214
|
+
pending_llm_started_at: Time.now.utc.iso8601(6).freeze
|
|
215
|
+
)
|
|
216
|
+
submit_provider_dispatch_preparation(operation)
|
|
217
|
+
nil
|
|
218
|
+
rescue => error
|
|
219
|
+
event_sink.post(:llm_setup_failed, translated(error))
|
|
220
|
+
nil
|
|
221
|
+
end
|
|
222
|
+
|
|
223
|
+
# @api private
|
|
224
|
+
def prepare_tool_dispatch(invocation, event_sink:)
|
|
58
225
|
runtime = Phronomy::Runtime.instance
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
226
|
+
event_loop = runtime.event_loop
|
|
227
|
+
assert_event_loop!(event_loop)
|
|
228
|
+
state = event_loop.agent_execution_state(invocation.execution_id)
|
|
229
|
+
validate_live_session!(state, invocation, event_sink.fsm_session_id)
|
|
230
|
+
unless event_loop.fsm_session_state(event_sink.fsm_session_id) == :dispatching_tools
|
|
231
|
+
raise Phronomy::Error,
|
|
232
|
+
"Tool dispatch preparation requires the owning FSMSession to be in :dispatching_tools"
|
|
233
|
+
end
|
|
234
|
+
|
|
235
|
+
tool_batch_snapshot = RecoverySupport.build_tool_batch_snapshot(invocation)
|
|
236
|
+
if tool_batch_snapshot.empty?
|
|
237
|
+
raise Phronomy::Error, "Tool dispatch preparation requires a non-empty Tool batch"
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
operation = ToolDispatchPreparationCommand.new(
|
|
241
|
+
execution_id: state.execution_id,
|
|
242
|
+
fsm_session_id: event_sink.fsm_session_id.to_s.freeze,
|
|
243
|
+
expected_execution_revision: state.execution.execution_revision,
|
|
244
|
+
root: @agent.agent_root,
|
|
245
|
+
execution: state.execution,
|
|
246
|
+
runtime_snapshot: invocation.runtime_snapshot,
|
|
247
|
+
tool_batch_snapshot: tool_batch_snapshot
|
|
248
|
+
)
|
|
249
|
+
submit_tool_dispatch_preparation(operation)
|
|
250
|
+
nil
|
|
251
|
+
rescue => error
|
|
252
|
+
event_sink.post(:tool_setup_failed, translated(error))
|
|
253
|
+
nil
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def submit_provider_dispatch_preparation(operation)
|
|
257
|
+
runtime = Phronomy::Runtime.instance
|
|
258
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
259
|
+
perform_provider_dispatch_preparation(operation)
|
|
260
|
+
end
|
|
261
|
+
task.on_complete do |result, error|
|
|
262
|
+
ready = ProviderDispatchPreparationReady.new(
|
|
263
|
+
coordinator: self,
|
|
264
|
+
operation: operation,
|
|
265
|
+
result: result,
|
|
266
|
+
error: error
|
|
267
|
+
)
|
|
268
|
+
unless post_control(runtime, ready)
|
|
269
|
+
Phronomy.configuration.logger&.warn(
|
|
270
|
+
"[Phronomy] EventLoop rejected Provider dispatch preparation result for " \
|
|
271
|
+
"#{operation.execution_id}"
|
|
65
272
|
)
|
|
66
|
-
|
|
67
|
-
return result_task
|
|
273
|
+
end
|
|
68
274
|
end
|
|
275
|
+
end
|
|
69
276
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
277
|
+
def submit_tool_dispatch_preparation(operation)
|
|
278
|
+
runtime = Phronomy::Runtime.instance
|
|
279
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
280
|
+
perform_tool_dispatch_preparation(operation)
|
|
281
|
+
end
|
|
282
|
+
task.on_complete do |result, error|
|
|
283
|
+
ready = ToolDispatchPreparationReady.new(
|
|
284
|
+
coordinator: self,
|
|
285
|
+
operation: operation,
|
|
286
|
+
result: result,
|
|
287
|
+
error: error
|
|
288
|
+
)
|
|
289
|
+
unless post_control(runtime, ready)
|
|
290
|
+
Phronomy.configuration.logger&.warn(
|
|
291
|
+
"[Phronomy] EventLoop rejected Tool dispatch preparation result for " \
|
|
292
|
+
"#{operation.execution_id}"
|
|
76
293
|
)
|
|
77
|
-
|
|
78
|
-
return result_task
|
|
294
|
+
end
|
|
79
295
|
end
|
|
296
|
+
end
|
|
80
297
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
298
|
+
def submit_provider_dispatch_preparation_reconciliation(operation, uncertainty)
|
|
299
|
+
# simplecov:disable
|
|
300
|
+
runtime = Phronomy::Runtime.instance
|
|
301
|
+
command = ProviderDispatchPreparationReconciliationCommand.new(
|
|
302
|
+
operation: operation,
|
|
303
|
+
intended_result: uncertainty.intended_result,
|
|
304
|
+
original_error: uncertainty.original_error
|
|
305
|
+
)
|
|
306
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
307
|
+
perform_provider_dispatch_preparation_reconciliation(command)
|
|
88
308
|
end
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
309
|
+
task.on_complete do |result, error|
|
|
310
|
+
ready = ProviderDispatchPreparationReconciliationReady.new(
|
|
311
|
+
coordinator: self,
|
|
312
|
+
command: command,
|
|
313
|
+
result: result,
|
|
314
|
+
error: error
|
|
315
|
+
)
|
|
316
|
+
unless post_control(runtime, ready)
|
|
317
|
+
Phronomy.configuration.logger&.warn(
|
|
318
|
+
"[Phronomy] EventLoop rejected Provider dispatch reconciliation result for #{operation.execution_id}"
|
|
319
|
+
)
|
|
93
320
|
end
|
|
321
|
+
end
|
|
322
|
+
rescue => error
|
|
323
|
+
mark_barrier_recovery_required(operation, error)
|
|
324
|
+
# simplecov:enable
|
|
325
|
+
end
|
|
94
326
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
327
|
+
def submit_tool_dispatch_preparation_reconciliation(operation, uncertainty)
|
|
328
|
+
# simplecov:disable
|
|
329
|
+
runtime = Phronomy::Runtime.instance
|
|
330
|
+
command = ToolDispatchPreparationReconciliationCommand.new(
|
|
331
|
+
operation: operation,
|
|
332
|
+
intended_result: uncertainty.intended_result,
|
|
333
|
+
original_error: uncertainty.original_error
|
|
334
|
+
)
|
|
335
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
336
|
+
perform_tool_dispatch_preparation_reconciliation(command)
|
|
337
|
+
end
|
|
338
|
+
task.on_complete do |result, error|
|
|
339
|
+
ready = ToolDispatchPreparationReconciliationReady.new(
|
|
340
|
+
coordinator: self,
|
|
341
|
+
command: command,
|
|
342
|
+
result: result,
|
|
343
|
+
error: error
|
|
344
|
+
)
|
|
345
|
+
unless post_control(runtime, ready)
|
|
346
|
+
Phronomy.configuration.logger&.warn(
|
|
347
|
+
"[Phronomy] EventLoop rejected Tool dispatch reconciliation result for #{operation.execution_id}"
|
|
101
348
|
)
|
|
102
|
-
rescue => start_error
|
|
103
|
-
commit = runtime.offload.submit(on_full: :raise) do
|
|
104
|
-
commit_failed(
|
|
105
|
-
prepared_activation,
|
|
106
|
-
prepared_activation.invocation,
|
|
107
|
-
start_error
|
|
108
|
-
)
|
|
109
|
-
end
|
|
110
|
-
commit.on_complete do |terminal, commit_error|
|
|
111
|
-
if commit_error
|
|
112
|
-
fail_task(result_task, commit_error)
|
|
113
|
-
else
|
|
114
|
-
runtime.__agent_activations.delete(execution_id)
|
|
115
|
-
fail_task(result_task, terminal.fetch(:error))
|
|
116
|
-
end
|
|
117
|
-
end
|
|
118
349
|
end
|
|
119
350
|
end
|
|
120
|
-
result_task
|
|
121
351
|
rescue => error
|
|
122
|
-
|
|
123
|
-
|
|
352
|
+
mark_barrier_recovery_required(operation, error)
|
|
353
|
+
# simplecov:enable
|
|
124
354
|
end
|
|
125
355
|
|
|
126
|
-
#
|
|
127
|
-
#
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
journal_records = @agent.send(:_journal_records_snapshot)
|
|
133
|
-
manifest = manifest_ref = updated = nil
|
|
356
|
+
# Every control message is delivered by EventLoop. This method is the only
|
|
357
|
+
# coordinator entry point allowed to advance Phronomy-managed live state.
|
|
358
|
+
# @api private
|
|
359
|
+
def deliver_on_event_loop(command)
|
|
360
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
361
|
+
assert_event_loop!(event_loop)
|
|
134
362
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
)
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
363
|
+
case command
|
|
364
|
+
when StartCommand
|
|
365
|
+
begin_start_on_event_loop(command)
|
|
366
|
+
when InitialPreparationReady
|
|
367
|
+
apply_initial_preparation_on_event_loop(command)
|
|
368
|
+
when InitialPreparationRecoveryReady
|
|
369
|
+
apply_initial_preparation_recovery_on_event_loop(command)
|
|
370
|
+
when ProviderDispatchPreparationReady
|
|
371
|
+
apply_provider_dispatch_preparation_on_event_loop(command)
|
|
372
|
+
when ToolDispatchPreparationReady
|
|
373
|
+
apply_tool_dispatch_preparation_on_event_loop(command)
|
|
374
|
+
when ProviderDispatchPreparationReconciliationReady
|
|
375
|
+
apply_provider_dispatch_preparation_reconciliation_on_event_loop(command)
|
|
376
|
+
when ToolDispatchPreparationReconciliationReady
|
|
377
|
+
apply_tool_dispatch_preparation_reconciliation_on_event_loop(command)
|
|
378
|
+
when ResumeCommand
|
|
379
|
+
begin_resume_on_event_loop(command)
|
|
380
|
+
when ResumeCommitReady
|
|
381
|
+
apply_resume_commit_on_event_loop(command)
|
|
382
|
+
when TerminalCommitReady
|
|
383
|
+
apply_terminal_commit_on_event_loop(command)
|
|
384
|
+
when DeferredTerminalCommand
|
|
385
|
+
resume_deferred_terminal_on_event_loop(command)
|
|
386
|
+
else
|
|
387
|
+
raise Phronomy::Error, "unknown Agent control command: #{command.class}"
|
|
388
|
+
end
|
|
389
|
+
end
|
|
390
|
+
|
|
391
|
+
private
|
|
392
|
+
|
|
393
|
+
# ----------------------------------------------------------------------
|
|
394
|
+
# Initial preparation
|
|
395
|
+
# ----------------------------------------------------------------------
|
|
396
|
+
|
|
397
|
+
def begin_start_on_event_loop(request)
|
|
398
|
+
runtime = Phronomy::Runtime.instance
|
|
399
|
+
event_loop = runtime.event_loop
|
|
400
|
+
root = @agent.agent_root
|
|
401
|
+
if root.lifecycle_status == :closed
|
|
402
|
+
deliver_start_failure_on_event_loop(
|
|
403
|
+
request,
|
|
404
|
+
Phronomy::Error.new("agent is closed: #{@agent.agent_id}")
|
|
149
405
|
)
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
406
|
+
return
|
|
407
|
+
end
|
|
408
|
+
|
|
409
|
+
@agent.send(:__assert_live_agent!)
|
|
410
|
+
admitted = false
|
|
411
|
+
submitted = false
|
|
412
|
+
event_loop.admit_agent_execution(
|
|
413
|
+
@agent.agent_id,
|
|
414
|
+
owner_token: request.admission_token
|
|
415
|
+
)
|
|
416
|
+
admitted = true
|
|
417
|
+
|
|
418
|
+
# Ownership purge may begin on an application thread between the first
|
|
419
|
+
# live-owner check and EventLoop admission. Re-check after the slot is
|
|
420
|
+
# installed: once this check succeeds, purge observes the admission and
|
|
421
|
+
# must abort instead of deleting the Agent underneath this start.
|
|
422
|
+
@agent.send(:__assert_live_agent!)
|
|
423
|
+
|
|
424
|
+
operation = InitialPreparationCommand.new(
|
|
425
|
+
root: root,
|
|
426
|
+
journal_records: @agent.send(:_journal_records_snapshot),
|
|
427
|
+
input: request.input,
|
|
428
|
+
config: request.config,
|
|
429
|
+
preparation_replayable: initial_preparation_replayable?(
|
|
430
|
+
request.input,
|
|
431
|
+
request.config,
|
|
432
|
+
request.approval_policy
|
|
155
433
|
)
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
)
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
434
|
+
)
|
|
435
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
436
|
+
perform_initial_preparation(operation)
|
|
437
|
+
end
|
|
438
|
+
submitted = true
|
|
439
|
+
task.on_complete do |result, error|
|
|
440
|
+
ready = InitialPreparationReady.new(
|
|
441
|
+
coordinator: self,
|
|
442
|
+
request: request,
|
|
443
|
+
result: result,
|
|
444
|
+
error: error
|
|
445
|
+
)
|
|
446
|
+
fail_task(request.result_task, runtime_rejected_error(:initial_preparation)) unless
|
|
447
|
+
post_control(runtime, ready)
|
|
448
|
+
end
|
|
449
|
+
rescue => error
|
|
450
|
+
if admitted
|
|
451
|
+
if submitted
|
|
452
|
+
event_loop.mark_agent_admission_recovery_required(
|
|
453
|
+
@agent.agent_id,
|
|
454
|
+
owner_token: request.admission_token
|
|
174
455
|
)
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
456
|
+
else
|
|
457
|
+
event_loop.release_agent_execution_admission(
|
|
458
|
+
@agent.agent_id,
|
|
459
|
+
owner_token: request.admission_token
|
|
460
|
+
)
|
|
461
|
+
end
|
|
462
|
+
end
|
|
463
|
+
deliver_start_failure_on_event_loop(request, translated(error))
|
|
464
|
+
end
|
|
465
|
+
|
|
466
|
+
def perform_initial_preparation(operation)
|
|
467
|
+
begin
|
|
468
|
+
raw_message = @agent.send(:extract_message, operation.input)
|
|
469
|
+
rescue => error
|
|
470
|
+
return InitialPreparationResult.new(
|
|
471
|
+
execution: nil,
|
|
472
|
+
root: operation.root,
|
|
473
|
+
runtime_projection: nil,
|
|
474
|
+
filtered_input: nil,
|
|
475
|
+
config: operation.config,
|
|
476
|
+
appended_records: [].freeze,
|
|
477
|
+
error: translated(error),
|
|
478
|
+
admission_outcome: :not_established
|
|
180
479
|
)
|
|
181
480
|
end
|
|
182
481
|
|
|
183
|
-
|
|
184
|
-
|
|
482
|
+
begin
|
|
483
|
+
execution, active_root = admit_execution(
|
|
484
|
+
raw_message,
|
|
485
|
+
root: operation.root,
|
|
486
|
+
mode: (operation.config[:phronomy_recovery_mode] || :invoke).to_sym,
|
|
487
|
+
config: operation.config,
|
|
488
|
+
preparation_replayable: operation.preparation_replayable
|
|
489
|
+
)
|
|
490
|
+
rescue Phronomy::AgentBusyError => error
|
|
491
|
+
# A durable busy conflict proves that another nonterminal logical
|
|
492
|
+
# Execution already exists for this agent_id. Process loss does not
|
|
493
|
+
# make that execution terminal; keep Runtime admission fail-closed
|
|
494
|
+
# until ACS-15 recovery/reconciliation can establish current lineage.
|
|
495
|
+
return InitialPreparationResult.new(
|
|
496
|
+
execution: nil,
|
|
497
|
+
root: operation.root,
|
|
498
|
+
runtime_projection: nil,
|
|
499
|
+
filtered_input: nil,
|
|
500
|
+
config: operation.config,
|
|
501
|
+
appended_records: [].freeze,
|
|
502
|
+
error: translated(error),
|
|
503
|
+
admission_outcome: :recovery_required
|
|
504
|
+
)
|
|
505
|
+
rescue Phronomy::Persistence::ConflictError,
|
|
506
|
+
Phronomy::Persistence::NotFoundError,
|
|
507
|
+
Phronomy::Persistence::SerializationError,
|
|
508
|
+
ArgumentError,
|
|
509
|
+
Phronomy::ConfigurationError => error
|
|
510
|
+
return InitialPreparationResult.new(
|
|
511
|
+
execution: nil,
|
|
512
|
+
root: operation.root,
|
|
513
|
+
runtime_projection: nil,
|
|
514
|
+
filtered_input: nil,
|
|
515
|
+
config: operation.config,
|
|
516
|
+
appended_records: [].freeze,
|
|
517
|
+
error: translated(error),
|
|
518
|
+
admission_outcome: :not_established
|
|
519
|
+
)
|
|
520
|
+
rescue => error
|
|
521
|
+
return InitialPreparationResult.new(
|
|
522
|
+
execution: nil,
|
|
523
|
+
root: operation.root,
|
|
524
|
+
runtime_projection: nil,
|
|
525
|
+
filtered_input: nil,
|
|
526
|
+
config: operation.config,
|
|
527
|
+
appended_records: [].freeze,
|
|
528
|
+
error: translated(error),
|
|
529
|
+
admission_outcome: :outcome_unknown
|
|
530
|
+
)
|
|
531
|
+
end
|
|
185
532
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
533
|
+
perform_admitted_initial_preparation(
|
|
534
|
+
input: operation.input,
|
|
535
|
+
config: operation.config,
|
|
536
|
+
execution: execution,
|
|
537
|
+
active_root: active_root,
|
|
538
|
+
journal_records: operation.journal_records
|
|
539
|
+
)
|
|
192
540
|
end
|
|
193
541
|
|
|
194
|
-
def
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
542
|
+
def perform_admitted_initial_preparation(
|
|
543
|
+
input:,
|
|
544
|
+
config:,
|
|
545
|
+
execution:,
|
|
546
|
+
active_root:,
|
|
547
|
+
journal_records:
|
|
548
|
+
)
|
|
201
549
|
current_execution = execution
|
|
202
|
-
|
|
203
550
|
begin
|
|
204
|
-
effective_config =
|
|
551
|
+
effective_config = config
|
|
205
552
|
@agent.send(
|
|
206
553
|
:check_cancellation!,
|
|
207
554
|
effective_config,
|
|
@@ -215,46 +562,56 @@ module Phronomy
|
|
|
215
562
|
)
|
|
216
563
|
filtered_message = @agent.send(:extract_message, filtered_input)
|
|
217
564
|
manifest = manifest_ref = active_execution = nil
|
|
218
|
-
local_journal = @agent.send(:_journal_records_snapshot)
|
|
219
565
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
)
|
|
566
|
+
# Content-addressed source material may be written before the Policy call.
|
|
567
|
+
# No mutable Agent/Execution repository state is committed here.
|
|
568
|
+
filtered_ref = @agent.persistence.contents.put_text(filtered_message)
|
|
569
|
+
input_record = JournalRecord.new(
|
|
570
|
+
agent_id: @agent.agent_id,
|
|
571
|
+
execution_id: current_execution.execution_id,
|
|
572
|
+
kind: :external_message,
|
|
573
|
+
channel: :external,
|
|
574
|
+
role: :user,
|
|
575
|
+
content_ref: filtered_ref,
|
|
576
|
+
context_generation: active_root.transcript_generation,
|
|
577
|
+
context_candidate: true,
|
|
578
|
+
metadata: {"handoff_policy_category" => "current_request"}
|
|
579
|
+
)
|
|
580
|
+
staged = current_execution.with(
|
|
581
|
+
execution_revision: current_execution.execution_revision,
|
|
582
|
+
working_records: current_execution.working_records + [input_record],
|
|
583
|
+
metadata: current_execution.metadata.merge(
|
|
584
|
+
"current_input_ref" => filtered_ref,
|
|
585
|
+
"current_input_record_id" => input_record.record_id
|
|
241
586
|
)
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
587
|
+
)
|
|
588
|
+
assembler = ContextAssembler.new(
|
|
589
|
+
agent: @agent,
|
|
590
|
+
persistence: @agent.persistence,
|
|
591
|
+
journal_records: journal_records
|
|
592
|
+
)
|
|
593
|
+
prepared = assembler.prepare_initial(
|
|
594
|
+
input: filtered_input,
|
|
595
|
+
agent_root: active_root,
|
|
596
|
+
execution: staged,
|
|
597
|
+
config: effective_config,
|
|
598
|
+
patch: @agent.send(
|
|
599
|
+
:run_before_llm_input_hooks,
|
|
600
|
+
call_sequence: 1,
|
|
601
|
+
config: effective_config
|
|
257
602
|
)
|
|
603
|
+
)
|
|
604
|
+
@agent.send(
|
|
605
|
+
:check_cancellation!,
|
|
606
|
+
effective_config,
|
|
607
|
+
"invocation cancelled after context policy"
|
|
608
|
+
)
|
|
609
|
+
|
|
610
|
+
# Revalidate the durable base only after Application Policy returns.
|
|
611
|
+
# ContextPolicy never runs while this transaction is open.
|
|
612
|
+
@agent.persistence.transaction do |tx|
|
|
613
|
+
assert_local_durable_base!(tx, active_root)
|
|
614
|
+
manifest, manifest_ref = assembler.finalize(prepared, persistence: tx)
|
|
258
615
|
active_execution = staged.with(
|
|
259
616
|
status: :active,
|
|
260
617
|
phase: :calling_llm,
|
|
@@ -276,52 +633,94 @@ module Phronomy
|
|
|
276
633
|
agent: @agent,
|
|
277
634
|
persistence: @agent.persistence
|
|
278
635
|
).materialize(manifest: manifest, manifest_ref: manifest_ref)
|
|
279
|
-
|
|
636
|
+
InitialPreparationResult.new(
|
|
280
637
|
execution: active_execution,
|
|
638
|
+
root: active_root,
|
|
281
639
|
runtime_projection: projection,
|
|
282
640
|
filtered_input: filtered_input,
|
|
283
|
-
config: effective_config
|
|
641
|
+
config: effective_config,
|
|
642
|
+
appended_records: [].freeze,
|
|
643
|
+
error: nil,
|
|
644
|
+
admission_outcome: :active
|
|
284
645
|
)
|
|
285
646
|
rescue => error
|
|
286
|
-
commit_preparation_failure(
|
|
287
|
-
|
|
647
|
+
failure = commit_preparation_failure(
|
|
648
|
+
current_execution,
|
|
649
|
+
active_root,
|
|
650
|
+
error
|
|
651
|
+
)
|
|
652
|
+
InitialPreparationResult.new(
|
|
653
|
+
execution: failure.fetch(:execution),
|
|
654
|
+
root: failure.fetch(:root),
|
|
655
|
+
runtime_projection: nil,
|
|
656
|
+
filtered_input: nil,
|
|
657
|
+
config: config,
|
|
658
|
+
appended_records: failure.fetch(:appended_records),
|
|
659
|
+
error: failure.fetch(:error),
|
|
660
|
+
admission_outcome: :terminal
|
|
661
|
+
)
|
|
288
662
|
end
|
|
289
663
|
end
|
|
290
664
|
|
|
291
|
-
def admit_execution(
|
|
292
|
-
|
|
293
|
-
|
|
665
|
+
def admit_execution(
|
|
666
|
+
raw_message,
|
|
667
|
+
root:,
|
|
668
|
+
mode: :invoke,
|
|
669
|
+
config: {},
|
|
670
|
+
preparation_replayable: false
|
|
671
|
+
)
|
|
672
|
+
if root.lifecycle_status == :closed
|
|
673
|
+
raise Phronomy::Error,
|
|
674
|
+
"agent is closed: #{@agent.agent_id}"
|
|
675
|
+
end
|
|
294
676
|
|
|
295
677
|
execution = next_root = nil
|
|
678
|
+
mode = mode.to_sym
|
|
679
|
+
pending_llm_call_id = SecureRandom.uuid.to_s.freeze
|
|
680
|
+
pending_started_at = Time.now.utc.iso8601(6).freeze
|
|
681
|
+
|
|
296
682
|
@agent.persistence.transaction do |tx|
|
|
297
683
|
input_ref = tx.contents.put_text(raw_message)
|
|
684
|
+
durable_context_ref = if config.key?(:durable_context)
|
|
685
|
+
tx.contents.put_json(config.fetch(:durable_context))
|
|
686
|
+
end
|
|
298
687
|
input_record = JournalRecord.new(
|
|
299
688
|
agent_id: @agent.agent_id,
|
|
300
689
|
kind: :input_received,
|
|
301
690
|
channel: :external,
|
|
302
691
|
role: :user,
|
|
303
692
|
content_ref: input_ref,
|
|
304
|
-
correlation_id: thread_id,
|
|
305
693
|
context_generation: root.transcript_generation,
|
|
306
694
|
context_candidate: false
|
|
307
695
|
)
|
|
308
|
-
policy_descriptor = ContextPolicies::Default.new.descriptor
|
|
309
696
|
execution = AgentExecution.start(
|
|
310
697
|
agent_root: root,
|
|
311
698
|
input_record: input_record,
|
|
699
|
+
execution_id: config[:phronomy_reserved_execution_id] || SecureRandom.uuid,
|
|
312
700
|
metadata: {
|
|
313
|
-
"
|
|
701
|
+
"coordination" => config[:phronomy_coordination],
|
|
314
702
|
"current_input_ref" => input_ref,
|
|
315
|
-
"
|
|
703
|
+
"durable_context_ref" => durable_context_ref,
|
|
704
|
+
"preparation_replayable" => !!preparation_replayable,
|
|
705
|
+
RecoverySupport::CONTRACT_VERSION_KEY =>
|
|
706
|
+
RecoverySupport::CONTRACT_VERSION,
|
|
707
|
+
RecoverySupport::INVOCATION_MODE_KEY => mode.to_s,
|
|
708
|
+
RecoverySupport::PENDING_LLM_ID_KEY =>
|
|
709
|
+
pending_llm_call_id,
|
|
710
|
+
RecoverySupport::PENDING_LLM_STARTED_AT_KEY =>
|
|
711
|
+
pending_started_at
|
|
316
712
|
}.compact
|
|
317
713
|
)
|
|
318
714
|
input_record = JournalRecord.from_h(
|
|
319
|
-
input_record.to_h.merge(
|
|
715
|
+
input_record.to_h.merge(
|
|
716
|
+
"execution_id" => execution.execution_id
|
|
717
|
+
)
|
|
320
718
|
)
|
|
321
719
|
execution = execution.with(
|
|
322
720
|
execution_revision: 0,
|
|
323
721
|
working_records: [input_record]
|
|
324
722
|
)
|
|
723
|
+
validate_coordination_admission!(tx, execution)
|
|
325
724
|
tx.executions.create_active(execution)
|
|
326
725
|
next_root = root.with(
|
|
327
726
|
agent_revision: root.agent_revision + 1,
|
|
@@ -336,9 +735,70 @@ module Phronomy
|
|
|
336
735
|
[execution, next_root]
|
|
337
736
|
end
|
|
338
737
|
|
|
339
|
-
def
|
|
738
|
+
def validate_coordination_admission!(tx, execution)
|
|
739
|
+
owner = execution.metadata["coordination"]
|
|
740
|
+
return unless owner
|
|
741
|
+
case owner.fetch("kind")
|
|
742
|
+
when "team"
|
|
743
|
+
team = tx.team_executions.load(owner.fetch("team_execution_id"))
|
|
744
|
+
unless team.team_id == owner.fetch("team_id") && team.active? && !team.metadata["cancel_requested"]
|
|
745
|
+
raise Phronomy::CancellationError, "Team run is no longer dispatchable"
|
|
746
|
+
end
|
|
747
|
+
slot = if owner.fetch("slot") == "coordinator"
|
|
748
|
+
team.coordinator
|
|
749
|
+
else
|
|
750
|
+
assignment = team.assignments.find { |entry| entry.fetch("task_id") == owner.fetch("slot") }
|
|
751
|
+
worker = assignment && team.workers.fetch(assignment.fetch("worker"))
|
|
752
|
+
worker&.merge("execution_id" => assignment.fetch("execution_id"))
|
|
753
|
+
end
|
|
754
|
+
unless slot && slot.fetch("execution_id") == execution.execution_id && slot.fetch("agent_id") == execution.agent_id
|
|
755
|
+
raise Phronomy::Persistence::ConflictError, "Team reserved child identity mismatch"
|
|
756
|
+
end
|
|
757
|
+
when "subagent"
|
|
758
|
+
parent = tx.executions.load(owner.fetch("parent_execution_id"))
|
|
759
|
+
unless parent.agent_id == owner.fetch("parent_agent_id") && parent.active? && !parent.metadata["coordination_cancel_requested"]
|
|
760
|
+
raise Phronomy::CancellationError, "Parent run is no longer dispatchable"
|
|
761
|
+
end
|
|
762
|
+
snapshot = tx.contents.fetch_json(parent.metadata.fetch("multi_agent_coordination_ref"))
|
|
763
|
+
slot = snapshot.fetch("children").find { |entry| entry.fetch("slot") == owner.fetch("slot") }
|
|
764
|
+
unless slot && slot.fetch("agent_id") == execution.agent_id && slot.fetch("execution_id") == execution.execution_id
|
|
765
|
+
raise Phronomy::Persistence::ConflictError, "Parent reserved child identity mismatch"
|
|
766
|
+
end
|
|
767
|
+
when "handoff"
|
|
768
|
+
routing = tx.handoff_states.load(owner.fetch("main_agent_id"))
|
|
769
|
+
unless routing && routing.active_agent_id == execution.agent_id
|
|
770
|
+
raise Phronomy::Persistence::ConflictError, "Handoff responsibility changed before admission"
|
|
771
|
+
end
|
|
772
|
+
if Array(routing.metadata["cancelled_execution_ids"]).include?(execution.execution_id)
|
|
773
|
+
raise Phronomy::CancellationError, "Handoff reservation was cancelled"
|
|
774
|
+
end
|
|
775
|
+
if routing.phase != "stable" && routing.pending_target_execution_id != execution.execution_id
|
|
776
|
+
raise Phronomy::Persistence::ConflictError, "Handoff reserved Target identity mismatch"
|
|
777
|
+
end
|
|
778
|
+
else
|
|
779
|
+
raise Phronomy::ConfigurationError, "Unknown coordination owner kind"
|
|
780
|
+
end
|
|
781
|
+
end
|
|
782
|
+
|
|
783
|
+
def initial_preparation_replayable?(input, config, approval_policy)
|
|
784
|
+
return false unless input.is_a?(String)
|
|
785
|
+
return false unless approval_policy.nil?
|
|
786
|
+
if config.key?(:phronomy_handoff_bindings) ||
|
|
787
|
+
config.key?(:phronomy_handoff_context)
|
|
788
|
+
return false unless config[:phronomy_coordination]&.fetch("kind", nil) == "handoff"
|
|
789
|
+
end
|
|
790
|
+
|
|
791
|
+
invocation_context = config[:invocation_context]
|
|
792
|
+
return true unless invocation_context
|
|
793
|
+
|
|
794
|
+
%i[approval_policy redaction_policy token_budget].none? do |name|
|
|
795
|
+
invocation_context.respond_to?(name) &&
|
|
796
|
+
!invocation_context.public_send(name).nil?
|
|
797
|
+
end
|
|
798
|
+
end
|
|
799
|
+
|
|
800
|
+
def commit_preparation_failure(execution, root, error)
|
|
340
801
|
translated_error = translated(error)
|
|
341
|
-
root = @agent.agent_root
|
|
342
802
|
failed = next_root = appended = nil
|
|
343
803
|
|
|
344
804
|
@agent.persistence.transaction do |tx|
|
|
@@ -389,24 +849,95 @@ module Phronomy
|
|
|
389
849
|
root: next_root
|
|
390
850
|
)
|
|
391
851
|
end
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
852
|
+
{
|
|
853
|
+
error: translated_error,
|
|
854
|
+
execution: failed,
|
|
855
|
+
root: next_root,
|
|
856
|
+
appended_records: Array(appended).freeze
|
|
857
|
+
}.freeze
|
|
395
858
|
end
|
|
396
859
|
|
|
397
|
-
def
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
860
|
+
def apply_initial_preparation_on_event_loop(ready)
|
|
861
|
+
request = ready.request
|
|
862
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
863
|
+
if ready.error
|
|
864
|
+
event_loop.mark_agent_admission_recovery_required(
|
|
865
|
+
@agent.agent_id,
|
|
866
|
+
owner_token: request.admission_token
|
|
867
|
+
)
|
|
868
|
+
deliver_start_failure_on_event_loop(request, translated(ready.error))
|
|
869
|
+
return
|
|
870
|
+
end
|
|
871
|
+
|
|
872
|
+
result = ready.result
|
|
873
|
+
case result.admission_outcome
|
|
874
|
+
when :not_established
|
|
875
|
+
event_loop.release_agent_execution_admission(
|
|
876
|
+
@agent.agent_id,
|
|
877
|
+
owner_token: request.admission_token
|
|
878
|
+
)
|
|
879
|
+
deliver_start_failure_on_event_loop(request, result.error)
|
|
880
|
+
return
|
|
881
|
+
when :outcome_unknown, :recovery_required
|
|
882
|
+
event_loop.mark_agent_admission_recovery_required(
|
|
883
|
+
@agent.agent_id,
|
|
884
|
+
owner_token: request.admission_token
|
|
885
|
+
)
|
|
886
|
+
deliver_start_failure_on_event_loop(request, result.error)
|
|
887
|
+
return
|
|
888
|
+
when :active, :terminal
|
|
889
|
+
event_loop.bind_agent_execution_admission(
|
|
890
|
+
@agent.agent_id,
|
|
891
|
+
owner_token: request.admission_token,
|
|
892
|
+
execution_id: result.execution.execution_id
|
|
893
|
+
)
|
|
894
|
+
else
|
|
895
|
+
event_loop.mark_agent_admission_recovery_required(
|
|
896
|
+
@agent.agent_id,
|
|
897
|
+
owner_token: request.admission_token
|
|
898
|
+
)
|
|
899
|
+
raise Phronomy::Error,
|
|
900
|
+
"unknown initial admission outcome: #{result.admission_outcome.inspect}"
|
|
901
|
+
end
|
|
902
|
+
|
|
903
|
+
apply_agent_live_state(
|
|
904
|
+
root: result.root,
|
|
905
|
+
appended_records: result.appended_records
|
|
906
|
+
)
|
|
907
|
+
if result.admission_outcome == :terminal
|
|
908
|
+
event_loop.release_agent_execution_admission(
|
|
909
|
+
@agent.agent_id,
|
|
910
|
+
execution_id: result.execution.execution_id
|
|
911
|
+
)
|
|
912
|
+
deliver_start_failure_on_event_loop(request, result.error)
|
|
913
|
+
return
|
|
914
|
+
end
|
|
915
|
+
|
|
916
|
+
register_initial_session_on_event_loop(request, result)
|
|
917
|
+
rescue => error
|
|
918
|
+
# Preparation may already be durably active. Terminalize through another
|
|
919
|
+
# offloaded durable operation rather than releasing the admission blindly.
|
|
920
|
+
begin_terminal_without_session_on_event_loop(
|
|
921
|
+
request: request,
|
|
922
|
+
prepared: ready.result,
|
|
923
|
+
error: error
|
|
404
924
|
)
|
|
925
|
+
end
|
|
926
|
+
|
|
927
|
+
def register_initial_session_on_event_loop(request, prepared)
|
|
405
928
|
runtime = Phronomy::Runtime.instance
|
|
406
|
-
runtime.__agent_activations.register(activation)
|
|
407
929
|
event_loop = runtime.event_loop
|
|
930
|
+
Phronomy::Tracing::Automatic.observe_task(
|
|
931
|
+
request.result_task,
|
|
932
|
+
"agent.execution",
|
|
933
|
+
input: request.input,
|
|
934
|
+
agent_id: @agent.agent_id,
|
|
935
|
+
execution_id: prepared.execution.execution_id,
|
|
936
|
+
mode: request.mode,
|
|
937
|
+
**@agent.send(:_build_caller_meta, prepared.config)
|
|
938
|
+
)
|
|
408
939
|
effective_config = prepared.config.merge(
|
|
409
|
-
|
|
940
|
+
phronomy_execution_coordinator: self,
|
|
410
941
|
phronomy_runtime_projection: prepared.runtime_projection,
|
|
411
942
|
phronomy_filtered_input: prepared.filtered_input,
|
|
412
943
|
execution_id: prepared.execution.execution_id
|
|
@@ -415,128 +946,1384 @@ module Phronomy
|
|
|
415
946
|
agent: @agent,
|
|
416
947
|
input: prepared.filtered_input,
|
|
417
948
|
config: effective_config,
|
|
418
|
-
approval_policy: approval_policy,
|
|
419
|
-
approval_listener: approval_listener,
|
|
420
|
-
mode: mode,
|
|
421
|
-
on_event:
|
|
949
|
+
approval_policy: request.approval_policy,
|
|
950
|
+
approval_listener: request.approval_listener,
|
|
951
|
+
mode: request.mode,
|
|
952
|
+
on_event: request.on_event,
|
|
422
953
|
runtime: runtime
|
|
423
954
|
)
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
955
|
+
event_loop.install_agent_execution(
|
|
956
|
+
execution_id: prepared.execution.execution_id,
|
|
957
|
+
agent: @agent,
|
|
958
|
+
coordinator: self,
|
|
959
|
+
execution: prepared.execution,
|
|
960
|
+
runtime_projection: prepared.runtime_projection,
|
|
961
|
+
base_manifest: prepared.runtime_projection.manifest,
|
|
962
|
+
invocation: session.context,
|
|
963
|
+
fsm_session_id: session.id
|
|
964
|
+
)
|
|
965
|
+
event_loop.register_agent_completion_waiter(
|
|
966
|
+
prepared.execution.execution_id,
|
|
967
|
+
request.result_task
|
|
968
|
+
)
|
|
969
|
+
source_task = Phronomy::Task.deferred(name: "#{request.result_task.name}-source")
|
|
427
970
|
source_task.on_complete do |invocation, error|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
result_task,
|
|
971
|
+
finish_on_event_loop(
|
|
972
|
+
prepared.execution.execution_id,
|
|
973
|
+
request.result_task,
|
|
431
974
|
invocation || session.context,
|
|
432
|
-
error
|
|
975
|
+
error,
|
|
976
|
+
fsm_session_id: session.id
|
|
433
977
|
)
|
|
434
978
|
end
|
|
435
979
|
event_loop.register(session, completion: source_task)
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
coordinator: self,
|
|
442
|
-
application_listener: on_event
|
|
443
|
-
)
|
|
444
|
-
finish(activation, result_task, activation.invocation, error)
|
|
980
|
+
# simplecov:disable
|
|
981
|
+
rescue => _error
|
|
982
|
+
event_loop&.release_agent_execution(prepared.execution.execution_id) if
|
|
983
|
+
event_loop&.current? && event_loop.agent_execution_state(prepared.execution.execution_id)
|
|
984
|
+
raise
|
|
445
985
|
end
|
|
986
|
+
# simplecov:enable
|
|
446
987
|
|
|
447
|
-
def
|
|
988
|
+
def deliver_start_failure_on_event_loop(request, error)
|
|
989
|
+
callback_error = @agent.send(
|
|
990
|
+
:_deliver_stream_event,
|
|
991
|
+
request.on_event,
|
|
992
|
+
StreamEvent.new(
|
|
993
|
+
type: terminal_event_type(error),
|
|
994
|
+
payload: {error: error}
|
|
995
|
+
)
|
|
996
|
+
)
|
|
997
|
+
settle_after_terminal(
|
|
998
|
+
request.result_task,
|
|
999
|
+
callback_error,
|
|
1000
|
+
terminal_event_type(error),
|
|
1001
|
+
nil,
|
|
1002
|
+
error
|
|
1003
|
+
)
|
|
1004
|
+
end
|
|
1005
|
+
|
|
1006
|
+
# Restarts only the replay-safe initial preparation region of an already
|
|
1007
|
+
# durably admitted logical Execution. Runtime-only caller state is not
|
|
1008
|
+
# reconstructed here; the durable preparation inputs are authoritative.
|
|
1009
|
+
# @api private
|
|
1010
|
+
def start_initial_preparation_recovery_on_event_loop(
|
|
1011
|
+
execution,
|
|
1012
|
+
result_task,
|
|
1013
|
+
load_completion:
|
|
1014
|
+
)
|
|
448
1015
|
runtime = Phronomy::Runtime.instance
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
1016
|
+
event_loop = runtime.event_loop
|
|
1017
|
+
assert_event_loop!(event_loop)
|
|
1018
|
+
state = event_loop.agent_execution_state(execution.execution_id)
|
|
1019
|
+
unless state && state.agent.equal?(@agent) &&
|
|
1020
|
+
state.execution.execution_revision == execution.execution_revision &&
|
|
1021
|
+
state.execution.status == :preparing &&
|
|
1022
|
+
state.execution.phase.to_sym == :preparing
|
|
1023
|
+
error = Phronomy::ExecutionRehydrationRequiredError.new(
|
|
1024
|
+
"stale :preparing Recovery installation for #{execution.execution_id}"
|
|
1025
|
+
)
|
|
1026
|
+
fail_task(result_task, error)
|
|
1027
|
+
load_completion.fail(error)
|
|
1028
|
+
return
|
|
1029
|
+
end
|
|
1030
|
+
|
|
1031
|
+
root = @agent.agent_root
|
|
1032
|
+
journal_records = @agent.send(:_journal_records_snapshot)
|
|
1033
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
1034
|
+
perform_initial_preparation_recovery(
|
|
1035
|
+
execution,
|
|
1036
|
+
root: root,
|
|
1037
|
+
journal_records: journal_records
|
|
462
1038
|
)
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
1039
|
+
end
|
|
1040
|
+
task.on_complete do |result, error|
|
|
1041
|
+
ready = InitialPreparationRecoveryReady.new(
|
|
1042
|
+
coordinator: self,
|
|
1043
|
+
execution_id: execution.execution_id.to_s.freeze,
|
|
1044
|
+
expected_execution_revision: execution.execution_revision,
|
|
1045
|
+
result_task: result_task,
|
|
1046
|
+
load_completion: load_completion,
|
|
1047
|
+
result: result,
|
|
1048
|
+
error: error
|
|
1049
|
+
)
|
|
1050
|
+
unless post_control(runtime, ready)
|
|
1051
|
+
rejected = runtime_rejected_error(:initial_preparation_recovery)
|
|
1052
|
+
fail_task(result_task, rejected)
|
|
1053
|
+
load_completion.fail(rejected)
|
|
1054
|
+
end
|
|
1055
|
+
end
|
|
1056
|
+
rescue => error
|
|
1057
|
+
# simplecov:disable
|
|
1058
|
+
begin
|
|
1059
|
+
if defined?(event_loop) && event_loop.current?
|
|
1060
|
+
release_initial_preparation_recovery_runtime_state(
|
|
1061
|
+
event_loop,
|
|
1062
|
+
execution.execution_id
|
|
466
1063
|
)
|
|
467
1064
|
end
|
|
1065
|
+
rescue
|
|
1066
|
+
nil
|
|
468
1067
|
end
|
|
469
|
-
|
|
470
|
-
fail_task(result_task,
|
|
1068
|
+
translated_error = translated(error)
|
|
1069
|
+
fail_task(result_task, translated_error)
|
|
1070
|
+
load_completion.fail(translated_error)
|
|
1071
|
+
# simplecov:enable
|
|
471
1072
|
end
|
|
472
1073
|
|
|
473
|
-
|
|
1074
|
+
def perform_initial_preparation_recovery(
|
|
1075
|
+
execution,
|
|
1076
|
+
root:,
|
|
1077
|
+
journal_records:
|
|
1078
|
+
)
|
|
1079
|
+
unless execution.metadata["preparation_replayable"] == true
|
|
1080
|
+
raise Phronomy::ExecutionRehydrationRequiredError,
|
|
1081
|
+
"execution #{execution.execution_id} has no replay-safe initial preparation contract"
|
|
1082
|
+
end
|
|
474
1083
|
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
1084
|
+
input_ref = execution.metadata.fetch("current_input_ref")
|
|
1085
|
+
input = @agent.persistence.contents.fetch_text(input_ref)
|
|
1086
|
+
mode = (
|
|
1087
|
+
execution.metadata[RecoverySupport::INVOCATION_MODE_KEY] || "invoke"
|
|
1088
|
+
).to_sym
|
|
1089
|
+
config = {phronomy_recovery_mode: mode}.merge(@agent.__coordination_config)
|
|
1090
|
+
|
|
1091
|
+
if execution.metadata.key?("durable_context_ref")
|
|
1092
|
+
durable_context_ref = execution.metadata.fetch("durable_context_ref")
|
|
1093
|
+
durable_context = @agent.persistence.contents.fetch_json(
|
|
1094
|
+
durable_context_ref
|
|
1095
|
+
)
|
|
1096
|
+
unless durable_context.is_a?(Hash)
|
|
1097
|
+
raise Phronomy::ExecutionRehydrationRequiredError,
|
|
1098
|
+
"execution #{execution.execution_id} has a non-Hash durable_context"
|
|
1099
|
+
end
|
|
1100
|
+
Phronomy::Agent::Immutable.validate_canonical_json!(
|
|
1101
|
+
durable_context,
|
|
1102
|
+
label: "Recovered durable_context"
|
|
1103
|
+
)
|
|
1104
|
+
config[:durable_context] =
|
|
1105
|
+
Phronomy::Agent::Immutable.copy(durable_context)
|
|
1106
|
+
end
|
|
1107
|
+
|
|
1108
|
+
perform_admitted_initial_preparation(
|
|
1109
|
+
input: input,
|
|
1110
|
+
config: config.freeze,
|
|
1111
|
+
execution: execution,
|
|
1112
|
+
active_root: root,
|
|
1113
|
+
journal_records: journal_records
|
|
1114
|
+
)
|
|
1115
|
+
end
|
|
1116
|
+
|
|
1117
|
+
def apply_initial_preparation_recovery_on_event_loop(ready)
|
|
1118
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1119
|
+
state = event_loop.agent_execution_state(ready.execution_id)
|
|
1120
|
+
unless state && state.agent.equal?(@agent) &&
|
|
1121
|
+
state.execution.execution_revision == ready.expected_execution_revision &&
|
|
1122
|
+
state.execution.status == :preparing &&
|
|
1123
|
+
state.execution.phase.to_sym == :preparing
|
|
1124
|
+
error = Phronomy::ExecutionRehydrationRequiredError.new(
|
|
1125
|
+
"stale initial preparation Recovery result for #{ready.execution_id}"
|
|
1126
|
+
)
|
|
1127
|
+
fail_task(ready.result_task, error)
|
|
1128
|
+
ready.load_completion.fail(error)
|
|
1129
|
+
return
|
|
1130
|
+
end
|
|
1131
|
+
|
|
1132
|
+
if ready.error
|
|
1133
|
+
release_initial_preparation_recovery_runtime_state(
|
|
1134
|
+
event_loop,
|
|
1135
|
+
ready.execution_id
|
|
1136
|
+
)
|
|
1137
|
+
error = translated(ready.error)
|
|
1138
|
+
fail_task(ready.result_task, error)
|
|
1139
|
+
ready.load_completion.fail(error)
|
|
1140
|
+
return
|
|
1141
|
+
end
|
|
1142
|
+
|
|
1143
|
+
result = ready.result
|
|
1144
|
+
case result.admission_outcome
|
|
1145
|
+
when :terminal
|
|
1146
|
+
apply_agent_live_state(
|
|
1147
|
+
root: result.root,
|
|
1148
|
+
appended_records: result.appended_records
|
|
1149
|
+
)
|
|
1150
|
+
event_loop.replace_agent_execution(
|
|
1151
|
+
ready.execution_id,
|
|
1152
|
+
execution: result.execution
|
|
1153
|
+
)
|
|
1154
|
+
event_loop.release_agent_execution(ready.execution_id)
|
|
1155
|
+
event_loop.release_agent_execution_admission(
|
|
1156
|
+
@agent.agent_id,
|
|
1157
|
+
execution_id: ready.execution_id
|
|
1158
|
+
)
|
|
1159
|
+
event_type = terminal_event_type(result.error)
|
|
1160
|
+
callback_error = deliver_terminal(
|
|
1161
|
+
@agent.send(:_phronomy_event_listener),
|
|
1162
|
+
event_type,
|
|
1163
|
+
error: result.error
|
|
485
1164
|
)
|
|
486
1165
|
settle_after_terminal(
|
|
487
|
-
|
|
1166
|
+
ready.result_task,
|
|
488
1167
|
callback_error,
|
|
489
|
-
|
|
1168
|
+
event_type,
|
|
490
1169
|
nil,
|
|
491
|
-
|
|
1170
|
+
result.error
|
|
1171
|
+
)
|
|
1172
|
+
ready.load_completion.complete(@agent)
|
|
1173
|
+
when :active
|
|
1174
|
+
apply_agent_live_state(
|
|
1175
|
+
root: result.root,
|
|
1176
|
+
appended_records: result.appended_records
|
|
1177
|
+
)
|
|
1178
|
+
event_loop.release_agent_execution(ready.execution_id)
|
|
1179
|
+
mode = (
|
|
1180
|
+
result.execution.metadata[RecoverySupport::INVOCATION_MODE_KEY] ||
|
|
1181
|
+
"invoke"
|
|
1182
|
+
).to_sym
|
|
1183
|
+
request = StartCommand.new(
|
|
1184
|
+
coordinator: self,
|
|
1185
|
+
input: result.filtered_input,
|
|
1186
|
+
config: result.config,
|
|
1187
|
+
mode: mode,
|
|
1188
|
+
approval_policy: nil,
|
|
1189
|
+
approval_listener: nil,
|
|
1190
|
+
on_event: @agent.send(:_phronomy_event_listener),
|
|
1191
|
+
result_task: ready.result_task,
|
|
1192
|
+
admission_token: Object.new.freeze
|
|
492
1193
|
)
|
|
493
|
-
|
|
494
|
-
|
|
1194
|
+
begin
|
|
1195
|
+
register_initial_session_on_event_loop(request, result)
|
|
1196
|
+
rescue => error
|
|
1197
|
+
begin_terminal_without_session_on_event_loop(
|
|
1198
|
+
request: request,
|
|
1199
|
+
prepared: result,
|
|
1200
|
+
error: error
|
|
1201
|
+
)
|
|
1202
|
+
end
|
|
1203
|
+
ready.load_completion.complete(@agent)
|
|
495
1204
|
else
|
|
496
|
-
|
|
1205
|
+
error = Phronomy::ExecutionRehydrationRequiredError.new(
|
|
1206
|
+
"unexpected initial preparation Recovery outcome: #{result.admission_outcome.inspect}"
|
|
1207
|
+
)
|
|
1208
|
+
release_initial_preparation_recovery_runtime_state(
|
|
1209
|
+
event_loop,
|
|
1210
|
+
ready.execution_id
|
|
1211
|
+
)
|
|
1212
|
+
fail_task(ready.result_task, error)
|
|
1213
|
+
ready.load_completion.fail(error)
|
|
1214
|
+
end
|
|
1215
|
+
rescue => error
|
|
1216
|
+
# simplecov:disable
|
|
1217
|
+
begin
|
|
1218
|
+
release_initial_preparation_recovery_runtime_state(
|
|
1219
|
+
event_loop,
|
|
1220
|
+
ready.execution_id
|
|
1221
|
+
)
|
|
1222
|
+
rescue
|
|
1223
|
+
nil
|
|
1224
|
+
end
|
|
1225
|
+
translated_error = translated(error)
|
|
1226
|
+
fail_task(ready.result_task, translated_error)
|
|
1227
|
+
ready.load_completion.fail(translated_error)
|
|
1228
|
+
# simplecov:enable
|
|
1229
|
+
end
|
|
1230
|
+
|
|
1231
|
+
def release_initial_preparation_recovery_runtime_state(
|
|
1232
|
+
event_loop,
|
|
1233
|
+
execution_id
|
|
1234
|
+
)
|
|
1235
|
+
state = event_loop.agent_execution_state(execution_id)
|
|
1236
|
+
event_loop.release_agent_execution(execution_id) if state
|
|
1237
|
+
event_loop.release_agent_execution_admission(
|
|
1238
|
+
@agent.agent_id,
|
|
1239
|
+
execution_id: execution_id
|
|
1240
|
+
)
|
|
1241
|
+
end
|
|
1242
|
+
|
|
1243
|
+
# ----------------------------------------------------------------------
|
|
1244
|
+
# External-operation causal durable barriers
|
|
1245
|
+
# ----------------------------------------------------------------------
|
|
1246
|
+
|
|
1247
|
+
def perform_provider_dispatch_preparation(operation)
|
|
1248
|
+
metadata = operation.execution.metadata.dup
|
|
1249
|
+
metadata.delete(RecoverySupport::TOOL_BATCH_METADATA_KEY)
|
|
1250
|
+
metadata.delete(RecoverySupport::RECOVERY_METADATA_KEY)
|
|
1251
|
+
metadata[RecoverySupport::PENDING_LLM_ID_KEY] =
|
|
1252
|
+
operation.pending_llm_call_id
|
|
1253
|
+
metadata[RecoverySupport::PENDING_LLM_STARTED_AT_KEY] =
|
|
1254
|
+
operation.pending_llm_started_at
|
|
1255
|
+
metadata[RecoverySupport::CONTRACT_VERSION_KEY] =
|
|
1256
|
+
RecoverySupport::CONTRACT_VERSION
|
|
1257
|
+
staged_execution = operation.execution.with(
|
|
1258
|
+
execution_revision: operation.execution.execution_revision,
|
|
1259
|
+
metadata: metadata
|
|
1260
|
+
)
|
|
1261
|
+
|
|
1262
|
+
manifest = manifest_ref = updated = intended = nil
|
|
1263
|
+
|
|
1264
|
+
# Encode the immutable runtime snapshot first. This transaction may add
|
|
1265
|
+
# content-addressed blobs, but it does not advance Execution state.
|
|
1266
|
+
encoded_records = call_records = nil
|
|
1267
|
+
@agent.persistence.transaction do |tx|
|
|
1268
|
+
assert_local_durable_base!(tx, operation.root)
|
|
1269
|
+
encoded_records, call_records = encode_runtime_records(
|
|
1270
|
+
staged_execution,
|
|
1271
|
+
tx: tx,
|
|
1272
|
+
snapshot: operation.runtime_snapshot,
|
|
1273
|
+
context_candidate: true,
|
|
1274
|
+
agent_root: operation.root
|
|
1275
|
+
)
|
|
1276
|
+
end
|
|
1277
|
+
staged = staged_execution.with(
|
|
1278
|
+
execution_revision: staged_execution.execution_revision,
|
|
1279
|
+
phase: :preparing_llm_call,
|
|
1280
|
+
working_records: staged_execution.working_records + encoded_records,
|
|
1281
|
+
llm_calls: staged_execution.llm_calls + call_records
|
|
1282
|
+
)
|
|
1283
|
+
patch = @agent.send(
|
|
1284
|
+
:run_before_llm_input_hooks,
|
|
1285
|
+
call_sequence: staged.llm_calls.length + 1,
|
|
1286
|
+
config: operation.invocation_config
|
|
1287
|
+
)
|
|
1288
|
+
assembler = ContextAssembler.new(
|
|
1289
|
+
agent: @agent,
|
|
1290
|
+
persistence: @agent.persistence,
|
|
1291
|
+
journal_records: operation.journal_records
|
|
1292
|
+
)
|
|
1293
|
+
prepared = assembler.prepare_followup(
|
|
1294
|
+
base_manifest: operation.base_manifest,
|
|
1295
|
+
agent_root: operation.root,
|
|
1296
|
+
execution: staged,
|
|
1297
|
+
config: operation.invocation_config,
|
|
1298
|
+
patch: patch
|
|
1299
|
+
)
|
|
1300
|
+
@agent.send(
|
|
1301
|
+
:check_cancellation!,
|
|
1302
|
+
operation.invocation_config,
|
|
1303
|
+
"invocation cancelled after context policy"
|
|
1304
|
+
)
|
|
1305
|
+
|
|
1306
|
+
begin
|
|
1307
|
+
@agent.persistence.transaction do |tx|
|
|
1308
|
+
assert_local_durable_base!(tx, operation.root)
|
|
1309
|
+
manifest, manifest_ref = assembler.finalize(prepared, persistence: tx)
|
|
1310
|
+
refs = Array(staged.metadata["manifest_refs"]) + [manifest_ref]
|
|
1311
|
+
updated = staged.with(
|
|
1312
|
+
phase: :calling_llm,
|
|
1313
|
+
metadata: staged.metadata.merge(
|
|
1314
|
+
"manifest_ref" => manifest_ref,
|
|
1315
|
+
"manifest_refs" => refs
|
|
1316
|
+
)
|
|
1317
|
+
)
|
|
1318
|
+
updated = @agent.__prepare_coordination_record(updated, tx: tx)
|
|
1319
|
+
tx.executions.save(
|
|
1320
|
+
operation.execution.execution_id,
|
|
1321
|
+
expected_revision: operation.execution.execution_revision,
|
|
1322
|
+
execution: updated
|
|
1323
|
+
)
|
|
1324
|
+
intended = ProviderDispatchPreparationResult.new(
|
|
1325
|
+
execution: updated,
|
|
1326
|
+
runtime_projection: nil,
|
|
1327
|
+
error: nil
|
|
1328
|
+
)
|
|
1329
|
+
end
|
|
1330
|
+
rescue => caught
|
|
1331
|
+
raise if known_durable_preparation_failure?(caught) || intended.nil?
|
|
1332
|
+
|
|
1333
|
+
raise PreparationOutcomeUnknownError.new(caught, intended)
|
|
1334
|
+
end
|
|
1335
|
+
|
|
1336
|
+
projection = materialization_error = nil
|
|
1337
|
+
begin
|
|
1338
|
+
projection = RubyLLMMaterializer.new(
|
|
1339
|
+
agent: @agent,
|
|
1340
|
+
persistence: @agent.persistence
|
|
1341
|
+
).materialize(manifest: manifest, manifest_ref: manifest_ref)
|
|
1342
|
+
rescue => error
|
|
1343
|
+
materialization_error = error
|
|
497
1344
|
end
|
|
1345
|
+
ProviderDispatchPreparationResult.new(
|
|
1346
|
+
execution: updated,
|
|
1347
|
+
runtime_projection: projection,
|
|
1348
|
+
error: materialization_error
|
|
1349
|
+
)
|
|
498
1350
|
end
|
|
499
1351
|
|
|
500
|
-
|
|
1352
|
+
def perform_tool_dispatch_preparation(operation)
|
|
1353
|
+
intended = nil
|
|
1354
|
+
begin
|
|
1355
|
+
@agent.persistence.transaction do |tx|
|
|
1356
|
+
assert_local_durable_base!(tx, operation.root)
|
|
1357
|
+
encoded_records, call_records = encode_runtime_records(
|
|
1358
|
+
operation.execution,
|
|
1359
|
+
tx: tx,
|
|
1360
|
+
snapshot: operation.runtime_snapshot,
|
|
1361
|
+
context_candidate: true,
|
|
1362
|
+
agent_root: operation.root
|
|
1363
|
+
)
|
|
1364
|
+
metadata = operation.execution.metadata.dup
|
|
1365
|
+
metadata.delete(RecoverySupport::PENDING_LLM_ID_KEY)
|
|
1366
|
+
metadata.delete(RecoverySupport::PENDING_LLM_STARTED_AT_KEY)
|
|
1367
|
+
metadata.delete(RecoverySupport::RECOVERY_METADATA_KEY)
|
|
1368
|
+
metadata.delete("framework_calls_pending")
|
|
1369
|
+
metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY] =
|
|
1370
|
+
RecoverySupport.canonical_copy(operation.tool_batch_snapshot)
|
|
1371
|
+
metadata[RecoverySupport::CONTRACT_VERSION_KEY] =
|
|
1372
|
+
RecoverySupport::CONTRACT_VERSION
|
|
1373
|
+
updated = operation.execution.with(
|
|
1374
|
+
phase: :dispatching_tools,
|
|
1375
|
+
working_records: operation.execution.working_records + encoded_records,
|
|
1376
|
+
llm_calls: operation.execution.llm_calls + call_records,
|
|
1377
|
+
metadata: metadata
|
|
1378
|
+
)
|
|
1379
|
+
updated = @agent.__prepare_coordination_record(updated, tx: tx)
|
|
1380
|
+
tx.executions.save(
|
|
1381
|
+
operation.execution.execution_id,
|
|
1382
|
+
expected_revision: operation.execution.execution_revision,
|
|
1383
|
+
execution: updated
|
|
1384
|
+
)
|
|
1385
|
+
intended = ToolDispatchPreparationResult.new(execution: updated)
|
|
1386
|
+
end
|
|
1387
|
+
rescue => caught
|
|
1388
|
+
raise if known_durable_preparation_failure?(caught) || intended.nil?
|
|
1389
|
+
|
|
1390
|
+
raise PreparationOutcomeUnknownError.new(caught, intended)
|
|
1391
|
+
end
|
|
1392
|
+
intended
|
|
1393
|
+
end
|
|
1394
|
+
|
|
1395
|
+
def apply_provider_dispatch_preparation_on_event_loop(ready)
|
|
1396
|
+
operation = ready.operation
|
|
1397
|
+
state = authoritative_state_for_operation(
|
|
1398
|
+
execution_id: operation.execution_id,
|
|
1399
|
+
fsm_session_id: operation.fsm_session_id,
|
|
1400
|
+
expected_execution_revision: operation.expected_execution_revision,
|
|
1401
|
+
expected_fsm_state: :calling_llm
|
|
1402
|
+
)
|
|
1403
|
+
return unless state
|
|
1404
|
+
|
|
1405
|
+
if ready.error
|
|
1406
|
+
if ready.error.is_a?(PreparationOutcomeUnknownError)
|
|
1407
|
+
submit_provider_dispatch_preparation_reconciliation(
|
|
1408
|
+
operation,
|
|
1409
|
+
ready.error
|
|
1410
|
+
)
|
|
1411
|
+
else
|
|
1412
|
+
state.invocation.event_sink.post(:llm_setup_failed, translated(ready.error))
|
|
1413
|
+
end
|
|
1414
|
+
return
|
|
1415
|
+
end
|
|
1416
|
+
|
|
1417
|
+
apply_confirmed_provider_dispatch_preparation_on_event_loop(
|
|
1418
|
+
operation,
|
|
1419
|
+
ready.result,
|
|
1420
|
+
state
|
|
1421
|
+
)
|
|
1422
|
+
rescue => error
|
|
1423
|
+
state&.invocation&.event_sink&.post(:llm_setup_failed, translated(error))
|
|
1424
|
+
end
|
|
1425
|
+
|
|
1426
|
+
def apply_tool_dispatch_preparation_on_event_loop(ready)
|
|
1427
|
+
operation = ready.operation
|
|
1428
|
+
state = authoritative_state_for_operation(
|
|
1429
|
+
execution_id: operation.execution_id,
|
|
1430
|
+
fsm_session_id: operation.fsm_session_id,
|
|
1431
|
+
expected_execution_revision: operation.expected_execution_revision,
|
|
1432
|
+
expected_fsm_state: :dispatching_tools
|
|
1433
|
+
)
|
|
1434
|
+
return unless state
|
|
1435
|
+
|
|
1436
|
+
if ready.error
|
|
1437
|
+
if ready.error.is_a?(PreparationOutcomeUnknownError)
|
|
1438
|
+
submit_tool_dispatch_preparation_reconciliation(
|
|
1439
|
+
operation,
|
|
1440
|
+
ready.error
|
|
1441
|
+
)
|
|
1442
|
+
else
|
|
1443
|
+
state.invocation.event_sink.post(:tool_setup_failed, translated(ready.error))
|
|
1444
|
+
end
|
|
1445
|
+
return
|
|
1446
|
+
end
|
|
1447
|
+
|
|
1448
|
+
apply_confirmed_tool_dispatch_preparation_on_event_loop(
|
|
1449
|
+
operation,
|
|
1450
|
+
ready.result,
|
|
1451
|
+
state
|
|
1452
|
+
)
|
|
1453
|
+
rescue => error
|
|
1454
|
+
state&.invocation&.event_sink&.post(:tool_setup_failed, translated(error))
|
|
1455
|
+
end
|
|
1456
|
+
|
|
1457
|
+
def apply_confirmed_provider_dispatch_preparation_on_event_loop(
|
|
1458
|
+
operation,
|
|
1459
|
+
result,
|
|
1460
|
+
state
|
|
1461
|
+
)
|
|
1462
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1463
|
+
if result.runtime_projection
|
|
1464
|
+
event_loop.replace_agent_execution(
|
|
1465
|
+
operation.execution_id,
|
|
1466
|
+
execution: result.execution,
|
|
1467
|
+
runtime_projection: result.runtime_projection
|
|
1468
|
+
)
|
|
1469
|
+
else
|
|
1470
|
+
event_loop.replace_agent_execution(
|
|
1471
|
+
operation.execution_id,
|
|
1472
|
+
execution: result.execution
|
|
1473
|
+
)
|
|
1474
|
+
end
|
|
1475
|
+
state.invocation.acknowledge_runtime_snapshot(operation.runtime_snapshot)
|
|
1476
|
+
|
|
1477
|
+
if result.error
|
|
1478
|
+
state.invocation.event_sink.post(:llm_setup_failed, translated(result.error))
|
|
1479
|
+
return
|
|
1480
|
+
end
|
|
1481
|
+
|
|
1482
|
+
Agent::AgentInvocationSessionBuilder.start_prepared_provider_call(
|
|
1483
|
+
agent: @agent,
|
|
1484
|
+
runtime: Phronomy::Runtime.instance,
|
|
1485
|
+
event_sink: state.invocation.event_sink,
|
|
1486
|
+
invocation: state.invocation,
|
|
1487
|
+
projection: result.runtime_projection,
|
|
1488
|
+
streaming: operation.streaming
|
|
1489
|
+
)
|
|
1490
|
+
end
|
|
1491
|
+
|
|
1492
|
+
def apply_confirmed_tool_dispatch_preparation_on_event_loop(
|
|
1493
|
+
operation,
|
|
1494
|
+
result,
|
|
1495
|
+
state
|
|
1496
|
+
)
|
|
1497
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1498
|
+
event_loop.replace_agent_execution(
|
|
1499
|
+
operation.execution_id,
|
|
1500
|
+
execution: result.execution
|
|
1501
|
+
)
|
|
1502
|
+
state.invocation.acknowledge_runtime_snapshot(operation.runtime_snapshot)
|
|
1503
|
+
Agent::AgentInvocationSessionBuilder.start_prepared_tool_dispatch(
|
|
1504
|
+
runtime: Phronomy::Runtime.instance,
|
|
1505
|
+
event_sink: state.invocation.event_sink,
|
|
1506
|
+
invocation: state.invocation
|
|
1507
|
+
)
|
|
1508
|
+
end
|
|
1509
|
+
|
|
1510
|
+
def perform_provider_dispatch_preparation_reconciliation(command)
|
|
1511
|
+
# simplecov:disable
|
|
1512
|
+
disposition, current = preparation_reconciliation_state(
|
|
1513
|
+
command.operation,
|
|
1514
|
+
command.intended_result.execution
|
|
1515
|
+
)
|
|
1516
|
+
preparation_result = nil
|
|
1517
|
+
if disposition == :committed
|
|
1518
|
+
projection = materialization_error = nil
|
|
1519
|
+
begin
|
|
1520
|
+
_manifest, projection = RecoverySupport.materialize_projection(
|
|
1521
|
+
@agent,
|
|
1522
|
+
current.metadata.fetch("manifest_ref")
|
|
1523
|
+
)
|
|
1524
|
+
rescue => error
|
|
1525
|
+
materialization_error = error
|
|
1526
|
+
end
|
|
1527
|
+
preparation_result = ProviderDispatchPreparationResult.new(
|
|
1528
|
+
execution: current,
|
|
1529
|
+
runtime_projection: projection,
|
|
1530
|
+
error: materialization_error
|
|
1531
|
+
)
|
|
1532
|
+
end
|
|
1533
|
+
ProviderDispatchPreparationReconciliationResult.new(
|
|
1534
|
+
disposition: disposition,
|
|
1535
|
+
preparation_result: preparation_result
|
|
1536
|
+
)
|
|
1537
|
+
end
|
|
1538
|
+
|
|
1539
|
+
def perform_tool_dispatch_preparation_reconciliation(command)
|
|
1540
|
+
disposition, current = preparation_reconciliation_state(
|
|
1541
|
+
command.operation,
|
|
1542
|
+
command.intended_result.execution
|
|
1543
|
+
)
|
|
1544
|
+
preparation_result = if disposition == :committed
|
|
1545
|
+
ToolDispatchPreparationResult.new(execution: current)
|
|
1546
|
+
end
|
|
1547
|
+
ToolDispatchPreparationReconciliationResult.new(
|
|
1548
|
+
disposition: disposition,
|
|
1549
|
+
preparation_result: preparation_result
|
|
1550
|
+
)
|
|
1551
|
+
end
|
|
1552
|
+
|
|
1553
|
+
def apply_provider_dispatch_preparation_reconciliation_on_event_loop(ready)
|
|
1554
|
+
command = ready.command
|
|
1555
|
+
operation = command.operation
|
|
1556
|
+
state = authoritative_state_for_operation(
|
|
1557
|
+
execution_id: operation.execution_id,
|
|
1558
|
+
fsm_session_id: operation.fsm_session_id,
|
|
1559
|
+
expected_execution_revision: operation.expected_execution_revision,
|
|
1560
|
+
expected_fsm_state: :calling_llm
|
|
1561
|
+
)
|
|
1562
|
+
return unless state
|
|
1563
|
+
|
|
1564
|
+
if ready.error
|
|
1565
|
+
mark_barrier_recovery_required(operation, ready.error)
|
|
1566
|
+
return
|
|
1567
|
+
end
|
|
1568
|
+
|
|
1569
|
+
case ready.result.disposition
|
|
1570
|
+
when :committed
|
|
1571
|
+
apply_confirmed_provider_dispatch_preparation_on_event_loop(
|
|
1572
|
+
operation,
|
|
1573
|
+
ready.result.preparation_result,
|
|
1574
|
+
state
|
|
1575
|
+
)
|
|
1576
|
+
when :not_committed
|
|
1577
|
+
state.invocation.event_sink.post(
|
|
1578
|
+
:llm_setup_failed,
|
|
1579
|
+
translated(command.original_error)
|
|
1580
|
+
)
|
|
1581
|
+
when :conflict
|
|
1582
|
+
mark_barrier_recovery_required(operation, command.original_error)
|
|
1583
|
+
else
|
|
1584
|
+
raise Phronomy::Error,
|
|
1585
|
+
"unknown Provider dispatch reconciliation disposition: " \
|
|
1586
|
+
"#{ready.result.disposition.inspect}"
|
|
1587
|
+
end
|
|
1588
|
+
rescue => error
|
|
1589
|
+
mark_barrier_recovery_required(operation, error)
|
|
1590
|
+
end
|
|
1591
|
+
|
|
1592
|
+
def apply_tool_dispatch_preparation_reconciliation_on_event_loop(ready)
|
|
1593
|
+
command = ready.command
|
|
1594
|
+
operation = command.operation
|
|
1595
|
+
state = authoritative_state_for_operation(
|
|
1596
|
+
execution_id: operation.execution_id,
|
|
1597
|
+
fsm_session_id: operation.fsm_session_id,
|
|
1598
|
+
expected_execution_revision: operation.expected_execution_revision,
|
|
1599
|
+
expected_fsm_state: :dispatching_tools
|
|
1600
|
+
)
|
|
1601
|
+
return unless state
|
|
1602
|
+
|
|
1603
|
+
if ready.error
|
|
1604
|
+
mark_barrier_recovery_required(operation, ready.error)
|
|
1605
|
+
return
|
|
1606
|
+
end
|
|
1607
|
+
|
|
1608
|
+
case ready.result.disposition
|
|
1609
|
+
when :committed
|
|
1610
|
+
apply_confirmed_tool_dispatch_preparation_on_event_loop(
|
|
1611
|
+
operation,
|
|
1612
|
+
ready.result.preparation_result,
|
|
1613
|
+
state
|
|
1614
|
+
)
|
|
1615
|
+
when :not_committed
|
|
1616
|
+
state.invocation.event_sink.post(
|
|
1617
|
+
:tool_setup_failed,
|
|
1618
|
+
translated(command.original_error)
|
|
1619
|
+
)
|
|
1620
|
+
when :conflict
|
|
1621
|
+
mark_barrier_recovery_required(operation, command.original_error)
|
|
1622
|
+
else
|
|
1623
|
+
raise Phronomy::Error,
|
|
1624
|
+
"unknown Tool dispatch reconciliation disposition: " \
|
|
1625
|
+
"#{ready.result.disposition.inspect}"
|
|
1626
|
+
end
|
|
1627
|
+
rescue => error
|
|
1628
|
+
mark_barrier_recovery_required(operation, error)
|
|
1629
|
+
# simplecov:enable
|
|
1630
|
+
end
|
|
1631
|
+
|
|
1632
|
+
# Worker-only Persistence readback for causal-barrier F1 reconciliation.
|
|
1633
|
+
# EventLoop submits one operation-specific ReconciliationCommand and only
|
|
1634
|
+
# applies this result after it returns through the normal control path.
|
|
1635
|
+
def preparation_reconciliation_state(operation, intended_execution)
|
|
1636
|
+
current = @agent.persistence.executions.load(operation.execution_id)
|
|
1637
|
+
if current.execution_revision == intended_execution.execution_revision &&
|
|
1638
|
+
current.to_h == intended_execution.to_h
|
|
1639
|
+
return [:committed, current]
|
|
1640
|
+
end
|
|
1641
|
+
if current.execution_revision == operation.execution.execution_revision &&
|
|
1642
|
+
current.to_h == operation.execution.to_h
|
|
1643
|
+
return [:not_committed, current]
|
|
1644
|
+
end
|
|
1645
|
+
|
|
1646
|
+
[:conflict, current]
|
|
1647
|
+
end
|
|
1648
|
+
|
|
1649
|
+
def known_durable_preparation_failure?(error)
|
|
1650
|
+
error.is_a?(Phronomy::Persistence::ConflictError) ||
|
|
1651
|
+
error.is_a?(Phronomy::Persistence::NotFoundError) ||
|
|
1652
|
+
error.is_a?(Phronomy::Persistence::SerializationError) ||
|
|
1653
|
+
error.is_a?(Phronomy::Persistence::UnsupportedBackendError) ||
|
|
1654
|
+
error.is_a?(ArgumentError) ||
|
|
1655
|
+
error.is_a?(Phronomy::ConfigurationError)
|
|
1656
|
+
end
|
|
1657
|
+
|
|
1658
|
+
def mark_barrier_recovery_required(operation, error)
|
|
1659
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1660
|
+
event_loop.mark_agent_execution_admission(
|
|
1661
|
+
@agent.agent_id,
|
|
1662
|
+
execution_id: operation.execution_id,
|
|
1663
|
+
state: :recovery_required
|
|
1664
|
+
)
|
|
1665
|
+
Phronomy.configuration.logger&.warn(
|
|
1666
|
+
"[Phronomy] causal durable barrier requires recovery for " \
|
|
1667
|
+
"#{operation.execution_id}: #{error.class}: #{error.message}"
|
|
1668
|
+
)
|
|
1669
|
+
end
|
|
1670
|
+
|
|
1671
|
+
# ----------------------------------------------------------------------
|
|
1672
|
+
# Approval resume
|
|
1673
|
+
# ----------------------------------------------------------------------
|
|
1674
|
+
|
|
1675
|
+
def begin_resume_on_event_loop(request)
|
|
1676
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1677
|
+
state = event_loop.agent_execution_state(request.execution_id)
|
|
1678
|
+
if state&.agent&.equal?(@agent) && state&.invocation
|
|
1679
|
+
@recovery_resume_snapshot_mutex ||= Mutex.new
|
|
1680
|
+
snapshot = RecoverySupport.build_tool_batch_snapshot(
|
|
1681
|
+
state.invocation
|
|
1682
|
+
)
|
|
1683
|
+
@recovery_resume_snapshot_mutex.synchronize do
|
|
1684
|
+
@recovery_resume_snapshots ||= {}
|
|
1685
|
+
@recovery_resume_snapshots[request.execution_id.to_s] =
|
|
1686
|
+
snapshot
|
|
1687
|
+
end
|
|
1688
|
+
end
|
|
1689
|
+
|
|
1690
|
+
runtime = Phronomy::Runtime.instance
|
|
1691
|
+
event_loop = runtime.event_loop
|
|
1692
|
+
state = event_loop.agent_execution_state(request.execution_id)
|
|
1693
|
+
unless state
|
|
1694
|
+
fail_task(
|
|
1695
|
+
request.result_task,
|
|
1696
|
+
Phronomy::ExecutionRehydrationRequiredError.new(
|
|
1697
|
+
"no live execution owner for #{request.execution_id}; durable rehydration is required"
|
|
1698
|
+
)
|
|
1699
|
+
)
|
|
1700
|
+
return
|
|
1701
|
+
end
|
|
1702
|
+
|
|
1703
|
+
unless state.agent.equal?(@agent) && state.execution.status == :suspended
|
|
1704
|
+
fail_task(
|
|
1705
|
+
request.result_task,
|
|
1706
|
+
ArgumentError.new(
|
|
1707
|
+
"execution is not a suspended execution of this agent: #{request.execution_id}"
|
|
1708
|
+
)
|
|
1709
|
+
)
|
|
1710
|
+
return
|
|
1711
|
+
end
|
|
1712
|
+
|
|
1713
|
+
approval = state.execution.approval_request || {}
|
|
1714
|
+
current_request_id = approval["id"] || approval[:id]
|
|
1715
|
+
unless current_request_id.to_s == request.approval_request_id
|
|
1716
|
+
fail_task(
|
|
1717
|
+
request.result_task,
|
|
1718
|
+
ArgumentError.new(
|
|
1719
|
+
"approval request does not match execution #{request.execution_id}"
|
|
1720
|
+
)
|
|
1721
|
+
)
|
|
1722
|
+
return
|
|
1723
|
+
end
|
|
1724
|
+
|
|
1725
|
+
operation = ResumeCommitCommand.new(
|
|
1726
|
+
execution_id: state.execution_id,
|
|
1727
|
+
expected_execution_revision: state.execution.execution_revision,
|
|
1728
|
+
root: @agent.agent_root,
|
|
1729
|
+
execution: state.execution,
|
|
1730
|
+
approval_request_id: request.approval_request_id,
|
|
1731
|
+
approved: request.approved
|
|
1732
|
+
)
|
|
1733
|
+
resume_transition_started = false
|
|
1734
|
+
submitted = false
|
|
1735
|
+
event_loop.mark_agent_execution_admission(
|
|
1736
|
+
@agent.agent_id,
|
|
1737
|
+
execution_id: state.execution_id,
|
|
1738
|
+
state: :resuming
|
|
1739
|
+
)
|
|
1740
|
+
resume_transition_started = true
|
|
1741
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
1742
|
+
perform_resume_commit(operation)
|
|
1743
|
+
end
|
|
1744
|
+
submitted = true
|
|
1745
|
+
task.on_complete do |result, error|
|
|
1746
|
+
ready = ResumeCommitReady.new(
|
|
1747
|
+
coordinator: self,
|
|
1748
|
+
request: request,
|
|
1749
|
+
operation: operation,
|
|
1750
|
+
result: result,
|
|
1751
|
+
error: error
|
|
1752
|
+
)
|
|
1753
|
+
fail_task(request.result_task, runtime_rejected_error(:resume_commit)) unless
|
|
1754
|
+
post_control(runtime, ready)
|
|
1755
|
+
end
|
|
1756
|
+
rescue => error
|
|
1757
|
+
if resume_transition_started
|
|
1758
|
+
event_loop.mark_agent_execution_admission(
|
|
1759
|
+
@agent.agent_id,
|
|
1760
|
+
execution_id: state.execution_id,
|
|
1761
|
+
state: submitted ? :recovery_required : :suspended
|
|
1762
|
+
)
|
|
1763
|
+
end
|
|
1764
|
+
fail_task(request.result_task, translated(error))
|
|
1765
|
+
raise unless resume_transition_started
|
|
1766
|
+
end
|
|
1767
|
+
|
|
1768
|
+
def perform_resume_commit(operation)
|
|
1769
|
+
snapshot = nil
|
|
1770
|
+
@recovery_resume_snapshot_mutex&.synchronize do
|
|
1771
|
+
snapshot = @recovery_resume_snapshots&.delete(
|
|
1772
|
+
operation.execution_id.to_s
|
|
1773
|
+
)
|
|
1774
|
+
end
|
|
1775
|
+
|
|
1776
|
+
if snapshot
|
|
1777
|
+
staged_execution = RecoverySupport.with_recovery_metadata(
|
|
1778
|
+
operation.execution,
|
|
1779
|
+
RecoverySupport::TOOL_BATCH_METADATA_KEY => snapshot
|
|
1780
|
+
)
|
|
1781
|
+
operation = operation.class.new(
|
|
1782
|
+
**operation.to_h.merge(execution: staged_execution)
|
|
1783
|
+
)
|
|
1784
|
+
end
|
|
1785
|
+
|
|
1786
|
+
current = operation.execution
|
|
1787
|
+
current_root = operation.root
|
|
1788
|
+
request = current.approval_request || {}
|
|
1789
|
+
request_id = request["id"] || request[:id]
|
|
1790
|
+
unless request_id.to_s == operation.approval_request_id
|
|
1791
|
+
raise ArgumentError,
|
|
1792
|
+
"approval request does not match execution #{current.execution_id}"
|
|
1793
|
+
end
|
|
1794
|
+
|
|
1795
|
+
updated = next_root = nil
|
|
1796
|
+
@agent.persistence.transaction do |tx|
|
|
1797
|
+
decision_ref = tx.contents.put_json(
|
|
1798
|
+
"approval_request_id" => request_id.to_s,
|
|
1799
|
+
"approved" => operation.approved
|
|
1800
|
+
)
|
|
1801
|
+
decision_record = JournalRecord.new(
|
|
1802
|
+
agent_id: @agent.agent_id,
|
|
1803
|
+
execution_id: current.execution_id,
|
|
1804
|
+
kind: :approval_decided,
|
|
1805
|
+
channel: :approval,
|
|
1806
|
+
content_ref: decision_ref,
|
|
1807
|
+
context_generation: current_root.transcript_generation,
|
|
1808
|
+
context_candidate: false
|
|
1809
|
+
)
|
|
1810
|
+
updated = current.with(
|
|
1811
|
+
status: :active,
|
|
1812
|
+
phase: :resuming,
|
|
1813
|
+
working_records: current.working_records + [decision_record],
|
|
1814
|
+
approval_request: request.merge("approved" => operation.approved)
|
|
1815
|
+
)
|
|
1816
|
+
tx.executions.save(
|
|
1817
|
+
current.execution_id,
|
|
1818
|
+
expected_revision: current.execution_revision,
|
|
1819
|
+
execution: updated
|
|
1820
|
+
)
|
|
1821
|
+
next_root = current_root.with(
|
|
1822
|
+
agent_revision: current_root.agent_revision + 1,
|
|
1823
|
+
lifecycle_status: :active
|
|
1824
|
+
)
|
|
1825
|
+
tx.agents.save(
|
|
1826
|
+
current_root.agent_id,
|
|
1827
|
+
expected_revision: current_root.agent_revision,
|
|
1828
|
+
root: next_root
|
|
1829
|
+
)
|
|
1830
|
+
end
|
|
1831
|
+
ResumeCommitResult.new(execution: updated, root: next_root)
|
|
1832
|
+
end
|
|
1833
|
+
|
|
1834
|
+
def apply_resume_commit_on_event_loop(ready)
|
|
1835
|
+
request = ready.request
|
|
1836
|
+
operation = ready.operation
|
|
1837
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1838
|
+
state = event_loop.agent_execution_state(operation.execution_id)
|
|
1839
|
+
unless state && state.agent.equal?(@agent) &&
|
|
1840
|
+
state.execution.execution_revision == operation.expected_execution_revision &&
|
|
1841
|
+
state.execution.status == :suspended
|
|
1842
|
+
fail_task(
|
|
1843
|
+
request.result_task,
|
|
1844
|
+
Phronomy::Error.new(
|
|
1845
|
+
"stale approval resume result for #{operation.execution_id}"
|
|
1846
|
+
)
|
|
1847
|
+
)
|
|
1848
|
+
return
|
|
1849
|
+
end
|
|
1850
|
+
|
|
1851
|
+
if ready.error
|
|
1852
|
+
event_loop.mark_agent_execution_admission(
|
|
1853
|
+
@agent.agent_id,
|
|
1854
|
+
execution_id: operation.execution_id,
|
|
1855
|
+
state: :recovery_required
|
|
1856
|
+
)
|
|
1857
|
+
fail_task(request.result_task, translated(ready.error))
|
|
1858
|
+
return
|
|
1859
|
+
end
|
|
1860
|
+
|
|
1861
|
+
apply_agent_live_state(root: ready.result.root, appended_records: [])
|
|
1862
|
+
event_loop.mark_agent_execution_admission(
|
|
1863
|
+
@agent.agent_id,
|
|
1864
|
+
execution_id: operation.execution_id,
|
|
1865
|
+
state: :executing
|
|
1866
|
+
)
|
|
1867
|
+
event_loop.replace_agent_execution(
|
|
1868
|
+
operation.execution_id,
|
|
1869
|
+
execution: ready.result.execution
|
|
1870
|
+
)
|
|
1871
|
+
event_loop.register_agent_completion_waiter(
|
|
1872
|
+
operation.execution_id,
|
|
1873
|
+
request.result_task
|
|
1874
|
+
)
|
|
1875
|
+
trace_config = state.invocation ?
|
|
1876
|
+
state.invocation.config.merge(request.config) : request.config
|
|
1877
|
+
trace_mode = state.invocation&.mode ||
|
|
1878
|
+
(
|
|
1879
|
+
ready.result.execution.metadata[
|
|
1880
|
+
RecoverySupport::INVOCATION_MODE_KEY
|
|
1881
|
+
] || "invoke"
|
|
1882
|
+
).to_sym
|
|
1883
|
+
Phronomy::Tracing::Automatic.observe_task(
|
|
1884
|
+
request.result_task,
|
|
1885
|
+
"agent.execution",
|
|
1886
|
+
agent_id: @agent.agent_id,
|
|
1887
|
+
execution_id: operation.execution_id,
|
|
1888
|
+
mode: trace_mode,
|
|
1889
|
+
**@agent.send(:_build_caller_meta, trace_config)
|
|
1890
|
+
)
|
|
1891
|
+
start_resume_on_event_loop(
|
|
1892
|
+
operation.execution_id,
|
|
1893
|
+
request.result_task,
|
|
1894
|
+
approved: request.approved,
|
|
1895
|
+
config: request.config
|
|
1896
|
+
)
|
|
1897
|
+
rescue => error
|
|
1898
|
+
state = event_loop&.agent_execution_state(operation.execution_id) if event_loop&.current?
|
|
1899
|
+
if state
|
|
1900
|
+
begin_terminal_commit_on_event_loop(
|
|
1901
|
+
state,
|
|
1902
|
+
request.result_task,
|
|
1903
|
+
state.invocation,
|
|
1904
|
+
error,
|
|
1905
|
+
fsm_session_id: state.fsm_session_id
|
|
1906
|
+
)
|
|
1907
|
+
else
|
|
1908
|
+
fail_task(request.result_task, translated(error))
|
|
1909
|
+
end
|
|
1910
|
+
end
|
|
1911
|
+
|
|
1912
|
+
def start_resume_on_event_loop(execution_id, result_task, approved:, config:)
|
|
1913
|
+
runtime = Phronomy::Runtime.instance
|
|
1914
|
+
event_loop = runtime.event_loop
|
|
1915
|
+
state = event_loop.agent_execution_state(execution_id)
|
|
1916
|
+
invocation = state.invocation
|
|
1917
|
+
invocation.merge_config!(config)
|
|
1918
|
+
invocation.begin_approval_resume!(approved: approved)
|
|
1919
|
+
parent_session = Agent::AgentInvocationSessionBuilder.build_for_resume(
|
|
1920
|
+
agent_invocation: invocation,
|
|
1921
|
+
resume_event: :resume,
|
|
1922
|
+
resume_phase: :suspended,
|
|
1923
|
+
runtime: runtime
|
|
1924
|
+
)
|
|
1925
|
+
event_loop.replace_agent_execution(
|
|
1926
|
+
execution_id,
|
|
1927
|
+
invocation: invocation,
|
|
1928
|
+
fsm_session_id: parent_session.id
|
|
1929
|
+
)
|
|
1930
|
+
source_task = Phronomy::Task.deferred(name: "#{result_task.name}-source")
|
|
1931
|
+
source_task.on_complete do |completed, error|
|
|
1932
|
+
finish_on_event_loop(
|
|
1933
|
+
execution_id,
|
|
1934
|
+
result_task,
|
|
1935
|
+
completed || parent_session.context,
|
|
1936
|
+
error,
|
|
1937
|
+
fsm_session_id: parent_session.id
|
|
1938
|
+
)
|
|
1939
|
+
end
|
|
1940
|
+
event_loop.register(parent_session, completion: source_task)
|
|
1941
|
+
invocation.tool_invocations.each do |child|
|
|
1942
|
+
child_session = if child.awaiting_approval?
|
|
1943
|
+
Agent::ToolInvocationSessionBuilder.build_for_resume(
|
|
1944
|
+
tool_invocation: child,
|
|
1945
|
+
parent_event_sink: parent_session.event_sink,
|
|
1946
|
+
resume_event: approved ? :approve : :reject,
|
|
1947
|
+
resume_phase: :awaiting_approval,
|
|
1948
|
+
runtime: runtime
|
|
1949
|
+
)
|
|
1950
|
+
elsif !approved && child.authorized?
|
|
1951
|
+
Agent::ToolInvocationSessionBuilder.build_for_resume(
|
|
1952
|
+
tool_invocation: child,
|
|
1953
|
+
parent_event_sink: parent_session.event_sink,
|
|
1954
|
+
resume_event: :cancel,
|
|
1955
|
+
resume_phase: :authorized,
|
|
1956
|
+
runtime: runtime
|
|
1957
|
+
)
|
|
1958
|
+
end
|
|
1959
|
+
register_child(event_loop, child, child_session, parent_session.event_sink) if child_session
|
|
1960
|
+
end
|
|
1961
|
+
end
|
|
1962
|
+
|
|
1963
|
+
def start_framework_tools_on_event_loop(execution_id, result_task)
|
|
1964
|
+
runtime = Phronomy::Runtime.instance
|
|
1965
|
+
event_loop = runtime.event_loop
|
|
1966
|
+
state = event_loop.agent_execution_state(execution_id)
|
|
1967
|
+
invocation = state.invocation
|
|
1968
|
+
session = AgentInvocationSessionBuilder.build_for_resume(agent_invocation: invocation,
|
|
1969
|
+
resume_event: :resume, resume_phase: :suspended, runtime: runtime)
|
|
1970
|
+
event_loop.replace_agent_execution(execution_id, invocation: invocation, fsm_session_id: session.id)
|
|
1971
|
+
event_loop.mark_agent_execution_admission(@agent.agent_id, execution_id: execution_id, state: :executing)
|
|
1972
|
+
source = Phronomy::Task.deferred(name: "framework-tool-recovery-source")
|
|
1973
|
+
source.on_complete do |result, error|
|
|
1974
|
+
finish_on_event_loop(execution_id, result_task, result || session.context, error, fsm_session_id: session.id)
|
|
1975
|
+
end
|
|
1976
|
+
event_loop.register(session, completion: source)
|
|
1977
|
+
invocation.tool_invocations.select(&:authorized?).each do |child|
|
|
1978
|
+
child_session = ToolInvocationSessionBuilder.build_for_resume(tool_invocation: child,
|
|
1979
|
+
parent_event_sink: session.event_sink, resume_event: :dispatch, resume_phase: :authorized, runtime: runtime)
|
|
1980
|
+
register_child(event_loop, child, child_session, session.event_sink)
|
|
1981
|
+
end
|
|
1982
|
+
end
|
|
1983
|
+
|
|
1984
|
+
def register_child(event_loop, child, session, parent_event_sink)
|
|
1985
|
+
completion = Phronomy::Task.deferred(name: "tool-session:#{child.id}")
|
|
1986
|
+
completion.on_complete do |_result, error|
|
|
1987
|
+
next unless error
|
|
1988
|
+
|
|
1989
|
+
child.mark_framework_failed!(error)
|
|
1990
|
+
parent_event_sink.post(:tool_failed, {tool_invocation_id: child.id})
|
|
1991
|
+
end
|
|
1992
|
+
event_loop.register(session, completion: completion)
|
|
1993
|
+
end
|
|
1994
|
+
|
|
1995
|
+
# ----------------------------------------------------------------------
|
|
1996
|
+
# Terminal durable barrier
|
|
1997
|
+
# ----------------------------------------------------------------------
|
|
501
1998
|
|
|
502
|
-
def
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
1999
|
+
def finish_on_event_loop(execution_id, result_task, invocation, error, fsm_session_id:)
|
|
2000
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
2001
|
+
assert_event_loop!(event_loop)
|
|
2002
|
+
state = event_loop.agent_execution_state(execution_id)
|
|
2003
|
+
return fail_task(result_task, runtime_rejected_error(:terminal)) unless state
|
|
2004
|
+
|
|
2005
|
+
unless state.fsm_session_id.to_s == fsm_session_id.to_s &&
|
|
2006
|
+
state.invocation.equal?(invocation)
|
|
2007
|
+
return fail_task(
|
|
2008
|
+
result_task,
|
|
2009
|
+
Phronomy::Error.new("stale Agent terminal callback for #{execution_id}")
|
|
2010
|
+
)
|
|
2011
|
+
end
|
|
2012
|
+
|
|
2013
|
+
terminal_error = error || invocation&.error
|
|
2014
|
+
unless event_loop.agent_execution_quiescent?(execution_id)
|
|
2015
|
+
wait_state = quiescence_sensitive_terminal_error?(terminal_error) ?
|
|
2016
|
+
:cancelling : :terminalizing
|
|
2017
|
+
event_loop.mark_agent_execution_admission(
|
|
2018
|
+
@agent.agent_id,
|
|
2019
|
+
execution_id: execution_id,
|
|
2020
|
+
state: wait_state
|
|
2021
|
+
)
|
|
2022
|
+
event_loop.defer_agent_terminal_until_quiescent(
|
|
2023
|
+
execution_id,
|
|
2024
|
+
DeferredTerminalCommand.new(
|
|
2025
|
+
coordinator: self,
|
|
2026
|
+
execution_id: execution_id.to_s.freeze,
|
|
2027
|
+
result_task: result_task,
|
|
2028
|
+
invocation: invocation,
|
|
2029
|
+
source_error: error,
|
|
2030
|
+
fsm_session_id: fsm_session_id.to_s.freeze
|
|
2031
|
+
)
|
|
2032
|
+
)
|
|
2033
|
+
return
|
|
2034
|
+
end
|
|
2035
|
+
|
|
2036
|
+
begin_terminal_commit_on_event_loop(
|
|
2037
|
+
state,
|
|
2038
|
+
result_task,
|
|
2039
|
+
invocation,
|
|
2040
|
+
error,
|
|
2041
|
+
fsm_session_id: fsm_session_id
|
|
2042
|
+
)
|
|
2043
|
+
end
|
|
2044
|
+
|
|
2045
|
+
def resume_deferred_terminal_on_event_loop(command)
|
|
2046
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
2047
|
+
state = event_loop.agent_execution_state(command.execution_id)
|
|
2048
|
+
unless state && state.agent.equal?(@agent) &&
|
|
2049
|
+
state.fsm_session_id.to_s == command.fsm_session_id.to_s &&
|
|
2050
|
+
state.invocation.equal?(command.invocation)
|
|
2051
|
+
Phronomy.configuration.logger&.warn(
|
|
2052
|
+
"[Phronomy] Dropped stale deferred terminal for #{command.execution_id}"
|
|
508
2053
|
)
|
|
509
|
-
return
|
|
2054
|
+
return
|
|
510
2055
|
end
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
2056
|
+
|
|
2057
|
+
unless event_loop.agent_execution_quiescent?(command.execution_id)
|
|
2058
|
+
raise Phronomy::Error,
|
|
2059
|
+
"deferred terminal resumed before quiescence for #{command.execution_id}"
|
|
514
2060
|
end
|
|
2061
|
+
|
|
2062
|
+
begin_terminal_commit_on_event_loop(
|
|
2063
|
+
state,
|
|
2064
|
+
command.result_task,
|
|
2065
|
+
command.invocation,
|
|
2066
|
+
command.source_error,
|
|
2067
|
+
fsm_session_id: command.fsm_session_id
|
|
2068
|
+
)
|
|
2069
|
+
end
|
|
2070
|
+
|
|
2071
|
+
def quiescence_sensitive_terminal_error?(error)
|
|
2072
|
+
error.is_a?(Phronomy::CancellationError) ||
|
|
2073
|
+
error.is_a?(Phronomy::TimeoutError)
|
|
2074
|
+
end
|
|
2075
|
+
|
|
2076
|
+
def begin_terminal_commit_on_event_loop(
|
|
2077
|
+
state,
|
|
2078
|
+
result_task,
|
|
2079
|
+
invocation,
|
|
2080
|
+
source_error,
|
|
2081
|
+
fsm_session_id:
|
|
2082
|
+
)
|
|
515
2083
|
if invocation&.phase == :suspended
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
2084
|
+
snapshot = RecoverySupport.build_tool_batch_snapshot(invocation)
|
|
2085
|
+
staged_execution = RecoverySupport.with_recovery_metadata(
|
|
2086
|
+
state.execution,
|
|
2087
|
+
RecoverySupport::TOOL_BATCH_METADATA_KEY => snapshot
|
|
2088
|
+
)
|
|
2089
|
+
state = state.class.new(
|
|
2090
|
+
**state.to_h.merge(execution: staged_execution)
|
|
2091
|
+
)
|
|
2092
|
+
end
|
|
2093
|
+
|
|
2094
|
+
operation = build_terminal_operation(
|
|
2095
|
+
execution: state.execution,
|
|
2096
|
+
root: @agent.agent_root,
|
|
2097
|
+
journal_records: @agent.send(:_journal_records_snapshot),
|
|
2098
|
+
invocation: invocation,
|
|
2099
|
+
source_error: source_error,
|
|
2100
|
+
fsm_session_id: fsm_session_id,
|
|
2101
|
+
state_required: true
|
|
2102
|
+
)
|
|
2103
|
+
delivery = TerminalDelivery.new(
|
|
2104
|
+
result_task: result_task,
|
|
2105
|
+
application_listener: invocation&.event_listener,
|
|
2106
|
+
approval_listener: invocation&.approval_listener,
|
|
2107
|
+
handoff_request: invocation&.handoff_request,
|
|
2108
|
+
handoff_manifest: state.runtime_projection&.manifest
|
|
2109
|
+
)
|
|
2110
|
+
submit_terminal_operation(operation, delivery)
|
|
2111
|
+
end
|
|
2112
|
+
|
|
2113
|
+
def begin_terminal_without_session_on_event_loop(request:, prepared:, error:)
|
|
2114
|
+
return deliver_start_failure_on_event_loop(request, translated(error)) unless prepared&.execution
|
|
2115
|
+
|
|
2116
|
+
operation = build_terminal_operation(
|
|
2117
|
+
execution: prepared.execution,
|
|
2118
|
+
root: prepared.root,
|
|
2119
|
+
journal_records: @agent.send(:_journal_records_snapshot),
|
|
2120
|
+
invocation: nil,
|
|
2121
|
+
source_error: error,
|
|
2122
|
+
fsm_session_id: nil,
|
|
2123
|
+
state_required: false
|
|
2124
|
+
)
|
|
2125
|
+
delivery = TerminalDelivery.new(
|
|
2126
|
+
result_task: request.result_task,
|
|
2127
|
+
application_listener: request.on_event,
|
|
2128
|
+
approval_listener: request.approval_listener,
|
|
2129
|
+
handoff_request: nil,
|
|
2130
|
+
handoff_manifest: nil
|
|
2131
|
+
)
|
|
2132
|
+
submit_terminal_operation(operation, delivery)
|
|
2133
|
+
end
|
|
2134
|
+
|
|
2135
|
+
def build_terminal_operation(
|
|
2136
|
+
execution:,
|
|
2137
|
+
root:,
|
|
2138
|
+
journal_records:,
|
|
2139
|
+
invocation:,
|
|
2140
|
+
source_error:,
|
|
2141
|
+
fsm_session_id:,
|
|
2142
|
+
state_required:
|
|
2143
|
+
)
|
|
2144
|
+
TerminalCommitCommand.new(
|
|
2145
|
+
execution_id: execution.execution_id.to_s.freeze,
|
|
2146
|
+
fsm_session_id: fsm_session_id&.to_s&.freeze,
|
|
2147
|
+
expected_execution_revision: execution.execution_revision,
|
|
2148
|
+
root: root,
|
|
2149
|
+
journal_records: journal_records,
|
|
2150
|
+
execution: execution,
|
|
2151
|
+
runtime_snapshot: invocation ? invocation.runtime_snapshot : empty_runtime_snapshot,
|
|
2152
|
+
terminal_view: terminal_view(invocation, source_error),
|
|
2153
|
+
state_required: state_required
|
|
2154
|
+
)
|
|
2155
|
+
end
|
|
2156
|
+
|
|
2157
|
+
def submit_terminal_operation(operation, delivery)
|
|
2158
|
+
runtime = Phronomy::Runtime.instance
|
|
2159
|
+
event_loop = runtime.event_loop
|
|
2160
|
+
terminal_transition_started = false
|
|
2161
|
+
event_loop.mark_agent_execution_admission(
|
|
2162
|
+
@agent.agent_id,
|
|
2163
|
+
execution_id: operation.execution_id,
|
|
2164
|
+
state: :terminalizing
|
|
2165
|
+
)
|
|
2166
|
+
terminal_transition_started = true
|
|
2167
|
+
event_loop.register_agent_completion_waiter(
|
|
2168
|
+
operation.execution_id,
|
|
2169
|
+
delivery.result_task
|
|
2170
|
+
)
|
|
2171
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
2172
|
+
# Only the operation-specific immutable durable snapshot crosses the
|
|
2173
|
+
# worker boundary. Task/listener delivery state stays outside it.
|
|
2174
|
+
compute_terminal(operation)
|
|
2175
|
+
end
|
|
2176
|
+
task.on_complete do |outcome, error|
|
|
2177
|
+
ready = TerminalCommitReady.new(
|
|
2178
|
+
coordinator: self,
|
|
2179
|
+
operation: operation,
|
|
2180
|
+
delivery: delivery,
|
|
2181
|
+
outcome: outcome,
|
|
2182
|
+
error: error
|
|
2183
|
+
)
|
|
2184
|
+
fail_task(delivery.result_task, runtime_rejected_error(:terminal_apply)) unless
|
|
2185
|
+
post_control(runtime, ready)
|
|
2186
|
+
end
|
|
2187
|
+
rescue => error
|
|
2188
|
+
if terminal_transition_started
|
|
2189
|
+
event_loop.mark_agent_execution_admission(
|
|
2190
|
+
@agent.agent_id,
|
|
2191
|
+
execution_id: operation.execution_id,
|
|
2192
|
+
state: :recovery_required
|
|
2193
|
+
)
|
|
2194
|
+
Phronomy.configuration.logger&.warn(
|
|
2195
|
+
"[Phronomy] Agent terminal transition requires recovery for " \
|
|
2196
|
+
"#{operation.execution_id}: #{error.class}: #{error.message}"
|
|
2197
|
+
)
|
|
2198
|
+
return
|
|
520
2199
|
end
|
|
521
|
-
raise invocation.block_error if invocation.input_blocked? || invocation.output_blocked?
|
|
522
|
-
raise invocation.error if invocation.error
|
|
523
2200
|
|
|
524
|
-
|
|
2201
|
+
fail_task(delivery.result_task, translated(error))
|
|
2202
|
+
raise
|
|
2203
|
+
end
|
|
2204
|
+
|
|
2205
|
+
def terminal_view(invocation, source_error)
|
|
2206
|
+
TerminalView.new(
|
|
2207
|
+
phase: invocation&.phase,
|
|
2208
|
+
output: invocation&.output,
|
|
2209
|
+
usage: invocation&.usage,
|
|
2210
|
+
approval_request: invocation&.approval_request,
|
|
2211
|
+
rejected: invocation ? !!invocation.rejected : false,
|
|
2212
|
+
input_blocked: invocation ? invocation.input_blocked? : false,
|
|
2213
|
+
output_blocked: invocation ? invocation.output_blocked? : false,
|
|
2214
|
+
block_error: invocation&.block_error,
|
|
2215
|
+
invocation_error: invocation&.error,
|
|
2216
|
+
handoff: handoff_terminal_view(invocation&.handoff_request),
|
|
2217
|
+
callback_failure: invocation&.callback_failure,
|
|
2218
|
+
source_error: source_error,
|
|
2219
|
+
cancel_requested: invocation&.config&.fetch(:cancellation_token, nil)&.cancelled? || false
|
|
2220
|
+
)
|
|
2221
|
+
end
|
|
2222
|
+
|
|
2223
|
+
def handoff_terminal_view(request)
|
|
2224
|
+
return unless request
|
|
2225
|
+
|
|
2226
|
+
HandoffTerminalView.new(
|
|
2227
|
+
target_agent_id: request.handoff.target_agent.agent_id.to_s.freeze,
|
|
2228
|
+
responsibility: request.responsibility.to_s.freeze,
|
|
2229
|
+
selection_intent: request.selection_intent.to_h do |category, included|
|
|
2230
|
+
[category.to_s.freeze, !!included]
|
|
2231
|
+
end.freeze,
|
|
2232
|
+
llm_call_id: request.llm_call_id&.to_s&.freeze,
|
|
2233
|
+
tool_call_id: request.tool_call_id&.to_s&.freeze,
|
|
2234
|
+
policy: Immutable.copy(request.handoff.policy.to_h)
|
|
2235
|
+
)
|
|
2236
|
+
end
|
|
2237
|
+
|
|
2238
|
+
def empty_runtime_snapshot
|
|
2239
|
+
{llm_results: [].freeze, runtime_events: [].freeze, active_call: nil}.freeze
|
|
2240
|
+
end
|
|
2241
|
+
|
|
2242
|
+
def compute_terminal(operation)
|
|
2243
|
+
view = operation.terminal_view
|
|
2244
|
+
error = view.callback_failure&.to_stream_callback_error || view.source_error ||
|
|
2245
|
+
view.block_error || view.invocation_error
|
|
2246
|
+
if operation.execution.metadata["multi_agent_coordination_ref"] &&
|
|
2247
|
+
(error.is_a?(Phronomy::ExecutionRehydrationRequiredError) || view.cancel_requested)
|
|
2248
|
+
waiting = commit_coordination_wait(operation, error)
|
|
2249
|
+
return waiting if waiting
|
|
2250
|
+
end
|
|
2251
|
+
raise error if error.is_a?(Phronomy::ExecutionRehydrationRequiredError)
|
|
2252
|
+
return commit_failed_outcome(operation, error) if error
|
|
2253
|
+
return commit_suspended(operation) if view.phase == :suspended
|
|
2254
|
+
commit_completed(operation)
|
|
525
2255
|
rescue => caught
|
|
526
|
-
|
|
527
|
-
{type: :failed, error: terminal.fetch(:error)}
|
|
2256
|
+
reconcile_terminal_error(operation, caught)
|
|
528
2257
|
end
|
|
529
2258
|
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
2259
|
+
# Reuse the Agent barrier to retain unfinished owned children. This is an
|
|
2260
|
+
# active AgentExecution snapshot, not a second scheduler or terminal state.
|
|
2261
|
+
def commit_coordination_wait(operation, error)
|
|
2262
|
+
current = operation.execution
|
|
2263
|
+
waiting = nil
|
|
2264
|
+
@agent.persistence.transaction do |tx|
|
|
2265
|
+
refreshed = @agent.__prepare_coordination_record(current, tx: tx)
|
|
2266
|
+
snapshot = tx.contents.fetch_json(refreshed.metadata.fetch("multi_agent_coordination_ref"))
|
|
2267
|
+
unresolved = snapshot.fetch("children").any? do |child|
|
|
2268
|
+
!%w[completed failed cancelled rejected blocked handed_off].include?(child.fetch("state")) &&
|
|
2269
|
+
!(operation.terminal_view.cancel_requested && child.fetch("state") == "reserved")
|
|
2270
|
+
end
|
|
2271
|
+
next unless unresolved
|
|
2272
|
+
waiting = current.with(metadata: refreshed.metadata.merge(
|
|
2273
|
+
"coordination_cancel_requested" => operation.terminal_view.cancel_requested || current.metadata["coordination_cancel_requested"] == true
|
|
2274
|
+
))
|
|
2275
|
+
tx.executions.save(current.execution_id, expected_revision: current.execution_revision, execution: waiting)
|
|
2276
|
+
end
|
|
2277
|
+
return unless waiting
|
|
2278
|
+
coordination_wait_outcome(operation, waiting, error)
|
|
2279
|
+
rescue => failure
|
|
2280
|
+
confirmed = @agent.persistence.executions.load(current.execution_id)
|
|
2281
|
+
raise failure unless waiting && confirmed.to_h == waiting.to_h
|
|
2282
|
+
coordination_wait_outcome(operation, confirmed, error)
|
|
2283
|
+
end
|
|
2284
|
+
|
|
2285
|
+
def coordination_wait_outcome(operation, execution, error)
|
|
2286
|
+
TerminalOutcome.new(type: :coordination_wait, execution: execution, root: operation.root,
|
|
2287
|
+
appended_records: [].freeze, result: nil,
|
|
2288
|
+
error: error.is_a?(Phronomy::ExecutionRehydrationRequiredError) ? error :
|
|
2289
|
+
Phronomy::ExecutionRehydrationRequiredError.new("Execution #{execution.execution_id} retains unfinished children for cancellation/recovery"),
|
|
2290
|
+
approval_request: nil)
|
|
2291
|
+
end
|
|
2292
|
+
|
|
2293
|
+
# F1: terminal atomicity alone does not establish commit outcome certainty.
|
|
2294
|
+
# Reuse an exact committed outcome; an absent/active/read-failed result is
|
|
2295
|
+
# never permission to write a second terminal transition.
|
|
2296
|
+
def reconcile_terminal_error(operation, error)
|
|
2297
|
+
confirmed = @agent.persistence.executions.load(operation.execution_id)
|
|
2298
|
+
raise error unless confirmed.terminal? &&
|
|
2299
|
+
confirmed.agent_id == @agent.agent_id &&
|
|
2300
|
+
confirmed.execution_revision == operation.expected_execution_revision + 1
|
|
2301
|
+
root = @agent.persistence.agents.load(@agent.agent_id)
|
|
2302
|
+
records = @agent.persistence.journals.read(@agent.agent_id,
|
|
2303
|
+
after: operation.root.journal_position, limit: root.journal_position - operation.root.journal_position)
|
|
2304
|
+
result = @agent.persistence.execution_result(confirmed.execution_id)
|
|
2305
|
+
failure = result[:error] && RecoverySupport.error_from_failure(result[:error])
|
|
2306
|
+
type = if failure
|
|
2307
|
+
:failed
|
|
2308
|
+
else
|
|
2309
|
+
((confirmed.status == :handed_off) ? :handed_off : :completed)
|
|
2310
|
+
end
|
|
2311
|
+
TerminalOutcome.new(type: type, execution: confirmed, root: root,
|
|
2312
|
+
appended_records: records.freeze,
|
|
2313
|
+
result: failure ? nil : result_base(confirmed, root).merge(output: result[:result]).freeze,
|
|
2314
|
+
error: failure, approval_request: nil)
|
|
2315
|
+
end
|
|
2316
|
+
|
|
2317
|
+
def commit_suspended(operation)
|
|
2318
|
+
current = operation.execution
|
|
2319
|
+
root = operation.root
|
|
2320
|
+
request = operation.terminal_view.approval_request
|
|
2321
|
+
runtime_snapshot = operation.runtime_snapshot
|
|
535
2322
|
suspended = next_root = nil
|
|
536
2323
|
|
|
537
2324
|
@agent.persistence.transaction do |tx|
|
|
538
2325
|
encoded_records, call_records = encode_runtime_records(
|
|
539
|
-
|
|
2326
|
+
current,
|
|
540
2327
|
tx: tx,
|
|
541
2328
|
snapshot: runtime_snapshot,
|
|
542
2329
|
context_candidate: true,
|
|
@@ -549,7 +2336,6 @@ module Phronomy
|
|
|
549
2336
|
kind: :approval_required,
|
|
550
2337
|
channel: :approval,
|
|
551
2338
|
content_ref: request_ref,
|
|
552
|
-
correlation_id: request.id.to_s,
|
|
553
2339
|
context_generation: root.transcript_generation,
|
|
554
2340
|
context_candidate: false
|
|
555
2341
|
)
|
|
@@ -575,31 +2361,49 @@ module Phronomy
|
|
|
575
2361
|
root: next_root
|
|
576
2362
|
)
|
|
577
2363
|
end
|
|
578
|
-
|
|
579
|
-
activation.replace_execution(suspended)
|
|
580
|
-
@agent.__replace_root(next_root)
|
|
581
|
-
dispatch_approval_listener(invocation, request)
|
|
582
|
-
result_base(suspended, next_root).merge(
|
|
2364
|
+
result = result_base(suspended, next_root).merge(
|
|
583
2365
|
suspended: true,
|
|
584
2366
|
approval_request: request
|
|
585
2367
|
)
|
|
2368
|
+
TerminalOutcome.new(
|
|
2369
|
+
type: :suspended,
|
|
2370
|
+
execution: suspended,
|
|
2371
|
+
root: next_root,
|
|
2372
|
+
appended_records: [].freeze,
|
|
2373
|
+
result: result.freeze,
|
|
2374
|
+
error: nil,
|
|
2375
|
+
approval_request: request
|
|
2376
|
+
)
|
|
586
2377
|
end
|
|
587
2378
|
|
|
588
|
-
def
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
2379
|
+
def synchronize_handoff_target(tx, execution)
|
|
2380
|
+
coordination = execution.metadata["coordination"]
|
|
2381
|
+
return unless coordination && coordination["kind"] == "handoff"
|
|
2382
|
+
routing = tx.handoff_states.load(coordination.fetch("main_agent_id"))
|
|
2383
|
+
return unless routing && routing.pending_target_execution_id == execution.execution_id && routing.phase != "stable"
|
|
2384
|
+
unless routing.active_agent_id == execution.agent_id
|
|
2385
|
+
raise Phronomy::Persistence::ConflictError, "Handoff Target owner mismatch"
|
|
2386
|
+
end
|
|
2387
|
+
tx.handoff_states.save(routing.main_agent_id, expected_revision: routing.handoff_revision,
|
|
2388
|
+
state: routing.with(phase: "stable"))
|
|
2389
|
+
end
|
|
2390
|
+
|
|
2391
|
+
def commit_completed(operation)
|
|
2392
|
+
current = operation.execution
|
|
2393
|
+
root = operation.root
|
|
2394
|
+
view = operation.terminal_view
|
|
2395
|
+
runtime_snapshot = operation.runtime_snapshot
|
|
2396
|
+
completed = next_root = appended = messages = nil
|
|
593
2397
|
|
|
594
2398
|
@agent.persistence.transaction do |tx|
|
|
595
2399
|
encoded_records, call_records = encode_runtime_records(
|
|
596
|
-
|
|
2400
|
+
current,
|
|
597
2401
|
tx: tx,
|
|
598
2402
|
snapshot: runtime_snapshot,
|
|
599
2403
|
context_candidate: true,
|
|
600
2404
|
agent_root: root
|
|
601
2405
|
)
|
|
602
|
-
output_ref = tx.contents.put_text(
|
|
2406
|
+
output_ref = tx.contents.put_text(view.output.to_s)
|
|
603
2407
|
final_output_record = JournalRecord.new(
|
|
604
2408
|
agent_id: @agent.agent_id,
|
|
605
2409
|
execution_id: current.execution_id,
|
|
@@ -613,7 +2417,7 @@ module Phronomy
|
|
|
613
2417
|
completed_record = JournalRecord.new(
|
|
614
2418
|
agent_id: @agent.agent_id,
|
|
615
2419
|
execution_id: current.execution_id,
|
|
616
|
-
kind:
|
|
2420
|
+
kind: view.rejected ? :execution_rejected : :execution_completed,
|
|
617
2421
|
channel: :audit,
|
|
618
2422
|
content_ref: output_ref,
|
|
619
2423
|
context_generation: root.transcript_generation,
|
|
@@ -627,14 +2431,16 @@ module Phronomy
|
|
|
627
2431
|
records: all_records
|
|
628
2432
|
)
|
|
629
2433
|
completed = current.with(
|
|
630
|
-
status:
|
|
2434
|
+
status: view.rejected ? :rejected : :completed,
|
|
631
2435
|
phase: :completed,
|
|
632
2436
|
working_records: [],
|
|
633
2437
|
llm_calls: current.llm_calls + call_records,
|
|
634
2438
|
approval_request: nil,
|
|
635
2439
|
result_ref: output_ref,
|
|
636
|
-
terminal_reason:
|
|
2440
|
+
terminal_reason: view.rejected ? "rejected" : "completed"
|
|
637
2441
|
)
|
|
2442
|
+
completed = @agent.__prepare_coordination_record(completed, tx: tx)
|
|
2443
|
+
synchronize_handoff_target(tx, completed)
|
|
638
2444
|
tx.executions.save(
|
|
639
2445
|
current.execution_id,
|
|
640
2446
|
expected_revision: current.execution_revision,
|
|
@@ -652,29 +2458,45 @@ module Phronomy
|
|
|
652
2458
|
expected_revision: root.agent_revision,
|
|
653
2459
|
root: next_root
|
|
654
2460
|
)
|
|
2461
|
+
|
|
2462
|
+
# Materialize the caller-facing transcript through the transaction
|
|
2463
|
+
# view before commit. If materialization fails, the terminal durable
|
|
2464
|
+
# transition rolls back instead of committing and then attempting a
|
|
2465
|
+
# second terminal transition from a stale execution revision.
|
|
2466
|
+
records_after_commit = operation.journal_records + Array(appended)
|
|
2467
|
+
messages = transcript_messages(
|
|
2468
|
+
next_root,
|
|
2469
|
+
records_after_commit,
|
|
2470
|
+
persistence: tx
|
|
2471
|
+
)
|
|
655
2472
|
end
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
output: invocation.output,
|
|
662
|
-
rejected: invocation.rejected || nil,
|
|
663
|
-
usage: invocation.usage,
|
|
664
|
-
messages: transcript_messages(next_root)
|
|
2473
|
+
result = result_base(completed, next_root).merge(
|
|
2474
|
+
output: view.output,
|
|
2475
|
+
rejected: view.rejected || nil,
|
|
2476
|
+
usage: view.usage,
|
|
2477
|
+
messages: messages
|
|
665
2478
|
).compact
|
|
2479
|
+
TerminalOutcome.new(
|
|
2480
|
+
type: :completed,
|
|
2481
|
+
execution: completed,
|
|
2482
|
+
root: next_root,
|
|
2483
|
+
appended_records: Array(appended).freeze,
|
|
2484
|
+
result: result.freeze,
|
|
2485
|
+
error: nil,
|
|
2486
|
+
approval_request: nil
|
|
2487
|
+
)
|
|
666
2488
|
end
|
|
667
2489
|
|
|
668
|
-
def
|
|
2490
|
+
def commit_failed_outcome(operation, error)
|
|
669
2491
|
translated_error = translated(error)
|
|
670
|
-
current =
|
|
671
|
-
root =
|
|
672
|
-
runtime_snapshot =
|
|
2492
|
+
current = operation.execution
|
|
2493
|
+
root = operation.root
|
|
2494
|
+
runtime_snapshot = operation.runtime_snapshot
|
|
673
2495
|
failed = next_root = appended = nil
|
|
674
2496
|
|
|
675
2497
|
@agent.persistence.transaction do |tx|
|
|
676
2498
|
encoded_records, call_records = encode_runtime_records(
|
|
677
|
-
|
|
2499
|
+
current,
|
|
678
2500
|
tx: tx,
|
|
679
2501
|
snapshot: runtime_snapshot,
|
|
680
2502
|
context_candidate: false,
|
|
@@ -712,6 +2534,8 @@ module Phronomy
|
|
|
712
2534
|
error_ref: error_ref,
|
|
713
2535
|
terminal_reason: translated_error.class.name
|
|
714
2536
|
)
|
|
2537
|
+
failed = @agent.__prepare_coordination_record(failed, tx: tx)
|
|
2538
|
+
synchronize_handoff_target(tx, failed)
|
|
715
2539
|
tx.executions.save(
|
|
716
2540
|
current.execution_id,
|
|
717
2541
|
expected_revision: current.execution_revision,
|
|
@@ -728,21 +2552,155 @@ module Phronomy
|
|
|
728
2552
|
root: next_root
|
|
729
2553
|
)
|
|
730
2554
|
end
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
2555
|
+
TerminalOutcome.new(
|
|
2556
|
+
type: :failed,
|
|
2557
|
+
execution: failed,
|
|
2558
|
+
root: next_root,
|
|
2559
|
+
appended_records: Array(appended).freeze,
|
|
2560
|
+
result: nil,
|
|
2561
|
+
error: translated_error,
|
|
2562
|
+
approval_request: nil
|
|
2563
|
+
)
|
|
2564
|
+
end
|
|
2565
|
+
|
|
2566
|
+
def apply_terminal_commit_on_event_loop(ready)
|
|
2567
|
+
operation = ready.operation
|
|
2568
|
+
delivery = ready.delivery
|
|
2569
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
2570
|
+
state = event_loop.agent_execution_state(operation.execution_id)
|
|
2571
|
+
|
|
2572
|
+
if operation.state_required
|
|
2573
|
+
unless state && state.agent.equal?(@agent) &&
|
|
2574
|
+
state.execution.execution_revision == operation.expected_execution_revision &&
|
|
2575
|
+
state.fsm_session_id.to_s == operation.fsm_session_id.to_s
|
|
2576
|
+
Phronomy.configuration.logger&.warn(
|
|
2577
|
+
"[Phronomy] Dropped stale Agent terminal result for #{operation.execution_id}"
|
|
2578
|
+
)
|
|
2579
|
+
return
|
|
2580
|
+
end
|
|
2581
|
+
end
|
|
2582
|
+
|
|
2583
|
+
if ready.error
|
|
2584
|
+
if operation.execution.metadata["coordination"] || operation.execution.metadata["multi_agent_coordination_ref"]
|
|
2585
|
+
event_loop.release_agent_execution(operation.execution_id) if state
|
|
2586
|
+
event_loop.release_agent_execution_admission(@agent.agent_id, execution_id: operation.execution_id)
|
|
2587
|
+
event_loop.take_agent_completion_waiters(operation.execution_id, fallback: delivery.result_task).each do |task|
|
|
2588
|
+
task.fail(ready.error)
|
|
2589
|
+
end
|
|
2590
|
+
return
|
|
2591
|
+
end
|
|
2592
|
+
event_loop.mark_agent_execution_admission(
|
|
2593
|
+
@agent.agent_id,
|
|
2594
|
+
execution_id: operation.execution_id,
|
|
2595
|
+
state: :recovery_required
|
|
2596
|
+
)
|
|
2597
|
+
Phronomy.configuration.logger&.warn(
|
|
2598
|
+
"[Phronomy] Agent terminal durable outcome requires recovery for " \
|
|
2599
|
+
"#{operation.execution_id}: #{ready.error.class}: #{ready.error.message}"
|
|
2600
|
+
)
|
|
2601
|
+
return
|
|
2602
|
+
end
|
|
2603
|
+
|
|
2604
|
+
outcome = ready.outcome
|
|
2605
|
+
apply_agent_live_state(
|
|
2606
|
+
root: outcome.root,
|
|
2607
|
+
appended_records: outcome.appended_records
|
|
2608
|
+
)
|
|
2609
|
+
if state
|
|
2610
|
+
state.invocation&.acknowledge_runtime_snapshot(operation.runtime_snapshot)
|
|
2611
|
+
event_loop.replace_agent_execution(
|
|
2612
|
+
operation.execution_id,
|
|
2613
|
+
execution: outcome.execution,
|
|
2614
|
+
fsm_session_id: (outcome.type == :suspended) ? nil : state.fsm_session_id
|
|
2615
|
+
)
|
|
2616
|
+
end
|
|
2617
|
+
|
|
2618
|
+
case outcome.type
|
|
2619
|
+
when :coordination_wait
|
|
2620
|
+
event_loop.release_agent_execution(operation.execution_id) if state
|
|
2621
|
+
event_loop.release_agent_execution_admission(@agent.agent_id, execution_id: operation.execution_id)
|
|
2622
|
+
event_loop.take_agent_completion_waiters(operation.execution_id, fallback: delivery.result_task).each { |task| task.fail(outcome.error) }
|
|
2623
|
+
when :suspended
|
|
2624
|
+
event_loop.mark_agent_execution_admission(
|
|
2625
|
+
@agent.agent_id,
|
|
2626
|
+
execution_id: operation.execution_id,
|
|
2627
|
+
state: :suspended
|
|
2628
|
+
)
|
|
2629
|
+
dispatch_approval_listener(delivery.approval_listener, outcome.approval_request)
|
|
2630
|
+
callback_error = deliver_terminal(
|
|
2631
|
+
delivery.application_listener,
|
|
2632
|
+
:approval_required,
|
|
2633
|
+
request: outcome.approval_request
|
|
2634
|
+
)
|
|
2635
|
+
report_nonterminal_callback_error(
|
|
2636
|
+
callback_error,
|
|
2637
|
+
:approval_required,
|
|
2638
|
+
outcome.result
|
|
2639
|
+
)
|
|
2640
|
+
Array(state&.invocation&.config&.fetch(:phronomy_exact_observers, [])).each do |task|
|
|
2641
|
+
task.fail(Phronomy::ExecutionRehydrationRequiredError.new("Execution #{operation.execution_id} requires approval"))
|
|
2642
|
+
end
|
|
2643
|
+
when :completed
|
|
2644
|
+
event_loop.release_agent_execution(operation.execution_id) if state
|
|
2645
|
+
event_loop.release_agent_execution_admission(
|
|
2646
|
+
@agent.agent_id,
|
|
2647
|
+
execution_id: operation.execution_id
|
|
2648
|
+
)
|
|
2649
|
+
callback_error = deliver_terminal(delivery.application_listener, :done, outcome.result)
|
|
2650
|
+
settle_execution_waiters(
|
|
2651
|
+
event_loop, operation.execution_id, delivery.result_task,
|
|
2652
|
+
callback_error, :done, outcome.result
|
|
2653
|
+
)
|
|
2654
|
+
when :handed_off
|
|
2655
|
+
event_loop.release_agent_execution(operation.execution_id) if state
|
|
2656
|
+
event_loop.release_agent_execution_admission(
|
|
2657
|
+
@agent.agent_id,
|
|
2658
|
+
execution_id: operation.execution_id
|
|
2659
|
+
)
|
|
2660
|
+
result = outcome.result.merge(
|
|
2661
|
+
handoff_request: delivery.handoff_request,
|
|
2662
|
+
_phronomy_handoff_manifest: delivery.handoff_manifest
|
|
2663
|
+
).freeze
|
|
2664
|
+
callback_error = deliver_terminal(delivery.application_listener, :handoff, result)
|
|
2665
|
+
settle_execution_waiters(
|
|
2666
|
+
event_loop, operation.execution_id, delivery.result_task,
|
|
2667
|
+
callback_error, :handoff, result
|
|
2668
|
+
)
|
|
2669
|
+
when :failed
|
|
2670
|
+
event_loop.release_agent_execution(operation.execution_id) if state
|
|
2671
|
+
event_loop.release_agent_execution_admission(
|
|
2672
|
+
@agent.agent_id,
|
|
2673
|
+
execution_id: operation.execution_id
|
|
2674
|
+
)
|
|
2675
|
+
type = terminal_event_type(outcome.error)
|
|
2676
|
+
callback_error = deliver_terminal(
|
|
2677
|
+
delivery.application_listener,
|
|
2678
|
+
type,
|
|
2679
|
+
error: outcome.error
|
|
2680
|
+
)
|
|
2681
|
+
settle_execution_waiters(
|
|
2682
|
+
event_loop, operation.execution_id, delivery.result_task,
|
|
2683
|
+
callback_error, type, nil, outcome.error
|
|
2684
|
+
)
|
|
2685
|
+
else
|
|
2686
|
+
fail_task(
|
|
2687
|
+
delivery.result_task,
|
|
2688
|
+
Phronomy::Error.new("unknown terminal outcome: #{outcome.type.inspect}")
|
|
2689
|
+
)
|
|
2690
|
+
end
|
|
736
2691
|
end
|
|
737
2692
|
|
|
2693
|
+
# ----------------------------------------------------------------------
|
|
2694
|
+
# Durable record encoding helpers
|
|
2695
|
+
# ----------------------------------------------------------------------
|
|
2696
|
+
|
|
738
2697
|
def encode_runtime_records(
|
|
739
|
-
|
|
2698
|
+
execution,
|
|
740
2699
|
tx:,
|
|
741
2700
|
snapshot:,
|
|
742
2701
|
context_candidate:,
|
|
743
2702
|
agent_root:
|
|
744
2703
|
)
|
|
745
|
-
execution = activation.execution
|
|
746
2704
|
root = agent_root
|
|
747
2705
|
records = []
|
|
748
2706
|
calls = []
|
|
@@ -846,6 +2804,9 @@ module Phronomy
|
|
|
846
2804
|
llm_call_id = payload[:llm_call_id] || payload["llm_call_id"]
|
|
847
2805
|
tool_call_id = payload.fetch(:tool_call_id).to_s
|
|
848
2806
|
tool_name = payload.fetch(:tool_name).to_s
|
|
2807
|
+
next if execution.working_records.any? do |record|
|
|
2808
|
+
record.kind == :tool_message && record.causation_id.to_s == tool_call_id && record.llm_call_id.to_s == llm_call_id.to_s
|
|
2809
|
+
end
|
|
849
2810
|
result_ref = put_runtime_content(tx, payload.fetch(:tool_result))
|
|
850
2811
|
records << JournalRecord.new(
|
|
851
2812
|
agent_id: @agent.agent_id,
|
|
@@ -889,168 +2850,6 @@ module Phronomy
|
|
|
889
2850
|
[records, calls]
|
|
890
2851
|
end
|
|
891
2852
|
|
|
892
|
-
def mark_resuming!(activation, approval_request_id:, approved:)
|
|
893
|
-
current = activation.execution
|
|
894
|
-
current_root = @agent.agent_root
|
|
895
|
-
unless current.agent_id == @agent.agent_id && current.status == :suspended
|
|
896
|
-
raise ArgumentError, "execution is not suspended: #{current.execution_id}"
|
|
897
|
-
end
|
|
898
|
-
|
|
899
|
-
request = current.approval_request || {}
|
|
900
|
-
request_id = request["id"] || request[:id]
|
|
901
|
-
unless request_id.to_s == approval_request_id.to_s
|
|
902
|
-
raise ArgumentError,
|
|
903
|
-
"approval request does not match execution #{current.execution_id}"
|
|
904
|
-
end
|
|
905
|
-
|
|
906
|
-
updated = next_root = nil
|
|
907
|
-
@agent.persistence.transaction do |tx|
|
|
908
|
-
decision_ref = tx.contents.put_json(
|
|
909
|
-
"approval_request_id" => request_id.to_s,
|
|
910
|
-
"approved" => !!approved
|
|
911
|
-
)
|
|
912
|
-
decision_record = JournalRecord.new(
|
|
913
|
-
agent_id: @agent.agent_id,
|
|
914
|
-
execution_id: current.execution_id,
|
|
915
|
-
kind: :approval_decided,
|
|
916
|
-
channel: :approval,
|
|
917
|
-
content_ref: decision_ref,
|
|
918
|
-
correlation_id: request_id.to_s,
|
|
919
|
-
context_generation: current_root.transcript_generation,
|
|
920
|
-
context_candidate: false
|
|
921
|
-
)
|
|
922
|
-
updated = current.with(
|
|
923
|
-
status: :active,
|
|
924
|
-
phase: :resuming,
|
|
925
|
-
working_records: current.working_records + [decision_record],
|
|
926
|
-
approval_request: request.merge("approved" => approved)
|
|
927
|
-
)
|
|
928
|
-
tx.executions.save(
|
|
929
|
-
current.execution_id,
|
|
930
|
-
expected_revision: current.execution_revision,
|
|
931
|
-
execution: updated
|
|
932
|
-
)
|
|
933
|
-
next_root = current_root.with(
|
|
934
|
-
agent_revision: current_root.agent_revision + 1,
|
|
935
|
-
lifecycle_status: :active
|
|
936
|
-
)
|
|
937
|
-
tx.agents.save(
|
|
938
|
-
current_root.agent_id,
|
|
939
|
-
expected_revision: current_root.agent_revision,
|
|
940
|
-
root: next_root
|
|
941
|
-
)
|
|
942
|
-
end
|
|
943
|
-
activation.replace_execution(updated)
|
|
944
|
-
@agent.__replace_root(next_root)
|
|
945
|
-
updated
|
|
946
|
-
end
|
|
947
|
-
|
|
948
|
-
def assert_local_durable_base!(tx, root)
|
|
949
|
-
tx.assert_agent_watermark!(
|
|
950
|
-
agent_id: root.agent_id,
|
|
951
|
-
agent_revision: root.agent_revision,
|
|
952
|
-
journal_position: root.journal_position
|
|
953
|
-
)
|
|
954
|
-
end
|
|
955
|
-
|
|
956
|
-
def transcript_messages(root)
|
|
957
|
-
materializer = RubyLLMMaterializer.new(
|
|
958
|
-
agent: @agent,
|
|
959
|
-
persistence: @agent.persistence
|
|
960
|
-
)
|
|
961
|
-
materializer.materialize_journal_records(
|
|
962
|
-
JournalProjection.new(
|
|
963
|
-
agent_root: root,
|
|
964
|
-
records: @agent.send(:_journal_records_snapshot)
|
|
965
|
-
).transcript_records
|
|
966
|
-
)
|
|
967
|
-
end
|
|
968
|
-
|
|
969
|
-
def deliver_execution_terminal(command)
|
|
970
|
-
activation = command.activation
|
|
971
|
-
registry = Phronomy::Runtime.instance.__agent_activations
|
|
972
|
-
if command.commit_error
|
|
973
|
-
callback_error = deliver_terminal(
|
|
974
|
-
activation,
|
|
975
|
-
:error,
|
|
976
|
-
error: command.commit_error
|
|
977
|
-
)
|
|
978
|
-
settle_after_terminal(
|
|
979
|
-
command.result_task,
|
|
980
|
-
callback_error,
|
|
981
|
-
:error,
|
|
982
|
-
nil,
|
|
983
|
-
command.commit_error
|
|
984
|
-
)
|
|
985
|
-
return
|
|
986
|
-
end
|
|
987
|
-
|
|
988
|
-
outcome = command.outcome
|
|
989
|
-
case outcome.fetch(:type)
|
|
990
|
-
when :suspended
|
|
991
|
-
result = outcome.fetch(:result)
|
|
992
|
-
callback_error = deliver_terminal(
|
|
993
|
-
activation,
|
|
994
|
-
:approval_required,
|
|
995
|
-
request: result.fetch(:approval_request)
|
|
996
|
-
)
|
|
997
|
-
settle_after_terminal(
|
|
998
|
-
command.result_task,
|
|
999
|
-
callback_error,
|
|
1000
|
-
:approval_required,
|
|
1001
|
-
result
|
|
1002
|
-
)
|
|
1003
|
-
when :completed
|
|
1004
|
-
registry.delete(activation.execution_id)
|
|
1005
|
-
result = outcome.fetch(:result)
|
|
1006
|
-
callback_error = deliver_terminal(activation, :done, result)
|
|
1007
|
-
settle_after_terminal(
|
|
1008
|
-
command.result_task,
|
|
1009
|
-
callback_error,
|
|
1010
|
-
:done,
|
|
1011
|
-
result
|
|
1012
|
-
)
|
|
1013
|
-
when :failed
|
|
1014
|
-
registry.delete(activation.execution_id)
|
|
1015
|
-
error = outcome.fetch(:error)
|
|
1016
|
-
type = terminal_event_type(error)
|
|
1017
|
-
callback_error = deliver_terminal(activation, type, error: error)
|
|
1018
|
-
settle_after_terminal(
|
|
1019
|
-
command.result_task,
|
|
1020
|
-
callback_error,
|
|
1021
|
-
type,
|
|
1022
|
-
nil,
|
|
1023
|
-
error
|
|
1024
|
-
)
|
|
1025
|
-
end
|
|
1026
|
-
end
|
|
1027
|
-
|
|
1028
|
-
def settle_without_listener(activation, result_task, outcome, commit_error)
|
|
1029
|
-
registry = Phronomy::Runtime.instance.__agent_activations
|
|
1030
|
-
if commit_error
|
|
1031
|
-
return fail_task(result_task, commit_error)
|
|
1032
|
-
end
|
|
1033
|
-
|
|
1034
|
-
case outcome&.fetch(:type)
|
|
1035
|
-
when :suspended
|
|
1036
|
-
complete_task(result_task, outcome.fetch(:result))
|
|
1037
|
-
when :completed
|
|
1038
|
-
registry.delete(activation.execution_id)
|
|
1039
|
-
complete_task(result_task, outcome.fetch(:result))
|
|
1040
|
-
when :failed
|
|
1041
|
-
registry.delete(activation.execution_id)
|
|
1042
|
-
fail_task(result_task, outcome.fetch(:error))
|
|
1043
|
-
else
|
|
1044
|
-
registry.delete(activation.execution_id)
|
|
1045
|
-
fail_task(
|
|
1046
|
-
result_task,
|
|
1047
|
-
Phronomy::RuntimeShutdownError.new(
|
|
1048
|
-
"EventLoop is not accepting terminal delivery"
|
|
1049
|
-
)
|
|
1050
|
-
)
|
|
1051
|
-
end
|
|
1052
|
-
end
|
|
1053
|
-
|
|
1054
2853
|
def assistant_message_ref(tx, outcome)
|
|
1055
2854
|
return unless outcome
|
|
1056
2855
|
|
|
@@ -1084,64 +2883,76 @@ module Phronomy
|
|
|
1084
2883
|
tx.contents.put_text(value) : tx.contents.put_json(json_value(value))
|
|
1085
2884
|
end
|
|
1086
2885
|
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
2886
|
+
# ----------------------------------------------------------------------
|
|
2887
|
+
# EventLoop validation/apply helpers
|
|
2888
|
+
# ----------------------------------------------------------------------
|
|
2889
|
+
|
|
2890
|
+
def authoritative_state_for_operation(
|
|
2891
|
+
execution_id:,
|
|
2892
|
+
fsm_session_id:,
|
|
2893
|
+
expected_execution_revision:,
|
|
2894
|
+
expected_fsm_state:
|
|
2895
|
+
)
|
|
2896
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
2897
|
+
state = event_loop.agent_execution_state(execution_id)
|
|
2898
|
+
authoritative = state &&
|
|
2899
|
+
state.agent.equal?(@agent) &&
|
|
2900
|
+
state.execution.execution_revision == expected_execution_revision &&
|
|
2901
|
+
state.fsm_session_id.to_s == fsm_session_id.to_s &&
|
|
2902
|
+
event_loop.fsm_session_state(fsm_session_id) == expected_fsm_state
|
|
2903
|
+
return state if authoritative
|
|
2904
|
+
|
|
2905
|
+
Phronomy.configuration.logger&.warn(
|
|
2906
|
+
"[Phronomy] Dropped stale Agent operation result: " \
|
|
2907
|
+
"execution_id=#{execution_id} fsm_session_id=#{fsm_session_id}"
|
|
1098
2908
|
)
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
invocation.tool_invocations.each do |child|
|
|
1111
|
-
child_session = if child.awaiting_approval?
|
|
1112
|
-
Agent::ToolInvocationSessionBuilder.build_for_resume(
|
|
1113
|
-
tool_invocation: child,
|
|
1114
|
-
resume_event: approved ? :approve : :reject,
|
|
1115
|
-
resume_phase: :awaiting_approval,
|
|
1116
|
-
runtime: runtime
|
|
1117
|
-
)
|
|
1118
|
-
elsif !approved && child.authorized?
|
|
1119
|
-
Agent::ToolInvocationSessionBuilder.build_for_resume(
|
|
1120
|
-
tool_invocation: child,
|
|
1121
|
-
resume_event: :cancel,
|
|
1122
|
-
resume_phase: :authorized,
|
|
1123
|
-
runtime: runtime
|
|
1124
|
-
)
|
|
1125
|
-
end
|
|
1126
|
-
register_child(event_loop, runtime, child, child_session) if child_session
|
|
2909
|
+
nil
|
|
2910
|
+
rescue
|
|
2911
|
+
nil
|
|
2912
|
+
end
|
|
2913
|
+
|
|
2914
|
+
def validate_live_session!(state, invocation, fsm_session_id)
|
|
2915
|
+
unless state && state.agent.equal?(@agent) &&
|
|
2916
|
+
state.invocation.equal?(invocation) &&
|
|
2917
|
+
state.fsm_session_id.to_s == fsm_session_id.to_s
|
|
2918
|
+
raise Phronomy::Error,
|
|
2919
|
+
"Agent operation does not belong to the current live FSMSession"
|
|
1127
2920
|
end
|
|
1128
2921
|
end
|
|
1129
2922
|
|
|
1130
|
-
def
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
2923
|
+
def apply_agent_live_state(root:, appended_records:)
|
|
2924
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
2925
|
+
assert_event_loop!(event_loop)
|
|
2926
|
+
@agent.send(:_append_journal_records, appended_records)
|
|
2927
|
+
@agent.__replace_root(root)
|
|
2928
|
+
end
|
|
1134
2929
|
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
2930
|
+
def assert_event_loop!(event_loop)
|
|
2931
|
+
return if event_loop.current?
|
|
2932
|
+
|
|
2933
|
+
raise Phronomy::Error,
|
|
2934
|
+
"ExecutionCoordinator live-state apply must run on EventLoop"
|
|
2935
|
+
end
|
|
2936
|
+
|
|
2937
|
+
def assert_local_durable_base!(tx, root)
|
|
2938
|
+
tx.assert_agent_watermark!(
|
|
2939
|
+
agent_id: root.agent_id,
|
|
2940
|
+
agent_revision: root.agent_revision,
|
|
2941
|
+
journal_position: root.journal_position
|
|
2942
|
+
)
|
|
2943
|
+
end
|
|
2944
|
+
|
|
2945
|
+
def transcript_messages(root, journal_records, persistence: @agent.persistence)
|
|
2946
|
+
materializer = RubyLLMMaterializer.new(
|
|
2947
|
+
agent: @agent,
|
|
2948
|
+
persistence: persistence
|
|
2949
|
+
)
|
|
2950
|
+
materializer.materialize_journal_records(
|
|
2951
|
+
JournalProjection.new(
|
|
2952
|
+
agent_root: root,
|
|
2953
|
+
records: journal_records
|
|
2954
|
+
).transcript_records
|
|
2955
|
+
)
|
|
1145
2956
|
end
|
|
1146
2957
|
|
|
1147
2958
|
def result_base(execution, root)
|
|
@@ -1154,16 +2965,7 @@ module Phronomy
|
|
|
1154
2965
|
}
|
|
1155
2966
|
end
|
|
1156
2967
|
|
|
1157
|
-
def
|
|
1158
|
-
descriptor_hash = execution.metadata.fetch("context_policy") do
|
|
1159
|
-
ContextPolicies::Default.new.descriptor.to_h
|
|
1160
|
-
end
|
|
1161
|
-
descriptor = ContextPolicyDescriptor.from_h(descriptor_hash)
|
|
1162
|
-
ContextPolicyRegistry.default.resolve(descriptor)
|
|
1163
|
-
end
|
|
1164
|
-
|
|
1165
|
-
def deliver_terminal(activation, type, payload)
|
|
1166
|
-
listener = activation.application_listener
|
|
2968
|
+
def deliver_terminal(listener, type, payload)
|
|
1167
2969
|
return nil unless listener
|
|
1168
2970
|
|
|
1169
2971
|
@agent.send(
|
|
@@ -1173,16 +2975,43 @@ module Phronomy
|
|
|
1173
2975
|
)
|
|
1174
2976
|
end
|
|
1175
2977
|
|
|
1176
|
-
def
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
2978
|
+
def dispatch_approval_listener(listener, request)
|
|
2979
|
+
return unless listener
|
|
2980
|
+
|
|
2981
|
+
Phronomy::Runtime.instance.offload.submit(on_full: :raise) do
|
|
2982
|
+
listener.call(request)
|
|
2983
|
+
end
|
|
2984
|
+
rescue => error
|
|
2985
|
+
Phronomy.configuration.logger&.warn(
|
|
2986
|
+
"[Phronomy] approval listener dispatch failed: #{error.class}: #{error.message}"
|
|
2987
|
+
)
|
|
2988
|
+
end
|
|
2989
|
+
|
|
2990
|
+
def settle_execution_waiters(
|
|
2991
|
+
event_loop, execution_id, fallback_task, callback_error, event_type,
|
|
2992
|
+
result, execution_error = nil
|
|
2993
|
+
)
|
|
2994
|
+
waiters = event_loop.take_agent_completion_waiters(
|
|
2995
|
+
execution_id,
|
|
2996
|
+
fallback: fallback_task
|
|
2997
|
+
)
|
|
2998
|
+
waiters.each do |task|
|
|
2999
|
+
settle_after_terminal(
|
|
3000
|
+
task, callback_error, event_type, result, execution_error
|
|
1183
3001
|
)
|
|
3002
|
+
end
|
|
3003
|
+
end
|
|
3004
|
+
|
|
3005
|
+
def report_nonterminal_callback_error(callback_error, event_type, result)
|
|
3006
|
+
return unless callback_error
|
|
3007
|
+
|
|
3008
|
+
@agent.send(
|
|
3009
|
+
:_report_stream_callback_error,
|
|
3010
|
+
callback_error,
|
|
3011
|
+
event: StreamEvent.new(type: event_type, payload: result),
|
|
3012
|
+
execution_id: result&.fetch(:execution_id, nil),
|
|
3013
|
+
callback_error_policy: Phronomy.configuration.stream_callback_error_policy
|
|
1184
3014
|
)
|
|
1185
|
-
fail_task(result_task, error) unless posted
|
|
1186
3015
|
end
|
|
1187
3016
|
|
|
1188
3017
|
def settle_after_terminal(
|
|
@@ -1201,7 +3030,7 @@ module Phronomy
|
|
|
1201
3030
|
type: event_type,
|
|
1202
3031
|
payload: result || {error: execution_error}
|
|
1203
3032
|
),
|
|
1204
|
-
|
|
3033
|
+
execution_id: result&.fetch(:execution_id, nil),
|
|
1205
3034
|
callback_error_policy: policy
|
|
1206
3035
|
)
|
|
1207
3036
|
if policy == :fail_task && execution_error.nil?
|
|
@@ -1216,8 +3045,15 @@ module Phronomy
|
|
|
1216
3045
|
)
|
|
1217
3046
|
end
|
|
1218
3047
|
end
|
|
1219
|
-
execution_error
|
|
1220
|
-
|
|
3048
|
+
if execution_error
|
|
3049
|
+
if execution_error.is_a?(Phronomy::CancellationError)
|
|
3050
|
+
result_task.cancel!(execution_error)
|
|
3051
|
+
else
|
|
3052
|
+
fail_task(result_task, execution_error)
|
|
3053
|
+
end
|
|
3054
|
+
else
|
|
3055
|
+
complete_task(result_task, result)
|
|
3056
|
+
end
|
|
1221
3057
|
end
|
|
1222
3058
|
|
|
1223
3059
|
def terminal_event_type(error)
|
|
@@ -1227,15 +3063,6 @@ module Phronomy
|
|
|
1227
3063
|
:error
|
|
1228
3064
|
end
|
|
1229
3065
|
|
|
1230
|
-
def dispatch_approval_listener(invocation, request)
|
|
1231
|
-
listener = invocation.approval_listener
|
|
1232
|
-
return unless listener
|
|
1233
|
-
|
|
1234
|
-
Phronomy::Runtime.instance.offload.submit(on_full: :raise) do
|
|
1235
|
-
listener.call(request)
|
|
1236
|
-
end
|
|
1237
|
-
end
|
|
1238
|
-
|
|
1239
3066
|
def json_value(value)
|
|
1240
3067
|
case value
|
|
1241
3068
|
when Hash
|
|
@@ -1277,6 +3104,24 @@ module Phronomy
|
|
|
1277
3104
|
}.fetch(status)
|
|
1278
3105
|
end
|
|
1279
3106
|
|
|
3107
|
+
def post_control(runtime, command)
|
|
3108
|
+
runtime.event_loop.post(
|
|
3109
|
+
Phronomy::Event.new(
|
|
3110
|
+
type: :agent_control,
|
|
3111
|
+
target_id: Phronomy::EventLoop::SYSTEM_CHANNEL_ID,
|
|
3112
|
+
payload: {command: command}
|
|
3113
|
+
)
|
|
3114
|
+
)
|
|
3115
|
+
rescue Phronomy::RuntimeShutdownError
|
|
3116
|
+
false
|
|
3117
|
+
end
|
|
3118
|
+
|
|
3119
|
+
def runtime_rejected_error(operation)
|
|
3120
|
+
Phronomy::RuntimeShutdownError.new(
|
|
3121
|
+
"EventLoop is not accepting Agent #{operation} control delivery"
|
|
3122
|
+
)
|
|
3123
|
+
end
|
|
3124
|
+
|
|
1280
3125
|
def translated(error)
|
|
1281
3126
|
@agent.send(:_translated_error, error)
|
|
1282
3127
|
end
|