phronomy 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.mutant.yml +3 -4
- data/CHANGELOG.md +200 -10
- data/CONTRIBUTING.md +81 -9
- data/README.md +15 -6
- data/VERIFY.sh +587 -0
- data/benchmark/bench_agent_invoke.rb +2 -2
- data/benchmark/bench_context_assembler.rb +39 -68
- data/benchmark/bench_regression.rb +2 -2
- data/docs/architecture/agent-context.md +174 -0
- data/docs/architecture/before-llm-input.md +78 -0
- data/docs/architecture/context-management.md +232 -0
- data/docs/architecture/knowledge-and-rag.md +130 -0
- data/docs/architecture/multi-agent-handoff.md +152 -0
- data/docs/architecture/persistence.md +175 -0
- data/docs/architecture/removed/agent-context.md +72 -0
- data/docs/architecture/security-boundaries.md +173 -0
- data/docs/architecture/tracing.md +194 -0
- data/docs/architecture.md +82 -0
- data/docs/archive/design/archived/04_api_design.md +507 -0
- data/docs/archive/design/archived/09_guardrails.md +186 -0
- data/docs/archive/design/archived/17_rails_integration.md +175 -0
- data/docs/archive/design/historical/00_design_philosophy.md +122 -0
- data/docs/archive/design/historical/01_rubyllm_evaluation.md +178 -0
- data/docs/archive/design/historical/06_design_decisions.md +143 -0
- data/docs/changelog/0.14-and-earlier.md +1 -1
- data/docs/decisions/001-rubyllm-as-provider-layer.md +6 -1
- data/docs/decisions/002-workflow-context-immutability.md +26 -1
- data/docs/decisions/006-no-built-in-guardrails.md +2 -1
- data/docs/decisions/012-canonical-execution-log-and-context-policy.md +120 -38
- data/docs/decisions/014-unified-persistence-durable-state.md +9 -2
- data/docs/decisions/016-semantic-multi-agent-handoff.md +112 -0
- data/docs/decisions/017-design-authority-and-adr-governance.md +200 -0
- data/docs/decisions/018-durability-guarantees-and-failure-model.md +488 -0
- data/docs/decisions/019-filter-contract-and-security-boundaries.md +229 -0
- data/docs/decisions/020-canonical-workflow-instance-identity.md +177 -0
- data/docs/decisions/021-generic-agent-invocation-identity-removal.md +119 -0
- data/docs/decisions/022-agent-execution-parent-identity-and-runtime-routing-boundary.md +193 -0
- data/docs/decisions/023-fsm-session-incarnation-identity-and-routing.md +139 -0
- data/docs/decisions/024-event-loop-single-writer-agent-runtime.md +188 -0
- data/docs/decisions/025-process-local-agent-ownership-and-runtime-admission.md +249 -0
- data/docs/decisions/026-workflow-runtime-admission-and-durable-terminal-barrier.md +257 -0
- data/docs/decisions/027-llm-adapter-provider-boundary.md +93 -0
- data/docs/decisions/README.md +172 -0
- data/docs/features.md +31 -11
- data/docs/getting-started.md +77 -45
- data/docs/migrations/0.19.md +14 -7
- data/docs/migrations/0.22.md +390 -0
- data/docs/persistence-backends.md +88 -38
- data/docs/runtime-and-concurrency.md +227 -33
- data/examples/README.md +13 -0
- data/lib/phronomy/agent/agent_execution.rb +19 -15
- data/lib/phronomy/agent/agent_invocation.rb +288 -93
- data/lib/phronomy/agent/agent_invocation_session_builder.rb +236 -202
- data/lib/phronomy/agent/agent_root.rb +3 -3
- data/lib/phronomy/agent/approval_evaluation_request.rb +37 -19
- data/lib/phronomy/agent/async_event_api.rb +145 -72
- data/lib/phronomy/agent/base.rb +388 -181
- data/lib/phronomy/agent/concerns/before_llm_input.rb +1 -1
- data/lib/phronomy/agent/context_assembler.rb +437 -178
- data/lib/phronomy/agent/context_candidate_resolver.rb +2 -2
- data/lib/phronomy/agent/context_plan.rb +18 -13
- data/lib/phronomy/agent/context_plan_validator.rb +246 -88
- data/lib/phronomy/agent/context_policies/default.rb +123 -34
- data/lib/phronomy/agent/context_policy.rb +109 -3
- data/lib/phronomy/agent/context_policy_input.rb +244 -0
- data/lib/phronomy/agent/context_policy_input_builder.rb +241 -0
- data/lib/phronomy/agent/execution_coordinator.rb +1975 -587
- data/lib/phronomy/agent/journal_record.rb +17 -4
- data/lib/phronomy/agent/llm_input_build_context.rb +1 -1
- data/lib/phronomy/agent/llm_input_manifest.rb +277 -2
- data/lib/phronomy/agent/llm_operation_result.rb +12 -7
- data/lib/phronomy/agent/phase_machine_builder.rb +19 -7
- data/lib/phronomy/agent/provider_call_outcome.rb +23 -7
- data/lib/phronomy/agent/recovery_coordinator/continuation.rb +271 -0
- data/lib/phronomy/agent/recovery_coordinator/installation.rb +427 -0
- data/lib/phronomy/agent/recovery_coordinator/resolution.rb +635 -0
- data/lib/phronomy/agent/recovery_coordinator.rb +211 -0
- data/lib/phronomy/agent/recovery_support.rb +512 -0
- data/lib/phronomy/agent/ruby_llm_materializer.rb +22 -13
- data/lib/phronomy/agent/selection/candidate.rb +53 -0
- data/lib/phronomy/agent/selection/constraint.rb +49 -0
- data/lib/phronomy/agent/shared_state.rb +38 -1
- data/lib/phronomy/agent/tool_approval_request.rb +33 -5
- data/lib/phronomy/agent/tool_definition_set.rb +49 -3
- data/lib/phronomy/agent/tool_invocation.rb +336 -102
- data/lib/phronomy/agent/tool_invocation_session_builder.rb +49 -45
- data/lib/phronomy/agent.rb +20 -2
- data/lib/phronomy/agent_already_exists_error.rb +5 -0
- data/lib/phronomy/agent_purged_error.rb +5 -0
- data/lib/phronomy/engine/concurrency/offload_pool.rb +17 -3
- data/lib/phronomy/engine/concurrency/physical_completion_task.rb +135 -0
- data/lib/phronomy/engine/event_loop.rb +622 -63
- data/lib/phronomy/engine/fsm_session.rb +194 -21
- data/lib/phronomy/engine/runtime/agent_ownership_registry.rb +352 -0
- data/lib/phronomy/engine/runtime.rb +77 -20
- data/lib/phronomy/generator_verifier.rb +12 -14
- data/lib/phronomy/invocation_context.rb +9 -29
- data/lib/phronomy/multi_agent/admission_registry.rb +51 -0
- data/lib/phronomy/multi_agent/coordination_state.rb +18 -0
- data/lib/phronomy/multi_agent/coordinator.rb +154 -0
- data/lib/phronomy/multi_agent/execution_coordinator.rb +116 -0
- data/lib/phronomy/multi_agent/fan_out_invocation.rb +24 -33
- data/lib/phronomy/multi_agent/fan_out_session_builder.rb +12 -19
- data/lib/phronomy/multi_agent/handoff.rb +24 -45
- data/lib/phronomy/multi_agent/handoff_capability_factory.rb +87 -0
- data/lib/phronomy/multi_agent/handoff_context.rb +95 -0
- data/lib/phronomy/multi_agent/handoff_policy.rb +137 -0
- data/lib/phronomy/multi_agent/handoff_projection.rb +191 -0
- data/lib/phronomy/multi_agent/handoff_request.rb +45 -0
- data/lib/phronomy/multi_agent/orchestrator.rb +12 -15
- data/lib/phronomy/multi_agent/runner.rb +98 -0
- data/lib/phronomy/persistence/durable_codec.rb +646 -0
- data/lib/phronomy/persistence/durable_record.rb +117 -0
- data/lib/phronomy/persistence/in_memory.rb +210 -134
- data/lib/phronomy/persistence/migration/initial_format_migration.rb +226 -0
- data/lib/phronomy/persistence/repository_facades.rb +316 -0
- data/lib/phronomy/persistence.rb +81 -41
- data/lib/phronomy/recovery.rb +186 -0
- data/lib/phronomy/testing/persistence_contract/a_journal_repository.rb +2 -2
- data/lib/phronomy/testing/persistence_contract/a_persistence_backend.rb +1 -1
- data/lib/phronomy/testing/persistence_contract/a_workflow_state_repository.rb +19 -19
- data/lib/phronomy/testing/persistence_contract/an_agent_repository.rb +3 -3
- data/lib/phronomy/testing/persistence_contract/an_execution_repository.rb +5 -5
- data/lib/phronomy/tracing/automatic.rb +176 -0
- data/lib/phronomy/tracing/base.rb +11 -2
- data/lib/phronomy/tracing/langfuse_tracer.rb +20 -12
- data/lib/phronomy/version.rb +1 -1
- data/lib/phronomy/workflow.rb +3 -6
- data/lib/phronomy/workflow_context.rb +14 -5
- data/lib/phronomy/workflow_recovery.rb +123 -0
- data/lib/phronomy/workflow_runner.rb +468 -256
- data/lib/phronomy.rb +6 -0
- data/scripts/api_snapshot.rb +12 -0
- data/sig/phronomy/agent.rbs +209 -7
- data/sig/phronomy/multi_agent.rbs +39 -0
- data/sig/phronomy/persistence.rbs +62 -4
- data/sig/phronomy/runtime.rbs +1 -4
- data/sig/phronomy/workflow.rbs +2 -2
- data/sig/phronomy.rbs +10 -0
- metadata +65 -17
- data/examples/workflows/agent_event_mapping.rb +0 -101
- data/examples/workflows/generic_task_event_mapping.rb +0 -66
- data/lib/phronomy/agent/activation_registry.rb +0 -28
- data/lib/phronomy/agent/agent_execution_activation.rb +0 -172
- data/lib/phronomy/agent/context_candidate.rb +0 -47
- data/lib/phronomy/agent/context_parts/budget/token_budget_packer.rb +0 -53
- data/lib/phronomy/agent/context_parts/requirements/required_context_resolver.rb +0 -56
- data/lib/phronomy/agent/context_parts/selectors/recent_first_selector.rb +0 -30
- data/lib/phronomy/agent/context_parts/unit_builders/dependency_aware_unit_builder.rb +0 -118
- data/lib/phronomy/agent/context_policy_descriptor.rb +0 -49
- data/lib/phronomy/agent/context_policy_registry.rb +0 -46
- data/lib/phronomy/agent/context_request.rb +0 -35
- data/lib/phronomy/agent/context_selection_unit.rb +0 -38
- data/lib/phronomy/agent/derived_content_spec.rb +0 -34
- data/lib/phronomy/agent/runner.rb +0 -97
|
@@ -1,48 +1,154 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "time"
|
|
4
|
+
require "securerandom"
|
|
4
5
|
|
|
5
6
|
module Phronomy
|
|
6
7
|
module Agent
|
|
7
8
|
class ExecutionCoordinator
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
9
|
+
# External/API -> EventLoop control messages.
|
|
10
|
+
StartCommand = Data.define(
|
|
11
|
+
:coordinator, :input, :config, :mode,
|
|
12
|
+
:approval_policy, :approval_listener, :on_event, :result_task,
|
|
13
|
+
:admission_token
|
|
14
|
+
)
|
|
15
|
+
ResumeCommand = Data.define(
|
|
16
|
+
:coordinator, :execution_id, :approval_request_id,
|
|
17
|
+
:approved, :config, :result_task
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
# EventLoop -> Offload operation-specific immutable snapshots.
|
|
21
|
+
InitialPreparationCommand = Data.define(
|
|
22
|
+
:root, :journal_records, :input, :config
|
|
23
|
+
)
|
|
24
|
+
ProviderDispatchPreparationCommand = Data.define(
|
|
25
|
+
:execution_id, :fsm_session_id, :expected_execution_revision,
|
|
26
|
+
:root, :journal_records, :execution, :base_manifest,
|
|
27
|
+
:invocation_config, :runtime_snapshot, :streaming,
|
|
28
|
+
:pending_llm_call_id, :pending_llm_started_at
|
|
29
|
+
)
|
|
30
|
+
ToolDispatchPreparationCommand = Data.define(
|
|
31
|
+
:execution_id, :fsm_session_id, :expected_execution_revision,
|
|
32
|
+
:root, :execution, :runtime_snapshot, :tool_batch_snapshot
|
|
33
|
+
)
|
|
34
|
+
ProviderDispatchPreparationReconciliationCommand = Data.define(
|
|
35
|
+
:operation, :intended_result, :original_error
|
|
36
|
+
)
|
|
37
|
+
ToolDispatchPreparationReconciliationCommand = Data.define(
|
|
38
|
+
:operation, :intended_result, :original_error
|
|
39
|
+
)
|
|
40
|
+
ResumeCommitCommand = Data.define(
|
|
41
|
+
:execution_id, :expected_execution_revision,
|
|
42
|
+
:root, :execution, :approval_request_id, :approved
|
|
43
|
+
)
|
|
44
|
+
HandoffTerminalView = Data.define(
|
|
45
|
+
:target_agent_id, :responsibility, :selection_intent,
|
|
46
|
+
:llm_call_id, :tool_call_id
|
|
47
|
+
)
|
|
48
|
+
TerminalView = Data.define(
|
|
49
|
+
:phase, :output, :usage, :approval_request, :rejected,
|
|
50
|
+
:input_blocked, :output_blocked, :block_error,
|
|
51
|
+
:invocation_error, :handoff, :callback_failure, :source_error
|
|
52
|
+
)
|
|
53
|
+
TerminalCommitCommand = Data.define(
|
|
54
|
+
:execution_id, :fsm_session_id, :expected_execution_revision,
|
|
55
|
+
:root, :journal_records, :execution, :runtime_snapshot,
|
|
56
|
+
:terminal_view, :state_required
|
|
57
|
+
)
|
|
58
|
+
TerminalDelivery = Data.define(
|
|
59
|
+
:result_task, :application_listener, :approval_listener,
|
|
60
|
+
:handoff_request, :handoff_manifest
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# Offload -> EventLoop operation results. No live Agent/Invocation object is
|
|
64
|
+
# mutated by a worker; these values are validated and applied on EventLoop.
|
|
65
|
+
InitialPreparationResult = Data.define(
|
|
66
|
+
:execution, :root, :runtime_projection, :filtered_input,
|
|
67
|
+
:config, :appended_records, :error, :admission_outcome
|
|
68
|
+
)
|
|
69
|
+
ProviderDispatchPreparationResult = Data.define(:execution, :runtime_projection, :error)
|
|
70
|
+
ToolDispatchPreparationResult = Data.define(:execution)
|
|
71
|
+
ProviderDispatchPreparationReconciliationResult = Data.define(
|
|
72
|
+
:disposition, :preparation_result
|
|
73
|
+
)
|
|
74
|
+
ToolDispatchPreparationReconciliationResult = Data.define(
|
|
75
|
+
:disposition, :preparation_result
|
|
76
|
+
)
|
|
77
|
+
ResumeCommitResult = Data.define(:execution, :root)
|
|
78
|
+
TerminalOutcome = Data.define(
|
|
79
|
+
:type, :execution, :root, :appended_records,
|
|
80
|
+
:result, :error, :approval_request
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
InitialPreparationReady = Data.define(
|
|
84
|
+
:coordinator, :request, :result, :error
|
|
85
|
+
)
|
|
86
|
+
ProviderDispatchPreparationReady = Data.define(
|
|
87
|
+
:coordinator, :operation, :result, :error
|
|
88
|
+
)
|
|
89
|
+
ToolDispatchPreparationReady = Data.define(
|
|
90
|
+
:coordinator, :operation, :result, :error
|
|
91
|
+
)
|
|
92
|
+
ProviderDispatchPreparationReconciliationReady = Data.define(
|
|
93
|
+
:coordinator, :command, :result, :error
|
|
94
|
+
)
|
|
95
|
+
ToolDispatchPreparationReconciliationReady = Data.define(
|
|
96
|
+
:coordinator, :command, :result, :error
|
|
97
|
+
)
|
|
98
|
+
ResumeCommitReady = Data.define(
|
|
99
|
+
:coordinator, :request, :operation, :result, :error
|
|
100
|
+
)
|
|
101
|
+
TerminalCommitReady = Data.define(
|
|
102
|
+
:coordinator, :operation, :delivery, :outcome, :error
|
|
103
|
+
)
|
|
104
|
+
DeferredTerminalCommand = Data.define(
|
|
105
|
+
:coordinator, :execution_id, :result_task, :invocation,
|
|
106
|
+
:source_error, :fsm_session_id
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
class PreparationOutcomeUnknownError < Phronomy::Error
|
|
110
|
+
attr_reader :original_error, :intended_result
|
|
111
|
+
|
|
112
|
+
def initialize(original_error, intended_result)
|
|
113
|
+
@original_error = original_error
|
|
114
|
+
@intended_result = intended_result
|
|
115
|
+
super(
|
|
116
|
+
"durable dispatch preparation outcome is uncertain: " \
|
|
117
|
+
"#{original_error.class}: #{original_error.message}"
|
|
118
|
+
)
|
|
119
|
+
set_backtrace(original_error.backtrace)
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
private_constant :PreparationOutcomeUnknownError
|
|
11
123
|
|
|
12
124
|
def initialize(agent)
|
|
13
125
|
@agent = agent
|
|
14
126
|
end
|
|
15
127
|
|
|
16
128
|
def start(
|
|
17
|
-
input,
|
|
129
|
+
input, config: {}, mode: :invoke,
|
|
18
130
|
approval_policy: nil, approval_listener: nil, on_event: nil
|
|
19
131
|
)
|
|
132
|
+
@agent.send(:__assert_live_agent!)
|
|
133
|
+
@agent.send(:_reject_removed_generic_identity_keys!, config)
|
|
20
134
|
result_task = Phronomy::Task.deferred(name: "agent-#{@agent.agent_id}-#{mode}")
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
approval_listener: approval_listener,
|
|
35
|
-
on_event: on_event
|
|
36
|
-
)
|
|
37
|
-
end
|
|
135
|
+
command = StartCommand.new(
|
|
136
|
+
coordinator: self,
|
|
137
|
+
input: input,
|
|
138
|
+
config: config.dup.freeze,
|
|
139
|
+
mode: mode.to_sym,
|
|
140
|
+
approval_policy: approval_policy,
|
|
141
|
+
approval_listener: approval_listener,
|
|
142
|
+
on_event: on_event,
|
|
143
|
+
result_task: result_task,
|
|
144
|
+
admission_token: Object.new.freeze
|
|
145
|
+
)
|
|
146
|
+
unless post_control(Phronomy::Runtime.instance, command)
|
|
147
|
+
fail_task(result_task, runtime_rejected_error(:start))
|
|
38
148
|
end
|
|
39
149
|
result_task
|
|
40
150
|
rescue => error
|
|
41
|
-
if defined?(
|
|
42
|
-
post_prep_failure(runtime, on_event, result_task, translated(error))
|
|
43
|
-
else
|
|
44
|
-
fail_task(result_task, translated(error))
|
|
45
|
-
end
|
|
151
|
+
fail_task(result_task, translated(error)) if defined?(result_task) && result_task
|
|
46
152
|
result_task
|
|
47
153
|
end
|
|
48
154
|
|
|
@@ -52,162 +158,374 @@ module Phronomy
|
|
|
52
158
|
approved:,
|
|
53
159
|
config: {}
|
|
54
160
|
)
|
|
161
|
+
@agent.send(:__assert_live_agent!)
|
|
162
|
+
@agent.send(:_reject_removed_generic_identity_keys!, config)
|
|
55
163
|
result_task = Phronomy::Task.deferred(
|
|
56
164
|
name: "agent-approval-resume:#{execution_id}"
|
|
57
165
|
)
|
|
166
|
+
command = ResumeCommand.new(
|
|
167
|
+
coordinator: self,
|
|
168
|
+
execution_id: execution_id.to_s.freeze,
|
|
169
|
+
approval_request_id: approval_request_id.to_s.freeze,
|
|
170
|
+
approved: !!approved,
|
|
171
|
+
config: config.dup.freeze,
|
|
172
|
+
result_task: result_task
|
|
173
|
+
)
|
|
174
|
+
unless post_control(Phronomy::Runtime.instance, command)
|
|
175
|
+
fail_task(result_task, runtime_rejected_error(:resume))
|
|
176
|
+
end
|
|
177
|
+
result_task
|
|
178
|
+
rescue => error
|
|
179
|
+
fail_task(result_task, translated(error)) if defined?(result_task) && result_task
|
|
180
|
+
result_task
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# Called only from an Agent FSMSession external-operation dispatch state on
|
|
184
|
+
# EventLoop. Each method captures one operation-specific immutable snapshot
|
|
185
|
+
# and sends only that snapshot to OffloadPool.
|
|
186
|
+
# @api private
|
|
187
|
+
def prepare_provider_dispatch(invocation, event_sink:, streaming:)
|
|
188
|
+
runtime = Phronomy::Runtime.instance
|
|
189
|
+
event_loop = runtime.event_loop
|
|
190
|
+
assert_event_loop!(event_loop)
|
|
191
|
+
state = event_loop.agent_execution_state(invocation.execution_id)
|
|
192
|
+
validate_live_session!(state, invocation, event_sink.fsm_session_id)
|
|
193
|
+
unless event_loop.fsm_session_state(event_sink.fsm_session_id) == :calling_llm
|
|
194
|
+
raise Phronomy::Error,
|
|
195
|
+
"Provider dispatch preparation requires the owning FSMSession to be in :calling_llm"
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
operation = ProviderDispatchPreparationCommand.new(
|
|
199
|
+
execution_id: state.execution_id,
|
|
200
|
+
fsm_session_id: event_sink.fsm_session_id.to_s.freeze,
|
|
201
|
+
expected_execution_revision: state.execution.execution_revision,
|
|
202
|
+
root: @agent.agent_root,
|
|
203
|
+
journal_records: @agent.send(:_journal_records_snapshot),
|
|
204
|
+
execution: state.execution,
|
|
205
|
+
base_manifest: state.base_manifest,
|
|
206
|
+
invocation_config: invocation.config.dup.freeze,
|
|
207
|
+
runtime_snapshot: invocation.runtime_snapshot,
|
|
208
|
+
streaming: !!streaming,
|
|
209
|
+
pending_llm_call_id: SecureRandom.uuid.to_s.freeze,
|
|
210
|
+
pending_llm_started_at: Time.now.utc.iso8601(6).freeze
|
|
211
|
+
)
|
|
212
|
+
submit_provider_dispatch_preparation(operation)
|
|
213
|
+
nil
|
|
214
|
+
rescue => error
|
|
215
|
+
event_sink.post(:llm_setup_failed, translated(error))
|
|
216
|
+
nil
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
# @api private
|
|
220
|
+
def prepare_tool_dispatch(invocation, event_sink:)
|
|
58
221
|
runtime = Phronomy::Runtime.instance
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
222
|
+
event_loop = runtime.event_loop
|
|
223
|
+
assert_event_loop!(event_loop)
|
|
224
|
+
state = event_loop.agent_execution_state(invocation.execution_id)
|
|
225
|
+
validate_live_session!(state, invocation, event_sink.fsm_session_id)
|
|
226
|
+
unless event_loop.fsm_session_state(event_sink.fsm_session_id) == :dispatching_tools
|
|
227
|
+
raise Phronomy::Error,
|
|
228
|
+
"Tool dispatch preparation requires the owning FSMSession to be in :dispatching_tools"
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
tool_batch_snapshot = RecoverySupport.build_tool_batch_snapshot(invocation)
|
|
232
|
+
if tool_batch_snapshot.empty?
|
|
233
|
+
raise Phronomy::Error, "Tool dispatch preparation requires a non-empty Tool batch"
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
operation = ToolDispatchPreparationCommand.new(
|
|
237
|
+
execution_id: state.execution_id,
|
|
238
|
+
fsm_session_id: event_sink.fsm_session_id.to_s.freeze,
|
|
239
|
+
expected_execution_revision: state.execution.execution_revision,
|
|
240
|
+
root: @agent.agent_root,
|
|
241
|
+
execution: state.execution,
|
|
242
|
+
runtime_snapshot: invocation.runtime_snapshot,
|
|
243
|
+
tool_batch_snapshot: tool_batch_snapshot
|
|
244
|
+
)
|
|
245
|
+
submit_tool_dispatch_preparation(operation)
|
|
246
|
+
nil
|
|
247
|
+
rescue => error
|
|
248
|
+
event_sink.post(:tool_setup_failed, translated(error))
|
|
249
|
+
nil
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
def submit_provider_dispatch_preparation(operation)
|
|
253
|
+
runtime = Phronomy::Runtime.instance
|
|
254
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
255
|
+
perform_provider_dispatch_preparation(operation)
|
|
256
|
+
end
|
|
257
|
+
task.on_complete do |result, error|
|
|
258
|
+
ready = ProviderDispatchPreparationReady.new(
|
|
259
|
+
coordinator: self,
|
|
260
|
+
operation: operation,
|
|
261
|
+
result: result,
|
|
262
|
+
error: error
|
|
263
|
+
)
|
|
264
|
+
unless post_control(runtime, ready)
|
|
265
|
+
Phronomy.configuration.logger&.warn(
|
|
266
|
+
"[Phronomy] EventLoop rejected Provider dispatch preparation result for " \
|
|
267
|
+
"#{operation.execution_id}"
|
|
65
268
|
)
|
|
66
|
-
|
|
67
|
-
return result_task
|
|
269
|
+
end
|
|
68
270
|
end
|
|
271
|
+
end
|
|
69
272
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
273
|
+
def submit_tool_dispatch_preparation(operation)
|
|
274
|
+
runtime = Phronomy::Runtime.instance
|
|
275
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
276
|
+
perform_tool_dispatch_preparation(operation)
|
|
277
|
+
end
|
|
278
|
+
task.on_complete do |result, error|
|
|
279
|
+
ready = ToolDispatchPreparationReady.new(
|
|
280
|
+
coordinator: self,
|
|
281
|
+
operation: operation,
|
|
282
|
+
result: result,
|
|
283
|
+
error: error
|
|
284
|
+
)
|
|
285
|
+
unless post_control(runtime, ready)
|
|
286
|
+
Phronomy.configuration.logger&.warn(
|
|
287
|
+
"[Phronomy] EventLoop rejected Tool dispatch preparation result for " \
|
|
288
|
+
"#{operation.execution_id}"
|
|
76
289
|
)
|
|
77
|
-
|
|
78
|
-
return result_task
|
|
290
|
+
end
|
|
79
291
|
end
|
|
292
|
+
end
|
|
80
293
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
294
|
+
def submit_provider_dispatch_preparation_reconciliation(operation, uncertainty)
|
|
295
|
+
# simplecov:disable
|
|
296
|
+
runtime = Phronomy::Runtime.instance
|
|
297
|
+
command = ProviderDispatchPreparationReconciliationCommand.new(
|
|
298
|
+
operation: operation,
|
|
299
|
+
intended_result: uncertainty.intended_result,
|
|
300
|
+
original_error: uncertainty.original_error
|
|
301
|
+
)
|
|
302
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
303
|
+
perform_provider_dispatch_preparation_reconciliation(command)
|
|
88
304
|
end
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
305
|
+
task.on_complete do |result, error|
|
|
306
|
+
ready = ProviderDispatchPreparationReconciliationReady.new(
|
|
307
|
+
coordinator: self,
|
|
308
|
+
command: command,
|
|
309
|
+
result: result,
|
|
310
|
+
error: error
|
|
311
|
+
)
|
|
312
|
+
unless post_control(runtime, ready)
|
|
313
|
+
Phronomy.configuration.logger&.warn(
|
|
314
|
+
"[Phronomy] EventLoop rejected Provider dispatch reconciliation result for #{operation.execution_id}"
|
|
315
|
+
)
|
|
93
316
|
end
|
|
317
|
+
end
|
|
318
|
+
rescue => error
|
|
319
|
+
mark_barrier_recovery_required(operation, error)
|
|
320
|
+
# simplecov:enable
|
|
321
|
+
end
|
|
94
322
|
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
323
|
+
def submit_tool_dispatch_preparation_reconciliation(operation, uncertainty)
|
|
324
|
+
# simplecov:disable
|
|
325
|
+
runtime = Phronomy::Runtime.instance
|
|
326
|
+
command = ToolDispatchPreparationReconciliationCommand.new(
|
|
327
|
+
operation: operation,
|
|
328
|
+
intended_result: uncertainty.intended_result,
|
|
329
|
+
original_error: uncertainty.original_error
|
|
330
|
+
)
|
|
331
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
332
|
+
perform_tool_dispatch_preparation_reconciliation(command)
|
|
333
|
+
end
|
|
334
|
+
task.on_complete do |result, error|
|
|
335
|
+
ready = ToolDispatchPreparationReconciliationReady.new(
|
|
336
|
+
coordinator: self,
|
|
337
|
+
command: command,
|
|
338
|
+
result: result,
|
|
339
|
+
error: error
|
|
340
|
+
)
|
|
341
|
+
unless post_control(runtime, ready)
|
|
342
|
+
Phronomy.configuration.logger&.warn(
|
|
343
|
+
"[Phronomy] EventLoop rejected Tool dispatch reconciliation result for #{operation.execution_id}"
|
|
101
344
|
)
|
|
102
|
-
rescue => start_error
|
|
103
|
-
commit = runtime.offload.submit(on_full: :raise) do
|
|
104
|
-
commit_failed(
|
|
105
|
-
prepared_activation,
|
|
106
|
-
prepared_activation.invocation,
|
|
107
|
-
start_error
|
|
108
|
-
)
|
|
109
|
-
end
|
|
110
|
-
commit.on_complete do |terminal, commit_error|
|
|
111
|
-
if commit_error
|
|
112
|
-
fail_task(result_task, commit_error)
|
|
113
|
-
else
|
|
114
|
-
runtime.__agent_activations.delete(execution_id)
|
|
115
|
-
fail_task(result_task, terminal.fetch(:error))
|
|
116
|
-
end
|
|
117
|
-
end
|
|
118
345
|
end
|
|
119
346
|
end
|
|
120
|
-
result_task
|
|
121
347
|
rescue => error
|
|
122
|
-
|
|
123
|
-
|
|
348
|
+
mark_barrier_recovery_required(operation, error)
|
|
349
|
+
# simplecov:enable
|
|
124
350
|
end
|
|
125
351
|
|
|
126
|
-
#
|
|
127
|
-
#
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
journal_records = @agent.send(:_journal_records_snapshot)
|
|
133
|
-
manifest = manifest_ref = updated = nil
|
|
352
|
+
# Every control message is delivered by EventLoop. This method is the only
|
|
353
|
+
# coordinator entry point allowed to advance Phronomy-managed live state.
|
|
354
|
+
# @api private
|
|
355
|
+
def deliver_on_event_loop(command)
|
|
356
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
357
|
+
assert_event_loop!(event_loop)
|
|
134
358
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
)
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
)
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
359
|
+
case command
|
|
360
|
+
when StartCommand
|
|
361
|
+
begin_start_on_event_loop(command)
|
|
362
|
+
when InitialPreparationReady
|
|
363
|
+
apply_initial_preparation_on_event_loop(command)
|
|
364
|
+
when ProviderDispatchPreparationReady
|
|
365
|
+
apply_provider_dispatch_preparation_on_event_loop(command)
|
|
366
|
+
when ToolDispatchPreparationReady
|
|
367
|
+
apply_tool_dispatch_preparation_on_event_loop(command)
|
|
368
|
+
when ProviderDispatchPreparationReconciliationReady
|
|
369
|
+
apply_provider_dispatch_preparation_reconciliation_on_event_loop(command)
|
|
370
|
+
when ToolDispatchPreparationReconciliationReady
|
|
371
|
+
apply_tool_dispatch_preparation_reconciliation_on_event_loop(command)
|
|
372
|
+
when ResumeCommand
|
|
373
|
+
begin_resume_on_event_loop(command)
|
|
374
|
+
when ResumeCommitReady
|
|
375
|
+
apply_resume_commit_on_event_loop(command)
|
|
376
|
+
when TerminalCommitReady
|
|
377
|
+
apply_terminal_commit_on_event_loop(command)
|
|
378
|
+
when DeferredTerminalCommand
|
|
379
|
+
resume_deferred_terminal_on_event_loop(command)
|
|
380
|
+
else
|
|
381
|
+
raise Phronomy::Error, "unknown Agent control command: #{command.class}"
|
|
382
|
+
end
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
private
|
|
386
|
+
|
|
387
|
+
# ----------------------------------------------------------------------
|
|
388
|
+
# Initial preparation
|
|
389
|
+
# ----------------------------------------------------------------------
|
|
390
|
+
|
|
391
|
+
def begin_start_on_event_loop(request)
|
|
392
|
+
runtime = Phronomy::Runtime.instance
|
|
393
|
+
event_loop = runtime.event_loop
|
|
394
|
+
root = @agent.agent_root
|
|
395
|
+
if root.lifecycle_status == :closed
|
|
396
|
+
deliver_start_failure_on_event_loop(
|
|
397
|
+
request,
|
|
398
|
+
Phronomy::Error.new("agent is closed: #{@agent.agent_id}")
|
|
155
399
|
)
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
400
|
+
return
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
@agent.send(:__assert_live_agent!)
|
|
404
|
+
admitted = false
|
|
405
|
+
submitted = false
|
|
406
|
+
event_loop.admit_agent_execution(
|
|
407
|
+
@agent.agent_id,
|
|
408
|
+
owner_token: request.admission_token
|
|
409
|
+
)
|
|
410
|
+
admitted = true
|
|
411
|
+
|
|
412
|
+
# Ownership purge may begin on an application thread between the first
|
|
413
|
+
# live-owner check and EventLoop admission. Re-check after the slot is
|
|
414
|
+
# installed: once this check succeeds, purge observes the admission and
|
|
415
|
+
# must abort instead of deleting the Agent underneath this start.
|
|
416
|
+
@agent.send(:__assert_live_agent!)
|
|
417
|
+
|
|
418
|
+
operation = InitialPreparationCommand.new(
|
|
419
|
+
root: root,
|
|
420
|
+
journal_records: @agent.send(:_journal_records_snapshot),
|
|
421
|
+
input: request.input,
|
|
422
|
+
config: request.config
|
|
423
|
+
)
|
|
424
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
425
|
+
perform_initial_preparation(operation)
|
|
426
|
+
end
|
|
427
|
+
submitted = true
|
|
428
|
+
task.on_complete do |result, error|
|
|
429
|
+
ready = InitialPreparationReady.new(
|
|
430
|
+
coordinator: self,
|
|
431
|
+
request: request,
|
|
432
|
+
result: result,
|
|
433
|
+
error: error
|
|
434
|
+
)
|
|
435
|
+
fail_task(request.result_task, runtime_rejected_error(:initial_preparation)) unless
|
|
436
|
+
post_control(runtime, ready)
|
|
437
|
+
end
|
|
438
|
+
rescue => error
|
|
439
|
+
if admitted
|
|
440
|
+
if submitted
|
|
441
|
+
event_loop.mark_agent_admission_recovery_required(
|
|
442
|
+
@agent.agent_id,
|
|
443
|
+
owner_token: request.admission_token
|
|
174
444
|
)
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
445
|
+
else
|
|
446
|
+
event_loop.release_agent_execution_admission(
|
|
447
|
+
@agent.agent_id,
|
|
448
|
+
owner_token: request.admission_token
|
|
449
|
+
)
|
|
450
|
+
end
|
|
181
451
|
end
|
|
452
|
+
deliver_start_failure_on_event_loop(request, translated(error))
|
|
453
|
+
end
|
|
182
454
|
|
|
183
|
-
|
|
184
|
-
|
|
455
|
+
def perform_initial_preparation(operation)
|
|
456
|
+
begin
|
|
457
|
+
raw_message = @agent.send(:extract_message, operation.input)
|
|
458
|
+
rescue => error
|
|
459
|
+
return InitialPreparationResult.new(
|
|
460
|
+
execution: nil,
|
|
461
|
+
root: operation.root,
|
|
462
|
+
runtime_projection: nil,
|
|
463
|
+
filtered_input: nil,
|
|
464
|
+
config: operation.config,
|
|
465
|
+
appended_records: [].freeze,
|
|
466
|
+
error: translated(error),
|
|
467
|
+
admission_outcome: :not_established
|
|
468
|
+
)
|
|
469
|
+
end
|
|
185
470
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
471
|
+
begin
|
|
472
|
+
execution, active_root = admit_execution(
|
|
473
|
+
raw_message,
|
|
474
|
+
root: operation.root,
|
|
475
|
+
mode: (operation.config[:phronomy_recovery_mode] || :invoke).to_sym
|
|
476
|
+
)
|
|
477
|
+
rescue Phronomy::AgentBusyError => error
|
|
478
|
+
# A durable busy conflict proves that another nonterminal logical
|
|
479
|
+
# Execution already exists for this agent_id. Process loss does not
|
|
480
|
+
# make that execution terminal; keep Runtime admission fail-closed
|
|
481
|
+
# until ACS-15 recovery/reconciliation can establish current lineage.
|
|
482
|
+
return InitialPreparationResult.new(
|
|
483
|
+
execution: nil,
|
|
484
|
+
root: operation.root,
|
|
485
|
+
runtime_projection: nil,
|
|
486
|
+
filtered_input: nil,
|
|
487
|
+
config: operation.config,
|
|
488
|
+
appended_records: [].freeze,
|
|
489
|
+
error: translated(error),
|
|
490
|
+
admission_outcome: :recovery_required
|
|
491
|
+
)
|
|
492
|
+
rescue Phronomy::Persistence::ConflictError,
|
|
493
|
+
Phronomy::Persistence::NotFoundError,
|
|
494
|
+
Phronomy::Persistence::SerializationError,
|
|
495
|
+
ArgumentError,
|
|
496
|
+
Phronomy::ConfigurationError => error
|
|
497
|
+
return InitialPreparationResult.new(
|
|
498
|
+
execution: nil,
|
|
499
|
+
root: operation.root,
|
|
500
|
+
runtime_projection: nil,
|
|
501
|
+
filtered_input: nil,
|
|
502
|
+
config: operation.config,
|
|
503
|
+
appended_records: [].freeze,
|
|
504
|
+
error: translated(error),
|
|
505
|
+
admission_outcome: :not_established
|
|
506
|
+
)
|
|
507
|
+
rescue => error
|
|
508
|
+
return InitialPreparationResult.new(
|
|
509
|
+
execution: nil,
|
|
510
|
+
root: operation.root,
|
|
511
|
+
runtime_projection: nil,
|
|
512
|
+
filtered_input: nil,
|
|
513
|
+
config: operation.config,
|
|
514
|
+
appended_records: [].freeze,
|
|
515
|
+
error: translated(error),
|
|
516
|
+
admission_outcome: :outcome_unknown
|
|
517
|
+
)
|
|
518
|
+
end
|
|
193
519
|
|
|
194
|
-
def prepare(input, thread_id:, config:)
|
|
195
|
-
raw_message = @agent.send(:extract_message, input)
|
|
196
|
-
execution, active_root = admit_execution(
|
|
197
|
-
raw_message,
|
|
198
|
-
thread_id: thread_id
|
|
199
|
-
)
|
|
200
|
-
@agent.__replace_root(active_root)
|
|
201
520
|
current_execution = execution
|
|
202
|
-
|
|
203
521
|
begin
|
|
204
|
-
effective_config =
|
|
522
|
+
effective_config = operation.config
|
|
205
523
|
@agent.send(
|
|
206
524
|
:check_cancellation!,
|
|
207
525
|
effective_config,
|
|
208
526
|
"invocation cancelled before input filtering"
|
|
209
527
|
)
|
|
210
|
-
filtered_input = @agent.send(:run_input_filters!, input)
|
|
528
|
+
filtered_input = @agent.send(:run_input_filters!, operation.input)
|
|
211
529
|
@agent.send(
|
|
212
530
|
:check_cancellation!,
|
|
213
531
|
effective_config,
|
|
@@ -215,46 +533,56 @@ module Phronomy
|
|
|
215
533
|
)
|
|
216
534
|
filtered_message = @agent.send(:extract_message, filtered_input)
|
|
217
535
|
manifest = manifest_ref = active_execution = nil
|
|
218
|
-
local_journal = @agent.send(:_journal_records_snapshot)
|
|
219
536
|
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
)
|
|
537
|
+
# Content-addressed source material may be written before the Policy call.
|
|
538
|
+
# No mutable Agent/Execution repository state is committed here.
|
|
539
|
+
filtered_ref = @agent.persistence.contents.put_text(filtered_message)
|
|
540
|
+
input_record = JournalRecord.new(
|
|
541
|
+
agent_id: @agent.agent_id,
|
|
542
|
+
execution_id: current_execution.execution_id,
|
|
543
|
+
kind: :external_message,
|
|
544
|
+
channel: :external,
|
|
545
|
+
role: :user,
|
|
546
|
+
content_ref: filtered_ref,
|
|
547
|
+
context_generation: active_root.transcript_generation,
|
|
548
|
+
context_candidate: true,
|
|
549
|
+
metadata: {"handoff_policy_category" => "current_request"}
|
|
550
|
+
)
|
|
551
|
+
staged = current_execution.with(
|
|
552
|
+
execution_revision: current_execution.execution_revision,
|
|
553
|
+
working_records: current_execution.working_records + [input_record],
|
|
554
|
+
metadata: current_execution.metadata.merge(
|
|
555
|
+
"current_input_ref" => filtered_ref,
|
|
556
|
+
"current_input_record_id" => input_record.record_id
|
|
241
557
|
)
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
558
|
+
)
|
|
559
|
+
assembler = ContextAssembler.new(
|
|
560
|
+
agent: @agent,
|
|
561
|
+
persistence: @agent.persistence,
|
|
562
|
+
journal_records: operation.journal_records
|
|
563
|
+
)
|
|
564
|
+
prepared = assembler.prepare_initial(
|
|
565
|
+
input: filtered_input,
|
|
566
|
+
agent_root: active_root,
|
|
567
|
+
execution: staged,
|
|
568
|
+
config: effective_config,
|
|
569
|
+
patch: @agent.send(
|
|
570
|
+
:run_before_llm_input_hooks,
|
|
571
|
+
call_sequence: 1,
|
|
572
|
+
config: effective_config
|
|
257
573
|
)
|
|
574
|
+
)
|
|
575
|
+
@agent.send(
|
|
576
|
+
:check_cancellation!,
|
|
577
|
+
effective_config,
|
|
578
|
+
"invocation cancelled after context policy"
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
# Revalidate the durable base only after Application Policy returns.
|
|
582
|
+
# ContextPolicy never runs while this transaction is open.
|
|
583
|
+
@agent.persistence.transaction do |tx|
|
|
584
|
+
assert_local_durable_base!(tx, active_root)
|
|
585
|
+
manifest, manifest_ref = assembler.finalize(prepared, persistence: tx)
|
|
258
586
|
active_execution = staged.with(
|
|
259
587
|
status: :active,
|
|
260
588
|
phase: :calling_llm,
|
|
@@ -276,23 +604,46 @@ module Phronomy
|
|
|
276
604
|
agent: @agent,
|
|
277
605
|
persistence: @agent.persistence
|
|
278
606
|
).materialize(manifest: manifest, manifest_ref: manifest_ref)
|
|
279
|
-
|
|
607
|
+
InitialPreparationResult.new(
|
|
280
608
|
execution: active_execution,
|
|
609
|
+
root: active_root,
|
|
281
610
|
runtime_projection: projection,
|
|
282
611
|
filtered_input: filtered_input,
|
|
283
|
-
config: effective_config
|
|
612
|
+
config: effective_config,
|
|
613
|
+
appended_records: [].freeze,
|
|
614
|
+
error: nil,
|
|
615
|
+
admission_outcome: :active
|
|
284
616
|
)
|
|
285
617
|
rescue => error
|
|
286
|
-
commit_preparation_failure(
|
|
287
|
-
|
|
618
|
+
failure = commit_preparation_failure(
|
|
619
|
+
current_execution,
|
|
620
|
+
active_root,
|
|
621
|
+
error
|
|
622
|
+
)
|
|
623
|
+
InitialPreparationResult.new(
|
|
624
|
+
execution: failure.fetch(:execution),
|
|
625
|
+
root: failure.fetch(:root),
|
|
626
|
+
runtime_projection: nil,
|
|
627
|
+
filtered_input: nil,
|
|
628
|
+
config: operation.config,
|
|
629
|
+
appended_records: failure.fetch(:appended_records),
|
|
630
|
+
error: failure.fetch(:error),
|
|
631
|
+
admission_outcome: :terminal
|
|
632
|
+
)
|
|
288
633
|
end
|
|
289
634
|
end
|
|
290
635
|
|
|
291
|
-
def admit_execution(raw_message,
|
|
292
|
-
root
|
|
293
|
-
|
|
636
|
+
def admit_execution(raw_message, root:, mode: :invoke)
|
|
637
|
+
if root.lifecycle_status == :closed
|
|
638
|
+
raise Phronomy::Error,
|
|
639
|
+
"agent is closed: #{@agent.agent_id}"
|
|
640
|
+
end
|
|
294
641
|
|
|
295
642
|
execution = next_root = nil
|
|
643
|
+
mode = mode.to_sym
|
|
644
|
+
pending_llm_call_id = SecureRandom.uuid.to_s.freeze
|
|
645
|
+
pending_started_at = Time.now.utc.iso8601(6).freeze
|
|
646
|
+
|
|
296
647
|
@agent.persistence.transaction do |tx|
|
|
297
648
|
input_ref = tx.contents.put_text(raw_message)
|
|
298
649
|
input_record = JournalRecord.new(
|
|
@@ -301,22 +652,27 @@ module Phronomy
|
|
|
301
652
|
channel: :external,
|
|
302
653
|
role: :user,
|
|
303
654
|
content_ref: input_ref,
|
|
304
|
-
correlation_id: thread_id,
|
|
305
655
|
context_generation: root.transcript_generation,
|
|
306
656
|
context_candidate: false
|
|
307
657
|
)
|
|
308
|
-
policy_descriptor = ContextPolicies::Default.new.descriptor
|
|
309
658
|
execution = AgentExecution.start(
|
|
310
659
|
agent_root: root,
|
|
311
660
|
input_record: input_record,
|
|
312
661
|
metadata: {
|
|
313
|
-
"thread_id" => thread_id,
|
|
314
662
|
"current_input_ref" => input_ref,
|
|
315
|
-
|
|
663
|
+
RecoverySupport::CONTRACT_VERSION_KEY =>
|
|
664
|
+
RecoverySupport::CONTRACT_VERSION,
|
|
665
|
+
RecoverySupport::INVOCATION_MODE_KEY => mode.to_s,
|
|
666
|
+
RecoverySupport::PENDING_LLM_ID_KEY =>
|
|
667
|
+
pending_llm_call_id,
|
|
668
|
+
RecoverySupport::PENDING_LLM_STARTED_AT_KEY =>
|
|
669
|
+
pending_started_at
|
|
316
670
|
}.compact
|
|
317
671
|
)
|
|
318
672
|
input_record = JournalRecord.from_h(
|
|
319
|
-
input_record.to_h.merge(
|
|
673
|
+
input_record.to_h.merge(
|
|
674
|
+
"execution_id" => execution.execution_id
|
|
675
|
+
)
|
|
320
676
|
)
|
|
321
677
|
execution = execution.with(
|
|
322
678
|
execution_revision: 0,
|
|
@@ -336,9 +692,8 @@ module Phronomy
|
|
|
336
692
|
[execution, next_root]
|
|
337
693
|
end
|
|
338
694
|
|
|
339
|
-
def commit_preparation_failure(execution, error)
|
|
695
|
+
def commit_preparation_failure(execution, root, error)
|
|
340
696
|
translated_error = translated(error)
|
|
341
|
-
root = @agent.agent_root
|
|
342
697
|
failed = next_root = appended = nil
|
|
343
698
|
|
|
344
699
|
@agent.persistence.transaction do |tx|
|
|
@@ -389,24 +744,95 @@ module Phronomy
|
|
|
389
744
|
root: next_root
|
|
390
745
|
)
|
|
391
746
|
end
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
747
|
+
{
|
|
748
|
+
error: translated_error,
|
|
749
|
+
execution: failed,
|
|
750
|
+
root: next_root,
|
|
751
|
+
appended_records: Array(appended).freeze
|
|
752
|
+
}.freeze
|
|
395
753
|
end
|
|
396
754
|
|
|
397
|
-
def
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
755
|
+
def apply_initial_preparation_on_event_loop(ready)
|
|
756
|
+
request = ready.request
|
|
757
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
758
|
+
if ready.error
|
|
759
|
+
event_loop.mark_agent_admission_recovery_required(
|
|
760
|
+
@agent.agent_id,
|
|
761
|
+
owner_token: request.admission_token
|
|
762
|
+
)
|
|
763
|
+
deliver_start_failure_on_event_loop(request, translated(ready.error))
|
|
764
|
+
return
|
|
765
|
+
end
|
|
766
|
+
|
|
767
|
+
result = ready.result
|
|
768
|
+
case result.admission_outcome
|
|
769
|
+
when :not_established
|
|
770
|
+
event_loop.release_agent_execution_admission(
|
|
771
|
+
@agent.agent_id,
|
|
772
|
+
owner_token: request.admission_token
|
|
773
|
+
)
|
|
774
|
+
deliver_start_failure_on_event_loop(request, result.error)
|
|
775
|
+
return
|
|
776
|
+
when :outcome_unknown, :recovery_required
|
|
777
|
+
event_loop.mark_agent_admission_recovery_required(
|
|
778
|
+
@agent.agent_id,
|
|
779
|
+
owner_token: request.admission_token
|
|
780
|
+
)
|
|
781
|
+
deliver_start_failure_on_event_loop(request, result.error)
|
|
782
|
+
return
|
|
783
|
+
when :active, :terminal
|
|
784
|
+
event_loop.bind_agent_execution_admission(
|
|
785
|
+
@agent.agent_id,
|
|
786
|
+
owner_token: request.admission_token,
|
|
787
|
+
execution_id: result.execution.execution_id
|
|
788
|
+
)
|
|
789
|
+
else
|
|
790
|
+
event_loop.mark_agent_admission_recovery_required(
|
|
791
|
+
@agent.agent_id,
|
|
792
|
+
owner_token: request.admission_token
|
|
793
|
+
)
|
|
794
|
+
raise Phronomy::Error,
|
|
795
|
+
"unknown initial admission outcome: #{result.admission_outcome.inspect}"
|
|
796
|
+
end
|
|
797
|
+
|
|
798
|
+
apply_agent_live_state(
|
|
799
|
+
root: result.root,
|
|
800
|
+
appended_records: result.appended_records
|
|
801
|
+
)
|
|
802
|
+
if result.admission_outcome == :terminal
|
|
803
|
+
event_loop.release_agent_execution_admission(
|
|
804
|
+
@agent.agent_id,
|
|
805
|
+
execution_id: result.execution.execution_id
|
|
806
|
+
)
|
|
807
|
+
deliver_start_failure_on_event_loop(request, result.error)
|
|
808
|
+
return
|
|
809
|
+
end
|
|
810
|
+
|
|
811
|
+
register_initial_session_on_event_loop(request, result)
|
|
812
|
+
rescue => error
|
|
813
|
+
# Preparation may already be durably active. Terminalize through another
|
|
814
|
+
# offloaded durable operation rather than releasing the admission blindly.
|
|
815
|
+
begin_terminal_without_session_on_event_loop(
|
|
816
|
+
request: request,
|
|
817
|
+
prepared: ready.result,
|
|
818
|
+
error: error
|
|
404
819
|
)
|
|
820
|
+
end
|
|
821
|
+
|
|
822
|
+
def register_initial_session_on_event_loop(request, prepared)
|
|
405
823
|
runtime = Phronomy::Runtime.instance
|
|
406
|
-
runtime.__agent_activations.register(activation)
|
|
407
824
|
event_loop = runtime.event_loop
|
|
825
|
+
Phronomy::Tracing::Automatic.observe_task(
|
|
826
|
+
request.result_task,
|
|
827
|
+
"agent.execution",
|
|
828
|
+
input: request.input,
|
|
829
|
+
agent_id: @agent.agent_id,
|
|
830
|
+
execution_id: prepared.execution.execution_id,
|
|
831
|
+
mode: request.mode,
|
|
832
|
+
**@agent.send(:_build_caller_meta, prepared.config)
|
|
833
|
+
)
|
|
408
834
|
effective_config = prepared.config.merge(
|
|
409
|
-
|
|
835
|
+
phronomy_execution_coordinator: self,
|
|
410
836
|
phronomy_runtime_projection: prepared.runtime_projection,
|
|
411
837
|
phronomy_filtered_input: prepared.filtered_input,
|
|
412
838
|
execution_id: prepared.execution.execution_id
|
|
@@ -415,128 +841,1066 @@ module Phronomy
|
|
|
415
841
|
agent: @agent,
|
|
416
842
|
input: prepared.filtered_input,
|
|
417
843
|
config: effective_config,
|
|
418
|
-
approval_policy: approval_policy,
|
|
419
|
-
approval_listener: approval_listener,
|
|
420
|
-
mode: mode,
|
|
421
|
-
on_event:
|
|
844
|
+
approval_policy: request.approval_policy,
|
|
845
|
+
approval_listener: request.approval_listener,
|
|
846
|
+
mode: request.mode,
|
|
847
|
+
on_event: request.on_event,
|
|
422
848
|
runtime: runtime
|
|
423
849
|
)
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
850
|
+
event_loop.install_agent_execution(
|
|
851
|
+
execution_id: prepared.execution.execution_id,
|
|
852
|
+
agent: @agent,
|
|
853
|
+
coordinator: self,
|
|
854
|
+
execution: prepared.execution,
|
|
855
|
+
runtime_projection: prepared.runtime_projection,
|
|
856
|
+
base_manifest: prepared.runtime_projection.manifest,
|
|
857
|
+
invocation: session.context,
|
|
858
|
+
fsm_session_id: session.id
|
|
859
|
+
)
|
|
860
|
+
event_loop.register_agent_completion_waiter(
|
|
861
|
+
prepared.execution.execution_id,
|
|
862
|
+
request.result_task
|
|
863
|
+
)
|
|
864
|
+
source_task = Phronomy::Task.deferred(name: "#{request.result_task.name}-source")
|
|
427
865
|
source_task.on_complete do |invocation, error|
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
result_task,
|
|
866
|
+
finish_on_event_loop(
|
|
867
|
+
prepared.execution.execution_id,
|
|
868
|
+
request.result_task,
|
|
431
869
|
invocation || session.context,
|
|
432
|
-
error
|
|
870
|
+
error,
|
|
871
|
+
fsm_session_id: session.id
|
|
433
872
|
)
|
|
434
873
|
end
|
|
435
874
|
event_loop.register(session, completion: source_task)
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
coordinator: self,
|
|
442
|
-
application_listener: on_event
|
|
443
|
-
)
|
|
444
|
-
finish(activation, result_task, activation.invocation, error)
|
|
875
|
+
# simplecov:disable
|
|
876
|
+
rescue => _error
|
|
877
|
+
event_loop&.release_agent_execution(prepared.execution.execution_id) if
|
|
878
|
+
event_loop&.current? && event_loop.agent_execution_state(prepared.execution.execution_id)
|
|
879
|
+
raise
|
|
445
880
|
end
|
|
881
|
+
# simplecov:enable
|
|
446
882
|
|
|
447
|
-
def
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
self, activation, result_task, outcome, commit_error
|
|
455
|
-
)
|
|
456
|
-
posted = runtime.event_loop.post(
|
|
457
|
-
Phronomy::Event.new(
|
|
458
|
-
type: :agent_terminal_ready,
|
|
459
|
-
target_id: Phronomy::EventLoop::SYSTEM_CHANNEL_ID,
|
|
460
|
-
payload: {command: command}
|
|
461
|
-
)
|
|
883
|
+
def deliver_start_failure_on_event_loop(request, error)
|
|
884
|
+
callback_error = @agent.send(
|
|
885
|
+
:_deliver_stream_event,
|
|
886
|
+
request.on_event,
|
|
887
|
+
StreamEvent.new(
|
|
888
|
+
type: terminal_event_type(error),
|
|
889
|
+
payload: {error: error}
|
|
462
890
|
)
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
891
|
+
)
|
|
892
|
+
settle_after_terminal(
|
|
893
|
+
request.result_task,
|
|
894
|
+
callback_error,
|
|
895
|
+
terminal_event_type(error),
|
|
896
|
+
nil,
|
|
897
|
+
error
|
|
898
|
+
)
|
|
471
899
|
end
|
|
472
900
|
|
|
473
|
-
|
|
901
|
+
# ----------------------------------------------------------------------
|
|
902
|
+
# External-operation causal durable barriers
|
|
903
|
+
# ----------------------------------------------------------------------
|
|
904
|
+
|
|
905
|
+
def perform_provider_dispatch_preparation(operation)
|
|
906
|
+
metadata = operation.execution.metadata.dup
|
|
907
|
+
metadata.delete(RecoverySupport::TOOL_BATCH_METADATA_KEY)
|
|
908
|
+
metadata.delete(RecoverySupport::RECOVERY_METADATA_KEY)
|
|
909
|
+
metadata[RecoverySupport::PENDING_LLM_ID_KEY] =
|
|
910
|
+
operation.pending_llm_call_id
|
|
911
|
+
metadata[RecoverySupport::PENDING_LLM_STARTED_AT_KEY] =
|
|
912
|
+
operation.pending_llm_started_at
|
|
913
|
+
metadata[RecoverySupport::CONTRACT_VERSION_KEY] =
|
|
914
|
+
RecoverySupport::CONTRACT_VERSION
|
|
915
|
+
staged_execution = operation.execution.with(
|
|
916
|
+
execution_revision: operation.execution.execution_revision,
|
|
917
|
+
metadata: metadata
|
|
918
|
+
)
|
|
474
919
|
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
callback_error,
|
|
489
|
-
terminal_event_type(command.error),
|
|
490
|
-
nil,
|
|
491
|
-
command.error
|
|
920
|
+
manifest = manifest_ref = updated = intended = nil
|
|
921
|
+
|
|
922
|
+
# Encode the immutable runtime snapshot first. This transaction may add
|
|
923
|
+
# content-addressed blobs, but it does not advance Execution state.
|
|
924
|
+
encoded_records = call_records = nil
|
|
925
|
+
@agent.persistence.transaction do |tx|
|
|
926
|
+
assert_local_durable_base!(tx, operation.root)
|
|
927
|
+
encoded_records, call_records = encode_runtime_records(
|
|
928
|
+
staged_execution,
|
|
929
|
+
tx: tx,
|
|
930
|
+
snapshot: operation.runtime_snapshot,
|
|
931
|
+
context_candidate: true,
|
|
932
|
+
agent_root: operation.root
|
|
492
933
|
)
|
|
493
|
-
when AgentTerminalCommand
|
|
494
|
-
deliver_execution_terminal(command)
|
|
495
|
-
else
|
|
496
|
-
raise Phronomy::Error, "unknown terminal command: #{command.class}"
|
|
497
934
|
end
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
935
|
+
staged = staged_execution.with(
|
|
936
|
+
execution_revision: staged_execution.execution_revision,
|
|
937
|
+
phase: :preparing_llm_call,
|
|
938
|
+
working_records: staged_execution.working_records + encoded_records,
|
|
939
|
+
llm_calls: staged_execution.llm_calls + call_records
|
|
940
|
+
)
|
|
941
|
+
patch = @agent.send(
|
|
942
|
+
:run_before_llm_input_hooks,
|
|
943
|
+
call_sequence: staged.llm_calls.length + 1,
|
|
944
|
+
config: operation.invocation_config
|
|
945
|
+
)
|
|
946
|
+
assembler = ContextAssembler.new(
|
|
947
|
+
agent: @agent,
|
|
948
|
+
persistence: @agent.persistence,
|
|
949
|
+
journal_records: operation.journal_records
|
|
950
|
+
)
|
|
951
|
+
prepared = assembler.prepare_followup(
|
|
952
|
+
base_manifest: operation.base_manifest,
|
|
953
|
+
agent_root: operation.root,
|
|
954
|
+
execution: staged,
|
|
955
|
+
config: operation.invocation_config,
|
|
956
|
+
patch: patch
|
|
957
|
+
)
|
|
958
|
+
@agent.send(
|
|
959
|
+
:check_cancellation!,
|
|
960
|
+
operation.invocation_config,
|
|
961
|
+
"invocation cancelled after context policy"
|
|
962
|
+
)
|
|
963
|
+
|
|
964
|
+
begin
|
|
965
|
+
@agent.persistence.transaction do |tx|
|
|
966
|
+
assert_local_durable_base!(tx, operation.root)
|
|
967
|
+
manifest, manifest_ref = assembler.finalize(prepared, persistence: tx)
|
|
968
|
+
refs = Array(staged.metadata["manifest_refs"]) + [manifest_ref]
|
|
969
|
+
updated = staged.with(
|
|
970
|
+
phase: :calling_llm,
|
|
971
|
+
metadata: staged.metadata.merge(
|
|
972
|
+
"manifest_ref" => manifest_ref,
|
|
973
|
+
"manifest_refs" => refs
|
|
974
|
+
)
|
|
975
|
+
)
|
|
976
|
+
tx.executions.save(
|
|
977
|
+
operation.execution.execution_id,
|
|
978
|
+
expected_revision: operation.execution.execution_revision,
|
|
979
|
+
execution: updated
|
|
980
|
+
)
|
|
981
|
+
intended = ProviderDispatchPreparationResult.new(
|
|
982
|
+
execution: updated,
|
|
983
|
+
runtime_projection: nil,
|
|
984
|
+
error: nil
|
|
985
|
+
)
|
|
986
|
+
end
|
|
987
|
+
rescue => caught
|
|
988
|
+
raise if known_durable_preparation_failure?(caught) || intended.nil?
|
|
989
|
+
|
|
990
|
+
raise PreparationOutcomeUnknownError.new(caught, intended)
|
|
991
|
+
end
|
|
992
|
+
|
|
993
|
+
projection = materialization_error = nil
|
|
994
|
+
begin
|
|
995
|
+
projection = RubyLLMMaterializer.new(
|
|
996
|
+
agent: @agent,
|
|
997
|
+
persistence: @agent.persistence
|
|
998
|
+
).materialize(manifest: manifest, manifest_ref: manifest_ref)
|
|
999
|
+
rescue => error
|
|
1000
|
+
materialization_error = error
|
|
1001
|
+
end
|
|
1002
|
+
ProviderDispatchPreparationResult.new(
|
|
1003
|
+
execution: updated,
|
|
1004
|
+
runtime_projection: projection,
|
|
1005
|
+
error: materialization_error
|
|
1006
|
+
)
|
|
1007
|
+
end
|
|
1008
|
+
|
|
1009
|
+
def perform_tool_dispatch_preparation(operation)
|
|
1010
|
+
intended = nil
|
|
1011
|
+
begin
|
|
1012
|
+
@agent.persistence.transaction do |tx|
|
|
1013
|
+
assert_local_durable_base!(tx, operation.root)
|
|
1014
|
+
encoded_records, call_records = encode_runtime_records(
|
|
1015
|
+
operation.execution,
|
|
1016
|
+
tx: tx,
|
|
1017
|
+
snapshot: operation.runtime_snapshot,
|
|
1018
|
+
context_candidate: true,
|
|
1019
|
+
agent_root: operation.root
|
|
1020
|
+
)
|
|
1021
|
+
metadata = operation.execution.metadata.dup
|
|
1022
|
+
metadata.delete(RecoverySupport::PENDING_LLM_ID_KEY)
|
|
1023
|
+
metadata.delete(RecoverySupport::PENDING_LLM_STARTED_AT_KEY)
|
|
1024
|
+
metadata.delete(RecoverySupport::RECOVERY_METADATA_KEY)
|
|
1025
|
+
metadata[RecoverySupport::TOOL_BATCH_METADATA_KEY] =
|
|
1026
|
+
RecoverySupport.canonical_copy(operation.tool_batch_snapshot)
|
|
1027
|
+
metadata[RecoverySupport::CONTRACT_VERSION_KEY] =
|
|
1028
|
+
RecoverySupport::CONTRACT_VERSION
|
|
1029
|
+
updated = operation.execution.with(
|
|
1030
|
+
phase: :dispatching_tools,
|
|
1031
|
+
working_records: operation.execution.working_records + encoded_records,
|
|
1032
|
+
llm_calls: operation.execution.llm_calls + call_records,
|
|
1033
|
+
metadata: metadata
|
|
1034
|
+
)
|
|
1035
|
+
tx.executions.save(
|
|
1036
|
+
operation.execution.execution_id,
|
|
1037
|
+
expected_revision: operation.execution.execution_revision,
|
|
1038
|
+
execution: updated
|
|
1039
|
+
)
|
|
1040
|
+
intended = ToolDispatchPreparationResult.new(execution: updated)
|
|
1041
|
+
end
|
|
1042
|
+
rescue => caught
|
|
1043
|
+
raise if known_durable_preparation_failure?(caught) || intended.nil?
|
|
1044
|
+
|
|
1045
|
+
raise PreparationOutcomeUnknownError.new(caught, intended)
|
|
1046
|
+
end
|
|
1047
|
+
intended
|
|
1048
|
+
end
|
|
1049
|
+
|
|
1050
|
+
def apply_provider_dispatch_preparation_on_event_loop(ready)
|
|
1051
|
+
operation = ready.operation
|
|
1052
|
+
state = authoritative_state_for_operation(
|
|
1053
|
+
execution_id: operation.execution_id,
|
|
1054
|
+
fsm_session_id: operation.fsm_session_id,
|
|
1055
|
+
expected_execution_revision: operation.expected_execution_revision,
|
|
1056
|
+
expected_fsm_state: :calling_llm
|
|
1057
|
+
)
|
|
1058
|
+
return unless state
|
|
1059
|
+
|
|
1060
|
+
if ready.error
|
|
1061
|
+
if ready.error.is_a?(PreparationOutcomeUnknownError)
|
|
1062
|
+
submit_provider_dispatch_preparation_reconciliation(
|
|
1063
|
+
operation,
|
|
1064
|
+
ready.error
|
|
1065
|
+
)
|
|
1066
|
+
else
|
|
1067
|
+
state.invocation.event_sink.post(:llm_setup_failed, translated(ready.error))
|
|
1068
|
+
end
|
|
1069
|
+
return
|
|
1070
|
+
end
|
|
1071
|
+
|
|
1072
|
+
apply_confirmed_provider_dispatch_preparation_on_event_loop(
|
|
1073
|
+
operation,
|
|
1074
|
+
ready.result,
|
|
1075
|
+
state
|
|
1076
|
+
)
|
|
1077
|
+
rescue => error
|
|
1078
|
+
state&.invocation&.event_sink&.post(:llm_setup_failed, translated(error))
|
|
1079
|
+
end
|
|
1080
|
+
|
|
1081
|
+
def apply_tool_dispatch_preparation_on_event_loop(ready)
|
|
1082
|
+
operation = ready.operation
|
|
1083
|
+
state = authoritative_state_for_operation(
|
|
1084
|
+
execution_id: operation.execution_id,
|
|
1085
|
+
fsm_session_id: operation.fsm_session_id,
|
|
1086
|
+
expected_execution_revision: operation.expected_execution_revision,
|
|
1087
|
+
expected_fsm_state: :dispatching_tools
|
|
1088
|
+
)
|
|
1089
|
+
return unless state
|
|
1090
|
+
|
|
1091
|
+
if ready.error
|
|
1092
|
+
if ready.error.is_a?(PreparationOutcomeUnknownError)
|
|
1093
|
+
submit_tool_dispatch_preparation_reconciliation(
|
|
1094
|
+
operation,
|
|
1095
|
+
ready.error
|
|
1096
|
+
)
|
|
1097
|
+
else
|
|
1098
|
+
state.invocation.event_sink.post(:tool_setup_failed, translated(ready.error))
|
|
1099
|
+
end
|
|
1100
|
+
return
|
|
1101
|
+
end
|
|
1102
|
+
|
|
1103
|
+
apply_confirmed_tool_dispatch_preparation_on_event_loop(
|
|
1104
|
+
operation,
|
|
1105
|
+
ready.result,
|
|
1106
|
+
state
|
|
1107
|
+
)
|
|
1108
|
+
rescue => error
|
|
1109
|
+
state&.invocation&.event_sink&.post(:tool_setup_failed, translated(error))
|
|
1110
|
+
end
|
|
1111
|
+
|
|
1112
|
+
def apply_confirmed_provider_dispatch_preparation_on_event_loop(
|
|
1113
|
+
operation,
|
|
1114
|
+
result,
|
|
1115
|
+
state
|
|
1116
|
+
)
|
|
1117
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1118
|
+
if result.runtime_projection
|
|
1119
|
+
event_loop.replace_agent_execution(
|
|
1120
|
+
operation.execution_id,
|
|
1121
|
+
execution: result.execution,
|
|
1122
|
+
runtime_projection: result.runtime_projection
|
|
1123
|
+
)
|
|
1124
|
+
else
|
|
1125
|
+
event_loop.replace_agent_execution(
|
|
1126
|
+
operation.execution_id,
|
|
1127
|
+
execution: result.execution
|
|
1128
|
+
)
|
|
1129
|
+
end
|
|
1130
|
+
state.invocation.acknowledge_runtime_snapshot(operation.runtime_snapshot)
|
|
1131
|
+
|
|
1132
|
+
if result.error
|
|
1133
|
+
state.invocation.event_sink.post(:llm_setup_failed, translated(result.error))
|
|
1134
|
+
return
|
|
1135
|
+
end
|
|
1136
|
+
|
|
1137
|
+
Agent::AgentInvocationSessionBuilder.start_prepared_provider_call(
|
|
1138
|
+
agent: @agent,
|
|
1139
|
+
runtime: Phronomy::Runtime.instance,
|
|
1140
|
+
event_sink: state.invocation.event_sink,
|
|
1141
|
+
invocation: state.invocation,
|
|
1142
|
+
projection: result.runtime_projection,
|
|
1143
|
+
streaming: operation.streaming
|
|
1144
|
+
)
|
|
1145
|
+
end
|
|
1146
|
+
|
|
1147
|
+
def apply_confirmed_tool_dispatch_preparation_on_event_loop(
|
|
1148
|
+
operation,
|
|
1149
|
+
result,
|
|
1150
|
+
state
|
|
1151
|
+
)
|
|
1152
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1153
|
+
event_loop.replace_agent_execution(
|
|
1154
|
+
operation.execution_id,
|
|
1155
|
+
execution: result.execution
|
|
1156
|
+
)
|
|
1157
|
+
state.invocation.acknowledge_runtime_snapshot(operation.runtime_snapshot)
|
|
1158
|
+
Agent::AgentInvocationSessionBuilder.start_prepared_tool_dispatch(
|
|
1159
|
+
runtime: Phronomy::Runtime.instance,
|
|
1160
|
+
event_sink: state.invocation.event_sink,
|
|
1161
|
+
invocation: state.invocation
|
|
1162
|
+
)
|
|
1163
|
+
end
|
|
1164
|
+
|
|
1165
|
+
def perform_provider_dispatch_preparation_reconciliation(command)
|
|
1166
|
+
# simplecov:disable
|
|
1167
|
+
disposition, current = preparation_reconciliation_state(
|
|
1168
|
+
command.operation,
|
|
1169
|
+
command.intended_result.execution
|
|
1170
|
+
)
|
|
1171
|
+
preparation_result = nil
|
|
1172
|
+
if disposition == :committed
|
|
1173
|
+
projection = materialization_error = nil
|
|
1174
|
+
begin
|
|
1175
|
+
_manifest, projection = RecoverySupport.materialize_projection(
|
|
1176
|
+
@agent,
|
|
1177
|
+
current.metadata.fetch("manifest_ref")
|
|
1178
|
+
)
|
|
1179
|
+
rescue => error
|
|
1180
|
+
materialization_error = error
|
|
1181
|
+
end
|
|
1182
|
+
preparation_result = ProviderDispatchPreparationResult.new(
|
|
1183
|
+
execution: current,
|
|
1184
|
+
runtime_projection: projection,
|
|
1185
|
+
error: materialization_error
|
|
1186
|
+
)
|
|
1187
|
+
end
|
|
1188
|
+
ProviderDispatchPreparationReconciliationResult.new(
|
|
1189
|
+
disposition: disposition,
|
|
1190
|
+
preparation_result: preparation_result
|
|
1191
|
+
)
|
|
1192
|
+
end
|
|
1193
|
+
|
|
1194
|
+
def perform_tool_dispatch_preparation_reconciliation(command)
|
|
1195
|
+
disposition, current = preparation_reconciliation_state(
|
|
1196
|
+
command.operation,
|
|
1197
|
+
command.intended_result.execution
|
|
1198
|
+
)
|
|
1199
|
+
preparation_result = if disposition == :committed
|
|
1200
|
+
ToolDispatchPreparationResult.new(execution: current)
|
|
1201
|
+
end
|
|
1202
|
+
ToolDispatchPreparationReconciliationResult.new(
|
|
1203
|
+
disposition: disposition,
|
|
1204
|
+
preparation_result: preparation_result
|
|
1205
|
+
)
|
|
1206
|
+
end
|
|
1207
|
+
|
|
1208
|
+
def apply_provider_dispatch_preparation_reconciliation_on_event_loop(ready)
|
|
1209
|
+
command = ready.command
|
|
1210
|
+
operation = command.operation
|
|
1211
|
+
state = authoritative_state_for_operation(
|
|
1212
|
+
execution_id: operation.execution_id,
|
|
1213
|
+
fsm_session_id: operation.fsm_session_id,
|
|
1214
|
+
expected_execution_revision: operation.expected_execution_revision,
|
|
1215
|
+
expected_fsm_state: :calling_llm
|
|
1216
|
+
)
|
|
1217
|
+
return unless state
|
|
1218
|
+
|
|
1219
|
+
if ready.error
|
|
1220
|
+
mark_barrier_recovery_required(operation, ready.error)
|
|
1221
|
+
return
|
|
1222
|
+
end
|
|
1223
|
+
|
|
1224
|
+
case ready.result.disposition
|
|
1225
|
+
when :committed
|
|
1226
|
+
apply_confirmed_provider_dispatch_preparation_on_event_loop(
|
|
1227
|
+
operation,
|
|
1228
|
+
ready.result.preparation_result,
|
|
1229
|
+
state
|
|
1230
|
+
)
|
|
1231
|
+
when :not_committed
|
|
1232
|
+
state.invocation.event_sink.post(
|
|
1233
|
+
:llm_setup_failed,
|
|
1234
|
+
translated(command.original_error)
|
|
1235
|
+
)
|
|
1236
|
+
when :conflict
|
|
1237
|
+
mark_barrier_recovery_required(operation, command.original_error)
|
|
1238
|
+
else
|
|
1239
|
+
raise Phronomy::Error,
|
|
1240
|
+
"unknown Provider dispatch reconciliation disposition: " \
|
|
1241
|
+
"#{ready.result.disposition.inspect}"
|
|
1242
|
+
end
|
|
1243
|
+
rescue => error
|
|
1244
|
+
mark_barrier_recovery_required(operation, error)
|
|
1245
|
+
end
|
|
1246
|
+
|
|
1247
|
+
def apply_tool_dispatch_preparation_reconciliation_on_event_loop(ready)
|
|
1248
|
+
command = ready.command
|
|
1249
|
+
operation = command.operation
|
|
1250
|
+
state = authoritative_state_for_operation(
|
|
1251
|
+
execution_id: operation.execution_id,
|
|
1252
|
+
fsm_session_id: operation.fsm_session_id,
|
|
1253
|
+
expected_execution_revision: operation.expected_execution_revision,
|
|
1254
|
+
expected_fsm_state: :dispatching_tools
|
|
1255
|
+
)
|
|
1256
|
+
return unless state
|
|
1257
|
+
|
|
1258
|
+
if ready.error
|
|
1259
|
+
mark_barrier_recovery_required(operation, ready.error)
|
|
1260
|
+
return
|
|
1261
|
+
end
|
|
1262
|
+
|
|
1263
|
+
case ready.result.disposition
|
|
1264
|
+
when :committed
|
|
1265
|
+
apply_confirmed_tool_dispatch_preparation_on_event_loop(
|
|
1266
|
+
operation,
|
|
1267
|
+
ready.result.preparation_result,
|
|
1268
|
+
state
|
|
1269
|
+
)
|
|
1270
|
+
when :not_committed
|
|
1271
|
+
state.invocation.event_sink.post(
|
|
1272
|
+
:tool_setup_failed,
|
|
1273
|
+
translated(command.original_error)
|
|
1274
|
+
)
|
|
1275
|
+
when :conflict
|
|
1276
|
+
mark_barrier_recovery_required(operation, command.original_error)
|
|
1277
|
+
else
|
|
1278
|
+
raise Phronomy::Error,
|
|
1279
|
+
"unknown Tool dispatch reconciliation disposition: " \
|
|
1280
|
+
"#{ready.result.disposition.inspect}"
|
|
1281
|
+
end
|
|
1282
|
+
rescue => error
|
|
1283
|
+
mark_barrier_recovery_required(operation, error)
|
|
1284
|
+
# simplecov:enable
|
|
1285
|
+
end
|
|
1286
|
+
|
|
1287
|
+
# Worker-only Persistence readback for causal-barrier F1 reconciliation.
|
|
1288
|
+
# EventLoop submits one operation-specific ReconciliationCommand and only
|
|
1289
|
+
# applies this result after it returns through the normal control path.
|
|
1290
|
+
def preparation_reconciliation_state(operation, intended_execution)
|
|
1291
|
+
current = @agent.persistence.executions.load(operation.execution_id)
|
|
1292
|
+
if current.execution_revision == intended_execution.execution_revision &&
|
|
1293
|
+
current.to_h == intended_execution.to_h
|
|
1294
|
+
return [:committed, current]
|
|
1295
|
+
end
|
|
1296
|
+
if current.execution_revision == operation.execution.execution_revision &&
|
|
1297
|
+
current.to_h == operation.execution.to_h
|
|
1298
|
+
return [:not_committed, current]
|
|
1299
|
+
end
|
|
1300
|
+
|
|
1301
|
+
[:conflict, current]
|
|
1302
|
+
end
|
|
1303
|
+
|
|
1304
|
+
def known_durable_preparation_failure?(error)
|
|
1305
|
+
error.is_a?(Phronomy::Persistence::ConflictError) ||
|
|
1306
|
+
error.is_a?(Phronomy::Persistence::NotFoundError) ||
|
|
1307
|
+
error.is_a?(Phronomy::Persistence::SerializationError) ||
|
|
1308
|
+
error.is_a?(Phronomy::Persistence::UnsupportedBackendError) ||
|
|
1309
|
+
error.is_a?(ArgumentError) ||
|
|
1310
|
+
error.is_a?(Phronomy::ConfigurationError)
|
|
1311
|
+
end
|
|
1312
|
+
|
|
1313
|
+
def mark_barrier_recovery_required(operation, error)
|
|
1314
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1315
|
+
event_loop.mark_agent_execution_admission(
|
|
1316
|
+
@agent.agent_id,
|
|
1317
|
+
execution_id: operation.execution_id,
|
|
1318
|
+
state: :recovery_required
|
|
1319
|
+
)
|
|
1320
|
+
Phronomy.configuration.logger&.warn(
|
|
1321
|
+
"[Phronomy] causal durable barrier requires recovery for " \
|
|
1322
|
+
"#{operation.execution_id}: #{error.class}: #{error.message}"
|
|
1323
|
+
)
|
|
1324
|
+
end
|
|
1325
|
+
|
|
1326
|
+
# ----------------------------------------------------------------------
|
|
1327
|
+
# Approval resume
|
|
1328
|
+
# ----------------------------------------------------------------------
|
|
1329
|
+
|
|
1330
|
+
def begin_resume_on_event_loop(request)
|
|
1331
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1332
|
+
state = event_loop.agent_execution_state(request.execution_id)
|
|
1333
|
+
if state&.agent&.equal?(@agent) && state&.invocation
|
|
1334
|
+
@recovery_resume_snapshot_mutex ||= Mutex.new
|
|
1335
|
+
snapshot = RecoverySupport.build_tool_batch_snapshot(
|
|
1336
|
+
state.invocation
|
|
1337
|
+
)
|
|
1338
|
+
@recovery_resume_snapshot_mutex.synchronize do
|
|
1339
|
+
@recovery_resume_snapshots ||= {}
|
|
1340
|
+
@recovery_resume_snapshots[request.execution_id.to_s] =
|
|
1341
|
+
snapshot
|
|
1342
|
+
end
|
|
1343
|
+
end
|
|
1344
|
+
|
|
1345
|
+
runtime = Phronomy::Runtime.instance
|
|
1346
|
+
event_loop = runtime.event_loop
|
|
1347
|
+
state = event_loop.agent_execution_state(request.execution_id)
|
|
1348
|
+
unless state
|
|
1349
|
+
fail_task(
|
|
1350
|
+
request.result_task,
|
|
1351
|
+
Phronomy::ExecutionRehydrationRequiredError.new(
|
|
1352
|
+
"no live execution owner for #{request.execution_id}; durable rehydration is required"
|
|
1353
|
+
)
|
|
1354
|
+
)
|
|
1355
|
+
return
|
|
1356
|
+
end
|
|
1357
|
+
|
|
1358
|
+
unless state.agent.equal?(@agent) && state.execution.status == :suspended
|
|
1359
|
+
fail_task(
|
|
1360
|
+
request.result_task,
|
|
1361
|
+
ArgumentError.new(
|
|
1362
|
+
"execution is not a suspended execution of this agent: #{request.execution_id}"
|
|
1363
|
+
)
|
|
1364
|
+
)
|
|
1365
|
+
return
|
|
1366
|
+
end
|
|
1367
|
+
|
|
1368
|
+
approval = state.execution.approval_request || {}
|
|
1369
|
+
current_request_id = approval["id"] || approval[:id]
|
|
1370
|
+
unless current_request_id.to_s == request.approval_request_id
|
|
1371
|
+
fail_task(
|
|
1372
|
+
request.result_task,
|
|
1373
|
+
ArgumentError.new(
|
|
1374
|
+
"approval request does not match execution #{request.execution_id}"
|
|
1375
|
+
)
|
|
1376
|
+
)
|
|
1377
|
+
return
|
|
1378
|
+
end
|
|
1379
|
+
|
|
1380
|
+
operation = ResumeCommitCommand.new(
|
|
1381
|
+
execution_id: state.execution_id,
|
|
1382
|
+
expected_execution_revision: state.execution.execution_revision,
|
|
1383
|
+
root: @agent.agent_root,
|
|
1384
|
+
execution: state.execution,
|
|
1385
|
+
approval_request_id: request.approval_request_id,
|
|
1386
|
+
approved: request.approved
|
|
1387
|
+
)
|
|
1388
|
+
resume_transition_started = false
|
|
1389
|
+
submitted = false
|
|
1390
|
+
event_loop.mark_agent_execution_admission(
|
|
1391
|
+
@agent.agent_id,
|
|
1392
|
+
execution_id: state.execution_id,
|
|
1393
|
+
state: :resuming
|
|
1394
|
+
)
|
|
1395
|
+
resume_transition_started = true
|
|
1396
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
1397
|
+
perform_resume_commit(operation)
|
|
1398
|
+
end
|
|
1399
|
+
submitted = true
|
|
1400
|
+
task.on_complete do |result, error|
|
|
1401
|
+
ready = ResumeCommitReady.new(
|
|
1402
|
+
coordinator: self,
|
|
1403
|
+
request: request,
|
|
1404
|
+
operation: operation,
|
|
1405
|
+
result: result,
|
|
1406
|
+
error: error
|
|
1407
|
+
)
|
|
1408
|
+
fail_task(request.result_task, runtime_rejected_error(:resume_commit)) unless
|
|
1409
|
+
post_control(runtime, ready)
|
|
1410
|
+
end
|
|
1411
|
+
rescue => error
|
|
1412
|
+
if resume_transition_started
|
|
1413
|
+
event_loop.mark_agent_execution_admission(
|
|
1414
|
+
@agent.agent_id,
|
|
1415
|
+
execution_id: state.execution_id,
|
|
1416
|
+
state: submitted ? :recovery_required : :suspended
|
|
1417
|
+
)
|
|
1418
|
+
end
|
|
1419
|
+
fail_task(request.result_task, translated(error))
|
|
1420
|
+
raise unless resume_transition_started
|
|
1421
|
+
end
|
|
1422
|
+
|
|
1423
|
+
def perform_resume_commit(operation)
|
|
1424
|
+
snapshot = nil
|
|
1425
|
+
@recovery_resume_snapshot_mutex&.synchronize do
|
|
1426
|
+
snapshot = @recovery_resume_snapshots&.delete(
|
|
1427
|
+
operation.execution_id.to_s
|
|
1428
|
+
)
|
|
1429
|
+
end
|
|
1430
|
+
|
|
1431
|
+
if snapshot
|
|
1432
|
+
staged_execution = RecoverySupport.with_recovery_metadata(
|
|
1433
|
+
operation.execution,
|
|
1434
|
+
RecoverySupport::TOOL_BATCH_METADATA_KEY => snapshot
|
|
1435
|
+
)
|
|
1436
|
+
operation = operation.class.new(
|
|
1437
|
+
**operation.to_h.merge(execution: staged_execution)
|
|
1438
|
+
)
|
|
1439
|
+
end
|
|
1440
|
+
|
|
1441
|
+
current = operation.execution
|
|
1442
|
+
current_root = operation.root
|
|
1443
|
+
request = current.approval_request || {}
|
|
1444
|
+
request_id = request["id"] || request[:id]
|
|
1445
|
+
unless request_id.to_s == operation.approval_request_id
|
|
1446
|
+
raise ArgumentError,
|
|
1447
|
+
"approval request does not match execution #{current.execution_id}"
|
|
1448
|
+
end
|
|
1449
|
+
|
|
1450
|
+
updated = next_root = nil
|
|
1451
|
+
@agent.persistence.transaction do |tx|
|
|
1452
|
+
decision_ref = tx.contents.put_json(
|
|
1453
|
+
"approval_request_id" => request_id.to_s,
|
|
1454
|
+
"approved" => operation.approved
|
|
1455
|
+
)
|
|
1456
|
+
decision_record = JournalRecord.new(
|
|
1457
|
+
agent_id: @agent.agent_id,
|
|
1458
|
+
execution_id: current.execution_id,
|
|
1459
|
+
kind: :approval_decided,
|
|
1460
|
+
channel: :approval,
|
|
1461
|
+
content_ref: decision_ref,
|
|
1462
|
+
context_generation: current_root.transcript_generation,
|
|
1463
|
+
context_candidate: false
|
|
1464
|
+
)
|
|
1465
|
+
updated = current.with(
|
|
1466
|
+
status: :active,
|
|
1467
|
+
phase: :resuming,
|
|
1468
|
+
working_records: current.working_records + [decision_record],
|
|
1469
|
+
approval_request: request.merge("approved" => operation.approved)
|
|
1470
|
+
)
|
|
1471
|
+
tx.executions.save(
|
|
1472
|
+
current.execution_id,
|
|
1473
|
+
expected_revision: current.execution_revision,
|
|
1474
|
+
execution: updated
|
|
1475
|
+
)
|
|
1476
|
+
next_root = current_root.with(
|
|
1477
|
+
agent_revision: current_root.agent_revision + 1,
|
|
1478
|
+
lifecycle_status: :active
|
|
1479
|
+
)
|
|
1480
|
+
tx.agents.save(
|
|
1481
|
+
current_root.agent_id,
|
|
1482
|
+
expected_revision: current_root.agent_revision,
|
|
1483
|
+
root: next_root
|
|
1484
|
+
)
|
|
1485
|
+
end
|
|
1486
|
+
ResumeCommitResult.new(execution: updated, root: next_root)
|
|
1487
|
+
end
|
|
1488
|
+
|
|
1489
|
+
def apply_resume_commit_on_event_loop(ready)
|
|
1490
|
+
request = ready.request
|
|
1491
|
+
operation = ready.operation
|
|
1492
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1493
|
+
state = event_loop.agent_execution_state(operation.execution_id)
|
|
1494
|
+
unless state && state.agent.equal?(@agent) &&
|
|
1495
|
+
state.execution.execution_revision == operation.expected_execution_revision &&
|
|
1496
|
+
state.execution.status == :suspended
|
|
1497
|
+
fail_task(
|
|
1498
|
+
request.result_task,
|
|
1499
|
+
Phronomy::Error.new(
|
|
1500
|
+
"stale approval resume result for #{operation.execution_id}"
|
|
1501
|
+
)
|
|
1502
|
+
)
|
|
1503
|
+
return
|
|
1504
|
+
end
|
|
1505
|
+
|
|
1506
|
+
if ready.error
|
|
1507
|
+
event_loop.mark_agent_execution_admission(
|
|
1508
|
+
@agent.agent_id,
|
|
1509
|
+
execution_id: operation.execution_id,
|
|
1510
|
+
state: :recovery_required
|
|
1511
|
+
)
|
|
1512
|
+
fail_task(request.result_task, translated(ready.error))
|
|
1513
|
+
return
|
|
1514
|
+
end
|
|
1515
|
+
|
|
1516
|
+
apply_agent_live_state(root: ready.result.root, appended_records: [])
|
|
1517
|
+
event_loop.mark_agent_execution_admission(
|
|
1518
|
+
@agent.agent_id,
|
|
1519
|
+
execution_id: operation.execution_id,
|
|
1520
|
+
state: :executing
|
|
1521
|
+
)
|
|
1522
|
+
event_loop.replace_agent_execution(
|
|
1523
|
+
operation.execution_id,
|
|
1524
|
+
execution: ready.result.execution
|
|
1525
|
+
)
|
|
1526
|
+
event_loop.register_agent_completion_waiter(
|
|
1527
|
+
operation.execution_id,
|
|
1528
|
+
request.result_task
|
|
1529
|
+
)
|
|
1530
|
+
trace_config = state.invocation ?
|
|
1531
|
+
state.invocation.config.merge(request.config) : request.config
|
|
1532
|
+
trace_mode = state.invocation&.mode ||
|
|
1533
|
+
(
|
|
1534
|
+
ready.result.execution.metadata[
|
|
1535
|
+
RecoverySupport::INVOCATION_MODE_KEY
|
|
1536
|
+
] || "invoke"
|
|
1537
|
+
).to_sym
|
|
1538
|
+
Phronomy::Tracing::Automatic.observe_task(
|
|
1539
|
+
request.result_task,
|
|
1540
|
+
"agent.execution",
|
|
1541
|
+
agent_id: @agent.agent_id,
|
|
1542
|
+
execution_id: operation.execution_id,
|
|
1543
|
+
mode: trace_mode,
|
|
1544
|
+
**@agent.send(:_build_caller_meta, trace_config)
|
|
1545
|
+
)
|
|
1546
|
+
start_resume_on_event_loop(
|
|
1547
|
+
operation.execution_id,
|
|
1548
|
+
request.result_task,
|
|
1549
|
+
approved: request.approved,
|
|
1550
|
+
config: request.config
|
|
1551
|
+
)
|
|
1552
|
+
rescue => error
|
|
1553
|
+
state = event_loop&.agent_execution_state(operation.execution_id) if event_loop&.current?
|
|
1554
|
+
if state
|
|
1555
|
+
begin_terminal_commit_on_event_loop(
|
|
1556
|
+
state,
|
|
1557
|
+
request.result_task,
|
|
1558
|
+
state.invocation,
|
|
1559
|
+
error,
|
|
1560
|
+
fsm_session_id: state.fsm_session_id
|
|
1561
|
+
)
|
|
1562
|
+
else
|
|
1563
|
+
fail_task(request.result_task, translated(error))
|
|
1564
|
+
end
|
|
1565
|
+
end
|
|
1566
|
+
|
|
1567
|
+
def start_resume_on_event_loop(execution_id, result_task, approved:, config:)
|
|
1568
|
+
runtime = Phronomy::Runtime.instance
|
|
1569
|
+
event_loop = runtime.event_loop
|
|
1570
|
+
state = event_loop.agent_execution_state(execution_id)
|
|
1571
|
+
invocation = state.invocation
|
|
1572
|
+
invocation.merge_config!(config)
|
|
1573
|
+
invocation.begin_approval_resume!(approved: approved)
|
|
1574
|
+
parent_session = Agent::AgentInvocationSessionBuilder.build_for_resume(
|
|
1575
|
+
agent_invocation: invocation,
|
|
1576
|
+
resume_event: :resume,
|
|
1577
|
+
resume_phase: :suspended,
|
|
1578
|
+
runtime: runtime
|
|
1579
|
+
)
|
|
1580
|
+
event_loop.replace_agent_execution(
|
|
1581
|
+
execution_id,
|
|
1582
|
+
invocation: invocation,
|
|
1583
|
+
fsm_session_id: parent_session.id
|
|
1584
|
+
)
|
|
1585
|
+
source_task = Phronomy::Task.deferred(name: "#{result_task.name}-source")
|
|
1586
|
+
source_task.on_complete do |completed, error|
|
|
1587
|
+
finish_on_event_loop(
|
|
1588
|
+
execution_id,
|
|
1589
|
+
result_task,
|
|
1590
|
+
completed || parent_session.context,
|
|
1591
|
+
error,
|
|
1592
|
+
fsm_session_id: parent_session.id
|
|
1593
|
+
)
|
|
1594
|
+
end
|
|
1595
|
+
event_loop.register(parent_session, completion: source_task)
|
|
1596
|
+
invocation.tool_invocations.each do |child|
|
|
1597
|
+
child_session = if child.awaiting_approval?
|
|
1598
|
+
Agent::ToolInvocationSessionBuilder.build_for_resume(
|
|
1599
|
+
tool_invocation: child,
|
|
1600
|
+
parent_event_sink: parent_session.event_sink,
|
|
1601
|
+
resume_event: approved ? :approve : :reject,
|
|
1602
|
+
resume_phase: :awaiting_approval,
|
|
1603
|
+
runtime: runtime
|
|
1604
|
+
)
|
|
1605
|
+
elsif !approved && child.authorized?
|
|
1606
|
+
Agent::ToolInvocationSessionBuilder.build_for_resume(
|
|
1607
|
+
tool_invocation: child,
|
|
1608
|
+
parent_event_sink: parent_session.event_sink,
|
|
1609
|
+
resume_event: :cancel,
|
|
1610
|
+
resume_phase: :authorized,
|
|
1611
|
+
runtime: runtime
|
|
1612
|
+
)
|
|
1613
|
+
end
|
|
1614
|
+
register_child(event_loop, child, child_session, parent_session.event_sink) if child_session
|
|
1615
|
+
end
|
|
1616
|
+
end
|
|
1617
|
+
|
|
1618
|
+
def register_child(event_loop, child, session, parent_event_sink)
|
|
1619
|
+
completion = Phronomy::Task.deferred(name: "tool-session:#{child.id}")
|
|
1620
|
+
completion.on_complete do |_result, error|
|
|
1621
|
+
next unless error
|
|
1622
|
+
|
|
1623
|
+
child.mark_framework_failed!(error)
|
|
1624
|
+
parent_event_sink.post(:tool_failed, {tool_invocation_id: child.id})
|
|
1625
|
+
end
|
|
1626
|
+
event_loop.register(session, completion: completion)
|
|
1627
|
+
end
|
|
1628
|
+
|
|
1629
|
+
# ----------------------------------------------------------------------
|
|
1630
|
+
# Terminal durable barrier
|
|
1631
|
+
# ----------------------------------------------------------------------
|
|
1632
|
+
|
|
1633
|
+
def finish_on_event_loop(execution_id, result_task, invocation, error, fsm_session_id:)
|
|
1634
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1635
|
+
assert_event_loop!(event_loop)
|
|
1636
|
+
state = event_loop.agent_execution_state(execution_id)
|
|
1637
|
+
return fail_task(result_task, runtime_rejected_error(:terminal)) unless state
|
|
501
1638
|
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
1639
|
+
unless state.fsm_session_id.to_s == fsm_session_id.to_s &&
|
|
1640
|
+
state.invocation.equal?(invocation)
|
|
1641
|
+
return fail_task(
|
|
1642
|
+
result_task,
|
|
1643
|
+
Phronomy::Error.new("stale Agent terminal callback for #{execution_id}")
|
|
1644
|
+
)
|
|
1645
|
+
end
|
|
1646
|
+
|
|
1647
|
+
terminal_error = error || invocation&.error
|
|
1648
|
+
unless event_loop.agent_execution_quiescent?(execution_id)
|
|
1649
|
+
wait_state = quiescence_sensitive_terminal_error?(terminal_error) ?
|
|
1650
|
+
:cancelling : :terminalizing
|
|
1651
|
+
event_loop.mark_agent_execution_admission(
|
|
1652
|
+
@agent.agent_id,
|
|
1653
|
+
execution_id: execution_id,
|
|
1654
|
+
state: wait_state
|
|
1655
|
+
)
|
|
1656
|
+
event_loop.defer_agent_terminal_until_quiescent(
|
|
1657
|
+
execution_id,
|
|
1658
|
+
DeferredTerminalCommand.new(
|
|
1659
|
+
coordinator: self,
|
|
1660
|
+
execution_id: execution_id.to_s.freeze,
|
|
1661
|
+
result_task: result_task,
|
|
1662
|
+
invocation: invocation,
|
|
1663
|
+
source_error: error,
|
|
1664
|
+
fsm_session_id: fsm_session_id.to_s.freeze
|
|
1665
|
+
)
|
|
1666
|
+
)
|
|
1667
|
+
return
|
|
1668
|
+
end
|
|
1669
|
+
|
|
1670
|
+
begin_terminal_commit_on_event_loop(
|
|
1671
|
+
state,
|
|
1672
|
+
result_task,
|
|
1673
|
+
invocation,
|
|
1674
|
+
error,
|
|
1675
|
+
fsm_session_id: fsm_session_id
|
|
1676
|
+
)
|
|
1677
|
+
end
|
|
1678
|
+
|
|
1679
|
+
def resume_deferred_terminal_on_event_loop(command)
|
|
1680
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
1681
|
+
state = event_loop.agent_execution_state(command.execution_id)
|
|
1682
|
+
unless state && state.agent.equal?(@agent) &&
|
|
1683
|
+
state.fsm_session_id.to_s == command.fsm_session_id.to_s &&
|
|
1684
|
+
state.invocation.equal?(command.invocation)
|
|
1685
|
+
Phronomy.configuration.logger&.warn(
|
|
1686
|
+
"[Phronomy] Dropped stale deferred terminal for #{command.execution_id}"
|
|
508
1687
|
)
|
|
509
|
-
return
|
|
1688
|
+
return
|
|
510
1689
|
end
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
1690
|
+
|
|
1691
|
+
unless event_loop.agent_execution_quiescent?(command.execution_id)
|
|
1692
|
+
raise Phronomy::Error,
|
|
1693
|
+
"deferred terminal resumed before quiescence for #{command.execution_id}"
|
|
514
1694
|
end
|
|
1695
|
+
|
|
1696
|
+
begin_terminal_commit_on_event_loop(
|
|
1697
|
+
state,
|
|
1698
|
+
command.result_task,
|
|
1699
|
+
command.invocation,
|
|
1700
|
+
command.source_error,
|
|
1701
|
+
fsm_session_id: command.fsm_session_id
|
|
1702
|
+
)
|
|
1703
|
+
end
|
|
1704
|
+
|
|
1705
|
+
def quiescence_sensitive_terminal_error?(error)
|
|
1706
|
+
error.is_a?(Phronomy::CancellationError) ||
|
|
1707
|
+
error.is_a?(Phronomy::TimeoutError)
|
|
1708
|
+
end
|
|
1709
|
+
|
|
1710
|
+
def begin_terminal_commit_on_event_loop(
|
|
1711
|
+
state,
|
|
1712
|
+
result_task,
|
|
1713
|
+
invocation,
|
|
1714
|
+
source_error,
|
|
1715
|
+
fsm_session_id:
|
|
1716
|
+
)
|
|
515
1717
|
if invocation&.phase == :suspended
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
1718
|
+
snapshot = RecoverySupport.build_tool_batch_snapshot(invocation)
|
|
1719
|
+
staged_execution = RecoverySupport.with_recovery_metadata(
|
|
1720
|
+
state.execution,
|
|
1721
|
+
RecoverySupport::TOOL_BATCH_METADATA_KEY => snapshot
|
|
1722
|
+
)
|
|
1723
|
+
state = state.class.new(
|
|
1724
|
+
**state.to_h.merge(execution: staged_execution)
|
|
1725
|
+
)
|
|
1726
|
+
end
|
|
1727
|
+
|
|
1728
|
+
operation = build_terminal_operation(
|
|
1729
|
+
execution: state.execution,
|
|
1730
|
+
root: @agent.agent_root,
|
|
1731
|
+
journal_records: @agent.send(:_journal_records_snapshot),
|
|
1732
|
+
invocation: invocation,
|
|
1733
|
+
source_error: source_error,
|
|
1734
|
+
fsm_session_id: fsm_session_id,
|
|
1735
|
+
state_required: true
|
|
1736
|
+
)
|
|
1737
|
+
delivery = TerminalDelivery.new(
|
|
1738
|
+
result_task: result_task,
|
|
1739
|
+
application_listener: invocation&.event_listener,
|
|
1740
|
+
approval_listener: invocation&.approval_listener,
|
|
1741
|
+
handoff_request: invocation&.handoff_request,
|
|
1742
|
+
handoff_manifest: state.runtime_projection&.manifest
|
|
1743
|
+
)
|
|
1744
|
+
submit_terminal_operation(operation, delivery)
|
|
1745
|
+
end
|
|
1746
|
+
|
|
1747
|
+
def begin_terminal_without_session_on_event_loop(request:, prepared:, error:)
|
|
1748
|
+
return deliver_start_failure_on_event_loop(request, translated(error)) unless prepared&.execution
|
|
1749
|
+
|
|
1750
|
+
operation = build_terminal_operation(
|
|
1751
|
+
execution: prepared.execution,
|
|
1752
|
+
root: prepared.root,
|
|
1753
|
+
journal_records: @agent.send(:_journal_records_snapshot),
|
|
1754
|
+
invocation: nil,
|
|
1755
|
+
source_error: error,
|
|
1756
|
+
fsm_session_id: nil,
|
|
1757
|
+
state_required: false
|
|
1758
|
+
)
|
|
1759
|
+
delivery = TerminalDelivery.new(
|
|
1760
|
+
result_task: request.result_task,
|
|
1761
|
+
application_listener: request.on_event,
|
|
1762
|
+
approval_listener: request.approval_listener,
|
|
1763
|
+
handoff_request: nil,
|
|
1764
|
+
handoff_manifest: nil
|
|
1765
|
+
)
|
|
1766
|
+
submit_terminal_operation(operation, delivery)
|
|
1767
|
+
end
|
|
1768
|
+
|
|
1769
|
+
def build_terminal_operation(
|
|
1770
|
+
execution:,
|
|
1771
|
+
root:,
|
|
1772
|
+
journal_records:,
|
|
1773
|
+
invocation:,
|
|
1774
|
+
source_error:,
|
|
1775
|
+
fsm_session_id:,
|
|
1776
|
+
state_required:
|
|
1777
|
+
)
|
|
1778
|
+
TerminalCommitCommand.new(
|
|
1779
|
+
execution_id: execution.execution_id.to_s.freeze,
|
|
1780
|
+
fsm_session_id: fsm_session_id&.to_s&.freeze,
|
|
1781
|
+
expected_execution_revision: execution.execution_revision,
|
|
1782
|
+
root: root,
|
|
1783
|
+
journal_records: journal_records,
|
|
1784
|
+
execution: execution,
|
|
1785
|
+
runtime_snapshot: invocation ? invocation.runtime_snapshot : empty_runtime_snapshot,
|
|
1786
|
+
terminal_view: terminal_view(invocation, source_error),
|
|
1787
|
+
state_required: state_required
|
|
1788
|
+
)
|
|
1789
|
+
end
|
|
1790
|
+
|
|
1791
|
+
def submit_terminal_operation(operation, delivery)
|
|
1792
|
+
runtime = Phronomy::Runtime.instance
|
|
1793
|
+
event_loop = runtime.event_loop
|
|
1794
|
+
terminal_transition_started = false
|
|
1795
|
+
event_loop.mark_agent_execution_admission(
|
|
1796
|
+
@agent.agent_id,
|
|
1797
|
+
execution_id: operation.execution_id,
|
|
1798
|
+
state: :terminalizing
|
|
1799
|
+
)
|
|
1800
|
+
terminal_transition_started = true
|
|
1801
|
+
event_loop.register_agent_completion_waiter(
|
|
1802
|
+
operation.execution_id,
|
|
1803
|
+
delivery.result_task
|
|
1804
|
+
)
|
|
1805
|
+
task = runtime.offload.submit(on_full: :raise) do
|
|
1806
|
+
# Only the operation-specific immutable durable snapshot crosses the
|
|
1807
|
+
# worker boundary. Task/listener delivery state stays outside it.
|
|
1808
|
+
compute_terminal(operation)
|
|
520
1809
|
end
|
|
521
|
-
|
|
522
|
-
|
|
1810
|
+
task.on_complete do |outcome, error|
|
|
1811
|
+
ready = TerminalCommitReady.new(
|
|
1812
|
+
coordinator: self,
|
|
1813
|
+
operation: operation,
|
|
1814
|
+
delivery: delivery,
|
|
1815
|
+
outcome: outcome,
|
|
1816
|
+
error: error
|
|
1817
|
+
)
|
|
1818
|
+
fail_task(delivery.result_task, runtime_rejected_error(:terminal_apply)) unless
|
|
1819
|
+
post_control(runtime, ready)
|
|
1820
|
+
end
|
|
1821
|
+
rescue => error
|
|
1822
|
+
if terminal_transition_started
|
|
1823
|
+
event_loop.mark_agent_execution_admission(
|
|
1824
|
+
@agent.agent_id,
|
|
1825
|
+
execution_id: operation.execution_id,
|
|
1826
|
+
state: :recovery_required
|
|
1827
|
+
)
|
|
1828
|
+
Phronomy.configuration.logger&.warn(
|
|
1829
|
+
"[Phronomy] Agent terminal transition requires recovery for " \
|
|
1830
|
+
"#{operation.execution_id}: #{error.class}: #{error.message}"
|
|
1831
|
+
)
|
|
1832
|
+
return
|
|
1833
|
+
end
|
|
1834
|
+
|
|
1835
|
+
fail_task(delivery.result_task, translated(error))
|
|
1836
|
+
raise
|
|
1837
|
+
end
|
|
1838
|
+
|
|
1839
|
+
def terminal_view(invocation, source_error)
|
|
1840
|
+
TerminalView.new(
|
|
1841
|
+
phase: invocation&.phase,
|
|
1842
|
+
output: invocation&.output,
|
|
1843
|
+
usage: invocation&.usage,
|
|
1844
|
+
approval_request: invocation&.approval_request,
|
|
1845
|
+
rejected: invocation ? !!invocation.rejected : false,
|
|
1846
|
+
input_blocked: invocation ? invocation.input_blocked? : false,
|
|
1847
|
+
output_blocked: invocation ? invocation.output_blocked? : false,
|
|
1848
|
+
block_error: invocation&.block_error,
|
|
1849
|
+
invocation_error: invocation&.error,
|
|
1850
|
+
handoff: handoff_terminal_view(invocation&.handoff_request),
|
|
1851
|
+
callback_failure: invocation&.callback_failure,
|
|
1852
|
+
source_error: source_error
|
|
1853
|
+
)
|
|
1854
|
+
end
|
|
523
1855
|
|
|
524
|
-
|
|
1856
|
+
def handoff_terminal_view(request)
|
|
1857
|
+
return unless request
|
|
1858
|
+
|
|
1859
|
+
HandoffTerminalView.new(
|
|
1860
|
+
target_agent_id: request.handoff.target_agent.agent_id.to_s.freeze,
|
|
1861
|
+
responsibility: request.responsibility.to_s.freeze,
|
|
1862
|
+
selection_intent: request.selection_intent.to_h do |category, included|
|
|
1863
|
+
[category.to_s.freeze, !!included]
|
|
1864
|
+
end.freeze,
|
|
1865
|
+
llm_call_id: request.llm_call_id&.to_s&.freeze,
|
|
1866
|
+
tool_call_id: request.tool_call_id&.to_s&.freeze
|
|
1867
|
+
)
|
|
1868
|
+
end
|
|
1869
|
+
|
|
1870
|
+
def empty_runtime_snapshot
|
|
1871
|
+
{llm_results: [].freeze, runtime_events: [].freeze, active_call: nil}.freeze
|
|
1872
|
+
end
|
|
1873
|
+
|
|
1874
|
+
def compute_terminal(operation)
|
|
1875
|
+
view = operation.terminal_view
|
|
1876
|
+
if view.callback_failure
|
|
1877
|
+
return commit_failed_outcome(
|
|
1878
|
+
operation,
|
|
1879
|
+
view.callback_failure.to_stream_callback_error
|
|
1880
|
+
)
|
|
1881
|
+
end
|
|
1882
|
+
return commit_failed_outcome(operation, view.source_error) if view.source_error
|
|
1883
|
+
if view.phase == :suspended
|
|
1884
|
+
return commit_suspended(operation)
|
|
1885
|
+
end
|
|
1886
|
+
raise view.block_error if view.input_blocked || view.output_blocked
|
|
1887
|
+
raise view.invocation_error if view.invocation_error
|
|
1888
|
+
|
|
1889
|
+
commit_completed(operation)
|
|
525
1890
|
rescue => caught
|
|
526
|
-
|
|
527
|
-
{type: :failed, error: terminal.fetch(:error)}
|
|
1891
|
+
commit_failed_outcome(operation, caught)
|
|
528
1892
|
end
|
|
529
1893
|
|
|
530
|
-
def commit_suspended(
|
|
531
|
-
current =
|
|
532
|
-
root =
|
|
533
|
-
request =
|
|
534
|
-
runtime_snapshot =
|
|
1894
|
+
def commit_suspended(operation)
|
|
1895
|
+
current = operation.execution
|
|
1896
|
+
root = operation.root
|
|
1897
|
+
request = operation.terminal_view.approval_request
|
|
1898
|
+
runtime_snapshot = operation.runtime_snapshot
|
|
535
1899
|
suspended = next_root = nil
|
|
536
1900
|
|
|
537
1901
|
@agent.persistence.transaction do |tx|
|
|
538
1902
|
encoded_records, call_records = encode_runtime_records(
|
|
539
|
-
|
|
1903
|
+
current,
|
|
540
1904
|
tx: tx,
|
|
541
1905
|
snapshot: runtime_snapshot,
|
|
542
1906
|
context_candidate: true,
|
|
@@ -549,7 +1913,6 @@ module Phronomy
|
|
|
549
1913
|
kind: :approval_required,
|
|
550
1914
|
channel: :approval,
|
|
551
1915
|
content_ref: request_ref,
|
|
552
|
-
correlation_id: request.id.to_s,
|
|
553
1916
|
context_generation: root.transcript_generation,
|
|
554
1917
|
context_candidate: false
|
|
555
1918
|
)
|
|
@@ -575,31 +1938,37 @@ module Phronomy
|
|
|
575
1938
|
root: next_root
|
|
576
1939
|
)
|
|
577
1940
|
end
|
|
578
|
-
|
|
579
|
-
activation.replace_execution(suspended)
|
|
580
|
-
@agent.__replace_root(next_root)
|
|
581
|
-
dispatch_approval_listener(invocation, request)
|
|
582
|
-
result_base(suspended, next_root).merge(
|
|
1941
|
+
result = result_base(suspended, next_root).merge(
|
|
583
1942
|
suspended: true,
|
|
584
1943
|
approval_request: request
|
|
585
1944
|
)
|
|
1945
|
+
TerminalOutcome.new(
|
|
1946
|
+
type: :suspended,
|
|
1947
|
+
execution: suspended,
|
|
1948
|
+
root: next_root,
|
|
1949
|
+
appended_records: [].freeze,
|
|
1950
|
+
result: result.freeze,
|
|
1951
|
+
error: nil,
|
|
1952
|
+
approval_request: request
|
|
1953
|
+
)
|
|
586
1954
|
end
|
|
587
1955
|
|
|
588
|
-
def commit_completed(
|
|
589
|
-
current =
|
|
590
|
-
root =
|
|
591
|
-
|
|
592
|
-
|
|
1956
|
+
def commit_completed(operation)
|
|
1957
|
+
current = operation.execution
|
|
1958
|
+
root = operation.root
|
|
1959
|
+
view = operation.terminal_view
|
|
1960
|
+
runtime_snapshot = operation.runtime_snapshot
|
|
1961
|
+
completed = next_root = appended = messages = nil
|
|
593
1962
|
|
|
594
1963
|
@agent.persistence.transaction do |tx|
|
|
595
1964
|
encoded_records, call_records = encode_runtime_records(
|
|
596
|
-
|
|
1965
|
+
current,
|
|
597
1966
|
tx: tx,
|
|
598
1967
|
snapshot: runtime_snapshot,
|
|
599
1968
|
context_candidate: true,
|
|
600
1969
|
agent_root: root
|
|
601
1970
|
)
|
|
602
|
-
output_ref = tx.contents.put_text(
|
|
1971
|
+
output_ref = tx.contents.put_text(view.output.to_s)
|
|
603
1972
|
final_output_record = JournalRecord.new(
|
|
604
1973
|
agent_id: @agent.agent_id,
|
|
605
1974
|
execution_id: current.execution_id,
|
|
@@ -613,7 +1982,7 @@ module Phronomy
|
|
|
613
1982
|
completed_record = JournalRecord.new(
|
|
614
1983
|
agent_id: @agent.agent_id,
|
|
615
1984
|
execution_id: current.execution_id,
|
|
616
|
-
kind:
|
|
1985
|
+
kind: view.rejected ? :execution_rejected : :execution_completed,
|
|
617
1986
|
channel: :audit,
|
|
618
1987
|
content_ref: output_ref,
|
|
619
1988
|
context_generation: root.transcript_generation,
|
|
@@ -627,13 +1996,13 @@ module Phronomy
|
|
|
627
1996
|
records: all_records
|
|
628
1997
|
)
|
|
629
1998
|
completed = current.with(
|
|
630
|
-
status:
|
|
1999
|
+
status: view.rejected ? :rejected : :completed,
|
|
631
2000
|
phase: :completed,
|
|
632
2001
|
working_records: [],
|
|
633
2002
|
llm_calls: current.llm_calls + call_records,
|
|
634
2003
|
approval_request: nil,
|
|
635
2004
|
result_ref: output_ref,
|
|
636
|
-
terminal_reason:
|
|
2005
|
+
terminal_reason: view.rejected ? "rejected" : "completed"
|
|
637
2006
|
)
|
|
638
2007
|
tx.executions.save(
|
|
639
2008
|
current.execution_id,
|
|
@@ -652,29 +2021,45 @@ module Phronomy
|
|
|
652
2021
|
expected_revision: root.agent_revision,
|
|
653
2022
|
root: next_root
|
|
654
2023
|
)
|
|
2024
|
+
|
|
2025
|
+
# Materialize the caller-facing transcript through the transaction
|
|
2026
|
+
# view before commit. If materialization fails, the terminal durable
|
|
2027
|
+
# transition rolls back instead of committing and then attempting a
|
|
2028
|
+
# second terminal transition from a stale execution revision.
|
|
2029
|
+
records_after_commit = operation.journal_records + Array(appended)
|
|
2030
|
+
messages = transcript_messages(
|
|
2031
|
+
next_root,
|
|
2032
|
+
records_after_commit,
|
|
2033
|
+
persistence: tx
|
|
2034
|
+
)
|
|
655
2035
|
end
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
output: invocation.output,
|
|
662
|
-
rejected: invocation.rejected || nil,
|
|
663
|
-
usage: invocation.usage,
|
|
664
|
-
messages: transcript_messages(next_root)
|
|
2036
|
+
result = result_base(completed, next_root).merge(
|
|
2037
|
+
output: view.output,
|
|
2038
|
+
rejected: view.rejected || nil,
|
|
2039
|
+
usage: view.usage,
|
|
2040
|
+
messages: messages
|
|
665
2041
|
).compact
|
|
2042
|
+
TerminalOutcome.new(
|
|
2043
|
+
type: :completed,
|
|
2044
|
+
execution: completed,
|
|
2045
|
+
root: next_root,
|
|
2046
|
+
appended_records: Array(appended).freeze,
|
|
2047
|
+
result: result.freeze,
|
|
2048
|
+
error: nil,
|
|
2049
|
+
approval_request: nil
|
|
2050
|
+
)
|
|
666
2051
|
end
|
|
667
2052
|
|
|
668
|
-
def
|
|
2053
|
+
def commit_failed_outcome(operation, error)
|
|
669
2054
|
translated_error = translated(error)
|
|
670
|
-
current =
|
|
671
|
-
root =
|
|
672
|
-
runtime_snapshot =
|
|
2055
|
+
current = operation.execution
|
|
2056
|
+
root = operation.root
|
|
2057
|
+
runtime_snapshot = operation.runtime_snapshot
|
|
673
2058
|
failed = next_root = appended = nil
|
|
674
2059
|
|
|
675
2060
|
@agent.persistence.transaction do |tx|
|
|
676
2061
|
encoded_records, call_records = encode_runtime_records(
|
|
677
|
-
|
|
2062
|
+
current,
|
|
678
2063
|
tx: tx,
|
|
679
2064
|
snapshot: runtime_snapshot,
|
|
680
2065
|
context_candidate: false,
|
|
@@ -728,21 +2113,140 @@ module Phronomy
|
|
|
728
2113
|
root: next_root
|
|
729
2114
|
)
|
|
730
2115
|
end
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
2116
|
+
TerminalOutcome.new(
|
|
2117
|
+
type: :failed,
|
|
2118
|
+
execution: failed,
|
|
2119
|
+
root: next_root,
|
|
2120
|
+
appended_records: Array(appended).freeze,
|
|
2121
|
+
result: nil,
|
|
2122
|
+
error: translated_error,
|
|
2123
|
+
approval_request: nil
|
|
2124
|
+
)
|
|
2125
|
+
end
|
|
2126
|
+
|
|
2127
|
+
def apply_terminal_commit_on_event_loop(ready)
|
|
2128
|
+
operation = ready.operation
|
|
2129
|
+
delivery = ready.delivery
|
|
2130
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
2131
|
+
state = event_loop.agent_execution_state(operation.execution_id)
|
|
2132
|
+
|
|
2133
|
+
if operation.state_required
|
|
2134
|
+
unless state && state.agent.equal?(@agent) &&
|
|
2135
|
+
state.execution.execution_revision == operation.expected_execution_revision &&
|
|
2136
|
+
state.fsm_session_id.to_s == operation.fsm_session_id.to_s
|
|
2137
|
+
Phronomy.configuration.logger&.warn(
|
|
2138
|
+
"[Phronomy] Dropped stale Agent terminal result for #{operation.execution_id}"
|
|
2139
|
+
)
|
|
2140
|
+
return
|
|
2141
|
+
end
|
|
2142
|
+
end
|
|
2143
|
+
|
|
2144
|
+
if ready.error
|
|
2145
|
+
event_loop.mark_agent_execution_admission(
|
|
2146
|
+
@agent.agent_id,
|
|
2147
|
+
execution_id: operation.execution_id,
|
|
2148
|
+
state: :recovery_required
|
|
2149
|
+
)
|
|
2150
|
+
Phronomy.configuration.logger&.warn(
|
|
2151
|
+
"[Phronomy] Agent terminal durable outcome requires recovery for " \
|
|
2152
|
+
"#{operation.execution_id}: #{ready.error.class}: #{ready.error.message}"
|
|
2153
|
+
)
|
|
2154
|
+
return
|
|
2155
|
+
end
|
|
2156
|
+
|
|
2157
|
+
outcome = ready.outcome
|
|
2158
|
+
apply_agent_live_state(
|
|
2159
|
+
root: outcome.root,
|
|
2160
|
+
appended_records: outcome.appended_records
|
|
2161
|
+
)
|
|
2162
|
+
if state
|
|
2163
|
+
state.invocation&.acknowledge_runtime_snapshot(operation.runtime_snapshot)
|
|
2164
|
+
event_loop.replace_agent_execution(
|
|
2165
|
+
operation.execution_id,
|
|
2166
|
+
execution: outcome.execution,
|
|
2167
|
+
fsm_session_id: (outcome.type == :suspended) ? nil : state.fsm_session_id
|
|
2168
|
+
)
|
|
2169
|
+
end
|
|
2170
|
+
|
|
2171
|
+
case outcome.type
|
|
2172
|
+
when :suspended
|
|
2173
|
+
event_loop.mark_agent_execution_admission(
|
|
2174
|
+
@agent.agent_id,
|
|
2175
|
+
execution_id: operation.execution_id,
|
|
2176
|
+
state: :suspended
|
|
2177
|
+
)
|
|
2178
|
+
dispatch_approval_listener(delivery.approval_listener, outcome.approval_request)
|
|
2179
|
+
callback_error = deliver_terminal(
|
|
2180
|
+
delivery.application_listener,
|
|
2181
|
+
:approval_required,
|
|
2182
|
+
request: outcome.approval_request
|
|
2183
|
+
)
|
|
2184
|
+
report_nonterminal_callback_error(
|
|
2185
|
+
callback_error,
|
|
2186
|
+
:approval_required,
|
|
2187
|
+
outcome.result
|
|
2188
|
+
)
|
|
2189
|
+
when :completed
|
|
2190
|
+
event_loop.release_agent_execution(operation.execution_id) if state
|
|
2191
|
+
event_loop.release_agent_execution_admission(
|
|
2192
|
+
@agent.agent_id,
|
|
2193
|
+
execution_id: operation.execution_id
|
|
2194
|
+
)
|
|
2195
|
+
callback_error = deliver_terminal(delivery.application_listener, :done, outcome.result)
|
|
2196
|
+
settle_execution_waiters(
|
|
2197
|
+
event_loop, operation.execution_id, delivery.result_task,
|
|
2198
|
+
callback_error, :done, outcome.result
|
|
2199
|
+
)
|
|
2200
|
+
when :handed_off
|
|
2201
|
+
event_loop.release_agent_execution(operation.execution_id) if state
|
|
2202
|
+
event_loop.release_agent_execution_admission(
|
|
2203
|
+
@agent.agent_id,
|
|
2204
|
+
execution_id: operation.execution_id
|
|
2205
|
+
)
|
|
2206
|
+
result = outcome.result.merge(
|
|
2207
|
+
handoff_request: delivery.handoff_request,
|
|
2208
|
+
_phronomy_handoff_manifest: delivery.handoff_manifest
|
|
2209
|
+
).freeze
|
|
2210
|
+
callback_error = deliver_terminal(delivery.application_listener, :handoff, result)
|
|
2211
|
+
settle_execution_waiters(
|
|
2212
|
+
event_loop, operation.execution_id, delivery.result_task,
|
|
2213
|
+
callback_error, :handoff, result
|
|
2214
|
+
)
|
|
2215
|
+
when :failed
|
|
2216
|
+
event_loop.release_agent_execution(operation.execution_id) if state
|
|
2217
|
+
event_loop.release_agent_execution_admission(
|
|
2218
|
+
@agent.agent_id,
|
|
2219
|
+
execution_id: operation.execution_id
|
|
2220
|
+
)
|
|
2221
|
+
type = terminal_event_type(outcome.error)
|
|
2222
|
+
callback_error = deliver_terminal(
|
|
2223
|
+
delivery.application_listener,
|
|
2224
|
+
type,
|
|
2225
|
+
error: outcome.error
|
|
2226
|
+
)
|
|
2227
|
+
settle_execution_waiters(
|
|
2228
|
+
event_loop, operation.execution_id, delivery.result_task,
|
|
2229
|
+
callback_error, type, nil, outcome.error
|
|
2230
|
+
)
|
|
2231
|
+
else
|
|
2232
|
+
fail_task(
|
|
2233
|
+
delivery.result_task,
|
|
2234
|
+
Phronomy::Error.new("unknown terminal outcome: #{outcome.type.inspect}")
|
|
2235
|
+
)
|
|
2236
|
+
end
|
|
736
2237
|
end
|
|
737
2238
|
|
|
2239
|
+
# ----------------------------------------------------------------------
|
|
2240
|
+
# Durable record encoding helpers
|
|
2241
|
+
# ----------------------------------------------------------------------
|
|
2242
|
+
|
|
738
2243
|
def encode_runtime_records(
|
|
739
|
-
|
|
2244
|
+
execution,
|
|
740
2245
|
tx:,
|
|
741
2246
|
snapshot:,
|
|
742
2247
|
context_candidate:,
|
|
743
2248
|
agent_root:
|
|
744
2249
|
)
|
|
745
|
-
execution = activation.execution
|
|
746
2250
|
root = agent_root
|
|
747
2251
|
records = []
|
|
748
2252
|
calls = []
|
|
@@ -889,168 +2393,6 @@ module Phronomy
|
|
|
889
2393
|
[records, calls]
|
|
890
2394
|
end
|
|
891
2395
|
|
|
892
|
-
def mark_resuming!(activation, approval_request_id:, approved:)
|
|
893
|
-
current = activation.execution
|
|
894
|
-
current_root = @agent.agent_root
|
|
895
|
-
unless current.agent_id == @agent.agent_id && current.status == :suspended
|
|
896
|
-
raise ArgumentError, "execution is not suspended: #{current.execution_id}"
|
|
897
|
-
end
|
|
898
|
-
|
|
899
|
-
request = current.approval_request || {}
|
|
900
|
-
request_id = request["id"] || request[:id]
|
|
901
|
-
unless request_id.to_s == approval_request_id.to_s
|
|
902
|
-
raise ArgumentError,
|
|
903
|
-
"approval request does not match execution #{current.execution_id}"
|
|
904
|
-
end
|
|
905
|
-
|
|
906
|
-
updated = next_root = nil
|
|
907
|
-
@agent.persistence.transaction do |tx|
|
|
908
|
-
decision_ref = tx.contents.put_json(
|
|
909
|
-
"approval_request_id" => request_id.to_s,
|
|
910
|
-
"approved" => !!approved
|
|
911
|
-
)
|
|
912
|
-
decision_record = JournalRecord.new(
|
|
913
|
-
agent_id: @agent.agent_id,
|
|
914
|
-
execution_id: current.execution_id,
|
|
915
|
-
kind: :approval_decided,
|
|
916
|
-
channel: :approval,
|
|
917
|
-
content_ref: decision_ref,
|
|
918
|
-
correlation_id: request_id.to_s,
|
|
919
|
-
context_generation: current_root.transcript_generation,
|
|
920
|
-
context_candidate: false
|
|
921
|
-
)
|
|
922
|
-
updated = current.with(
|
|
923
|
-
status: :active,
|
|
924
|
-
phase: :resuming,
|
|
925
|
-
working_records: current.working_records + [decision_record],
|
|
926
|
-
approval_request: request.merge("approved" => approved)
|
|
927
|
-
)
|
|
928
|
-
tx.executions.save(
|
|
929
|
-
current.execution_id,
|
|
930
|
-
expected_revision: current.execution_revision,
|
|
931
|
-
execution: updated
|
|
932
|
-
)
|
|
933
|
-
next_root = current_root.with(
|
|
934
|
-
agent_revision: current_root.agent_revision + 1,
|
|
935
|
-
lifecycle_status: :active
|
|
936
|
-
)
|
|
937
|
-
tx.agents.save(
|
|
938
|
-
current_root.agent_id,
|
|
939
|
-
expected_revision: current_root.agent_revision,
|
|
940
|
-
root: next_root
|
|
941
|
-
)
|
|
942
|
-
end
|
|
943
|
-
activation.replace_execution(updated)
|
|
944
|
-
@agent.__replace_root(next_root)
|
|
945
|
-
updated
|
|
946
|
-
end
|
|
947
|
-
|
|
948
|
-
def assert_local_durable_base!(tx, root)
|
|
949
|
-
tx.assert_agent_watermark!(
|
|
950
|
-
agent_id: root.agent_id,
|
|
951
|
-
agent_revision: root.agent_revision,
|
|
952
|
-
journal_position: root.journal_position
|
|
953
|
-
)
|
|
954
|
-
end
|
|
955
|
-
|
|
956
|
-
def transcript_messages(root)
|
|
957
|
-
materializer = RubyLLMMaterializer.new(
|
|
958
|
-
agent: @agent,
|
|
959
|
-
persistence: @agent.persistence
|
|
960
|
-
)
|
|
961
|
-
materializer.materialize_journal_records(
|
|
962
|
-
JournalProjection.new(
|
|
963
|
-
agent_root: root,
|
|
964
|
-
records: @agent.send(:_journal_records_snapshot)
|
|
965
|
-
).transcript_records
|
|
966
|
-
)
|
|
967
|
-
end
|
|
968
|
-
|
|
969
|
-
def deliver_execution_terminal(command)
|
|
970
|
-
activation = command.activation
|
|
971
|
-
registry = Phronomy::Runtime.instance.__agent_activations
|
|
972
|
-
if command.commit_error
|
|
973
|
-
callback_error = deliver_terminal(
|
|
974
|
-
activation,
|
|
975
|
-
:error,
|
|
976
|
-
error: command.commit_error
|
|
977
|
-
)
|
|
978
|
-
settle_after_terminal(
|
|
979
|
-
command.result_task,
|
|
980
|
-
callback_error,
|
|
981
|
-
:error,
|
|
982
|
-
nil,
|
|
983
|
-
command.commit_error
|
|
984
|
-
)
|
|
985
|
-
return
|
|
986
|
-
end
|
|
987
|
-
|
|
988
|
-
outcome = command.outcome
|
|
989
|
-
case outcome.fetch(:type)
|
|
990
|
-
when :suspended
|
|
991
|
-
result = outcome.fetch(:result)
|
|
992
|
-
callback_error = deliver_terminal(
|
|
993
|
-
activation,
|
|
994
|
-
:approval_required,
|
|
995
|
-
request: result.fetch(:approval_request)
|
|
996
|
-
)
|
|
997
|
-
settle_after_terminal(
|
|
998
|
-
command.result_task,
|
|
999
|
-
callback_error,
|
|
1000
|
-
:approval_required,
|
|
1001
|
-
result
|
|
1002
|
-
)
|
|
1003
|
-
when :completed
|
|
1004
|
-
registry.delete(activation.execution_id)
|
|
1005
|
-
result = outcome.fetch(:result)
|
|
1006
|
-
callback_error = deliver_terminal(activation, :done, result)
|
|
1007
|
-
settle_after_terminal(
|
|
1008
|
-
command.result_task,
|
|
1009
|
-
callback_error,
|
|
1010
|
-
:done,
|
|
1011
|
-
result
|
|
1012
|
-
)
|
|
1013
|
-
when :failed
|
|
1014
|
-
registry.delete(activation.execution_id)
|
|
1015
|
-
error = outcome.fetch(:error)
|
|
1016
|
-
type = terminal_event_type(error)
|
|
1017
|
-
callback_error = deliver_terminal(activation, type, error: error)
|
|
1018
|
-
settle_after_terminal(
|
|
1019
|
-
command.result_task,
|
|
1020
|
-
callback_error,
|
|
1021
|
-
type,
|
|
1022
|
-
nil,
|
|
1023
|
-
error
|
|
1024
|
-
)
|
|
1025
|
-
end
|
|
1026
|
-
end
|
|
1027
|
-
|
|
1028
|
-
def settle_without_listener(activation, result_task, outcome, commit_error)
|
|
1029
|
-
registry = Phronomy::Runtime.instance.__agent_activations
|
|
1030
|
-
if commit_error
|
|
1031
|
-
return fail_task(result_task, commit_error)
|
|
1032
|
-
end
|
|
1033
|
-
|
|
1034
|
-
case outcome&.fetch(:type)
|
|
1035
|
-
when :suspended
|
|
1036
|
-
complete_task(result_task, outcome.fetch(:result))
|
|
1037
|
-
when :completed
|
|
1038
|
-
registry.delete(activation.execution_id)
|
|
1039
|
-
complete_task(result_task, outcome.fetch(:result))
|
|
1040
|
-
when :failed
|
|
1041
|
-
registry.delete(activation.execution_id)
|
|
1042
|
-
fail_task(result_task, outcome.fetch(:error))
|
|
1043
|
-
else
|
|
1044
|
-
registry.delete(activation.execution_id)
|
|
1045
|
-
fail_task(
|
|
1046
|
-
result_task,
|
|
1047
|
-
Phronomy::RuntimeShutdownError.new(
|
|
1048
|
-
"EventLoop is not accepting terminal delivery"
|
|
1049
|
-
)
|
|
1050
|
-
)
|
|
1051
|
-
end
|
|
1052
|
-
end
|
|
1053
|
-
|
|
1054
2396
|
def assistant_message_ref(tx, outcome)
|
|
1055
2397
|
return unless outcome
|
|
1056
2398
|
|
|
@@ -1084,64 +2426,76 @@ module Phronomy
|
|
|
1084
2426
|
tx.contents.put_text(value) : tx.contents.put_json(json_value(value))
|
|
1085
2427
|
end
|
|
1086
2428
|
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
2429
|
+
# ----------------------------------------------------------------------
|
|
2430
|
+
# EventLoop validation/apply helpers
|
|
2431
|
+
# ----------------------------------------------------------------------
|
|
2432
|
+
|
|
2433
|
+
def authoritative_state_for_operation(
|
|
2434
|
+
execution_id:,
|
|
2435
|
+
fsm_session_id:,
|
|
2436
|
+
expected_execution_revision:,
|
|
2437
|
+
expected_fsm_state:
|
|
2438
|
+
)
|
|
2439
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
2440
|
+
state = event_loop.agent_execution_state(execution_id)
|
|
2441
|
+
authoritative = state &&
|
|
2442
|
+
state.agent.equal?(@agent) &&
|
|
2443
|
+
state.execution.execution_revision == expected_execution_revision &&
|
|
2444
|
+
state.fsm_session_id.to_s == fsm_session_id.to_s &&
|
|
2445
|
+
event_loop.fsm_session_state(fsm_session_id) == expected_fsm_state
|
|
2446
|
+
return state if authoritative
|
|
2447
|
+
|
|
2448
|
+
Phronomy.configuration.logger&.warn(
|
|
2449
|
+
"[Phronomy] Dropped stale Agent operation result: " \
|
|
2450
|
+
"execution_id=#{execution_id} fsm_session_id=#{fsm_session_id}"
|
|
1098
2451
|
)
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
invocation.tool_invocations.each do |child|
|
|
1111
|
-
child_session = if child.awaiting_approval?
|
|
1112
|
-
Agent::ToolInvocationSessionBuilder.build_for_resume(
|
|
1113
|
-
tool_invocation: child,
|
|
1114
|
-
resume_event: approved ? :approve : :reject,
|
|
1115
|
-
resume_phase: :awaiting_approval,
|
|
1116
|
-
runtime: runtime
|
|
1117
|
-
)
|
|
1118
|
-
elsif !approved && child.authorized?
|
|
1119
|
-
Agent::ToolInvocationSessionBuilder.build_for_resume(
|
|
1120
|
-
tool_invocation: child,
|
|
1121
|
-
resume_event: :cancel,
|
|
1122
|
-
resume_phase: :authorized,
|
|
1123
|
-
runtime: runtime
|
|
1124
|
-
)
|
|
1125
|
-
end
|
|
1126
|
-
register_child(event_loop, runtime, child, child_session) if child_session
|
|
2452
|
+
nil
|
|
2453
|
+
rescue
|
|
2454
|
+
nil
|
|
2455
|
+
end
|
|
2456
|
+
|
|
2457
|
+
def validate_live_session!(state, invocation, fsm_session_id)
|
|
2458
|
+
unless state && state.agent.equal?(@agent) &&
|
|
2459
|
+
state.invocation.equal?(invocation) &&
|
|
2460
|
+
state.fsm_session_id.to_s == fsm_session_id.to_s
|
|
2461
|
+
raise Phronomy::Error,
|
|
2462
|
+
"Agent operation does not belong to the current live FSMSession"
|
|
1127
2463
|
end
|
|
1128
2464
|
end
|
|
1129
2465
|
|
|
1130
|
-
def
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
2466
|
+
def apply_agent_live_state(root:, appended_records:)
|
|
2467
|
+
event_loop = Phronomy::Runtime.instance.event_loop
|
|
2468
|
+
assert_event_loop!(event_loop)
|
|
2469
|
+
@agent.send(:_append_journal_records, appended_records)
|
|
2470
|
+
@agent.__replace_root(root)
|
|
2471
|
+
end
|
|
1134
2472
|
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
2473
|
+
def assert_event_loop!(event_loop)
|
|
2474
|
+
return if event_loop.current?
|
|
2475
|
+
|
|
2476
|
+
raise Phronomy::Error,
|
|
2477
|
+
"ExecutionCoordinator live-state apply must run on EventLoop"
|
|
2478
|
+
end
|
|
2479
|
+
|
|
2480
|
+
def assert_local_durable_base!(tx, root)
|
|
2481
|
+
tx.assert_agent_watermark!(
|
|
2482
|
+
agent_id: root.agent_id,
|
|
2483
|
+
agent_revision: root.agent_revision,
|
|
2484
|
+
journal_position: root.journal_position
|
|
2485
|
+
)
|
|
2486
|
+
end
|
|
2487
|
+
|
|
2488
|
+
def transcript_messages(root, journal_records, persistence: @agent.persistence)
|
|
2489
|
+
materializer = RubyLLMMaterializer.new(
|
|
2490
|
+
agent: @agent,
|
|
2491
|
+
persistence: persistence
|
|
2492
|
+
)
|
|
2493
|
+
materializer.materialize_journal_records(
|
|
2494
|
+
JournalProjection.new(
|
|
2495
|
+
agent_root: root,
|
|
2496
|
+
records: journal_records
|
|
2497
|
+
).transcript_records
|
|
2498
|
+
)
|
|
1145
2499
|
end
|
|
1146
2500
|
|
|
1147
2501
|
def result_base(execution, root)
|
|
@@ -1154,16 +2508,7 @@ module Phronomy
|
|
|
1154
2508
|
}
|
|
1155
2509
|
end
|
|
1156
2510
|
|
|
1157
|
-
def
|
|
1158
|
-
descriptor_hash = execution.metadata.fetch("context_policy") do
|
|
1159
|
-
ContextPolicies::Default.new.descriptor.to_h
|
|
1160
|
-
end
|
|
1161
|
-
descriptor = ContextPolicyDescriptor.from_h(descriptor_hash)
|
|
1162
|
-
ContextPolicyRegistry.default.resolve(descriptor)
|
|
1163
|
-
end
|
|
1164
|
-
|
|
1165
|
-
def deliver_terminal(activation, type, payload)
|
|
1166
|
-
listener = activation.application_listener
|
|
2511
|
+
def deliver_terminal(listener, type, payload)
|
|
1167
2512
|
return nil unless listener
|
|
1168
2513
|
|
|
1169
2514
|
@agent.send(
|
|
@@ -1173,16 +2518,43 @@ module Phronomy
|
|
|
1173
2518
|
)
|
|
1174
2519
|
end
|
|
1175
2520
|
|
|
1176
|
-
def
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
2521
|
+
def dispatch_approval_listener(listener, request)
|
|
2522
|
+
return unless listener
|
|
2523
|
+
|
|
2524
|
+
Phronomy::Runtime.instance.offload.submit(on_full: :raise) do
|
|
2525
|
+
listener.call(request)
|
|
2526
|
+
end
|
|
2527
|
+
rescue => error
|
|
2528
|
+
Phronomy.configuration.logger&.warn(
|
|
2529
|
+
"[Phronomy] approval listener dispatch failed: #{error.class}: #{error.message}"
|
|
2530
|
+
)
|
|
2531
|
+
end
|
|
2532
|
+
|
|
2533
|
+
def settle_execution_waiters(
|
|
2534
|
+
event_loop, execution_id, fallback_task, callback_error, event_type,
|
|
2535
|
+
result, execution_error = nil
|
|
2536
|
+
)
|
|
2537
|
+
waiters = event_loop.take_agent_completion_waiters(
|
|
2538
|
+
execution_id,
|
|
2539
|
+
fallback: fallback_task
|
|
2540
|
+
)
|
|
2541
|
+
waiters.each do |task|
|
|
2542
|
+
settle_after_terminal(
|
|
2543
|
+
task, callback_error, event_type, result, execution_error
|
|
1183
2544
|
)
|
|
2545
|
+
end
|
|
2546
|
+
end
|
|
2547
|
+
|
|
2548
|
+
def report_nonterminal_callback_error(callback_error, event_type, result)
|
|
2549
|
+
return unless callback_error
|
|
2550
|
+
|
|
2551
|
+
@agent.send(
|
|
2552
|
+
:_report_stream_callback_error,
|
|
2553
|
+
callback_error,
|
|
2554
|
+
event: StreamEvent.new(type: event_type, payload: result),
|
|
2555
|
+
execution_id: result&.fetch(:execution_id, nil),
|
|
2556
|
+
callback_error_policy: Phronomy.configuration.stream_callback_error_policy
|
|
1184
2557
|
)
|
|
1185
|
-
fail_task(result_task, error) unless posted
|
|
1186
2558
|
end
|
|
1187
2559
|
|
|
1188
2560
|
def settle_after_terminal(
|
|
@@ -1201,7 +2573,7 @@ module Phronomy
|
|
|
1201
2573
|
type: event_type,
|
|
1202
2574
|
payload: result || {error: execution_error}
|
|
1203
2575
|
),
|
|
1204
|
-
|
|
2576
|
+
execution_id: result&.fetch(:execution_id, nil),
|
|
1205
2577
|
callback_error_policy: policy
|
|
1206
2578
|
)
|
|
1207
2579
|
if policy == :fail_task && execution_error.nil?
|
|
@@ -1216,8 +2588,15 @@ module Phronomy
|
|
|
1216
2588
|
)
|
|
1217
2589
|
end
|
|
1218
2590
|
end
|
|
1219
|
-
execution_error
|
|
1220
|
-
|
|
2591
|
+
if execution_error
|
|
2592
|
+
if execution_error.is_a?(Phronomy::CancellationError)
|
|
2593
|
+
result_task.cancel!(execution_error)
|
|
2594
|
+
else
|
|
2595
|
+
fail_task(result_task, execution_error)
|
|
2596
|
+
end
|
|
2597
|
+
else
|
|
2598
|
+
complete_task(result_task, result)
|
|
2599
|
+
end
|
|
1221
2600
|
end
|
|
1222
2601
|
|
|
1223
2602
|
def terminal_event_type(error)
|
|
@@ -1227,15 +2606,6 @@ module Phronomy
|
|
|
1227
2606
|
:error
|
|
1228
2607
|
end
|
|
1229
2608
|
|
|
1230
|
-
def dispatch_approval_listener(invocation, request)
|
|
1231
|
-
listener = invocation.approval_listener
|
|
1232
|
-
return unless listener
|
|
1233
|
-
|
|
1234
|
-
Phronomy::Runtime.instance.offload.submit(on_full: :raise) do
|
|
1235
|
-
listener.call(request)
|
|
1236
|
-
end
|
|
1237
|
-
end
|
|
1238
|
-
|
|
1239
2609
|
def json_value(value)
|
|
1240
2610
|
case value
|
|
1241
2611
|
when Hash
|
|
@@ -1277,6 +2647,24 @@ module Phronomy
|
|
|
1277
2647
|
}.fetch(status)
|
|
1278
2648
|
end
|
|
1279
2649
|
|
|
2650
|
+
def post_control(runtime, command)
|
|
2651
|
+
runtime.event_loop.post(
|
|
2652
|
+
Phronomy::Event.new(
|
|
2653
|
+
type: :agent_control,
|
|
2654
|
+
target_id: Phronomy::EventLoop::SYSTEM_CHANNEL_ID,
|
|
2655
|
+
payload: {command: command}
|
|
2656
|
+
)
|
|
2657
|
+
)
|
|
2658
|
+
rescue Phronomy::RuntimeShutdownError
|
|
2659
|
+
false
|
|
2660
|
+
end
|
|
2661
|
+
|
|
2662
|
+
def runtime_rejected_error(operation)
|
|
2663
|
+
Phronomy::RuntimeShutdownError.new(
|
|
2664
|
+
"EventLoop is not accepting Agent #{operation} control delivery"
|
|
2665
|
+
)
|
|
2666
|
+
end
|
|
2667
|
+
|
|
1280
2668
|
def translated(error)
|
|
1281
2669
|
@agent.send(:_translated_error, error)
|
|
1282
2670
|
end
|