phronomy 0.22.0 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.mutant.yml +3 -4
- data/CHANGELOG.md +200 -10
- data/CONTRIBUTING.md +81 -9
- data/README.md +15 -6
- data/VERIFY.sh +587 -0
- data/benchmark/bench_agent_invoke.rb +2 -2
- data/benchmark/bench_context_assembler.rb +39 -68
- data/benchmark/bench_regression.rb +2 -2
- data/docs/architecture/agent-context.md +174 -0
- data/docs/architecture/before-llm-input.md +78 -0
- data/docs/architecture/context-management.md +232 -0
- data/docs/architecture/knowledge-and-rag.md +130 -0
- data/docs/architecture/multi-agent-handoff.md +152 -0
- data/docs/architecture/persistence.md +175 -0
- data/docs/architecture/removed/agent-context.md +72 -0
- data/docs/architecture/security-boundaries.md +173 -0
- data/docs/architecture/tracing.md +194 -0
- data/docs/architecture.md +82 -0
- data/docs/archive/design/archived/04_api_design.md +507 -0
- data/docs/archive/design/archived/09_guardrails.md +186 -0
- data/docs/archive/design/archived/17_rails_integration.md +175 -0
- data/docs/archive/design/historical/00_design_philosophy.md +122 -0
- data/docs/archive/design/historical/01_rubyllm_evaluation.md +178 -0
- data/docs/archive/design/historical/06_design_decisions.md +143 -0
- data/docs/changelog/0.14-and-earlier.md +1 -1
- data/docs/decisions/001-rubyllm-as-provider-layer.md +6 -1
- data/docs/decisions/002-workflow-context-immutability.md +26 -1
- data/docs/decisions/006-no-built-in-guardrails.md +2 -1
- data/docs/decisions/012-canonical-execution-log-and-context-policy.md +120 -38
- data/docs/decisions/014-unified-persistence-durable-state.md +9 -2
- data/docs/decisions/016-semantic-multi-agent-handoff.md +112 -0
- data/docs/decisions/017-design-authority-and-adr-governance.md +200 -0
- data/docs/decisions/018-durability-guarantees-and-failure-model.md +488 -0
- data/docs/decisions/019-filter-contract-and-security-boundaries.md +229 -0
- data/docs/decisions/020-canonical-workflow-instance-identity.md +177 -0
- data/docs/decisions/021-generic-agent-invocation-identity-removal.md +119 -0
- data/docs/decisions/022-agent-execution-parent-identity-and-runtime-routing-boundary.md +193 -0
- data/docs/decisions/023-fsm-session-incarnation-identity-and-routing.md +139 -0
- data/docs/decisions/024-event-loop-single-writer-agent-runtime.md +188 -0
- data/docs/decisions/025-process-local-agent-ownership-and-runtime-admission.md +249 -0
- data/docs/decisions/026-workflow-runtime-admission-and-durable-terminal-barrier.md +257 -0
- data/docs/decisions/027-llm-adapter-provider-boundary.md +93 -0
- data/docs/decisions/README.md +172 -0
- data/docs/features.md +31 -11
- data/docs/getting-started.md +77 -45
- data/docs/migrations/0.19.md +14 -7
- data/docs/migrations/0.22.md +390 -0
- data/docs/persistence-backends.md +88 -38
- data/docs/runtime-and-concurrency.md +227 -33
- data/examples/README.md +13 -0
- data/lib/phronomy/agent/agent_execution.rb +19 -15
- data/lib/phronomy/agent/agent_invocation.rb +288 -93
- data/lib/phronomy/agent/agent_invocation_session_builder.rb +236 -202
- data/lib/phronomy/agent/agent_root.rb +3 -3
- data/lib/phronomy/agent/approval_evaluation_request.rb +37 -19
- data/lib/phronomy/agent/async_event_api.rb +145 -72
- data/lib/phronomy/agent/base.rb +388 -181
- data/lib/phronomy/agent/concerns/before_llm_input.rb +1 -1
- data/lib/phronomy/agent/context_assembler.rb +437 -178
- data/lib/phronomy/agent/context_candidate_resolver.rb +2 -2
- data/lib/phronomy/agent/context_plan.rb +18 -13
- data/lib/phronomy/agent/context_plan_validator.rb +246 -88
- data/lib/phronomy/agent/context_policies/default.rb +123 -34
- data/lib/phronomy/agent/context_policy.rb +109 -3
- data/lib/phronomy/agent/context_policy_input.rb +244 -0
- data/lib/phronomy/agent/context_policy_input_builder.rb +241 -0
- data/lib/phronomy/agent/execution_coordinator.rb +1975 -587
- data/lib/phronomy/agent/journal_record.rb +17 -4
- data/lib/phronomy/agent/llm_input_build_context.rb +1 -1
- data/lib/phronomy/agent/llm_input_manifest.rb +277 -2
- data/lib/phronomy/agent/llm_operation_result.rb +12 -7
- data/lib/phronomy/agent/phase_machine_builder.rb +19 -7
- data/lib/phronomy/agent/provider_call_outcome.rb +23 -7
- data/lib/phronomy/agent/recovery_coordinator/continuation.rb +271 -0
- data/lib/phronomy/agent/recovery_coordinator/installation.rb +427 -0
- data/lib/phronomy/agent/recovery_coordinator/resolution.rb +635 -0
- data/lib/phronomy/agent/recovery_coordinator.rb +211 -0
- data/lib/phronomy/agent/recovery_support.rb +512 -0
- data/lib/phronomy/agent/ruby_llm_materializer.rb +22 -13
- data/lib/phronomy/agent/selection/candidate.rb +53 -0
- data/lib/phronomy/agent/selection/constraint.rb +49 -0
- data/lib/phronomy/agent/shared_state.rb +38 -1
- data/lib/phronomy/agent/tool_approval_request.rb +33 -5
- data/lib/phronomy/agent/tool_definition_set.rb +49 -3
- data/lib/phronomy/agent/tool_invocation.rb +336 -102
- data/lib/phronomy/agent/tool_invocation_session_builder.rb +49 -45
- data/lib/phronomy/agent.rb +20 -2
- data/lib/phronomy/agent_already_exists_error.rb +5 -0
- data/lib/phronomy/agent_purged_error.rb +5 -0
- data/lib/phronomy/engine/concurrency/offload_pool.rb +17 -3
- data/lib/phronomy/engine/concurrency/physical_completion_task.rb +135 -0
- data/lib/phronomy/engine/event_loop.rb +622 -63
- data/lib/phronomy/engine/fsm_session.rb +194 -21
- data/lib/phronomy/engine/runtime/agent_ownership_registry.rb +352 -0
- data/lib/phronomy/engine/runtime.rb +77 -20
- data/lib/phronomy/generator_verifier.rb +12 -14
- data/lib/phronomy/invocation_context.rb +9 -29
- data/lib/phronomy/multi_agent/admission_registry.rb +51 -0
- data/lib/phronomy/multi_agent/coordination_state.rb +18 -0
- data/lib/phronomy/multi_agent/coordinator.rb +154 -0
- data/lib/phronomy/multi_agent/execution_coordinator.rb +116 -0
- data/lib/phronomy/multi_agent/fan_out_invocation.rb +24 -33
- data/lib/phronomy/multi_agent/fan_out_session_builder.rb +12 -19
- data/lib/phronomy/multi_agent/handoff.rb +24 -45
- data/lib/phronomy/multi_agent/handoff_capability_factory.rb +87 -0
- data/lib/phronomy/multi_agent/handoff_context.rb +95 -0
- data/lib/phronomy/multi_agent/handoff_policy.rb +137 -0
- data/lib/phronomy/multi_agent/handoff_projection.rb +191 -0
- data/lib/phronomy/multi_agent/handoff_request.rb +45 -0
- data/lib/phronomy/multi_agent/orchestrator.rb +12 -15
- data/lib/phronomy/multi_agent/runner.rb +98 -0
- data/lib/phronomy/persistence/durable_codec.rb +646 -0
- data/lib/phronomy/persistence/durable_record.rb +117 -0
- data/lib/phronomy/persistence/in_memory.rb +210 -134
- data/lib/phronomy/persistence/migration/initial_format_migration.rb +226 -0
- data/lib/phronomy/persistence/repository_facades.rb +316 -0
- data/lib/phronomy/persistence.rb +81 -41
- data/lib/phronomy/recovery.rb +186 -0
- data/lib/phronomy/testing/persistence_contract/a_journal_repository.rb +2 -2
- data/lib/phronomy/testing/persistence_contract/a_persistence_backend.rb +1 -1
- data/lib/phronomy/testing/persistence_contract/a_workflow_state_repository.rb +19 -19
- data/lib/phronomy/testing/persistence_contract/an_agent_repository.rb +3 -3
- data/lib/phronomy/testing/persistence_contract/an_execution_repository.rb +5 -5
- data/lib/phronomy/tracing/automatic.rb +176 -0
- data/lib/phronomy/tracing/base.rb +11 -2
- data/lib/phronomy/tracing/langfuse_tracer.rb +20 -12
- data/lib/phronomy/version.rb +1 -1
- data/lib/phronomy/workflow.rb +3 -6
- data/lib/phronomy/workflow_context.rb +14 -5
- data/lib/phronomy/workflow_recovery.rb +123 -0
- data/lib/phronomy/workflow_runner.rb +468 -256
- data/lib/phronomy.rb +6 -0
- data/scripts/api_snapshot.rb +12 -0
- data/sig/phronomy/agent.rbs +209 -7
- data/sig/phronomy/multi_agent.rbs +39 -0
- data/sig/phronomy/persistence.rbs +62 -4
- data/sig/phronomy/runtime.rbs +1 -4
- data/sig/phronomy/workflow.rbs +2 -2
- data/sig/phronomy.rbs +10 -0
- metadata +65 -17
- data/examples/workflows/agent_event_mapping.rb +0 -101
- data/examples/workflows/generic_task_event_mapping.rb +0 -66
- data/lib/phronomy/agent/activation_registry.rb +0 -28
- data/lib/phronomy/agent/agent_execution_activation.rb +0 -172
- data/lib/phronomy/agent/context_candidate.rb +0 -47
- data/lib/phronomy/agent/context_parts/budget/token_budget_packer.rb +0 -53
- data/lib/phronomy/agent/context_parts/requirements/required_context_resolver.rb +0 -56
- data/lib/phronomy/agent/context_parts/selectors/recent_first_selector.rb +0 -30
- data/lib/phronomy/agent/context_parts/unit_builders/dependency_aware_unit_builder.rb +0 -118
- data/lib/phronomy/agent/context_policy_descriptor.rb +0 -49
- data/lib/phronomy/agent/context_policy_registry.rb +0 -46
- data/lib/phronomy/agent/context_request.rb +0 -35
- data/lib/phronomy/agent/context_selection_unit.rb +0 -38
- data/lib/phronomy/agent/derived_content_spec.rb +0 -34
- data/lib/phronomy/agent/runner.rb +0 -97
|
@@ -4,19 +4,56 @@ module Phronomy
|
|
|
4
4
|
# Runtime-owned FIFO event loop for FSMSession instances.
|
|
5
5
|
#
|
|
6
6
|
# EventLoop owns the framework's sole control-plane OS thread. All session
|
|
7
|
-
# lifecycle progression
|
|
7
|
+
# lifecycle progression and Phronomy-managed live execution-state mutation
|
|
8
|
+
# happens by short event dispatches on this thread.
|
|
8
9
|
class EventLoop
|
|
9
10
|
SYSTEM_CHANNEL_ID = "__event_loop__"
|
|
10
11
|
|
|
11
12
|
QUEUE_BACKLOG_WARNING_THRESHOLD = 1_000
|
|
12
13
|
QUEUE_BACKLOG_WARNING_INTERVAL_SECONDS = 60.0
|
|
13
14
|
|
|
14
|
-
TERMINAL_MANAGEMENT_EVENTS = %i[finished halted error].freeze
|
|
15
|
+
TERMINAL_MANAGEMENT_EVENTS = %i[finished halted error recovery_required].freeze
|
|
15
16
|
private_constant :TERMINAL_MANAGEMENT_EVENTS
|
|
16
17
|
|
|
17
18
|
STOP = Object.new.freeze
|
|
18
19
|
WAKE = Object.new.freeze
|
|
19
|
-
|
|
20
|
+
UNSET = Object.new.freeze
|
|
21
|
+
private_constant :STOP, :WAKE, :UNSET
|
|
22
|
+
|
|
23
|
+
# Immutable EventLoop-owned value. The map containing these records is the
|
|
24
|
+
# mutable authority; records are replaced rather than mutated in place.
|
|
25
|
+
AgentExecutionState = Data.define(
|
|
26
|
+
:execution_id,
|
|
27
|
+
:agent,
|
|
28
|
+
:coordinator,
|
|
29
|
+
:execution,
|
|
30
|
+
:runtime_projection,
|
|
31
|
+
:base_manifest,
|
|
32
|
+
:invocation,
|
|
33
|
+
:fsm_session_id
|
|
34
|
+
)
|
|
35
|
+
private_constant :AgentExecutionState
|
|
36
|
+
|
|
37
|
+
# Read-only process-local lookup view used by approval/live-owner APIs.
|
|
38
|
+
# It intentionally exposes no mutable execution, invocation, or projection.
|
|
39
|
+
AgentExecutionOwner = Data.define(:execution_id, :agent, :coordinator, :status)
|
|
40
|
+
private_constant :AgentExecutionOwner
|
|
41
|
+
|
|
42
|
+
# EventLoop-owned process-local top-level execution admission. This is
|
|
43
|
+
# separate from AgentExecutionState because admission begins before the
|
|
44
|
+
# durable AgentExecution exists. owner_token is coordination-only and is
|
|
45
|
+
# never a semantic result-authority identifier.
|
|
46
|
+
AgentAdmission = Data.define(:agent_id, :owner_token, :execution_id, :state)
|
|
47
|
+
private_constant :AgentAdmission
|
|
48
|
+
|
|
49
|
+
# EventLoop-owned process-local Workflow execution-segment admission. The
|
|
50
|
+
# owner token is a Runtime coordination capability, not a domain identity or
|
|
51
|
+
# routing identity. fsm_session_id is bound only after durable load/hydration
|
|
52
|
+
# completes and a concrete FSMSession has been constructed.
|
|
53
|
+
WorkflowAdmission = Data.define(
|
|
54
|
+
:workflow_instance_id, :owner_token, :fsm_session_id, :state
|
|
55
|
+
)
|
|
56
|
+
private_constant :WorkflowAdmission
|
|
20
57
|
|
|
21
58
|
def initialize(runtime:)
|
|
22
59
|
@runtime = runtime
|
|
@@ -28,8 +65,13 @@ module Phronomy
|
|
|
28
65
|
|
|
29
66
|
@fsms = {}
|
|
30
67
|
@waiting = {}
|
|
31
|
-
@
|
|
68
|
+
@admitted_fsm_session_ids = Set.new
|
|
32
69
|
@workflow_admissions = {}
|
|
70
|
+
@agent_admissions = {}
|
|
71
|
+
@agent_executions = {}
|
|
72
|
+
@agent_completion_waiters = Hash.new { |hash, key| hash[key] = [] }
|
|
73
|
+
@agent_inflight_work = Hash.new { |hash, key| hash[key] = {} }
|
|
74
|
+
@agent_deferred_terminals = {}
|
|
33
75
|
|
|
34
76
|
@lifecycle_mutex = Mutex.new
|
|
35
77
|
@idle_cond = ConditionVariable.new
|
|
@@ -88,17 +130,17 @@ module Phronomy
|
|
|
88
130
|
|
|
89
131
|
@lifecycle_mutex.synchronize do
|
|
90
132
|
ensure_accepting_registrations!
|
|
91
|
-
if @
|
|
133
|
+
if @admitted_fsm_session_ids.include?(fsm_session.id)
|
|
92
134
|
raise Phronomy::Error,
|
|
93
135
|
"FSMSession #{fsm_session.id.inspect} is already registered"
|
|
94
136
|
end
|
|
95
137
|
|
|
96
|
-
@
|
|
138
|
+
@admitted_fsm_session_ids.add(fsm_session.id)
|
|
97
139
|
@outstanding_sessions += 1
|
|
98
140
|
begin
|
|
99
141
|
queued_depth = enqueue([event, monotonic_nanoseconds])
|
|
100
142
|
rescue
|
|
101
|
-
@
|
|
143
|
+
@admitted_fsm_session_ids.delete(fsm_session.id)
|
|
102
144
|
@outstanding_sessions -= 1
|
|
103
145
|
@idle_cond.broadcast if runtime_idle_locked?
|
|
104
146
|
raise
|
|
@@ -114,16 +156,16 @@ module Phronomy
|
|
|
114
156
|
accepted = @lifecycle_mutex.synchronize do
|
|
115
157
|
next false unless accepting_events?
|
|
116
158
|
|
|
117
|
-
|
|
159
|
+
terminal_fsm_session_id = nil
|
|
118
160
|
if terminal_management_event?(event)
|
|
119
|
-
|
|
120
|
-
@
|
|
161
|
+
terminal_fsm_session_id = event.payload.fetch(:fsm_session_id)
|
|
162
|
+
@admitted_fsm_session_ids.delete(terminal_fsm_session_id)
|
|
121
163
|
end
|
|
122
164
|
|
|
123
165
|
begin
|
|
124
166
|
queued_depth = enqueue([event, monotonic_nanoseconds])
|
|
125
167
|
rescue
|
|
126
|
-
@
|
|
168
|
+
@admitted_fsm_session_ids.add(terminal_fsm_session_id) if terminal_fsm_session_id
|
|
127
169
|
raise
|
|
128
170
|
end
|
|
129
171
|
true
|
|
@@ -142,7 +184,7 @@ module Phronomy
|
|
|
142
184
|
queued_depth = nil
|
|
143
185
|
accepted = @lifecycle_mutex.synchronize do
|
|
144
186
|
next false unless accepting_events?
|
|
145
|
-
next false unless @
|
|
187
|
+
next false unless @admitted_fsm_session_ids.include?(event.target_id)
|
|
146
188
|
|
|
147
189
|
queued_depth = enqueue([event, monotonic_nanoseconds])
|
|
148
190
|
true
|
|
@@ -153,36 +195,452 @@ module Phronomy
|
|
|
153
195
|
true
|
|
154
196
|
end
|
|
155
197
|
|
|
156
|
-
#
|
|
157
|
-
#
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
198
|
+
# Process-local read-only admission check used by destructive Agent lifecycle
|
|
199
|
+
# operations. The mutable admission map itself remains EventLoop-owned.
|
|
200
|
+
def agent_execution_admitted?(agent_id)
|
|
201
|
+
@lifecycle_mutex.synchronize { @agent_admissions.key?(agent_id.to_s) }
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
# Reserves the one top-level logical execution slot for agent_id before any
|
|
205
|
+
# Persistence execution admission is attempted.
|
|
206
|
+
# @api private
|
|
207
|
+
def admit_agent_execution(agent_id, owner_token:)
|
|
208
|
+
assert_event_loop_thread!
|
|
209
|
+
key = agent_id.to_s
|
|
210
|
+
raise ArgumentError, "agent_id must not be empty" if key.empty?
|
|
211
|
+
raise ArgumentError, "owner_token is required" unless owner_token
|
|
212
|
+
|
|
213
|
+
@lifecycle_mutex.synchronize do
|
|
214
|
+
ensure_accepting_registrations!
|
|
215
|
+
if @agent_admissions.key?(key)
|
|
216
|
+
raise Phronomy::AgentBusyError,
|
|
217
|
+
"Agent #{key.inspect} already has a nonterminal top-level execution"
|
|
218
|
+
end
|
|
219
|
+
@agent_admissions[key] = AgentAdmission.new(
|
|
220
|
+
agent_id: key.freeze,
|
|
221
|
+
owner_token: owner_token,
|
|
222
|
+
execution_id: nil,
|
|
223
|
+
state: :admitting
|
|
224
|
+
)
|
|
225
|
+
end
|
|
226
|
+
true
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
# Binds a successful durable AgentExecution identity to the earlier
|
|
230
|
+
# process-local admission.
|
|
231
|
+
# @api private
|
|
232
|
+
def bind_agent_execution_admission(agent_id, owner_token:, execution_id:)
|
|
233
|
+
assert_event_loop_thread!
|
|
234
|
+
key = agent_id.to_s
|
|
235
|
+
execution_key = execution_id.to_s
|
|
236
|
+
@lifecycle_mutex.synchronize do
|
|
237
|
+
current = @agent_admissions.fetch(key) do
|
|
238
|
+
raise Phronomy::Error, "Agent #{key.inspect} has no Runtime admission"
|
|
239
|
+
end
|
|
240
|
+
unless current.owner_token.equal?(owner_token) && current.execution_id.nil?
|
|
241
|
+
raise Phronomy::Error, "stale Agent admission bind for #{key.inspect}"
|
|
242
|
+
end
|
|
243
|
+
@agent_admissions[key] = AgentAdmission.new(
|
|
244
|
+
agent_id: current.agent_id,
|
|
245
|
+
owner_token: current.owner_token,
|
|
246
|
+
execution_id: execution_key.freeze,
|
|
247
|
+
state: :executing
|
|
248
|
+
)
|
|
249
|
+
end
|
|
250
|
+
true
|
|
251
|
+
end
|
|
252
|
+
|
|
253
|
+
# @api private
|
|
254
|
+
def mark_agent_execution_admission(agent_id, execution_id:, state:)
|
|
255
|
+
assert_event_loop_thread!
|
|
256
|
+
key = agent_id.to_s
|
|
257
|
+
execution_key = execution_id.to_s
|
|
258
|
+
next_state = state.to_sym
|
|
259
|
+
unless %i[executing suspended resuming cancelling terminalizing recovery_required].include?(next_state)
|
|
260
|
+
raise ArgumentError, "unsupported Agent admission state: #{next_state.inspect}"
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
@lifecycle_mutex.synchronize do
|
|
264
|
+
current = @agent_admissions.fetch(key) do
|
|
265
|
+
raise Phronomy::Error, "Agent #{key.inspect} has no Runtime admission"
|
|
266
|
+
end
|
|
267
|
+
unless current.execution_id.to_s == execution_key
|
|
268
|
+
raise Phronomy::Error, "stale Agent admission state update for #{key.inspect}"
|
|
269
|
+
end
|
|
270
|
+
@agent_admissions[key] = AgentAdmission.new(
|
|
271
|
+
agent_id: current.agent_id,
|
|
272
|
+
owner_token: current.owner_token,
|
|
273
|
+
execution_id: current.execution_id,
|
|
274
|
+
state: next_state
|
|
275
|
+
)
|
|
276
|
+
@idle_cond.broadcast if runtime_idle_locked?
|
|
277
|
+
end
|
|
278
|
+
true
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
# @api private
|
|
282
|
+
def mark_agent_admission_recovery_required(agent_id, owner_token:)
|
|
283
|
+
assert_event_loop_thread!
|
|
284
|
+
key = agent_id.to_s
|
|
285
|
+
@lifecycle_mutex.synchronize do
|
|
286
|
+
current = @agent_admissions.fetch(key) do
|
|
287
|
+
raise Phronomy::Error, "Agent #{key.inspect} has no Runtime admission"
|
|
288
|
+
end
|
|
289
|
+
unless current.owner_token.equal?(owner_token)
|
|
290
|
+
raise Phronomy::Error, "stale Agent admission recovery update for #{key.inspect}"
|
|
291
|
+
end
|
|
292
|
+
@agent_admissions[key] = AgentAdmission.new(
|
|
293
|
+
agent_id: current.agent_id,
|
|
294
|
+
owner_token: current.owner_token,
|
|
295
|
+
execution_id: current.execution_id,
|
|
296
|
+
state: :recovery_required
|
|
297
|
+
)
|
|
298
|
+
@idle_cond.broadcast if runtime_idle_locked?
|
|
299
|
+
end
|
|
300
|
+
true
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
# Owner-aware release. Pre-durable failures release by owner_token; durable
|
|
304
|
+
# terminal outcomes release by execution_id.
|
|
305
|
+
# @api private
|
|
306
|
+
def release_agent_execution_admission(agent_id, owner_token: nil, execution_id: nil)
|
|
307
|
+
assert_event_loop_thread!
|
|
308
|
+
key = agent_id.to_s
|
|
309
|
+
@lifecycle_mutex.synchronize do
|
|
310
|
+
current = @agent_admissions[key]
|
|
311
|
+
next false unless current
|
|
312
|
+
|
|
313
|
+
authoritative = if execution_id
|
|
314
|
+
current.execution_id.to_s == execution_id.to_s
|
|
315
|
+
elsif owner_token
|
|
316
|
+
current.owner_token.equal?(owner_token)
|
|
317
|
+
else
|
|
318
|
+
false
|
|
319
|
+
end
|
|
320
|
+
next false unless authoritative
|
|
321
|
+
|
|
322
|
+
@agent_admissions.delete(key)
|
|
323
|
+
@idle_cond.broadcast if runtime_idle_locked?
|
|
324
|
+
true
|
|
325
|
+
end
|
|
326
|
+
end
|
|
327
|
+
|
|
328
|
+
# Process-local read-only owner lookup. Mutable Agent execution state never
|
|
329
|
+
# crosses this boundary; external callers receive only routing/ownership data.
|
|
330
|
+
def agent_execution_owner(execution_id)
|
|
331
|
+
key = execution_id.to_s
|
|
332
|
+
@lifecycle_mutex.synchronize do
|
|
333
|
+
state = @agent_executions[key]
|
|
334
|
+
next nil unless state
|
|
335
|
+
|
|
336
|
+
AgentExecutionOwner.new(
|
|
337
|
+
execution_id: key.freeze,
|
|
338
|
+
agent: state.agent,
|
|
339
|
+
coordinator: state.coordinator,
|
|
340
|
+
status: state.execution.status
|
|
341
|
+
)
|
|
342
|
+
end
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
# EventLoop-only accessors below form the live Agent execution authority.
|
|
346
|
+
# Offload workers receive operation-specific immutable snapshots instead.
|
|
347
|
+
# @api private
|
|
348
|
+
def agent_execution_state(execution_id)
|
|
349
|
+
assert_event_loop_thread!
|
|
350
|
+
@agent_executions[execution_id.to_s]
|
|
351
|
+
end
|
|
352
|
+
|
|
353
|
+
# @api private
|
|
354
|
+
def install_agent_execution(
|
|
355
|
+
execution_id:,
|
|
356
|
+
agent:,
|
|
357
|
+
coordinator:,
|
|
358
|
+
execution:,
|
|
359
|
+
runtime_projection:,
|
|
360
|
+
base_manifest:,
|
|
361
|
+
invocation:,
|
|
362
|
+
fsm_session_id:
|
|
363
|
+
)
|
|
364
|
+
assert_event_loop_thread!
|
|
365
|
+
key = execution_id.to_s
|
|
366
|
+
state = AgentExecutionState.new(
|
|
367
|
+
execution_id: key.freeze,
|
|
368
|
+
agent: agent,
|
|
369
|
+
coordinator: coordinator,
|
|
370
|
+
execution: execution,
|
|
371
|
+
runtime_projection: runtime_projection,
|
|
372
|
+
base_manifest: base_manifest,
|
|
373
|
+
invocation: invocation,
|
|
374
|
+
fsm_session_id: fsm_session_id&.to_s&.freeze
|
|
375
|
+
)
|
|
376
|
+
@lifecycle_mutex.synchronize do
|
|
377
|
+
if @agent_executions.key?(key)
|
|
378
|
+
raise Phronomy::Error, "Agent execution #{key.inspect} is already live"
|
|
379
|
+
end
|
|
380
|
+
@agent_executions[key] = state
|
|
381
|
+
end
|
|
382
|
+
state
|
|
383
|
+
end
|
|
384
|
+
|
|
385
|
+
# @api private
|
|
386
|
+
def replace_agent_execution(
|
|
387
|
+
execution_id,
|
|
388
|
+
execution: UNSET,
|
|
389
|
+
runtime_projection: UNSET,
|
|
390
|
+
invocation: UNSET,
|
|
391
|
+
fsm_session_id: UNSET
|
|
392
|
+
)
|
|
393
|
+
assert_event_loop_thread!
|
|
394
|
+
key = execution_id.to_s
|
|
395
|
+
@lifecycle_mutex.synchronize do
|
|
396
|
+
current = @agent_executions.fetch(key) do
|
|
397
|
+
raise Phronomy::Error, "Agent execution #{key.inspect} is not live"
|
|
398
|
+
end
|
|
399
|
+
updated = AgentExecutionState.new(
|
|
400
|
+
execution_id: current.execution_id,
|
|
401
|
+
agent: current.agent,
|
|
402
|
+
coordinator: current.coordinator,
|
|
403
|
+
execution: execution.equal?(UNSET) ? current.execution : execution,
|
|
404
|
+
runtime_projection: runtime_projection.equal?(UNSET) ?
|
|
405
|
+
current.runtime_projection : runtime_projection,
|
|
406
|
+
base_manifest: current.base_manifest,
|
|
407
|
+
invocation: invocation.equal?(UNSET) ? current.invocation : invocation,
|
|
408
|
+
fsm_session_id: fsm_session_id.equal?(UNSET) ?
|
|
409
|
+
current.fsm_session_id : fsm_session_id&.to_s&.freeze
|
|
410
|
+
)
|
|
411
|
+
@agent_executions[key] = updated
|
|
412
|
+
updated
|
|
413
|
+
end
|
|
414
|
+
end
|
|
415
|
+
|
|
416
|
+
# Registers a caller-facing Task that observes the authoritative terminal
|
|
417
|
+
# outcome of one logical Agent execution. Waiters are Runtime-only and are
|
|
418
|
+
# never persisted or rehydrated.
|
|
419
|
+
# @api private
|
|
420
|
+
def register_agent_completion_waiter(execution_id, task)
|
|
421
|
+
assert_event_loop_thread!
|
|
422
|
+
unless task.is_a?(Phronomy::Task)
|
|
423
|
+
raise ArgumentError, "Agent completion waiter must be a Phronomy::Task"
|
|
424
|
+
end
|
|
425
|
+
|
|
426
|
+
key = execution_id.to_s
|
|
427
|
+
@lifecycle_mutex.synchronize do
|
|
428
|
+
waiters = @agent_completion_waiters[key]
|
|
429
|
+
waiters << task unless waiters.include?(task)
|
|
430
|
+
end
|
|
431
|
+
task
|
|
432
|
+
end
|
|
433
|
+
|
|
434
|
+
# Atomically detaches all process-local completion waiters at authoritative
|
|
435
|
+
# terminal delivery. A fallback Task is included for pre-install terminal
|
|
436
|
+
# paths that never acquired a live execution directory entry.
|
|
437
|
+
# @api private
|
|
438
|
+
def take_agent_completion_waiters(execution_id, fallback: nil)
|
|
439
|
+
assert_event_loop_thread!
|
|
440
|
+
key = execution_id.to_s
|
|
441
|
+
@lifecycle_mutex.synchronize do
|
|
442
|
+
waiters = @agent_completion_waiters.delete(key) || []
|
|
443
|
+
waiters << fallback if fallback && !waiters.include?(fallback)
|
|
444
|
+
waiters
|
|
445
|
+
end
|
|
446
|
+
end
|
|
447
|
+
|
|
448
|
+
# Registers one execution-owned asynchronous operation for physical
|
|
449
|
+
# quiescence supervision. OffloadPool tasks expose a private physical
|
|
450
|
+
# completion signal; custom asynchronous handles are required to make their
|
|
451
|
+
# ordinary completion mean that no residual execution-affecting work remains.
|
|
452
|
+
# @api private
|
|
453
|
+
def supervise_agent_operation(execution_id, operation)
|
|
454
|
+
assert_event_loop_thread!
|
|
455
|
+
key = execution_id.to_s
|
|
456
|
+
unless operation.respond_to?(:on_complete)
|
|
457
|
+
raise ArgumentError, "supervised operation must expose on_complete"
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
physically_done = if operation.respond_to?(:physical_complete?)
|
|
461
|
+
operation.physical_complete?
|
|
462
|
+
elsif operation.respond_to?(:done?)
|
|
463
|
+
operation.done?
|
|
464
|
+
else
|
|
465
|
+
false
|
|
466
|
+
end
|
|
467
|
+
return operation if physically_done
|
|
468
|
+
|
|
469
|
+
token = Object.new.freeze
|
|
470
|
+
@lifecycle_mutex.synchronize do
|
|
471
|
+
unless @agent_executions.key?(key)
|
|
472
|
+
raise Phronomy::Error, "Agent execution #{key.inspect} is not live"
|
|
473
|
+
end
|
|
474
|
+
@agent_inflight_work[key][token] = true
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
callback = lambda do
|
|
478
|
+
accepted = post(
|
|
479
|
+
Phronomy::Event.new(
|
|
480
|
+
type: :agent_physical_work_completed,
|
|
481
|
+
target_id: SYSTEM_CHANNEL_ID,
|
|
482
|
+
payload: {execution_id: key, token: token}.freeze
|
|
483
|
+
)
|
|
484
|
+
)
|
|
485
|
+
unless accepted
|
|
486
|
+
Phronomy.configuration.logger&.warn(
|
|
487
|
+
"[Phronomy] EventLoop rejected physical-completion delivery for #{key}"
|
|
488
|
+
)
|
|
489
|
+
end
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
if operation.respond_to?(:on_physical_complete)
|
|
493
|
+
operation.on_physical_complete(&callback)
|
|
494
|
+
else
|
|
495
|
+
operation.on_complete { |_value, _error| callback.call }
|
|
496
|
+
end
|
|
497
|
+
operation
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
# @api private
|
|
501
|
+
def agent_execution_quiescent?(execution_id)
|
|
502
|
+
assert_event_loop_thread!
|
|
503
|
+
key = execution_id.to_s
|
|
504
|
+
@lifecycle_mutex.synchronize do
|
|
505
|
+
work = @agent_inflight_work.fetch(key, nil)
|
|
506
|
+
work.nil? || work.empty?
|
|
507
|
+
end
|
|
508
|
+
end
|
|
509
|
+
|
|
510
|
+
# Holds exactly one terminal continuation while cancellation/deadline has
|
|
511
|
+
# revoked result authority but execution-owned physical work is still live.
|
|
512
|
+
# @api private
|
|
513
|
+
def defer_agent_terminal_until_quiescent(execution_id, command)
|
|
514
|
+
assert_event_loop_thread!
|
|
515
|
+
key = execution_id.to_s
|
|
516
|
+
@lifecycle_mutex.synchronize do
|
|
517
|
+
if @agent_deferred_terminals.key?(key)
|
|
518
|
+
raise Phronomy::Error, "Agent execution #{key.inspect} already has a deferred terminal"
|
|
519
|
+
end
|
|
520
|
+
@agent_deferred_terminals[key] = command
|
|
521
|
+
end
|
|
522
|
+
true
|
|
523
|
+
end
|
|
524
|
+
|
|
525
|
+
# @api private
|
|
526
|
+
def agent_inflight_work_count(execution_id)
|
|
527
|
+
key = execution_id.to_s
|
|
528
|
+
@lifecycle_mutex.synchronize do
|
|
529
|
+
(@agent_inflight_work.fetch(key, nil) || {}).size
|
|
530
|
+
end
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
# @api private
|
|
534
|
+
def release_agent_execution(execution_id)
|
|
535
|
+
assert_event_loop_thread!
|
|
536
|
+
key = execution_id.to_s
|
|
537
|
+
@lifecycle_mutex.synchronize do
|
|
538
|
+
work = @agent_inflight_work.fetch(key, nil)
|
|
539
|
+
unless work.nil? || work.empty?
|
|
540
|
+
raise Phronomy::Error,
|
|
541
|
+
"cannot release non-quiescent Agent execution #{key.inspect}"
|
|
542
|
+
end
|
|
543
|
+
if @agent_deferred_terminals.key?(key)
|
|
544
|
+
raise Phronomy::Error,
|
|
545
|
+
"cannot release Agent execution #{key.inspect} with deferred terminal work"
|
|
546
|
+
end
|
|
547
|
+
@agent_inflight_work.delete(key)
|
|
548
|
+
@agent_executions.delete(key)
|
|
549
|
+
end
|
|
550
|
+
end
|
|
551
|
+
|
|
552
|
+
# @api private
|
|
553
|
+
def fsm_session_state(fsm_session_id)
|
|
554
|
+
assert_event_loop_thread!
|
|
555
|
+
@fsms[fsm_session_id.to_s]&.current_state
|
|
556
|
+
end
|
|
557
|
+
|
|
558
|
+
# Reserves one logical Workflow execution segment before durable load or
|
|
559
|
+
# hydration. The owner token is independent from any later FSMSession id.
|
|
560
|
+
# @api private
|
|
561
|
+
def admit_workflow(workflow_instance_id, owner_token:)
|
|
562
|
+
assert_event_loop_thread!
|
|
563
|
+
key = workflow_instance_id.to_s
|
|
564
|
+
raise ArgumentError, "workflow_instance_id must not be empty" if key.empty?
|
|
565
|
+
raise ArgumentError, "owner_token is required" unless owner_token
|
|
164
566
|
|
|
165
567
|
@lifecycle_mutex.synchronize do
|
|
166
568
|
ensure_accepting_registrations!
|
|
167
|
-
|
|
168
|
-
if current_owner
|
|
569
|
+
if @workflow_admissions.key?(key)
|
|
169
570
|
raise Phronomy::Error,
|
|
170
|
-
"Workflow
|
|
171
|
-
|
|
571
|
+
"Workflow instance #{key.inspect} already has a live execution segment"
|
|
572
|
+
end
|
|
573
|
+
@workflow_admissions[key] = WorkflowAdmission.new(
|
|
574
|
+
workflow_instance_id: key.freeze,
|
|
575
|
+
owner_token: owner_token,
|
|
576
|
+
fsm_session_id: nil,
|
|
577
|
+
state: :admitting
|
|
578
|
+
)
|
|
579
|
+
end
|
|
580
|
+
true
|
|
581
|
+
end
|
|
582
|
+
|
|
583
|
+
# Binds the concrete routing identity after admission and durable hydration.
|
|
584
|
+
# @api private
|
|
585
|
+
def bind_workflow_session(workflow_instance_id, owner_token:, fsm_session_id:)
|
|
586
|
+
assert_event_loop_thread!
|
|
587
|
+
key = workflow_instance_id.to_s
|
|
588
|
+
fsm_key = fsm_session_id.to_s
|
|
589
|
+
raise ArgumentError, "fsm_session_id must not be empty" if fsm_key.empty?
|
|
590
|
+
|
|
591
|
+
@lifecycle_mutex.synchronize do
|
|
592
|
+
current = @workflow_admissions.fetch(key) do
|
|
593
|
+
raise Phronomy::Error, "Workflow instance #{key.inspect} has no Runtime admission"
|
|
594
|
+
end
|
|
595
|
+
unless current.owner_token.equal?(owner_token) && current.fsm_session_id.nil?
|
|
596
|
+
raise Phronomy::Error, "stale Workflow admission bind for #{key.inspect}"
|
|
597
|
+
end
|
|
598
|
+
@workflow_admissions[key] = WorkflowAdmission.new(
|
|
599
|
+
workflow_instance_id: current.workflow_instance_id,
|
|
600
|
+
owner_token: current.owner_token,
|
|
601
|
+
fsm_session_id: fsm_key.freeze,
|
|
602
|
+
state: :executing
|
|
603
|
+
)
|
|
604
|
+
end
|
|
605
|
+
true
|
|
606
|
+
end
|
|
607
|
+
|
|
608
|
+
# @api private
|
|
609
|
+
def mark_workflow_admission(workflow_instance_id, owner_token:, state:)
|
|
610
|
+
assert_event_loop_thread!
|
|
611
|
+
key = workflow_instance_id.to_s
|
|
612
|
+
next_state = state.to_sym
|
|
613
|
+
unless %i[executing persisting_terminal recovery_required].include?(next_state)
|
|
614
|
+
raise ArgumentError, "unsupported Workflow admission state: #{next_state.inspect}"
|
|
615
|
+
end
|
|
616
|
+
|
|
617
|
+
@lifecycle_mutex.synchronize do
|
|
618
|
+
current = @workflow_admissions.fetch(key) do
|
|
619
|
+
raise Phronomy::Error, "Workflow instance #{key.inspect} has no Runtime admission"
|
|
620
|
+
end
|
|
621
|
+
unless current.owner_token.equal?(owner_token)
|
|
622
|
+
raise Phronomy::Error, "stale Workflow admission update for #{key.inspect}"
|
|
172
623
|
end
|
|
173
|
-
@workflow_admissions[key] =
|
|
624
|
+
@workflow_admissions[key] = WorkflowAdmission.new(
|
|
625
|
+
workflow_instance_id: current.workflow_instance_id,
|
|
626
|
+
owner_token: current.owner_token,
|
|
627
|
+
fsm_session_id: current.fsm_session_id,
|
|
628
|
+
state: next_state
|
|
629
|
+
)
|
|
630
|
+
@idle_cond.broadcast if runtime_idle_locked?
|
|
174
631
|
end
|
|
175
632
|
true
|
|
176
633
|
end
|
|
177
634
|
|
|
178
|
-
#
|
|
179
|
-
#
|
|
180
|
-
#
|
|
181
|
-
def release_workflow(
|
|
182
|
-
|
|
183
|
-
|
|
635
|
+
# Owner-aware release. A competing or stale attempt cannot release the
|
|
636
|
+
# current Workflow execution segment.
|
|
637
|
+
# @api private
|
|
638
|
+
def release_workflow(workflow_instance_id, owner_token:)
|
|
639
|
+
assert_event_loop_thread!
|
|
640
|
+
key = workflow_instance_id.to_s
|
|
184
641
|
@lifecycle_mutex.synchronize do
|
|
185
|
-
|
|
642
|
+
current = @workflow_admissions[key]
|
|
643
|
+
next false unless current&.owner_token&.equal?(owner_token)
|
|
186
644
|
|
|
187
645
|
@workflow_admissions.delete(key)
|
|
188
646
|
@idle_cond.broadcast if runtime_idle_locked?
|
|
@@ -190,25 +648,40 @@ module Phronomy
|
|
|
190
648
|
end
|
|
191
649
|
end
|
|
192
650
|
|
|
193
|
-
|
|
194
|
-
|
|
651
|
+
# Read-only diagnostics used by internal tests and routing assertions.
|
|
652
|
+
def workflow_admission_owner(workflow_instance_id)
|
|
653
|
+
@lifecycle_mutex.synchronize do
|
|
654
|
+
@workflow_admissions[workflow_instance_id.to_s]&.owner_token
|
|
655
|
+
end
|
|
656
|
+
end
|
|
657
|
+
|
|
658
|
+
def workflow_admission_fsm_session_id(workflow_instance_id)
|
|
659
|
+
@lifecycle_mutex.synchronize do
|
|
660
|
+
@workflow_admissions[workflow_instance_id.to_s]&.fsm_session_id
|
|
661
|
+
end
|
|
195
662
|
end
|
|
196
663
|
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
664
|
+
def workflow_admission_state(workflow_instance_id)
|
|
665
|
+
@lifecycle_mutex.synchronize do
|
|
666
|
+
@workflow_admissions[workflow_instance_id.to_s]&.state
|
|
667
|
+
end
|
|
668
|
+
end
|
|
669
|
+
|
|
670
|
+
def post_to_workflow(workflow_instance_id:, event:, payload: nil)
|
|
200
671
|
queued_depth = nil
|
|
201
672
|
posted_event = nil
|
|
202
673
|
accepted = @lifecycle_mutex.synchronize do
|
|
203
674
|
next false unless accepting_events?
|
|
204
675
|
|
|
205
|
-
|
|
206
|
-
next false unless
|
|
207
|
-
|
|
676
|
+
admission = @workflow_admissions[workflow_instance_id.to_s]
|
|
677
|
+
next false unless admission&.state == :executing
|
|
678
|
+
fsm_session_id = admission.fsm_session_id
|
|
679
|
+
next false unless fsm_session_id
|
|
680
|
+
next false unless @admitted_fsm_session_ids.include?(fsm_session_id)
|
|
208
681
|
|
|
209
682
|
posted_event = Phronomy::Event.new(
|
|
210
683
|
type: event.to_sym,
|
|
211
|
-
target_id:
|
|
684
|
+
target_id: fsm_session_id,
|
|
212
685
|
payload: payload
|
|
213
686
|
)
|
|
214
687
|
queued_depth = enqueue([posted_event, monotonic_nanoseconds])
|
|
@@ -220,7 +693,6 @@ module Phronomy
|
|
|
220
693
|
true
|
|
221
694
|
end
|
|
222
695
|
|
|
223
|
-
# Interrupts the queue wait so EventLoop can recompute the next timer deadline.
|
|
224
696
|
def wake
|
|
225
697
|
@queue.push(WAKE)
|
|
226
698
|
true
|
|
@@ -228,8 +700,8 @@ module Phronomy
|
|
|
228
700
|
false
|
|
229
701
|
end
|
|
230
702
|
|
|
231
|
-
def
|
|
232
|
-
@lifecycle_mutex.synchronize { @
|
|
703
|
+
def admitted_fsm_session?(fsm_session_id)
|
|
704
|
+
@lifecycle_mutex.synchronize { @admitted_fsm_session_ids.include?(fsm_session_id) }
|
|
233
705
|
end
|
|
234
706
|
|
|
235
707
|
def current?
|
|
@@ -262,8 +734,6 @@ module Phronomy
|
|
|
262
734
|
end
|
|
263
735
|
end
|
|
264
736
|
|
|
265
|
-
# Sends STOP to the queue and joins the EventLoop thread.
|
|
266
|
-
# Assumes sessions have already been drained before this call.
|
|
267
737
|
def stop_and_join(deadline:)
|
|
268
738
|
@shutdown_mutex.synchronize do
|
|
269
739
|
return @shutdown_status if @shutdown_status
|
|
@@ -288,7 +758,6 @@ module Phronomy
|
|
|
288
758
|
end
|
|
289
759
|
end
|
|
290
760
|
|
|
291
|
-
# Legacy entry point kept for any callers that pass deadline:/cancel_grace:.
|
|
292
761
|
def shutdown(deadline:, cancel_grace: deadline)
|
|
293
762
|
stop_and_join(deadline: deadline)
|
|
294
763
|
end
|
|
@@ -297,7 +766,6 @@ module Phronomy
|
|
|
297
766
|
@thread&.alive? || false
|
|
298
767
|
end
|
|
299
768
|
|
|
300
|
-
# Compatibility for existing runtime/shutdown observers during migration.
|
|
301
769
|
alias_method :task_alive?, :thread_alive?
|
|
302
770
|
|
|
303
771
|
private
|
|
@@ -352,12 +820,9 @@ module Phronomy
|
|
|
352
820
|
def dispatch_management(event)
|
|
353
821
|
case event.type
|
|
354
822
|
when :finished, :halted, :error
|
|
355
|
-
|
|
356
|
-
session = @fsms.delete(
|
|
357
|
-
waiter = @waiting.delete(
|
|
358
|
-
# decrement before waking caller so wait_until_idle sees the control-plane
|
|
359
|
-
# session count immediately; Workflow durable admission may intentionally
|
|
360
|
-
# keep Runtime non-idle until its terminal save completes.
|
|
823
|
+
fsm_session_id = event.payload.fetch(:fsm_session_id)
|
|
824
|
+
session = @fsms.delete(fsm_session_id)
|
|
825
|
+
waiter = @waiting.delete(fsm_session_id)
|
|
361
826
|
decrement_outstanding if session
|
|
362
827
|
complete_waiter(waiter, event.payload.fetch(:result))
|
|
363
828
|
when :start
|
|
@@ -366,12 +831,21 @@ module Phronomy
|
|
|
366
831
|
@fsms[session.id] = session
|
|
367
832
|
@waiting[session.id] = waiter if waiter
|
|
368
833
|
session.start
|
|
369
|
-
when :agent_terminal_ready
|
|
834
|
+
when :agent_control, :agent_terminal_ready
|
|
835
|
+
# :agent_terminal_ready is retained as an internal migration-compatible
|
|
836
|
+
# dispatch name; ACS-11 emits the operation-neutral :agent_control event.
|
|
370
837
|
cmd = event.payload.fetch(:command)
|
|
371
838
|
cmd.coordinator.deliver_on_event_loop(cmd)
|
|
372
|
-
when :
|
|
839
|
+
when :agent_physical_work_completed
|
|
840
|
+
complete_agent_physical_work(event.payload)
|
|
841
|
+
when :workflow_control
|
|
373
842
|
cmd = event.payload.fetch(:command)
|
|
374
|
-
cmd.runner.
|
|
843
|
+
cmd.runner.deliver_on_event_loop(cmd)
|
|
844
|
+
when :recovery_required
|
|
845
|
+
fsm_session_id = event.payload.fetch(:fsm_session_id)
|
|
846
|
+
session = @fsms.delete(fsm_session_id)
|
|
847
|
+
decrement_outstanding if session
|
|
848
|
+
mark_workflow_recovery_required_for_session(fsm_session_id)
|
|
375
849
|
end
|
|
376
850
|
end
|
|
377
851
|
|
|
@@ -379,7 +853,43 @@ module Phronomy
|
|
|
379
853
|
event.target_id == SYSTEM_CHANNEL_ID &&
|
|
380
854
|
TERMINAL_MANAGEMENT_EVENTS.include?(event.type) &&
|
|
381
855
|
event.payload.is_a?(Hash) &&
|
|
382
|
-
event.payload.key?(:
|
|
856
|
+
event.payload.key?(:fsm_session_id)
|
|
857
|
+
end
|
|
858
|
+
|
|
859
|
+
def complete_agent_physical_work(payload)
|
|
860
|
+
key = payload.fetch(:execution_id).to_s
|
|
861
|
+
token = payload.fetch(:token)
|
|
862
|
+
deferred = @lifecycle_mutex.synchronize do
|
|
863
|
+
work = @agent_inflight_work.fetch(key, nil)
|
|
864
|
+
next nil unless work&.delete(token)
|
|
865
|
+
|
|
866
|
+
if work.empty?
|
|
867
|
+
@agent_inflight_work.delete(key)
|
|
868
|
+
command = @agent_deferred_terminals.delete(key)
|
|
869
|
+
@idle_cond.broadcast if runtime_idle_locked?
|
|
870
|
+
command
|
|
871
|
+
end
|
|
872
|
+
end
|
|
873
|
+
deferred&.coordinator&.deliver_on_event_loop(deferred)
|
|
874
|
+
true
|
|
875
|
+
end
|
|
876
|
+
|
|
877
|
+
def mark_workflow_recovery_required_for_session(fsm_session_id)
|
|
878
|
+
@lifecycle_mutex.synchronize do
|
|
879
|
+
key, admission = @workflow_admissions.find do |_workflow_instance_id, candidate|
|
|
880
|
+
candidate.fsm_session_id == fsm_session_id.to_s
|
|
881
|
+
end
|
|
882
|
+
return false unless admission
|
|
883
|
+
|
|
884
|
+
@workflow_admissions[key] = WorkflowAdmission.new(
|
|
885
|
+
workflow_instance_id: admission.workflow_instance_id,
|
|
886
|
+
owner_token: admission.owner_token,
|
|
887
|
+
fsm_session_id: nil,
|
|
888
|
+
state: :recovery_required
|
|
889
|
+
)
|
|
890
|
+
@idle_cond.broadcast if runtime_idle_locked?
|
|
891
|
+
end
|
|
892
|
+
true
|
|
383
893
|
end
|
|
384
894
|
|
|
385
895
|
def begin_stopping_if_idle
|
|
@@ -406,9 +916,19 @@ module Phronomy
|
|
|
406
916
|
@waiting.values.each { |waiter| complete_waiter(waiter, error) }
|
|
407
917
|
@waiting.clear
|
|
408
918
|
@fsms.clear
|
|
919
|
+
completion_waiters = @lifecycle_mutex.synchronize do
|
|
920
|
+
waiters = @agent_completion_waiters.values.flatten
|
|
921
|
+
@agent_completion_waiters.clear
|
|
922
|
+
@agent_inflight_work.clear
|
|
923
|
+
@agent_deferred_terminals.clear
|
|
924
|
+
waiters
|
|
925
|
+
end
|
|
926
|
+
completion_waiters.each { |waiter| complete_waiter(waiter, error) }
|
|
409
927
|
@lifecycle_mutex.synchronize do
|
|
410
|
-
@
|
|
928
|
+
@admitted_fsm_session_ids.clear
|
|
411
929
|
@workflow_admissions.clear
|
|
930
|
+
@agent_admissions.clear
|
|
931
|
+
@agent_executions.clear
|
|
412
932
|
@outstanding_sessions = 0
|
|
413
933
|
@idle_cond.broadcast
|
|
414
934
|
end
|
|
@@ -427,8 +947,10 @@ module Phronomy
|
|
|
427
947
|
def notify_unexpected_dispatcher_failure(error)
|
|
428
948
|
@lifecycle_mutex.synchronize do
|
|
429
949
|
@state = :failed
|
|
430
|
-
@
|
|
950
|
+
@admitted_fsm_session_ids.clear
|
|
431
951
|
@workflow_admissions.clear
|
|
952
|
+
@agent_admissions.clear
|
|
953
|
+
@agent_executions.clear
|
|
432
954
|
@idle_cond.broadcast
|
|
433
955
|
end
|
|
434
956
|
cleanup_abandoned_work(error)
|
|
@@ -445,6 +967,13 @@ module Phronomy
|
|
|
445
967
|
"EventLoop is #{@state}; new sessions are not accepted"
|
|
446
968
|
end
|
|
447
969
|
|
|
970
|
+
def assert_event_loop_thread!
|
|
971
|
+
return if current?
|
|
972
|
+
|
|
973
|
+
raise Phronomy::Error,
|
|
974
|
+
"Phronomy-managed live execution state may only be mutated on EventLoop"
|
|
975
|
+
end
|
|
976
|
+
|
|
448
977
|
def decrement_outstanding
|
|
449
978
|
@lifecycle_mutex.synchronize do
|
|
450
979
|
@outstanding_sessions -= 1 if @outstanding_sessions.positive?
|
|
@@ -453,7 +982,23 @@ module Phronomy
|
|
|
453
982
|
end
|
|
454
983
|
|
|
455
984
|
def runtime_idle_locked?
|
|
456
|
-
@outstanding_sessions.zero? &&
|
|
985
|
+
@outstanding_sessions.zero? &&
|
|
986
|
+
@agent_inflight_work.values.all?(&:empty?) &&
|
|
987
|
+
@agent_deferred_terminals.empty? &&
|
|
988
|
+
!workflow_admission_transition_in_progress_locked? &&
|
|
989
|
+
!agent_admission_transition_in_progress_locked?
|
|
990
|
+
end
|
|
991
|
+
|
|
992
|
+
def workflow_admission_transition_in_progress_locked?
|
|
993
|
+
@workflow_admissions.values.any? do |admission|
|
|
994
|
+
%i[admitting executing persisting_terminal].include?(admission.state)
|
|
995
|
+
end
|
|
996
|
+
end
|
|
997
|
+
|
|
998
|
+
def agent_admission_transition_in_progress_locked?
|
|
999
|
+
@agent_admissions.values.any? do |admission|
|
|
1000
|
+
%i[admitting executing resuming cancelling terminalizing].include?(admission.state)
|
|
1001
|
+
end
|
|
457
1002
|
end
|
|
458
1003
|
|
|
459
1004
|
def join_until(deadline)
|
|
@@ -465,12 +1010,26 @@ module Phronomy
|
|
|
465
1010
|
end
|
|
466
1011
|
|
|
467
1012
|
def finalize_terminated(status)
|
|
468
|
-
@lifecycle_mutex.synchronize do
|
|
1013
|
+
pending_waiters = @lifecycle_mutex.synchronize do
|
|
1014
|
+
waiters = @agent_completion_waiters.values.flatten
|
|
1015
|
+
@agent_completion_waiters.clear
|
|
1016
|
+
@agent_inflight_work.clear
|
|
1017
|
+
@agent_deferred_terminals.clear
|
|
469
1018
|
@state = :terminated
|
|
470
|
-
@
|
|
1019
|
+
@admitted_fsm_session_ids.clear
|
|
471
1020
|
@workflow_admissions.clear
|
|
1021
|
+
@agent_admissions.clear
|
|
1022
|
+
@agent_executions.clear
|
|
472
1023
|
@thread = nil unless @thread&.alive?
|
|
473
1024
|
@idle_cond.broadcast
|
|
1025
|
+
waiters
|
|
1026
|
+
end
|
|
1027
|
+
if pending_waiters.any?
|
|
1028
|
+
error = Phronomy::ExecutionRehydrationRequiredError.new(
|
|
1029
|
+
"Runtime terminated while Agent execution remained nonterminal; " \
|
|
1030
|
+
"process-local Task handles are not rehydrated"
|
|
1031
|
+
)
|
|
1032
|
+
pending_waiters.each { |waiter| complete_waiter(waiter, error) }
|
|
474
1033
|
end
|
|
475
1034
|
status
|
|
476
1035
|
end
|