@zq-silk/yui 0.13.3 → 0.13.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -8
- package/dist/cli/commandCatalog.js +22 -21
- package/dist/cli/interactionPolicy.js +0 -14
- package/dist/cli.js +42 -0
- package/dist/commands/executionAuditCommands.js +1 -1
- package/dist/commands/taskCommands.js +60 -326
- package/dist/commands/taskContextCommand.js +3 -14
- package/dist/commands/taskExecutionCommands.js +254 -0
- package/dist/commands/taskNextActionCommand.js +1 -3
- package/dist/commands/taskOverviewCommand.js +9 -2
- package/dist/commands/taskRoleRuntimeStatus.js +2 -25
- package/dist/controller/agentRuntimeObserver.js +4 -2
- package/dist/controller/clientRuntime.js +45 -2
- package/dist/controller/controller.js +6 -3
- package/dist/controller/fileSchedulerStoreAdapter.js +70 -188
- package/dist/controller/jobControl.js +3 -2
- package/dist/controller/runtime.js +6 -33
- package/dist/controller/runtimeEventProcessor.js +8 -4
- package/dist/controller/runtimeHookRunFence.js +4 -10
- package/dist/execution/executionHealth.js +8 -16
- package/dist/executor/agentExecutor.js +13 -14
- package/dist/executor/fileRoleLaunchPlanner.js +9 -16
- package/dist/lifecycle/exactRunTerminalization.js +24 -322
- package/dist/repository/taskWorkspaceCoordinator.js +0 -9
- package/dist/runtime/agentHost.js +20 -80
- package/dist/runtime/exactControlPlane.js +15 -9
- package/dist/runtime/providerContinuationReconciliationService.js +1 -1
- package/dist/runtime/providerRecoveryDecision.js +1 -1
- package/dist/runtime/providerRuntimeIdentity.js +25 -15
- package/dist/scheduler/activeRoleRunDelivery.js +1 -19
- package/dist/scheduler/leaderWakeupProcessor.js +12 -63
- package/dist/scheduler/ports.js +3 -2
- package/dist/scheduler/roleRunLiveness.js +4 -1
- package/dist/scheduler/roleRunStall.js +0 -2
- package/dist/scheduler/taskExecutionProjection.js +18 -1
- package/dist/scheduler/wakeupQueue.js +2 -1
- package/dist/storage/migration/productionRegistry.js +65 -0
- package/dist/storage/sqliteStore.js +10 -2
- package/dist/storage/taskStore.js +11 -3
- package/dist/task/completionReadiness.js +0 -67
- package/dist/task/nextAction.js +16 -32
- package/dist/task/task.js +38 -3
- package/dist/web/assets/client/i18n.js +0 -4
- package/dist/web/assets/client/view.js +0 -18
- package/dist/web/webSnapshot.js +7 -13
- package/package.json +1 -1
- package/dist/run/recoveryProjection.js +0 -252
- package/dist/runtime/conversationSwitch.js +0 -277
|
@@ -4,7 +4,6 @@ import { nativeSessionIdForLaunch } from "../runtime/preallocatedNativeSession.j
|
|
|
4
4
|
import { runtimeObservationFromTaskEvent } from "../runtime/runtimeObservation.js";
|
|
5
5
|
import { YUI_CONTROL_PLANE_DESCRIPTOR, YUI_TASK_RUNTIME_DESCRIPTOR, assertExactTaskRuntimeEnvironment, exactControlPlaneDigest, parseExactControlPlaneDescriptor, refreshReusedTaskRuntimeDescriptorSource } from "../runtime/exactControlPlane.js";
|
|
6
6
|
import { agentRunDeliveryReceiptId } from "../run/agentRun.js";
|
|
7
|
-
import { conversationReplacementBasis } from "../runtime/conversationSwitch.js";
|
|
8
7
|
/**
|
|
9
8
|
* Resolves turn identity from the current durable in-flight fence. The one
|
|
10
9
|
* exception is a Driver-declared startup Session Hook, which can arrive before
|
|
@@ -37,7 +36,9 @@ export function resolveRuntimeHookRunFence(environment, adapterId, payloadNative
|
|
|
37
36
|
}
|
|
38
37
|
const store = openCompatibleFileTaskStore(home);
|
|
39
38
|
const task = store.getTask(taskId);
|
|
40
|
-
if (task === null
|
|
39
|
+
if (task === null
|
|
40
|
+
|| task.status !== "active"
|
|
41
|
+
|| task.executionGate.state !== "enabled") {
|
|
41
42
|
throw new Error("Runtime observation Hook Task is not current and active.");
|
|
42
43
|
}
|
|
43
44
|
const role = store.getRole(taskId, roleName);
|
|
@@ -107,14 +108,7 @@ export function resolveRuntimeHookRunFence(environment, adapterId, payloadNative
|
|
|
107
108
|
&& sessions !== null
|
|
108
109
|
&& startupReservation
|
|
109
110
|
&& startupRun?.mode === "new"
|
|
110
|
-
&&
|
|
111
|
-
sessions,
|
|
112
|
-
events: store.listEvents(taskId),
|
|
113
|
-
mailbox: roleMailbox,
|
|
114
|
-
roleName,
|
|
115
|
-
runId: startupRun.id,
|
|
116
|
-
runMode: startupRun.mode
|
|
117
|
-
}) !== null;
|
|
111
|
+
&& (session.status === "stopped" || session.status === "broken");
|
|
118
112
|
const preallocatedStartup = options.startupSession === "preallocated"
|
|
119
113
|
&& expectedNativeSessionId !== undefined
|
|
120
114
|
&& (session === undefined || replacementStartup)
|
|
@@ -64,11 +64,6 @@ export function summarizeExecutionGroupHealth(input) {
|
|
|
64
64
|
}
|
|
65
65
|
/** Unresolved Lane recovery in deterministic operational priority order. */
|
|
66
66
|
export function actionableExecutionLaneRecoveries(groups) {
|
|
67
|
-
const priority = {
|
|
68
|
-
"terminate-exact-run": 0,
|
|
69
|
-
"retry-new-agent-run": 1,
|
|
70
|
-
diagnose: 2
|
|
71
|
-
};
|
|
72
67
|
return groups
|
|
73
68
|
.filter(({ resolution }) => resolution === undefined)
|
|
74
69
|
.flatMap((group) => group.laneSummaries.flatMap((lane) => {
|
|
@@ -76,9 +71,7 @@ export function actionableExecutionLaneRecoveries(groups) {
|
|
|
76
71
|
&& group.resources !== undefined
|
|
77
72
|
&& executionStageSpendClosed(group.resources))
|
|
78
73
|
return [];
|
|
79
|
-
if (lane.recovery !== "
|
|
80
|
-
&& lane.recovery !== "terminate-exact-run"
|
|
81
|
-
&& lane.recovery !== "retry-new-agent-run")
|
|
74
|
+
if (lane.recovery !== "retry-new-agent-run")
|
|
82
75
|
return [];
|
|
83
76
|
return [{
|
|
84
77
|
groupId: group.groupId,
|
|
@@ -87,8 +80,7 @@ export function actionableExecutionLaneRecoveries(groups) {
|
|
|
87
80
|
...(lane.runtimeHealth === undefined ? {} : { runtimeHealth: lane.runtimeHealth }),
|
|
88
81
|
recovery: lane.recovery
|
|
89
82
|
}];
|
|
90
|
-
}))
|
|
91
|
-
.sort((left, right) => priority[left.recovery] - priority[right.recovery]);
|
|
83
|
+
}));
|
|
92
84
|
}
|
|
93
85
|
function projectExecutionLaneHealth(lane, input, policy) {
|
|
94
86
|
const run = lane.runId === undefined
|
|
@@ -149,7 +141,7 @@ function projectExecutionLaneHealth(lane, input, policy) {
|
|
|
149
141
|
if (run === undefined) {
|
|
150
142
|
return projection(lane, {
|
|
151
143
|
runtimeHealth: "suspected-stalled",
|
|
152
|
-
recovery: "
|
|
144
|
+
recovery: "inspect",
|
|
153
145
|
resultReusable: false,
|
|
154
146
|
reason: "the running Lane has no exact AgentRun record",
|
|
155
147
|
evidence: ["execution-lineage-missing"]
|
|
@@ -167,7 +159,7 @@ function projectExecutionLaneHealth(lane, input, policy) {
|
|
|
167
159
|
if (run.status !== "active") {
|
|
168
160
|
return projection(lane, {
|
|
169
161
|
runtimeHealth: "suspected-stalled",
|
|
170
|
-
recovery: "
|
|
162
|
+
recovery: "inspect",
|
|
171
163
|
resultReusable: false,
|
|
172
164
|
reason: "the Lane is running but its exact AgentRun is terminal without a Lane result",
|
|
173
165
|
evidence: ["execution-lineage-inconsistent"]
|
|
@@ -189,7 +181,7 @@ function projectExecutionLaneHealth(lane, input, policy) {
|
|
|
189
181
|
&& observation.payload.failure?.runTerminal === true)) && !unsettledChildWork) {
|
|
190
182
|
return projection(lane, {
|
|
191
183
|
runtimeHealth: "confirmed-dead",
|
|
192
|
-
recovery: "
|
|
184
|
+
recovery: "inspect",
|
|
193
185
|
resultReusable: false,
|
|
194
186
|
reason: "the Provider reported an exact run-terminal failure",
|
|
195
187
|
evidence: ["provider-run-terminal"]
|
|
@@ -206,7 +198,7 @@ function projectExecutionLaneHealth(lane, input, policy) {
|
|
|
206
198
|
if (runtimeTerminalEvidence.length > 0 && !unsettledChildWork) {
|
|
207
199
|
return projection(lane, {
|
|
208
200
|
runtimeHealth: "confirmed-dead",
|
|
209
|
-
recovery: "
|
|
201
|
+
recovery: "inspect",
|
|
210
202
|
resultReusable: false,
|
|
211
203
|
reason: "the exact runtime host or Session is terminal and no unsettled child work remains",
|
|
212
204
|
evidence: runtimeTerminalEvidence
|
|
@@ -220,7 +212,7 @@ function projectExecutionLaneHealth(lane, input, policy) {
|
|
|
220
212
|
&& !unsettledChildWork) {
|
|
221
213
|
return projection(lane, {
|
|
222
214
|
runtimeHealth: "confirmed-dead",
|
|
223
|
-
recovery: "
|
|
215
|
+
recovery: "inspect",
|
|
224
216
|
resultReusable: false,
|
|
225
217
|
reason: "the exact Session and abnormal process exit independently confirm death",
|
|
226
218
|
evidence: ["native-session-terminal", `process-exit:${exit.classification}`]
|
|
@@ -229,7 +221,7 @@ function projectExecutionLaneHealth(lane, input, policy) {
|
|
|
229
221
|
if (isRoleRunStalled(input.events, run.id)) {
|
|
230
222
|
return projection(lane, {
|
|
231
223
|
runtimeHealth: "suspected-stalled",
|
|
232
|
-
recovery: "
|
|
224
|
+
recovery: "inspect",
|
|
233
225
|
resultReusable: false,
|
|
234
226
|
reason: `the durable progress clock has not advanced since ${latestStallProgressAt(input.events, run.id) ?? run.updatedAt}; no death proof exists`,
|
|
235
227
|
evidence: ["run-stalled"]
|
|
@@ -120,12 +120,8 @@ export function recordRoleAgentSession(set, input, now) {
|
|
|
120
120
|
validateRoleSessionSet(updated);
|
|
121
121
|
return updated;
|
|
122
122
|
}
|
|
123
|
-
/**
|
|
124
|
-
|
|
125
|
-
* Conversation selected by an already-authorized switch. Authorization is
|
|
126
|
-
* deliberately owned by the Controller caller, not inferred here.
|
|
127
|
-
*/
|
|
128
|
-
export function replaceTaskRoleAgentSessionForConversationSwitch(set, input, now) {
|
|
123
|
+
/** Atomically archives a disposable old native Session and binds its replacement. */
|
|
124
|
+
export function replaceTaskRoleAgentSession(set, input, now) {
|
|
129
125
|
validateRoleSessionSet(set);
|
|
130
126
|
const existing = set.sessions[input.agentId];
|
|
131
127
|
if (existing === undefined || existing.nativeSessionId === input.nativeSessionId) {
|
|
@@ -302,12 +298,15 @@ export function bindTaskRoleRun(set, fence, preparedAt, mode) {
|
|
|
302
298
|
throw new Error("Task Role session set already has an in-flight Run.");
|
|
303
299
|
}
|
|
304
300
|
const timestamp = requireDate(preparedAt, "Task Role Run preparedAt");
|
|
305
|
-
// A fresh
|
|
306
|
-
//
|
|
307
|
-
//
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
301
|
+
// A fresh Session follows physical cleanup and does not inherit execution
|
|
302
|
+
// fences from the disposable old Conversation. Runtime events retain its
|
|
303
|
+
// audit evidence; the new Provider binds a fresh control identity when it is
|
|
304
|
+
// observed. Resume keeps the existing exact Conversation fence.
|
|
305
|
+
const providerBinding = mode === "new"
|
|
306
|
+
? null
|
|
307
|
+
: set.providerBinding !== null
|
|
308
|
+
? rebindProviderRuntimeRun(set.providerBinding, normalized.runId)
|
|
309
|
+
: null;
|
|
311
310
|
const updated = {
|
|
312
311
|
...set,
|
|
313
312
|
inFlight: { ...normalized, preparedAt: timestamp },
|
|
@@ -666,8 +665,8 @@ export function validateRoleAgentSession(session, expectedAgentId = session.agen
|
|
|
666
665
|
if (session.effective.agentId !== agentId || session.effective.adapterId !== session.adapterId) {
|
|
667
666
|
throw new Error(`Role Agent session effective identity is inconsistent: ${agentId}.`);
|
|
668
667
|
}
|
|
669
|
-
// A restored opaque host may have no provider-native identity; its
|
|
670
|
-
// fence
|
|
668
|
+
// A restored opaque host may have no provider-native identity; retain its
|
|
669
|
+
// launch fence so it can be inspected or stopped without inventing identity.
|
|
671
670
|
if (session.nativeSessionId === undefined) {
|
|
672
671
|
if (session.launchId === undefined) {
|
|
673
672
|
throw new Error("Role Agent session requires a native Session or launch id.");
|
|
@@ -27,7 +27,6 @@ import { parseTaskRuntimeIsolationDescriptor, taskRuntimeIsolationEnvironment }
|
|
|
27
27
|
import { ResourceRegistrar } from "../resources/resourceRegistrar.js";
|
|
28
28
|
import { builtinAgentDriverRegistry, builtinDriverIdForAdapter } from "../runtime/builtinAgentDrivers.js";
|
|
29
29
|
import { managedRuntimeAdmission } from "../runtime/agentDriver.js";
|
|
30
|
-
import { freshConversationLaunchAllowed } from "../runtime/conversationSwitch.js";
|
|
31
30
|
import { currentProviderActivation } from "../runtime/providerRuntimeIdentity.js";
|
|
32
31
|
import { assertCodexLaunchOverridesAvailable, inspectCodexLaunchConfig } from "./codexConfigConflict.js";
|
|
33
32
|
/** Builds managed native Agent launches from the authoritative Task records. */
|
|
@@ -97,7 +96,10 @@ export class FileRoleLaunchPlanner {
|
|
|
97
96
|
refreshTaskRuntimeDescriptor(input) {
|
|
98
97
|
const task = this.store.getTask(input.taskId);
|
|
99
98
|
const role = this.store.getRole(input.taskId, input.roleName);
|
|
100
|
-
if (task === null
|
|
99
|
+
if (task === null
|
|
100
|
+
|| task.status !== "active"
|
|
101
|
+
|| task.executionGate.state !== "enabled"
|
|
102
|
+
|| role === null) {
|
|
101
103
|
throw new Error(`Task runtime is not current: ${input.taskId}/${input.roleName}.`);
|
|
102
104
|
}
|
|
103
105
|
const run = this.store.getActiveAgentRun(input.taskId, input.roleName);
|
|
@@ -137,8 +139,9 @@ export class FileRoleLaunchPlanner {
|
|
|
137
139
|
const task = this.store.getTask(input.taskId);
|
|
138
140
|
if (task === null)
|
|
139
141
|
throw new Error(`Task not found: ${input.taskId}.`);
|
|
140
|
-
if (task.status !== "active")
|
|
141
|
-
throw new Error(`Task is not
|
|
142
|
+
if (task.status !== "active" || task.executionGate.state !== "enabled") {
|
|
143
|
+
throw new Error(`Task execution is not enabled: ${input.taskId}.`);
|
|
144
|
+
}
|
|
142
145
|
const role = this.store.getRole(input.taskId, input.roleName);
|
|
143
146
|
if (role === null)
|
|
144
147
|
throw new Error(`Role not found: ${input.taskId}/${input.roleName}.`);
|
|
@@ -228,18 +231,8 @@ export class FileRoleLaunchPlanner {
|
|
|
228
231
|
throw new Error(`Task Role resume effective snapshot drifted: ${task.id}/${role.name}.`);
|
|
229
232
|
}
|
|
230
233
|
if (input.mode === "new" && sessionSet !== null
|
|
231
|
-
&&
|
|
232
|
-
|
|
233
|
-
events: this.store.listEvents(task.id),
|
|
234
|
-
mailbox: this.store.getWorkMailbox({
|
|
235
|
-
kind: "role",
|
|
236
|
-
taskId: task.id,
|
|
237
|
-
roleName: role.name
|
|
238
|
-
}),
|
|
239
|
-
roleName: role.name,
|
|
240
|
-
...(input.runId === undefined ? {} : { candidateRunId: input.runId })
|
|
241
|
-
})) {
|
|
242
|
-
throw new Error(`Fresh Provider Conversation is not authorized: ${task.id}/${role.name}.`);
|
|
234
|
+
&& activeLiveRoleAgentSession(sessionSet) !== null) {
|
|
235
|
+
throw new Error(`Task Role still has a live Session: ${task.id}/${role.name}.`);
|
|
243
236
|
}
|
|
244
237
|
return this.#compile(role, input, { scope: "task", taskId: task.id }, resolveTaskRoleSessionTitle(input.mode === "resume" ? existing?.title : undefined, task, role.name), input.mode === "resume" && compatibleExisting ? existing.nativeSessionId : undefined, runWorkspace, effective, {
|
|
245
238
|
purpose: activeRun?.purpose ?? "execution"
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import { isDeepStrictEqual } from "node:util";
|
|
2
2
|
import { completeProcessing } from "../coordination/workMailbox.js";
|
|
3
|
-
import { enqueueWork } from "../coordination/workMailboxQueue.js";
|
|
4
3
|
import { settleExactWorkExecution } from "../coordination/workMailboxQueue.js";
|
|
5
4
|
import { terminalizeTaskRoleRunSession } from "../executor/agentExecutor.js";
|
|
6
5
|
import { updateRoleStatus } from "../role/role.js";
|
|
7
|
-
import { createTaskEvent } from "../event/taskEvent.js";
|
|
8
6
|
import { finishReviewRound, updateReviewExecutionGroup } from "../review/reviewRound.js";
|
|
9
7
|
import { reconcileReviewFindingsAfterReview } from "../review/reviewFindingLedger.js";
|
|
10
8
|
import { agentRunDeliveryReceiptId, failAgentRun, withYieldReceipt, yieldAgentRun } from "../run/agentRun.js";
|
|
@@ -12,10 +10,9 @@ import { createYieldReceipt } from "../run/yieldReceipt.js";
|
|
|
12
10
|
import { recordExecutionLaneResult, resolveExecutionGroup } from "../execution/executionGroup.js";
|
|
13
11
|
import { isRuntimeLaunchReservation, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
14
12
|
import { runOwnsBlockingProviderContinuation } from "../runtime/runtimeContinuationProjection.js";
|
|
15
|
-
import {
|
|
16
|
-
import { latestRunDurableProgressAt, RUN_RECOVERY_APPLIED_EVENT, RUN_RECOVERY_REQUESTED_EVENT } from "../scheduler/roleRunStall.js";
|
|
13
|
+
import { latestRunDurableProgressAt } from "../scheduler/roleRunStall.js";
|
|
17
14
|
import { markTaskWakeConsumed } from "../scheduler/taskWake.js";
|
|
18
|
-
import { workItemExecutionGroupById,
|
|
15
|
+
import { workItemExecutionGroupById, updateWorkItemExecutionGroup } from "../workItem/workItem.js";
|
|
19
16
|
/**
|
|
20
17
|
* Validate every immutable identity and frozen Project head needed before a
|
|
21
18
|
* review Run can settle any mailbox or Round state. This is deliberately
|
|
@@ -249,23 +246,22 @@ export function retireExactActiveAgentRun(store, input, now) {
|
|
|
249
246
|
if (progress.progressAt !== input.expectedProgressAt) {
|
|
250
247
|
return { ...stateChanged("progress-fence-mismatch"), progressAt: progress.progressAt };
|
|
251
248
|
}
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
249
|
+
const sessions = store.getTaskRoleSessionSet(input.taskId, input.roleName);
|
|
250
|
+
const terminalInput = {
|
|
251
|
+
taskId: input.taskId,
|
|
252
|
+
roleName: input.roleName,
|
|
253
|
+
agentId: input.agentId,
|
|
254
|
+
runId: input.runId,
|
|
255
|
+
receiptId: agentRunDeliveryReceiptId(current),
|
|
256
|
+
...(input.nativeSessionId === undefined ? {} : { nativeSessionId: input.nativeSessionId }),
|
|
257
|
+
...(input.launchId === undefined ? {} : { launchId: input.launchId }),
|
|
258
|
+
settleFailedExecutionGroup: true,
|
|
259
|
+
outcome: { status: "failed", summary: input.reason }
|
|
260
|
+
};
|
|
261
|
+
if (!matchesSessionFence(sessions, terminalInput)
|
|
262
|
+
|| !matchesLaunchFence(store, sessions, terminalInput)) {
|
|
258
263
|
return stateChanged("session-or-launch-fence-mismatch");
|
|
259
|
-
const blocker = exactRecoveryExecutionBlocker(store, current);
|
|
260
|
-
if (blocker !== null) {
|
|
261
|
-
return {
|
|
262
|
-
disposition: "blocked",
|
|
263
|
-
run: current,
|
|
264
|
-
progressAt: progress.progressAt,
|
|
265
|
-
reason: blocker
|
|
266
|
-
};
|
|
267
264
|
}
|
|
268
|
-
const sessions = store.getTaskRoleSessionSet(input.taskId, input.roleName);
|
|
269
265
|
const session = sessions?.sessions[input.agentId];
|
|
270
266
|
const providerBinding = sessions?.providerBinding;
|
|
271
267
|
const providerTurn = providerBinding?.turn;
|
|
@@ -282,28 +278,10 @@ export function retireExactActiveAgentRun(store, input, now) {
|
|
|
282
278
|
reason: "runtime-not-terminal"
|
|
283
279
|
};
|
|
284
280
|
}
|
|
285
|
-
const terminal = terminalizeExactTaskRun(store,
|
|
286
|
-
taskId: input.taskId,
|
|
287
|
-
roleName: input.roleName,
|
|
288
|
-
agentId: input.agentId,
|
|
289
|
-
runId: input.runId,
|
|
290
|
-
receiptId: agentRunDeliveryReceiptId(current),
|
|
291
|
-
...(input.nativeSessionId === undefined ? {} : { nativeSessionId: input.nativeSessionId }),
|
|
292
|
-
...(input.launchId === undefined ? {} : { launchId: input.launchId }),
|
|
293
|
-
settleFailedExecutionGroup: true,
|
|
294
|
-
outcome: { status: "failed", summary: input.reason }
|
|
295
|
-
}, now);
|
|
281
|
+
const terminal = terminalizeExactTaskRun(store, terminalInput, now);
|
|
296
282
|
if (terminal.disposition !== "applied" || terminal.run === null) {
|
|
297
283
|
return stateChanged(terminal.reason ?? "terminalization-fence-mismatch");
|
|
298
284
|
}
|
|
299
|
-
if (terminal.run.purpose === "execution" && terminal.run.workItemId !== undefined) {
|
|
300
|
-
const item = store.getWorkItem(input.taskId, terminal.run.workItemId);
|
|
301
|
-
if (item !== null
|
|
302
|
-
&& !["completed", "failed", "retired"].includes(item.status)
|
|
303
|
-
&& !workItemOwnsUnresolvedExecutionLane(item, terminal.run.executionGroupId, terminal.run.executionLaneId)) {
|
|
304
|
-
store.saveWorkItem(input.taskId, updateWorkItemStatus(item, "failed", now, input.reason));
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
285
|
return {
|
|
308
286
|
disposition: "applied",
|
|
309
287
|
run: terminal.run,
|
|
@@ -342,12 +320,13 @@ export function terminalizeExactTaskRun(store, input, now) {
|
|
|
342
320
|
if (!matchesLaunchFence(store, sessions, input)) {
|
|
343
321
|
return obsolete(run, "launch-fence-mismatch");
|
|
344
322
|
}
|
|
345
|
-
if (
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
323
|
+
if (input.settleFailedExecutionGroup !== true
|
|
324
|
+
&& runOwnsBlockingProviderContinuation(store.listEvents(input.taskId), {
|
|
325
|
+
taskId: run.taskId,
|
|
326
|
+
roleName: run.roleName,
|
|
327
|
+
runId: run.id,
|
|
328
|
+
agentId: run.effective.agentId
|
|
329
|
+
})) {
|
|
351
330
|
return obsolete(run, "provider-continuation-writer-owned");
|
|
352
331
|
}
|
|
353
332
|
// Validate the exact ReviewRound, Candidate, stored workspace, and frozen
|
|
@@ -482,283 +461,6 @@ export function terminalizeExactTaskRun(store, input, now) {
|
|
|
482
461
|
settleLaunchReservation(store, sessions, input);
|
|
483
462
|
return { disposition: "applied", run: terminal };
|
|
484
463
|
}
|
|
485
|
-
/**
|
|
486
|
-
* Leader-controlled recovery boundary for one active AgentRun. This primitive
|
|
487
|
-
* validates every durable fence in one transaction and records only a
|
|
488
|
-
* structured request for same-Run diagnosis/retry. It never writes terminal
|
|
489
|
-
* bytes, retries a provider input, kills a host, or silently rebinds a native
|
|
490
|
-
* generation. Explicit termination is the sole action that changes Run state.
|
|
491
|
-
*/
|
|
492
|
-
export function recoverExactAgentRun(store, input) {
|
|
493
|
-
return store.transaction((tx) => recoverExactAgentRunInTransaction(tx, input));
|
|
494
|
-
}
|
|
495
|
-
/** Alias named after the existing exact terminalization primitive. */
|
|
496
|
-
export const recoverExactTaskRun = recoverExactAgentRun;
|
|
497
|
-
function recoverExactAgentRunInTransaction(store, input) {
|
|
498
|
-
const current = store.getAgentRun(input.taskId, input.runId);
|
|
499
|
-
const stateChanged = (reason) => ({
|
|
500
|
-
disposition: "state-changed",
|
|
501
|
-
action: input.action,
|
|
502
|
-
run: current,
|
|
503
|
-
...(current === null ? {} : { progressAt: latestRunDurableProgressAt(store, input.taskId, input.roleName, input.runId)?.progressAt }),
|
|
504
|
-
reason
|
|
505
|
-
});
|
|
506
|
-
const task = store.getTask(input.taskId);
|
|
507
|
-
if (task === null)
|
|
508
|
-
return stateChanged("task-missing");
|
|
509
|
-
if (task.status !== "active")
|
|
510
|
-
return stateChanged("task-terminal");
|
|
511
|
-
if (current === null)
|
|
512
|
-
return stateChanged("run-missing");
|
|
513
|
-
if (current.status !== "active")
|
|
514
|
-
return stateChanged("run-terminal");
|
|
515
|
-
if (current.taskId !== input.taskId || current.roleName !== input.roleName) {
|
|
516
|
-
return stateChanged("run-owner-mismatch");
|
|
517
|
-
}
|
|
518
|
-
if (current.effective.agentId !== input.agentId
|
|
519
|
-
|| current.effective.adapterId !== input.adapterId) {
|
|
520
|
-
return stateChanged("run-launch-identity-mismatch");
|
|
521
|
-
}
|
|
522
|
-
const role = store.getRole(input.taskId, input.roleName);
|
|
523
|
-
if (role === null)
|
|
524
|
-
return stateChanged("role-missing");
|
|
525
|
-
const active = current.executionGroupId !== undefined && current.executionLaneId !== undefined
|
|
526
|
-
? store.getActiveExecutionLaneRun(input.taskId, current.executionGroupId, current.executionLaneId)
|
|
527
|
-
: store.getActiveAgentRun(input.taskId, input.roleName);
|
|
528
|
-
if (active?.id !== current.id) {
|
|
529
|
-
return stateChanged("active-run-mismatch");
|
|
530
|
-
}
|
|
531
|
-
const progress = latestRunDurableProgressAt(store, input.taskId, input.roleName, input.runId);
|
|
532
|
-
if (progress === null)
|
|
533
|
-
return stateChanged("progress-unavailable");
|
|
534
|
-
if (progress.progressAt !== input.expectedProgressAt) {
|
|
535
|
-
return {
|
|
536
|
-
...stateChanged("progress-fence-mismatch"),
|
|
537
|
-
progressAt: progress.progressAt
|
|
538
|
-
};
|
|
539
|
-
}
|
|
540
|
-
// Acceptance is a durable delivery boundary, not a Leader assertion. An
|
|
541
|
-
// accepted request must match the Run's persisted receipt; an already
|
|
542
|
-
// delivered Run cannot be reclassified as provider-rejected.
|
|
543
|
-
if ((input.providerAcceptance === "accepted" && current.deliveredAt === undefined)
|
|
544
|
-
|| (input.providerAcceptance === "rejected" && current.deliveredAt !== undefined)) {
|
|
545
|
-
return {
|
|
546
|
-
disposition: "blocked",
|
|
547
|
-
action: input.action,
|
|
548
|
-
run: current,
|
|
549
|
-
progressAt: progress.progressAt,
|
|
550
|
-
reason: "provider-acceptance-mismatch"
|
|
551
|
-
};
|
|
552
|
-
}
|
|
553
|
-
if (!matchesRecoverySessionFence(store, input)) {
|
|
554
|
-
return stateChanged("session-or-launch-fence-mismatch");
|
|
555
|
-
}
|
|
556
|
-
const recoveryBlocker = exactRecoveryExecutionBlocker(store, current);
|
|
557
|
-
if (recoveryBlocker !== null) {
|
|
558
|
-
return {
|
|
559
|
-
disposition: "blocked",
|
|
560
|
-
action: input.action,
|
|
561
|
-
run: current,
|
|
562
|
-
progressAt: progress.progressAt,
|
|
563
|
-
reason: recoveryBlocker
|
|
564
|
-
};
|
|
565
|
-
}
|
|
566
|
-
if (input.action === "terminate") {
|
|
567
|
-
const sessions = store.getTaskRoleSessionSet(input.taskId, input.roleName);
|
|
568
|
-
const session = sessions?.sessions[input.agentId];
|
|
569
|
-
const providerTurn = sessions?.providerBinding?.turn;
|
|
570
|
-
const exactTerminalEvidence = session?.status === "stopped"
|
|
571
|
-
|| session?.status === "broken"
|
|
572
|
-
|| providerTurn?.status === "failed"
|
|
573
|
-
|| providerTurn?.status === "cancelled"
|
|
574
|
-
|| providerTurn?.status === "rejected";
|
|
575
|
-
if (!exactTerminalEvidence) {
|
|
576
|
-
return {
|
|
577
|
-
disposition: "blocked",
|
|
578
|
-
action: input.action,
|
|
579
|
-
run: current,
|
|
580
|
-
progressAt: progress.progressAt,
|
|
581
|
-
reason: "runtime-not-terminal"
|
|
582
|
-
};
|
|
583
|
-
}
|
|
584
|
-
}
|
|
585
|
-
if (input.providerAcceptance === "ambiguous"
|
|
586
|
-
&& input.action !== "diagnose") {
|
|
587
|
-
return {
|
|
588
|
-
disposition: "blocked",
|
|
589
|
-
action: input.action,
|
|
590
|
-
run: current,
|
|
591
|
-
progressAt: progress.progressAt,
|
|
592
|
-
reason: "provider-acceptance-ambiguous"
|
|
593
|
-
};
|
|
594
|
-
}
|
|
595
|
-
if (!hasRecoveryReason(input.reason)) {
|
|
596
|
-
return {
|
|
597
|
-
disposition: "blocked",
|
|
598
|
-
action: input.action,
|
|
599
|
-
run: current,
|
|
600
|
-
progressAt: progress.progressAt,
|
|
601
|
-
reason: "recovery-reason-required"
|
|
602
|
-
};
|
|
603
|
-
}
|
|
604
|
-
const eventPayload = {
|
|
605
|
-
runId: current.id,
|
|
606
|
-
roleName: current.roleName,
|
|
607
|
-
action: input.action,
|
|
608
|
-
providerAcceptance: input.providerAcceptance,
|
|
609
|
-
progressAt: progress.progressAt,
|
|
610
|
-
...(input.nativeSessionId === undefined ? {} : { nativeSessionId: input.nativeSessionId }),
|
|
611
|
-
...(input.launchId === undefined ? {} : { launchId: input.launchId }),
|
|
612
|
-
reason: input.reason
|
|
613
|
-
};
|
|
614
|
-
const events = store.listEvents(input.taskId);
|
|
615
|
-
const alreadyRequested = events.some((event) => (event.type === RUN_RECOVERY_REQUESTED_EVENT
|
|
616
|
-
&& event.payload.runId === current.id
|
|
617
|
-
&& event.payload.action === input.action
|
|
618
|
-
&& event.payload.progressAt === progress.progressAt
|
|
619
|
-
&& event.payload.nativeSessionId === input.nativeSessionId
|
|
620
|
-
&& event.payload.launchId === input.launchId));
|
|
621
|
-
if (input.action !== "terminate") {
|
|
622
|
-
if (!alreadyRequested) {
|
|
623
|
-
store.saveEvent(input.taskId, createTaskEvent(store.nextEventId(input.taskId), input.taskId, RUN_RECOVERY_REQUESTED_EVENT, eventPayload, input.now));
|
|
624
|
-
// A non-Leader request is surfaced through the existing Leader mailbox;
|
|
625
|
-
// it is not a Task Message and is coalesced by the mailbox reason.
|
|
626
|
-
if (input.roleName !== "leader") {
|
|
627
|
-
enqueueWork(store, { kind: "role", taskId: input.taskId, roleName: "leader" }, "run-recovery-requested", input.now, [{ type: "run", taskId: input.taskId, id: current.id }]);
|
|
628
|
-
}
|
|
629
|
-
}
|
|
630
|
-
return {
|
|
631
|
-
disposition: "applied",
|
|
632
|
-
action: input.action,
|
|
633
|
-
run: current,
|
|
634
|
-
progressAt: progress.progressAt,
|
|
635
|
-
requiresExplicitFollowup: true
|
|
636
|
-
};
|
|
637
|
-
}
|
|
638
|
-
const terminalization = terminalizeExactTaskRun(store, {
|
|
639
|
-
taskId: input.taskId,
|
|
640
|
-
roleName: input.roleName,
|
|
641
|
-
agentId: input.agentId,
|
|
642
|
-
runId: input.runId,
|
|
643
|
-
receiptId: agentRunDeliveryReceiptId(current),
|
|
644
|
-
...(input.nativeSessionId === undefined ? {} : { nativeSessionId: input.nativeSessionId }),
|
|
645
|
-
...(input.launchId === undefined ? {} : { launchId: input.launchId }),
|
|
646
|
-
outcome: { status: "failed", summary: input.reason }
|
|
647
|
-
}, input.now);
|
|
648
|
-
if (terminalization.disposition !== "applied" || terminalization.run === null) {
|
|
649
|
-
return stateChanged(terminalization.reason ?? "terminalization-fence-mismatch");
|
|
650
|
-
}
|
|
651
|
-
const terminal = terminalization.run;
|
|
652
|
-
if (terminal.purpose === "execution" && terminal.workItemId !== undefined) {
|
|
653
|
-
const item = store.getWorkItem(input.taskId, terminal.workItemId);
|
|
654
|
-
if (item !== null
|
|
655
|
-
&& !["completed", "failed", "retired"].includes(item.status)
|
|
656
|
-
&& !workItemOwnsUnresolvedExecutionLane(item, terminal.executionGroupId, terminal.executionLaneId)) {
|
|
657
|
-
store.saveWorkItem(input.taskId, updateWorkItemStatus(item, "failed", input.now, input.reason));
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
store.saveEvent(input.taskId, createTaskEvent(store.nextEventId(input.taskId), input.taskId, RUN_RECOVERY_APPLIED_EVENT, { ...eventPayload, status: "terminated" }, input.now));
|
|
661
|
-
if (input.roleName !== "leader") {
|
|
662
|
-
enqueueWork(store, { kind: "role", taskId: input.taskId, roleName: "leader" }, "run-recovery-terminated", input.now, [{ type: "run", taskId: input.taskId, id: input.runId }]);
|
|
663
|
-
}
|
|
664
|
-
return {
|
|
665
|
-
disposition: "applied",
|
|
666
|
-
action: input.action,
|
|
667
|
-
run: terminal,
|
|
668
|
-
progressAt: progress.progressAt
|
|
669
|
-
};
|
|
670
|
-
}
|
|
671
|
-
function exactRecoveryExecutionBlocker(store, run) {
|
|
672
|
-
const sessions = store.getTaskRoleSessionSet(run.taskId, run.roleName);
|
|
673
|
-
const binding = sessions?.providerBinding;
|
|
674
|
-
const mailbox = store.getWorkMailbox({
|
|
675
|
-
kind: "role",
|
|
676
|
-
taskId: run.taskId,
|
|
677
|
-
roleName: run.roleName
|
|
678
|
-
});
|
|
679
|
-
if (mailbox?.inputDelivery != null)
|
|
680
|
-
return "provider-input-delivery-unsettled";
|
|
681
|
-
if (runOwnsBlockingProviderContinuation(store.listEvents(run.taskId), {
|
|
682
|
-
taskId: run.taskId,
|
|
683
|
-
roleName: run.roleName,
|
|
684
|
-
runId: run.id,
|
|
685
|
-
agentId: run.effective.agentId
|
|
686
|
-
}))
|
|
687
|
-
return "provider-continuation-writer-owned";
|
|
688
|
-
if (runHasActiveRuntimeOperations(store.listEvents(run.taskId), {
|
|
689
|
-
taskId: run.taskId,
|
|
690
|
-
roleName: run.roleName,
|
|
691
|
-
runId: run.id,
|
|
692
|
-
agentId: run.effective.agentId
|
|
693
|
-
}))
|
|
694
|
-
return "provider-operation-active";
|
|
695
|
-
if (run.deliveredAt !== undefined
|
|
696
|
-
&& (binding === null || binding?.turn === null))
|
|
697
|
-
return "provider-turn-state-missing";
|
|
698
|
-
if (binding === null || binding === undefined)
|
|
699
|
-
return null;
|
|
700
|
-
if (["submitting", "accepted", "running", "delivery-unknown"].includes(binding.turn?.status ?? ""))
|
|
701
|
-
return "provider-turn-unsettled";
|
|
702
|
-
if (binding.authority.owner === "human" || binding.authority.owner === "unknown") {
|
|
703
|
-
return "provider-writer-authority-unavailable";
|
|
704
|
-
}
|
|
705
|
-
return null;
|
|
706
|
-
}
|
|
707
|
-
function matchesRecoverySessionFence(store, input) {
|
|
708
|
-
const sessions = store.getTaskRoleSessionSet(input.taskId, input.roleName);
|
|
709
|
-
// Recovery must never proceed without a durable Session fence. A missing
|
|
710
|
-
// nativeSessionId is supported for an opaque host, but its exact launchId
|
|
711
|
-
// is then the only identity that can fence the action.
|
|
712
|
-
if (sessions === null)
|
|
713
|
-
return false;
|
|
714
|
-
if (sessions.activeAgentId !== input.agentId)
|
|
715
|
-
return false;
|
|
716
|
-
const session = sessions.sessions[input.agentId];
|
|
717
|
-
if (session === undefined)
|
|
718
|
-
return false;
|
|
719
|
-
if (session.agentId !== input.agentId || session.adapterId !== input.adapterId)
|
|
720
|
-
return false;
|
|
721
|
-
// Preserve a dead Session's exact identity as the CAS fence; only same-Session
|
|
722
|
-
// retry is invalid once the native process is stopped or broken.
|
|
723
|
-
if ((session.status === "stopped" || session.status === "broken")
|
|
724
|
-
&& input.action === "retry")
|
|
725
|
-
return false;
|
|
726
|
-
const sessionNativeSessionId = session.nativeSessionId;
|
|
727
|
-
if (sessionNativeSessionId === undefined) {
|
|
728
|
-
if (input.nativeSessionId !== undefined)
|
|
729
|
-
return false;
|
|
730
|
-
}
|
|
731
|
-
else if (input.nativeSessionId !== sessionNativeSessionId) {
|
|
732
|
-
return false;
|
|
733
|
-
}
|
|
734
|
-
let launchMatches = false;
|
|
735
|
-
if (input.launchId !== undefined && session.launchId === input.launchId) {
|
|
736
|
-
launchMatches = true;
|
|
737
|
-
}
|
|
738
|
-
const mailbox = store.getWorkMailbox(runtimeLifecycleTarget({
|
|
739
|
-
scope: "task",
|
|
740
|
-
taskId: input.taskId,
|
|
741
|
-
roleName: input.roleName
|
|
742
|
-
}));
|
|
743
|
-
if (!launchMatches
|
|
744
|
-
&& input.launchId !== undefined
|
|
745
|
-
&& isRuntimeLaunchReservation(mailbox?.processing, input.launchId)) {
|
|
746
|
-
launchMatches = true;
|
|
747
|
-
}
|
|
748
|
-
// An opaque Session has no native identity to compare, so an exact launch
|
|
749
|
-
// identity is mandatory. Native Sessions may retain the older no-launch
|
|
750
|
-
// shape; their exact native identity remains a valid fence.
|
|
751
|
-
if (sessionNativeSessionId === undefined)
|
|
752
|
-
return launchMatches;
|
|
753
|
-
if (input.nativeSessionId === undefined)
|
|
754
|
-
return launchMatches;
|
|
755
|
-
return session.launchId === undefined
|
|
756
|
-
? input.launchId === undefined || launchMatches
|
|
757
|
-
: launchMatches;
|
|
758
|
-
}
|
|
759
|
-
function hasRecoveryReason(value) {
|
|
760
|
-
return typeof value === "string" && value.includes("\0") === false && value.trim().length > 0;
|
|
761
|
-
}
|
|
762
464
|
function matchesSessionFence(sessions, input) {
|
|
763
465
|
if (sessions === null)
|
|
764
466
|
return input.nativeSessionId === undefined;
|
|
@@ -1,7 +1,6 @@
|
|
|
1
1
|
import { isDeepStrictEqual } from "node:util";
|
|
2
2
|
import { retireConfirmedAbsentInactiveTaskRolePlaceholders } from "../executor/agentExecutor.js";
|
|
3
3
|
import { hasRuntimeLifecycleWork, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
4
|
-
import { blockingProviderContinuations } from "../runtime/runtimeContinuationProjection.js";
|
|
5
4
|
import { managedWorkspaceKey } from "../worktree/managedWorkspace.js";
|
|
6
5
|
import { acquireProjectMaintenanceLocks } from "./projectMaintenanceLock.js";
|
|
7
6
|
import { WorkspaceCleanupBlockedError } from "./taskWorkspacePreparer.js";
|
|
@@ -159,7 +158,6 @@ export class TaskWorkspaceCoordinator {
|
|
|
159
158
|
if (task.status !== "completed" && task.status !== "retired") {
|
|
160
159
|
throw new Error(`Task must be completed or retired before archive cleanup: ${task.id}.`);
|
|
161
160
|
}
|
|
162
|
-
this.#assertNoProviderContinuationWriters(task.id);
|
|
163
161
|
const managedWorkspaces = [...this.store.listManagedWorkspaces(task.id)]
|
|
164
162
|
.sort((left, right) => managedWorkspaceKey(left.owner)
|
|
165
163
|
.localeCompare(managedWorkspaceKey(right.owner)));
|
|
@@ -326,13 +324,6 @@ export class TaskWorkspaceCoordinator {
|
|
|
326
324
|
if (current === null || !isDeepStrictEqual(current, expected)) {
|
|
327
325
|
throw new WorkspaceCleanupBlockedError("task-changed", `task:${expected.id}`, true, `Task changed during archive cleanup: ${expected.id}.`);
|
|
328
326
|
}
|
|
329
|
-
this.#assertNoProviderContinuationWriters(expected.id);
|
|
330
|
-
}
|
|
331
|
-
#assertNoProviderContinuationWriters(taskId) {
|
|
332
|
-
const blockers = blockingProviderContinuations(this.store.listEvents(taskId));
|
|
333
|
-
if (blockers.length > 0) {
|
|
334
|
-
throw new WorkspaceCleanupBlockedError("active-run", `task:${taskId}`, true, `Task has Provider continuations that may still write its Workspace: ${taskId}.`);
|
|
335
|
-
}
|
|
336
327
|
}
|
|
337
328
|
#assertWorkItemRuntimeQuiescent(item) {
|
|
338
329
|
const activeRun = this.store.listAgentRuns(item.taskId)
|