@zq-silk/yui 0.13.7 → 0.13.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +40 -4
- package/README.md +23 -4
- package/dist/cli/commandCatalog.js +21 -2
- package/dist/cli/updatePorts.js +4 -4
- package/dist/cli.js +43 -14
- package/dist/commands/agentCommands.js +1 -1
- package/dist/commands/configCommands.js +1 -86
- package/dist/commands/executionAuditCommands.js +17 -16
- package/dist/commands/globalRoleCommands.js +4 -4
- package/dist/commands/sessionCommands.js +2 -6
- package/dist/commands/taskActor.js +1 -2
- package/dist/commands/taskCommands.js +93 -21
- package/dist/commands/taskContextCommand.js +1 -1
- package/dist/commands/taskExecutionCommands.js +0 -5
- package/dist/commands/taskOverviewCommand.js +5 -1
- package/dist/commands/taskRoleRuntimeStatus.js +3 -16
- package/dist/config/configCatalog.js +1 -6
- package/dist/config/yuiConfig.js +0 -79
- package/dist/context/sessionBootstrapManifest.js +26 -23
- package/dist/controller/clientRuntime.js +40 -3
- package/dist/controller/controller.js +29 -54
- package/dist/controller/fileSchedulerStoreAdapter.js +372 -1117
- package/dist/controller/runtime.js +12 -5
- package/dist/controller/runtimeHookRunFence.js +3 -9
- package/dist/controller/runtimeLaunchCoordinator.js +44 -67
- package/dist/controller/structuredProviderObservation.js +18 -5
- package/dist/coordination/workMailbox.js +4 -4
- package/dist/execution/executionHealth.js +1 -1
- package/dist/executor/agentExecutor.js +103 -93
- package/dist/executor/executorRegistry.js +8 -20
- package/dist/executor/fileRoleLaunchPlanner.js +26 -93
- package/dist/executor/turnCompletion.js +5 -5
- package/dist/lifecycle/exactRunTerminalization.js +2 -4
- package/dist/observability/executionAudit.js +40 -94
- package/dist/operator/operatorSessionHistory.js +7 -5
- package/dist/output/rolePresentation.js +0 -1
- package/dist/repository/taskWorkspacePreparer.js +0 -1
- package/dist/role/role.js +13 -21
- package/dist/run/agentRun.js +4 -54
- package/dist/runtime/agentDriver.js +2 -0
- package/dist/runtime/agentError.js +114 -0
- package/dist/runtime/agentHost.js +55 -82
- package/dist/runtime/builtinAgentDrivers.js +21 -9
- package/dist/runtime/builtinAgentErrorMappers.js +150 -0
- package/dist/runtime/exactControlPlane.js +6 -12
- package/dist/runtime/index.js +1 -2
- package/dist/runtime/launchBroker.js +5 -19
- package/dist/runtime/lifecycleReservation.js +20 -4
- package/dist/runtime/providerRuntimeIdentity.js +11 -19
- package/dist/runtime/runtimeBinding.js +0 -27
- package/dist/runtime/runtimeObservation.js +7 -16
- package/dist/runtime/runtimeSessionCandidate.js +3 -10
- package/dist/runtime/sessionLaunchRequest.js +1 -2
- package/dist/runtime/sessionReconciliation.js +2 -2
- package/dist/runtime/structuredProviderHost.js +44 -79
- package/dist/runtime/taskRuntimeIsolation.js +0 -7
- package/dist/runtime/tmuxAdapters.js +6 -49
- package/dist/scheduler/activeRoleRunDelivery.js +220 -178
- package/dist/scheduler/activeTaskProgress.js +1 -4
- package/dist/scheduler/leaderWakeupProcessor.js +123 -88
- package/dist/scheduler/roleRunLiveness.js +4 -1
- package/dist/scheduler/roleRunStall.js +10 -14
- package/dist/scheduler/wakeReason.js +4 -0
- package/dist/storage/migration/productionRegistry.js +475 -0
- package/dist/storage/sqliteSchema.js +54 -2
- package/dist/storage/sqliteStore.js +11 -44
- package/dist/storage/taskStore.js +8 -36
- package/dist/web/webSnapshot.js +3 -0
- package/i18n/README.zh-CN.md +11 -3
- package/package.json +1 -1
- package/skills/yui-leader/SKILL.md +30 -8
- package/skills/yui-operator/SKILL.md +14 -6
- package/skills/yui-runtime/SKILL.md +11 -8
- package/dist/lifecycle/providerErrorClass.js +0 -152
- package/dist/run/providerRetry.js +0 -226
- package/dist/run/providerRetryConfig.js +0 -27
- package/dist/runtime/providerErrorCodes.js +0 -278
- package/dist/runtime/providerRecoveryDecision.js +0 -55
|
@@ -2,6 +2,8 @@ import { selectedSchedulerRoles, selectedActiveSchedulerTasks } from "./ports.js
|
|
|
2
2
|
import { isSchedulerTaskWorkspaceReady } from "./ports.js";
|
|
3
3
|
import { formatAgentRunReceiptId } from "../task/taskRecordReference.js";
|
|
4
4
|
import { agentRunDeliveryReceiptId } from "../run/agentRun.js";
|
|
5
|
+
import { serializeRunBootstrapEnvelope, serializeRunHostRecoveryEnvelope } from "../context/runContextContract.js";
|
|
6
|
+
import { serializeWorkflowOutcomeRequestEnvelope } from "../run/runControlRequest.js";
|
|
5
7
|
import { effectiveLaunchSnapshotsCompatible, effectiveLaunchSnapshotsCompatibleForTaskSession } from "../executor/effectiveLaunch.js";
|
|
6
8
|
import { RuntimeLaunchError } from "../runtime/ports.js";
|
|
7
9
|
import { RuntimeLaunchFailure } from "../runtime/launchDiagnostics.js";
|
|
@@ -9,6 +11,7 @@ import { mailboxHasWork, nextPendingBatch } from "../coordination/workMailbox.js
|
|
|
9
11
|
import { runtimeObservationFromTaskEvent } from "../runtime/runtimeObservation.js";
|
|
10
12
|
import { projectProviderContinuations } from "../runtime/runtimeContinuationProjection.js";
|
|
11
13
|
import { RuntimeLifecycleBusyError } from "../runtime/lifecycleReservation.js";
|
|
14
|
+
import { serializeAgentErrorRaw } from "../runtime/agentError.js";
|
|
12
15
|
/**
|
|
13
16
|
* Delivers durable Work AgentRuns before liveness reconciliation. Task command
|
|
14
17
|
* handlers only record intent; this Controller path is the sole automated
|
|
@@ -20,7 +23,6 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
20
23
|
for (const role of selectedSchedulerRoles(store, task.id, selection)) {
|
|
21
24
|
const run = store.getActiveAgentRun(task.id, role.name);
|
|
22
25
|
if (run !== null && run.pushedAt !== undefined
|
|
23
|
-
&& run.providerRetry?.state !== "dispatching"
|
|
24
26
|
&& run.controlRequest?.state !== "dispatching") {
|
|
25
27
|
const continuation = await processActiveRunContinuation(store, delivery, task, role, run, now, inputDeliveryRecoveryCutoff);
|
|
26
28
|
if (continuation !== null)
|
|
@@ -34,7 +36,6 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
34
36
|
// no duplicate Enter while acceptance is still pending.
|
|
35
37
|
if (run === null
|
|
36
38
|
|| (run.pushedAt !== undefined
|
|
37
|
-
&& run.providerRetry?.state !== "dispatching"
|
|
38
39
|
&& run.controlRequest?.state !== "dispatching"))
|
|
39
40
|
continue;
|
|
40
41
|
const taskWorkspace = store.getTaskWorkspace(task.id);
|
|
@@ -48,6 +49,19 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
48
49
|
});
|
|
49
50
|
continue;
|
|
50
51
|
}
|
|
52
|
+
// Recovery can expose an unpushed Run while an older Provider Turn is
|
|
53
|
+
// still settling. Preserve the Run and its mailbox claim without even
|
|
54
|
+
// touching the Host; terminal observation will wake the next pass.
|
|
55
|
+
if (providerTurnIsUnsettled(store, task.id, role.name)) {
|
|
56
|
+
results.push({
|
|
57
|
+
taskId: task.id,
|
|
58
|
+
roleName: role.name,
|
|
59
|
+
runId: run.id,
|
|
60
|
+
status: "skipped",
|
|
61
|
+
reason: "not-ready"
|
|
62
|
+
});
|
|
63
|
+
continue;
|
|
64
|
+
}
|
|
51
65
|
const existingSession = store.getRoleSession(task.id, role.name, run.effective.agentId);
|
|
52
66
|
const receiptId = agentRunDeliveryReceiptId(run);
|
|
53
67
|
const target = { kind: "role", taskId: task.id, roleName: role.name };
|
|
@@ -82,16 +96,15 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
82
96
|
continue;
|
|
83
97
|
}
|
|
84
98
|
let prepared;
|
|
85
|
-
let preparedSession = existingSession;
|
|
86
99
|
let preStartFencePersisted = false;
|
|
87
100
|
let deliveryAttempted = false;
|
|
101
|
+
const launchMode = run.controlRequest?.state === "dispatching"
|
|
102
|
+
? "resume"
|
|
103
|
+
: run.mode;
|
|
88
104
|
try {
|
|
89
105
|
// Workflow-outcome control is always another Turn on the already-bound
|
|
90
|
-
//
|
|
91
|
-
//
|
|
92
|
-
const launchMode = run.controlRequest?.state === "dispatching"
|
|
93
|
-
? "resume"
|
|
94
|
-
: run.mode;
|
|
106
|
+
// Session. A fresh Session is selected only by the Run created after
|
|
107
|
+
// the prior Session was explicitly ended.
|
|
95
108
|
const nativeSessionId = launchMode === "resume"
|
|
96
109
|
? requireResumeSession(role, run.effective, existingSession)
|
|
97
110
|
: undefined;
|
|
@@ -108,30 +121,20 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
108
121
|
mode: launchMode,
|
|
109
122
|
runId: run.id,
|
|
110
123
|
...(nativeSessionId === undefined ? {} : { nativeSessionId }),
|
|
124
|
+
...(launchMode !== "resume" || existingSession?.launchId === undefined
|
|
125
|
+
? {}
|
|
126
|
+
: { hostActivationId: existingSession.launchId }),
|
|
111
127
|
beforeHostStart: (preflight) => {
|
|
112
128
|
persistPreStartFence(store, task, role, run, existingSession, preflight, now);
|
|
113
|
-
|
|
129
|
+
preflightSession(role, run.effective, existingSession, launchMode, preflight);
|
|
114
130
|
preStartFencePersisted = true;
|
|
115
131
|
}
|
|
116
132
|
});
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
//
|
|
120
|
-
// later readiness or aggregate-write failure is delivery uncertainty,
|
|
121
|
-
// not a launch failure: preserve the Run and its reservation for the
|
|
122
|
-
// matching provider Hook instead of terminalizing it.
|
|
123
|
-
deliveryAttempted = prepared.turnAcceptedDuringLaunch === true
|
|
124
|
-
|| prepared.turnDeliveryUnknownDuringLaunch === true;
|
|
125
|
-
// Preparation may already have an exact native Session identity while
|
|
126
|
-
// the provider is still starting. Persist that Session + the Run fence
|
|
127
|
-
// before awaiting readiness so a pre-input provider Hook can validate
|
|
128
|
-
// the complete generation (including receipt) instead of racing an
|
|
129
|
-
// in-memory boundary. A delivery adapter that cannot expose a
|
|
130
|
-
// pre-readiness Session falls back to the existing durable Session;
|
|
131
|
-
// fresh runtime-discovered providers intentionally persist `null`.
|
|
133
|
+
// Persist the Session identity before submitting any Turn. Host start
|
|
134
|
+
// and Session restore are complete lifecycle operations; neither owns
|
|
135
|
+
// nor implies the Run's first Provider write.
|
|
132
136
|
if (!preStartFencePersisted) {
|
|
133
|
-
const preparedFenceSession = validateLaunchSubmittedRecoverySession(role, run.effective, existingSession,
|
|
134
|
-
preparedSession = preparedFenceSession;
|
|
137
|
+
const preparedFenceSession = validateLaunchSubmittedRecoverySession(role, run.effective, existingSession, launchMode, prepared.session);
|
|
135
138
|
store.saveRoleRunPrepared({
|
|
136
139
|
task,
|
|
137
140
|
role,
|
|
@@ -142,11 +145,7 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
142
145
|
});
|
|
143
146
|
}
|
|
144
147
|
const ready = await delivery.waitUntilReady(prepared);
|
|
145
|
-
|
|
146
|
-
|| ready.prepared.turnAcceptedDuringLaunch === true
|
|
147
|
-
|| ready.prepared.turnDeliveryUnknownDuringLaunch === true;
|
|
148
|
-
const session = validateLaunchSubmittedRecoverySession(role, run.effective, existingSession, run.mode, ready.prepared, ready.session);
|
|
149
|
-
preparedSession = session;
|
|
148
|
+
const session = validateLaunchSubmittedRecoverySession(role, run.effective, existingSession, launchMode, ready.session);
|
|
150
149
|
store.saveRoleRunPrepared({
|
|
151
150
|
task,
|
|
152
151
|
role,
|
|
@@ -157,9 +156,13 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
157
156
|
: { launchId: ready.prepared.launchId }),
|
|
158
157
|
now
|
|
159
158
|
});
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
159
|
+
deliveryAttempted = true;
|
|
160
|
+
const outcome = await delivery.sendOnce({
|
|
161
|
+
delivery: ready,
|
|
162
|
+
receiptId,
|
|
163
|
+
text: initialRunTurnInput(run)
|
|
164
|
+
});
|
|
165
|
+
if (outcome === "busy" || outcome === "unavailable") {
|
|
163
166
|
delivery.forgetPrepared?.({
|
|
164
167
|
taskId: task.id,
|
|
165
168
|
roleName: role.name,
|
|
@@ -173,49 +176,26 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
173
176
|
roleName: role.name,
|
|
174
177
|
runId: run.id,
|
|
175
178
|
status: "skipped",
|
|
176
|
-
reason: "not-ready"
|
|
179
|
+
reason: outcome === "busy" ? "not-ready" : "runtime-unavailable"
|
|
177
180
|
});
|
|
178
181
|
continue;
|
|
179
182
|
}
|
|
180
|
-
if (
|
|
181
|
-
const
|
|
182
|
-
...roleRunDeliveryFailure(run, processing.batchId, session, ready.prepared.launchId),
|
|
183
|
-
summary: `Provider conclusively rejected the exact initial Run input: ${run.id}.`,
|
|
184
|
-
cleanupRequired: false,
|
|
185
|
-
now
|
|
186
|
-
});
|
|
187
|
-
if (persisted === "failed") {
|
|
188
|
-
delivery.forgetPrepared?.({
|
|
189
|
-
taskId: task.id,
|
|
190
|
-
roleName: role.name,
|
|
191
|
-
runId: run.id,
|
|
192
|
-
...(ready.prepared.launchId === undefined
|
|
193
|
-
? {}
|
|
194
|
-
: { launchId: ready.prepared.launchId })
|
|
195
|
-
});
|
|
196
|
-
}
|
|
197
|
-
results.push({
|
|
183
|
+
if (outcome === "rejected") {
|
|
184
|
+
const terminal = store.saveRoleRunDeliveryFailure({
|
|
198
185
|
taskId: task.id,
|
|
199
186
|
roleName: role.name,
|
|
187
|
+
agentId: run.effective.agentId,
|
|
188
|
+
adapterId: run.effective.adapterId,
|
|
200
189
|
runId: run.id,
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
terminalized: persisted === "failed",
|
|
204
|
-
...(persisted === "failed"
|
|
190
|
+
mailboxBatchId: processing.batchId,
|
|
191
|
+
...(session?.nativeSessionId === undefined
|
|
205
192
|
? {}
|
|
206
|
-
: {
|
|
207
|
-
});
|
|
208
|
-
continue;
|
|
209
|
-
}
|
|
210
|
-
if (ready.prepared.turnDeliveryUnknownDuringLaunch === true) {
|
|
211
|
-
store.saveRoleRunDelivery({
|
|
212
|
-
task,
|
|
213
|
-
role,
|
|
214
|
-
run,
|
|
215
|
-
session,
|
|
193
|
+
: { nativeSessionId: session.nativeSessionId }),
|
|
216
194
|
...(ready.prepared.launchId === undefined
|
|
217
195
|
? {}
|
|
218
196
|
: { launchId: ready.prepared.launchId }),
|
|
197
|
+
summary: "Provider rejected the Turn before acceptance; Agent recovery is required.",
|
|
198
|
+
cleanupRequired: false,
|
|
219
199
|
now
|
|
220
200
|
});
|
|
221
201
|
delivery.forgetPrepared?.({
|
|
@@ -230,15 +210,13 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
230
210
|
taskId: task.id,
|
|
231
211
|
roleName: role.name,
|
|
232
212
|
runId: run.id,
|
|
233
|
-
status: "skipped",
|
|
234
|
-
reason: "
|
|
213
|
+
status: terminal === "failed" ? "failed" : "skipped",
|
|
214
|
+
reason: "provider-rejected",
|
|
215
|
+
...(terminal === "failed" ? { terminalized: true } : {})
|
|
235
216
|
});
|
|
236
217
|
continue;
|
|
237
218
|
}
|
|
238
|
-
if (
|
|
239
|
-
// The newly-created Agent Host already submitted the exact Run Turn
|
|
240
|
-
// through Provider-native control. Persist launch transport success;
|
|
241
|
-
// only the matching structured acknowledgement marks it accepted.
|
|
219
|
+
if (outcome === "delivery-unknown") {
|
|
242
220
|
store.saveRoleRunDelivery({
|
|
243
221
|
task,
|
|
244
222
|
role,
|
|
@@ -261,11 +239,36 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
261
239
|
taskId: task.id,
|
|
262
240
|
roleName: role.name,
|
|
263
241
|
runId: run.id,
|
|
264
|
-
status: "
|
|
242
|
+
status: "skipped",
|
|
243
|
+
reason: "delivery-uncertain"
|
|
265
244
|
});
|
|
266
245
|
continue;
|
|
267
246
|
}
|
|
268
|
-
|
|
247
|
+
store.saveRoleRunDelivery({
|
|
248
|
+
task,
|
|
249
|
+
role,
|
|
250
|
+
run,
|
|
251
|
+
session,
|
|
252
|
+
...(ready.prepared.launchId === undefined
|
|
253
|
+
? {}
|
|
254
|
+
: { launchId: ready.prepared.launchId }),
|
|
255
|
+
now
|
|
256
|
+
});
|
|
257
|
+
delivery.forgetPrepared?.({
|
|
258
|
+
taskId: task.id,
|
|
259
|
+
roleName: role.name,
|
|
260
|
+
runId: run.id,
|
|
261
|
+
...(ready.prepared.launchId === undefined
|
|
262
|
+
? {}
|
|
263
|
+
: { launchId: ready.prepared.launchId })
|
|
264
|
+
});
|
|
265
|
+
results.push({
|
|
266
|
+
taskId: task.id,
|
|
267
|
+
roleName: role.name,
|
|
268
|
+
runId: run.id,
|
|
269
|
+
status: outcome === "sent" ? "delivered" : "already-delivered"
|
|
270
|
+
});
|
|
271
|
+
continue;
|
|
269
272
|
}
|
|
270
273
|
catch (error) {
|
|
271
274
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -290,8 +293,40 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
290
293
|
});
|
|
291
294
|
continue;
|
|
292
295
|
}
|
|
293
|
-
|
|
294
|
-
|
|
296
|
+
store.recordAgentError?.({
|
|
297
|
+
taskId: task.id,
|
|
298
|
+
roleName: role.name,
|
|
299
|
+
runId: run.id,
|
|
300
|
+
source: error instanceof RuntimeLaunchError || error instanceof RuntimeLaunchFailure
|
|
301
|
+
? "host"
|
|
302
|
+
: "yui",
|
|
303
|
+
phase: deliveryAttempted
|
|
304
|
+
? "turn-submit"
|
|
305
|
+
: launchMode === "new" ? "session-start" : "session-restore",
|
|
306
|
+
message,
|
|
307
|
+
raw: serializeAgentErrorRaw(error),
|
|
308
|
+
inputDisposition: deliveryAttempted ? "unknown" : "not-accepted"
|
|
309
|
+
}, now);
|
|
310
|
+
if (!deliveryAttempted) {
|
|
311
|
+
const failedSession = store.getRoleSession(task.id, role.name, run.effective.agentId);
|
|
312
|
+
const terminal = store.saveRoleRunDeliveryFailure({
|
|
313
|
+
taskId: task.id,
|
|
314
|
+
roleName: role.name,
|
|
315
|
+
agentId: run.effective.agentId,
|
|
316
|
+
adapterId: run.effective.adapterId,
|
|
317
|
+
runId: run.id,
|
|
318
|
+
mailboxBatchId: processing.batchId,
|
|
319
|
+
...(failedSession?.nativeSessionId === undefined
|
|
320
|
+
? {}
|
|
321
|
+
: { nativeSessionId: failedSession.nativeSessionId }),
|
|
322
|
+
...(prepared?.launchId === undefined && failedSession?.launchId === undefined
|
|
323
|
+
? {}
|
|
324
|
+
: { launchId: prepared?.launchId ?? failedSession.launchId }),
|
|
325
|
+
summary: `Agent Session preparation failed before Turn acceptance: ${message}`,
|
|
326
|
+
cleanupRequired: false,
|
|
327
|
+
now
|
|
328
|
+
});
|
|
329
|
+
if (terminal === "failed") {
|
|
295
330
|
delivery.forgetPrepared?.({
|
|
296
331
|
taskId: task.id,
|
|
297
332
|
roleName: role.name,
|
|
@@ -300,68 +335,48 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
300
335
|
? {}
|
|
301
336
|
: { launchId: prepared.launchId })
|
|
302
337
|
});
|
|
303
|
-
const persisted = store.saveExitedRoleRun({
|
|
304
|
-
task,
|
|
305
|
-
role,
|
|
306
|
-
run,
|
|
307
|
-
session: existingSession,
|
|
308
|
-
summary: error.message,
|
|
309
|
-
leaderRecovery: role.name === "leader" && error.diagnostic.kind === "config"
|
|
310
|
-
? "blocked"
|
|
311
|
-
: "automatic",
|
|
312
|
-
now
|
|
313
|
-
});
|
|
314
338
|
results.push({
|
|
315
339
|
taskId: task.id,
|
|
316
340
|
roleName: role.name,
|
|
317
341
|
runId: run.id,
|
|
318
342
|
status: "failed",
|
|
319
343
|
reason: "launch-failed",
|
|
320
|
-
error:
|
|
321
|
-
|
|
322
|
-
: message,
|
|
323
|
-
terminalized: persisted === "failed"
|
|
344
|
+
error: message,
|
|
345
|
+
terminalized: true
|
|
324
346
|
});
|
|
325
347
|
continue;
|
|
326
348
|
}
|
|
327
349
|
}
|
|
328
|
-
if (error instanceof
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
const writerAttached = error.reason === "writable-client";
|
|
332
|
-
results.push({
|
|
350
|
+
if (error instanceof RuntimeLaunchFailure) {
|
|
351
|
+
if (!deliveryAttempted) {
|
|
352
|
+
delivery.forgetPrepared?.({
|
|
333
353
|
taskId: task.id,
|
|
334
354
|
roleName: role.name,
|
|
335
355
|
runId: run.id,
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
...(writerAttached ? {} : { terminalFailure })
|
|
356
|
+
...(prepared?.launchId === undefined
|
|
357
|
+
? {}
|
|
358
|
+
: { launchId: prepared.launchId })
|
|
340
359
|
});
|
|
341
|
-
|
|
342
|
-
}
|
|
343
|
-
const persisted = store.saveRoleRunDeliveryFailure({
|
|
344
|
-
...terminalFailure,
|
|
345
|
-
now
|
|
346
|
-
});
|
|
347
|
-
if (persisted === "failed") {
|
|
348
|
-
delivery.forgetPrepared?.({
|
|
360
|
+
results.push({
|
|
349
361
|
taskId: task.id,
|
|
350
362
|
roleName: role.name,
|
|
351
363
|
runId: run.id,
|
|
352
|
-
|
|
364
|
+
status: "skipped",
|
|
365
|
+
reason: "launch-failed",
|
|
366
|
+
error: message
|
|
353
367
|
});
|
|
368
|
+
continue;
|
|
354
369
|
}
|
|
370
|
+
}
|
|
371
|
+
if (error instanceof RuntimeLaunchError) {
|
|
372
|
+
const writerAttached = error.reason === "writable-client";
|
|
355
373
|
results.push({
|
|
356
374
|
taskId: task.id,
|
|
357
375
|
roleName: role.name,
|
|
358
376
|
runId: run.id,
|
|
359
|
-
status: "
|
|
360
|
-
reason: "
|
|
361
|
-
error:
|
|
362
|
-
? "Run state changed during exact launch generation failure."
|
|
363
|
-
: message,
|
|
364
|
-
terminalized: persisted === "failed"
|
|
377
|
+
status: "skipped",
|
|
378
|
+
reason: writerAttached ? "writer-attached" : "runtime-unavailable",
|
|
379
|
+
error: message
|
|
365
380
|
});
|
|
366
381
|
continue;
|
|
367
382
|
}
|
|
@@ -374,23 +389,13 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
374
389
|
? {}
|
|
375
390
|
: { launchId: prepared.launchId })
|
|
376
391
|
});
|
|
377
|
-
const persisted = store.saveExitedRoleRun({
|
|
378
|
-
task,
|
|
379
|
-
role,
|
|
380
|
-
run,
|
|
381
|
-
session: existingSession,
|
|
382
|
-
summary: `Role Run could not start: ${message}`,
|
|
383
|
-
leaderRecovery: "automatic",
|
|
384
|
-
now
|
|
385
|
-
});
|
|
386
392
|
results.push({
|
|
387
393
|
taskId: task.id,
|
|
388
394
|
roleName: role.name,
|
|
389
395
|
runId: run.id,
|
|
390
|
-
status: "
|
|
391
|
-
reason: "
|
|
392
|
-
error:
|
|
393
|
-
terminalized: persisted === "failed"
|
|
396
|
+
status: "skipped",
|
|
397
|
+
reason: "runtime-unavailable",
|
|
398
|
+
error: message
|
|
394
399
|
});
|
|
395
400
|
continue;
|
|
396
401
|
}
|
|
@@ -400,19 +405,37 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
400
405
|
runId: run.id,
|
|
401
406
|
status: "failed",
|
|
402
407
|
reason: "delivery-uncertain",
|
|
403
|
-
error: message
|
|
404
|
-
terminalFailure: roleRunDeliveryFailure(run, processing.batchId, preparedSession, prepared?.launchId)
|
|
408
|
+
error: message
|
|
405
409
|
});
|
|
406
410
|
}
|
|
407
411
|
}
|
|
408
412
|
}
|
|
409
413
|
return results;
|
|
410
414
|
}
|
|
415
|
+
function providerTurnIsUnsettled(store, taskId, roleName) {
|
|
416
|
+
if (store.getRoleSession(taskId, roleName)?.status !== "active")
|
|
417
|
+
return false;
|
|
418
|
+
const turn = store.getTaskRoleSessionSet?.(taskId, roleName)?.providerBinding?.turn;
|
|
419
|
+
return turn !== null
|
|
420
|
+
&& turn !== undefined
|
|
421
|
+
&& ["submitting", "accepted", "running", "delivery-unknown"].includes(turn.status);
|
|
422
|
+
}
|
|
411
423
|
async function processActiveRunContinuation(store, delivery, task, role, run, now, inputDeliveryRecoveryCutoff) {
|
|
412
424
|
const target = { kind: "role", taskId: task.id, roleName: role.name };
|
|
413
425
|
const mailbox = store.getWorkMailbox(target);
|
|
414
426
|
if (mailbox === null || !mailboxHasWork(mailbox))
|
|
415
427
|
return null;
|
|
428
|
+
const blockedBy = latestContinuationPreparationFailure(store, task.id, run);
|
|
429
|
+
if (blockedBy !== null) {
|
|
430
|
+
return {
|
|
431
|
+
taskId: task.id,
|
|
432
|
+
roleName: role.name,
|
|
433
|
+
runId: run.id,
|
|
434
|
+
status: "skipped",
|
|
435
|
+
reason: "runtime-unavailable",
|
|
436
|
+
error: blockedBy.payload.message
|
|
437
|
+
};
|
|
438
|
+
}
|
|
416
439
|
const session = store.getRoleSession(task.id, role.name, run.effective.agentId);
|
|
417
440
|
if (session === null || session.launchId === undefined || !hasText(session.nativeSessionId)) {
|
|
418
441
|
return {
|
|
@@ -487,7 +510,7 @@ async function processActiveRunContinuation(store, delivery, task, role, run, no
|
|
|
487
510
|
}
|
|
488
511
|
if (pending === null || session.launchId === undefined)
|
|
489
512
|
return null;
|
|
490
|
-
if (session.status === "
|
|
513
|
+
if (session.status === "ended") {
|
|
491
514
|
return {
|
|
492
515
|
taskId: task.id,
|
|
493
516
|
roleName: role.name,
|
|
@@ -497,17 +520,9 @@ async function processActiveRunContinuation(store, delivery, task, role, run, no
|
|
|
497
520
|
};
|
|
498
521
|
}
|
|
499
522
|
// Managed input has one write path: a new structured Turn at a settled
|
|
500
|
-
// boundary.
|
|
523
|
+
// boundary. AgentHost owns Provider serialization; a busy send releases the
|
|
524
|
+
// claim and leaves the durable correction pending for the next pass.
|
|
501
525
|
const mode = "followup";
|
|
502
|
-
if (session.status === "running") {
|
|
503
|
-
return {
|
|
504
|
-
taskId: task.id,
|
|
505
|
-
roleName: role.name,
|
|
506
|
-
runId: run.id,
|
|
507
|
-
status: "skipped",
|
|
508
|
-
reason: "not-ready"
|
|
509
|
-
};
|
|
510
|
-
}
|
|
511
526
|
if (writer === null) {
|
|
512
527
|
return {
|
|
513
528
|
taskId: task.id,
|
|
@@ -530,11 +545,16 @@ async function processActiveRunContinuation(store, delivery, task, role, run, no
|
|
|
530
545
|
...(run.workspace === undefined ? {} : { managedWorkspace: run.workspace }),
|
|
531
546
|
mode: "resume",
|
|
532
547
|
runId: run.id,
|
|
533
|
-
nativeSessionId: session.nativeSessionId
|
|
548
|
+
nativeSessionId: session.nativeSessionId,
|
|
549
|
+
hostActivationId: session.launchId
|
|
534
550
|
});
|
|
535
551
|
const ready = await delivery.waitUntilReady(prepared);
|
|
536
|
-
const
|
|
537
|
-
|
|
552
|
+
const readyLaunchId = ready.prepared.launchId
|
|
553
|
+
?? ready.session?.launchId
|
|
554
|
+
?? session.launchId;
|
|
555
|
+
const readyNativeSessionId = ready.session?.nativeSessionId
|
|
556
|
+
?? session.nativeSessionId;
|
|
557
|
+
if (readyLaunchId === undefined || !hasText(readyNativeSessionId)) {
|
|
538
558
|
throw new Error("Continuation delivery has no Provider Activation fence.");
|
|
539
559
|
}
|
|
540
560
|
const providerFence = {
|
|
@@ -553,6 +573,15 @@ async function processActiveRunContinuation(store, delivery, task, role, run, no
|
|
|
553
573
|
});
|
|
554
574
|
if (claim.status === "empty")
|
|
555
575
|
return null;
|
|
576
|
+
if (claim.status === "busy") {
|
|
577
|
+
return {
|
|
578
|
+
taskId: task.id,
|
|
579
|
+
roleName: role.name,
|
|
580
|
+
runId: run.id,
|
|
581
|
+
status: "skipped",
|
|
582
|
+
reason: "not-ready"
|
|
583
|
+
};
|
|
584
|
+
}
|
|
556
585
|
if (claim.delivery.attemptId !== requestedBatchId || claim.status === "delivery") {
|
|
557
586
|
return {
|
|
558
587
|
taskId: task.id,
|
|
@@ -612,6 +641,18 @@ async function processActiveRunContinuation(store, delivery, task, role, run, no
|
|
|
612
641
|
};
|
|
613
642
|
}
|
|
614
643
|
catch (error) {
|
|
644
|
+
store.recordAgentError?.({
|
|
645
|
+
taskId: task.id,
|
|
646
|
+
roleName: role.name,
|
|
647
|
+
runId: run.id,
|
|
648
|
+
source: error instanceof RuntimeLaunchError || error instanceof RuntimeLaunchFailure
|
|
649
|
+
? "host"
|
|
650
|
+
: "yui",
|
|
651
|
+
phase: deliveryAttempted ? "turn-submit" : "session-restore",
|
|
652
|
+
message: error instanceof Error ? error.message : String(error),
|
|
653
|
+
raw: serializeAgentErrorRaw(error),
|
|
654
|
+
inputDisposition: deliveryAttempted ? "unknown" : "not-accepted"
|
|
655
|
+
}, now);
|
|
615
656
|
if (inputDelivery === undefined) {
|
|
616
657
|
// No durable input intent existed and no provider input method was
|
|
617
658
|
// called. Session preparation may be retried safely.
|
|
@@ -630,6 +671,24 @@ async function processActiveRunContinuation(store, delivery, task, role, run, no
|
|
|
630
671
|
};
|
|
631
672
|
}
|
|
632
673
|
}
|
|
674
|
+
/**
|
|
675
|
+
* A persisted, conclusively unaccepted continuation failure is the recovery
|
|
676
|
+
* fence for this exact Run. Periodic scheduler passes must not reinterpret it
|
|
677
|
+
* as permission to start another Host or submit the same pending input. A new
|
|
678
|
+
* Run has a new identity and therefore makes the Agent's explicit retry
|
|
679
|
+
* observable without adding a second writable recovery state.
|
|
680
|
+
*/
|
|
681
|
+
function latestContinuationPreparationFailure(store, taskId, run) {
|
|
682
|
+
if (run.deliveredAt === undefined)
|
|
683
|
+
return null;
|
|
684
|
+
const deliveredAt = Date.parse(run.deliveredAt);
|
|
685
|
+
const events = store.listEvents?.(taskId) ?? [];
|
|
686
|
+
return [...events].reverse().find((event) => (event.type === "runtime.agent-error"
|
|
687
|
+
&& event.payload.runId === run.id
|
|
688
|
+
&& ["session-restore", "turn-submit"].includes(event.payload.phase ?? "")
|
|
689
|
+
&& event.payload.inputDisposition === "not-accepted"
|
|
690
|
+
&& Date.parse(event.createdAt) >= deliveredAt)) ?? null;
|
|
691
|
+
}
|
|
633
692
|
async function reconcileStrandedInputDelivery(store, task, role, run, input, now) {
|
|
634
693
|
const target = { kind: "role", taskId: task.id, roleName: role.name };
|
|
635
694
|
const uncertain = (reason) => {
|
|
@@ -719,20 +778,6 @@ function boundedExcerpt(text) {
|
|
|
719
778
|
return joined;
|
|
720
779
|
return `${joined.slice(0, MAX_RESULT_SUMMARY_CHARS)}… [truncated]`;
|
|
721
780
|
}
|
|
722
|
-
function roleRunDeliveryFailure(run, mailboxBatchId, session, launchId) {
|
|
723
|
-
return {
|
|
724
|
-
taskId: run.taskId,
|
|
725
|
-
roleName: run.roleName,
|
|
726
|
-
agentId: run.effective.agentId,
|
|
727
|
-
adapterId: run.effective.adapterId,
|
|
728
|
-
runId: run.id,
|
|
729
|
-
mailboxBatchId,
|
|
730
|
-
...(session?.nativeSessionId === undefined
|
|
731
|
-
? {}
|
|
732
|
-
: { nativeSessionId: session.nativeSessionId }),
|
|
733
|
-
...(launchId === undefined ? {} : { launchId })
|
|
734
|
-
};
|
|
735
|
-
}
|
|
736
781
|
function requireResumeSession(role, effective, session) {
|
|
737
782
|
if (session === null || !hasText(session.nativeSessionId)) {
|
|
738
783
|
throw new Error(`Role resume has no fixed native session: ${role.taskId}/${role.name}.`);
|
|
@@ -769,32 +814,29 @@ function preflightSession(role, effective, existing, mode, preflight) {
|
|
|
769
814
|
nativeSessionId: preflight.nativeSessionId,
|
|
770
815
|
launchId: preflight.launchId,
|
|
771
816
|
...(preflight.sessionTitle === undefined ? {} : { title: preflight.sessionTitle }),
|
|
772
|
-
status: "
|
|
817
|
+
status: "active",
|
|
773
818
|
effective: preflight.effective
|
|
774
819
|
};
|
|
775
820
|
return validateRoleSession(role, effective, existing, mode, session);
|
|
776
821
|
}
|
|
777
|
-
|
|
778
|
-
* A Controller restart can recover an exact launch-submitted Run while its
|
|
779
|
-
* persistent Provider process is still alive. The Host intentionally does not
|
|
780
|
-
* re-plan or re-submit that Run and therefore may return no native identity;
|
|
781
|
-
* retain only the durable Session fenced to the same launch generation.
|
|
782
|
-
*/
|
|
783
|
-
function validateLaunchSubmittedRecoverySession(role, effective, existing, mode, prepared, session) {
|
|
784
|
-
if (session === null
|
|
785
|
-
&& (prepared.turnAcceptedDuringLaunch === true
|
|
786
|
-
|| prepared.turnDeliveryUnknownDuringLaunch === true
|
|
787
|
-
|| prepared.turnBusyDuringLaunch === true
|
|
788
|
-
|| prepared.turnRejectedDuringLaunch === true)
|
|
789
|
-
&& prepared.sessionStarted === false
|
|
790
|
-
&& existing?.launchId !== undefined
|
|
791
|
-
&& existing.launchId === prepared.launchId) {
|
|
792
|
-
return validateRoleSession(role, effective, existing, mode, existing);
|
|
793
|
-
}
|
|
822
|
+
function validateLaunchSubmittedRecoverySession(role, effective, existing, mode, session) {
|
|
794
823
|
return session === undefined
|
|
795
824
|
? existing
|
|
796
825
|
: validateRoleSession(role, effective, existing, mode, session);
|
|
797
826
|
}
|
|
827
|
+
function initialRunTurnInput(run) {
|
|
828
|
+
if (run.controlRequest?.state === "dispatching") {
|
|
829
|
+
return serializeWorkflowOutcomeRequestEnvelope({
|
|
830
|
+
taskId: run.taskId,
|
|
831
|
+
runId: run.id,
|
|
832
|
+
roleName: run.roleName,
|
|
833
|
+
request: run.controlRequest
|
|
834
|
+
});
|
|
835
|
+
}
|
|
836
|
+
return run.pushedAt === undefined
|
|
837
|
+
? serializeRunBootstrapEnvelope(run.bootstrapEnvelope)
|
|
838
|
+
: serializeRunHostRecoveryEnvelope(run.bootstrapEnvelope);
|
|
839
|
+
}
|
|
798
840
|
function validateRoleSession(role, effective, existing, mode, session) {
|
|
799
841
|
if (mode === "new" && session === null)
|
|
800
842
|
return null;
|
|
@@ -815,7 +857,7 @@ function validateRoleSession(role, effective, existing, mode, session) {
|
|
|
815
857
|
}
|
|
816
858
|
return {
|
|
817
859
|
...session,
|
|
818
|
-
status: "
|
|
860
|
+
status: "active",
|
|
819
861
|
...(mode === "resume" && existing !== null
|
|
820
862
|
? { effective: existing.effective }
|
|
821
863
|
: {})
|
|
@@ -25,8 +25,6 @@ export function repairOrphanedActiveTasks(store, now, selection) {
|
|
|
25
25
|
const run = store.getActiveAgentRun(task.id, role.name);
|
|
26
26
|
return run === null ? [] : [run];
|
|
27
27
|
});
|
|
28
|
-
const hasInFlightTurn = roles.some((role) => store.hasInFlightTurn(task.id, role.name));
|
|
29
|
-
const hasLeaderInFlightTurn = store.hasInFlightTurn(task.id, "leader");
|
|
30
28
|
const leaderTarget = { kind: "role", taskId: task.id, roleName: "leader" };
|
|
31
29
|
const leaderMailbox = store.getWorkMailbox(leaderTarget);
|
|
32
30
|
const projection = projectTaskExecution({
|
|
@@ -41,8 +39,7 @@ export function repairOrphanedActiveTasks(store, now, selection) {
|
|
|
41
39
|
&& !activeRuns.some(({ roleName }) => roleName === "leader")
|
|
42
40
|
&& activeRuns.some(({ roleName }) => roleName !== "leader")
|
|
43
41
|
&& projection.status === "waiting-on-agents";
|
|
44
|
-
if ((
|
|
45
|
-
|| store.hasOpenInputRequest(task.id)
|
|
42
|
+
if (store.hasOpenInputRequest(task.id)
|
|
46
43
|
|| store.getLeaderFailure(task.id) !== null
|
|
47
44
|
|| (projection.status !== "needs-leader-action" && !queuedAlongsideActiveSibling)
|
|
48
45
|
|| hasUnclaimedLeaderWork(store, task.id, leaderMailbox)) {
|