@zq-silk/yui 0.12.0 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -9
- package/dist/cli/commandCatalog.js +11 -6
- package/dist/cli/interactionPolicy.js +5 -6
- package/dist/cli/updateCommand.js +3 -1
- package/dist/cli/updateOrchestrator.js +173 -28
- package/dist/cli/updatePorts.js +137 -8
- package/dist/cli/upgradeCommand.js +19 -9
- package/dist/cli.js +51 -14
- package/dist/commands/configCommands.js +1 -1
- package/dist/commands/executionAuditCommands.js +2 -1
- package/dist/commands/taskCommands.js +80 -29
- package/dist/commands/taskContextCommand.js +6 -2
- package/dist/commands/taskRoleRuntimeStatus.js +31 -7
- package/dist/config/configCatalog.js +1 -1
- package/dist/controller/clientRuntime.js +38 -2
- package/dist/controller/controller.js +23 -15
- package/dist/controller/fileSchedulerStoreAdapter.js +248 -138
- package/dist/controller/runtime.js +56 -1
- package/dist/controller/runtimeHookRunFence.js +19 -4
- package/dist/controller/structuredProviderObservation.js +20 -3
- package/dist/core/controllerClient.js +20 -2
- package/dist/core/controllerServer.js +1 -0
- package/dist/executor/agentExecutor.js +48 -46
- package/dist/executor/fileRoleLaunchPlanner.js +94 -30
- package/dist/lifecycle/exactRunTerminalization.js +68 -3
- package/dist/observability/executionAudit.js +5 -0
- package/dist/release/runtimeRelease.js +20 -0
- package/dist/run/recoveryProjection.js +45 -6
- package/dist/runtime/agentHost.js +159 -85
- package/dist/runtime/conversationSwitch.js +277 -0
- package/dist/runtime/index.js +1 -1
- package/dist/runtime/launchBroker.js +12 -0
- package/dist/runtime/processExitOutbox.js +88 -0
- package/dist/runtime/providerRuntimeIdentity.js +29 -1
- package/dist/runtime/runtimeHealthPolicy.js +5 -5
- package/dist/runtime/runtimeObservation.js +15 -0
- package/dist/runtime/runtimeProjection.js +6 -7
- package/dist/runtime/tmuxAdapters.js +4 -1
- package/dist/scheduler/activeRoleRunDelivery.js +31 -196
- package/dist/scheduler/leaderWakeupProcessor.js +39 -133
- package/dist/scheduler/roleRunStall.js +53 -17
- package/dist/storage/sqliteSchema.js +57 -24
- package/dist/storage/sqliteStore.js +23 -4
- package/dist/storage/upgrade/homeClassification.js +52 -0
- package/dist/storage/upgrade/offlineUpgradeInventory.js +145 -7
- package/dist/storage/upgrade/upgradeOrchestrator.js +333 -12
- package/dist/task/nextAction.js +0 -34
- package/dist/web/webSnapshot.js +3 -1
- package/package.json +1 -1
- package/skills/yui-leader/SKILL.md +7 -4
- package/skills/yui-operator/SKILL.md +4 -4
- package/skills/yui-reviewer/SKILL.md +7 -4
- package/dist/lifecycle/taskRoleSessionReset.js +0 -118
|
@@ -1,20 +1,14 @@
|
|
|
1
1
|
import { selectedSchedulerRoles, selectedActiveSchedulerTasks } from "./ports.js";
|
|
2
2
|
import { isSchedulerTaskWorkspaceReady } from "./ports.js";
|
|
3
3
|
import { formatAgentRunReceiptId } from "../task/taskRecordReference.js";
|
|
4
|
-
import { prefixYuiTitleInput } from "../run/runIdentity.js";
|
|
5
|
-
import { resolveTaskRoleSessionTitle } from "../runtime/sessionTitle.js";
|
|
6
4
|
import { agentRunDeliveryReceiptId } from "../run/agentRun.js";
|
|
7
5
|
import { effectiveLaunchSnapshotsCompatible, effectiveLaunchSnapshotsCompatibleForTaskSession } from "../executor/effectiveLaunch.js";
|
|
8
6
|
import { RuntimeLaunchError } from "../runtime/ports.js";
|
|
9
7
|
import { RuntimeLaunchFailure } from "../runtime/launchDiagnostics.js";
|
|
10
|
-
import { builtinAgentDriverRegistry } from "../runtime/builtinAgentDrivers.js";
|
|
11
8
|
import { mailboxHasWork, nextPendingBatch } from "../coordination/workMailbox.js";
|
|
12
9
|
import { runtimeObservationFromTaskEvent } from "../runtime/runtimeObservation.js";
|
|
13
10
|
import { projectProviderContinuations } from "../runtime/runtimeContinuationProjection.js";
|
|
14
11
|
import { hasRuntimeLifecycleWork, RuntimeLifecycleBusyError, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
15
|
-
import { serializeRunBootstrapEnvelope } from "../context/runContextContract.js";
|
|
16
|
-
import { serializeProviderRetryEnvelope } from "../run/providerRetry.js";
|
|
17
|
-
import { serializeWorkflowOutcomeRequestEnvelope } from "../run/runControlRequest.js";
|
|
18
12
|
/**
|
|
19
13
|
* Delivers durable Work AgentRuns before liveness reconciliation. Task command
|
|
20
14
|
* handlers only record intent; this Controller path is the sole automated
|
|
@@ -109,9 +103,14 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
109
103
|
let preparedSession = existingSession;
|
|
110
104
|
let preStartFencePersisted = false;
|
|
111
105
|
let deliveryAttempted = false;
|
|
112
|
-
let providerSubmissionBegun = false;
|
|
113
106
|
try {
|
|
114
|
-
|
|
107
|
+
// Workflow-outcome control is always another Turn on the already-bound
|
|
108
|
+
// Conversation. Provider retry alone may carry an exact missing-
|
|
109
|
+
// Conversation decision and therefore retains its explicit new mode.
|
|
110
|
+
const launchMode = run.controlRequest?.state === "dispatching"
|
|
111
|
+
? "resume"
|
|
112
|
+
: run.mode;
|
|
113
|
+
const nativeSessionId = launchMode === "resume"
|
|
115
114
|
? requireResumeSession(role, run.effective, existingSession)
|
|
116
115
|
: undefined;
|
|
117
116
|
prepared = await delivery.prepareRoleSession({
|
|
@@ -124,12 +123,12 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
124
123
|
...(run.workspace === undefined
|
|
125
124
|
? {}
|
|
126
125
|
: { managedWorkspace: run.workspace }),
|
|
127
|
-
mode:
|
|
126
|
+
mode: launchMode,
|
|
128
127
|
runId: run.id,
|
|
129
128
|
...(nativeSessionId === undefined ? {} : { nativeSessionId }),
|
|
130
129
|
beforeHostStart: (preflight) => {
|
|
131
130
|
persistPreStartFence(store, task, role, run, existingSession, preflight, now);
|
|
132
|
-
preparedSession = preflightSession(role, run.effective, existingSession,
|
|
131
|
+
preparedSession = preflightSession(role, run.effective, existingSession, launchMode, preflight);
|
|
133
132
|
preStartFencePersisted = true;
|
|
134
133
|
}
|
|
135
134
|
});
|
|
@@ -177,20 +176,32 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
177
176
|
now
|
|
178
177
|
});
|
|
179
178
|
if (ready.prepared.turnRejectedDuringLaunch === true) {
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
? {}
|
|
186
|
-
: { launchId: ready.prepared.launchId })
|
|
179
|
+
const persisted = store.saveRoleRunDeliveryFailure({
|
|
180
|
+
...roleRunDeliveryFailure(run, processing.batchId, session, ready.prepared.launchId),
|
|
181
|
+
summary: `Provider conclusively rejected the exact initial Run input: ${run.id}.`,
|
|
182
|
+
cleanupRequired: false,
|
|
183
|
+
now
|
|
187
184
|
});
|
|
185
|
+
if (persisted === "failed") {
|
|
186
|
+
delivery.forgetPrepared?.({
|
|
187
|
+
taskId: task.id,
|
|
188
|
+
roleName: role.name,
|
|
189
|
+
runId: run.id,
|
|
190
|
+
...(ready.prepared.launchId === undefined
|
|
191
|
+
? {}
|
|
192
|
+
: { launchId: ready.prepared.launchId })
|
|
193
|
+
});
|
|
194
|
+
}
|
|
188
195
|
results.push({
|
|
189
196
|
taskId: task.id,
|
|
190
197
|
roleName: role.name,
|
|
191
198
|
runId: run.id,
|
|
192
|
-
status: "skipped",
|
|
193
|
-
reason: "
|
|
199
|
+
status: persisted === "failed" ? "failed" : "skipped",
|
|
200
|
+
reason: "provider-rejected",
|
|
201
|
+
terminalized: persisted === "failed",
|
|
202
|
+
...(persisted === "failed"
|
|
203
|
+
? {}
|
|
204
|
+
: { error: "Run state changed while recording exact Provider rejection." })
|
|
194
205
|
});
|
|
195
206
|
continue;
|
|
196
207
|
}
|
|
@@ -252,171 +263,10 @@ export async function processActiveRoleRunDeliveries(store, delivery, now, selec
|
|
|
252
263
|
});
|
|
253
264
|
continue;
|
|
254
265
|
}
|
|
255
|
-
|
|
256
|
-
// native event fires before the first prompt (e.g. Claude SessionStart),
|
|
257
|
-
// a freshly-started host must not be pushed until that provider-ready
|
|
258
|
-
// fact has been folded. Unsupported adapters (e.g. Codex) have no
|
|
259
|
-
// pre-input event, so their push proceeds and acceptance is confirmed
|
|
260
|
-
// only by the later exact provider-accepted fold. The gate reads the
|
|
261
|
-
// adapter capability and the durable ready projection — never a sleep,
|
|
262
|
-
// screen scrape, or pane/PID inference — and fails closed for a
|
|
263
|
-
// supported adapter whose readiness cannot be confirmed.
|
|
264
|
-
if (ready.prepared.sessionStarted
|
|
265
|
-
&& builtinAgentDriverRegistry().requireByAdapterId(run.effective.adapterId)
|
|
266
|
-
.capabilities.observation.preInputReadiness === "exact"
|
|
267
|
-
&& !providerReadyForPush(store, {
|
|
268
|
-
taskId: task.id,
|
|
269
|
-
roleName: role.name,
|
|
270
|
-
agentId: run.effective.agentId,
|
|
271
|
-
...(ready.prepared.launchId === undefined ? {} : { launchId: ready.prepared.launchId }),
|
|
272
|
-
...(session?.nativeSessionId === undefined
|
|
273
|
-
? {}
|
|
274
|
-
: { nativeSessionId: session.nativeSessionId })
|
|
275
|
-
})) {
|
|
276
|
-
results.push({
|
|
277
|
-
taskId: task.id,
|
|
278
|
-
roleName: role.name,
|
|
279
|
-
runId: run.id,
|
|
280
|
-
status: "skipped",
|
|
281
|
-
reason: "not-ready"
|
|
282
|
-
});
|
|
283
|
-
continue;
|
|
284
|
-
}
|
|
285
|
-
if (session === null || session.launchId === undefined
|
|
286
|
-
|| !hasText(session.nativeSessionId)
|
|
287
|
-
|| store.beginRoleRunProviderTurn?.({
|
|
288
|
-
taskId: task.id,
|
|
289
|
-
roleName: role.name,
|
|
290
|
-
runId: run.id,
|
|
291
|
-
agentId: run.effective.agentId,
|
|
292
|
-
launchId: session.launchId,
|
|
293
|
-
nativeSessionId: session.nativeSessionId,
|
|
294
|
-
attemptId: receiptId,
|
|
295
|
-
now
|
|
296
|
-
}) !== true) {
|
|
297
|
-
throw new Error("Provider Turn intent could not be durably fenced before delivery.");
|
|
298
|
-
}
|
|
299
|
-
providerSubmissionBegun = true;
|
|
300
|
-
deliveryAttempted = true;
|
|
301
|
-
const launchText = run.controlRequest?.state === "dispatching"
|
|
302
|
-
? serializeWorkflowOutcomeRequestEnvelope({
|
|
303
|
-
taskId: task.id,
|
|
304
|
-
runId: run.id,
|
|
305
|
-
roleName: role.name,
|
|
306
|
-
request: run.controlRequest
|
|
307
|
-
})
|
|
308
|
-
: run.providerRetry?.state === "dispatching"
|
|
309
|
-
? serializeProviderRetryEnvelope({
|
|
310
|
-
taskId: task.id,
|
|
311
|
-
runId: run.id,
|
|
312
|
-
roleName: role.name,
|
|
313
|
-
retry: run.providerRetry
|
|
314
|
-
})
|
|
315
|
-
: serializeRunBootstrapEnvelope(run.bootstrapEnvelope);
|
|
316
|
-
const outcome = await delivery.sendOnce({
|
|
317
|
-
delivery: ready,
|
|
318
|
-
receiptId,
|
|
319
|
-
text: prefixYuiTitleInput(launchText, resolveTaskRoleSessionTitle(session.title, task, role.name))
|
|
320
|
-
});
|
|
321
|
-
if (outcome === "busy" || outcome === "unavailable") {
|
|
322
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
323
|
-
taskId: task.id,
|
|
324
|
-
roleName: role.name,
|
|
325
|
-
runId: run.id,
|
|
326
|
-
attemptId: receiptId,
|
|
327
|
-
status: "rejected",
|
|
328
|
-
reason: outcome === "busy"
|
|
329
|
-
? "Agent Host was busy before Provider mutation."
|
|
330
|
-
: "Agent Host was unavailable before Provider mutation.",
|
|
331
|
-
now
|
|
332
|
-
});
|
|
333
|
-
providerSubmissionBegun = false;
|
|
334
|
-
results.push({
|
|
335
|
-
taskId: task.id,
|
|
336
|
-
roleName: role.name,
|
|
337
|
-
runId: run.id,
|
|
338
|
-
status: "skipped",
|
|
339
|
-
reason: outcome === "busy" ? "not-ready" : "runtime-unavailable",
|
|
340
|
-
terminalFailure: roleRunDeliveryFailure(run, processing.batchId, session, ready.prepared.launchId)
|
|
341
|
-
});
|
|
342
|
-
continue;
|
|
343
|
-
}
|
|
344
|
-
if (outcome === "rejected") {
|
|
345
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
346
|
-
taskId: task.id,
|
|
347
|
-
roleName: role.name,
|
|
348
|
-
runId: run.id,
|
|
349
|
-
attemptId: receiptId,
|
|
350
|
-
status: "rejected",
|
|
351
|
-
reason: "Provider returned an exact negative acknowledgement.",
|
|
352
|
-
now
|
|
353
|
-
});
|
|
354
|
-
providerSubmissionBegun = false;
|
|
355
|
-
results.push({
|
|
356
|
-
taskId: task.id,
|
|
357
|
-
roleName: role.name,
|
|
358
|
-
runId: run.id,
|
|
359
|
-
status: "skipped",
|
|
360
|
-
reason: "not-ready"
|
|
361
|
-
});
|
|
362
|
-
continue;
|
|
363
|
-
}
|
|
364
|
-
if (outcome === "delivery-unknown") {
|
|
365
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
366
|
-
taskId: task.id,
|
|
367
|
-
roleName: role.name,
|
|
368
|
-
runId: run.id,
|
|
369
|
-
attemptId: receiptId,
|
|
370
|
-
status: "delivery-unknown",
|
|
371
|
-
reason: "Provider Turn delivery is ambiguous; automatic retry is fenced.",
|
|
372
|
-
now
|
|
373
|
-
});
|
|
374
|
-
providerSubmissionBegun = false;
|
|
375
|
-
store.saveRoleRunDelivery({
|
|
376
|
-
task,
|
|
377
|
-
role,
|
|
378
|
-
run,
|
|
379
|
-
session,
|
|
380
|
-
...(ready.prepared.launchId === undefined
|
|
381
|
-
? {}
|
|
382
|
-
: { launchId: ready.prepared.launchId }),
|
|
383
|
-
now
|
|
384
|
-
});
|
|
385
|
-
results.push({
|
|
386
|
-
taskId: task.id,
|
|
387
|
-
roleName: role.name,
|
|
388
|
-
runId: run.id,
|
|
389
|
-
status: "skipped",
|
|
390
|
-
reason: "delivery-uncertain"
|
|
391
|
-
});
|
|
392
|
-
continue;
|
|
393
|
-
}
|
|
394
|
-
const status = outcome === "sent" ? "delivered" : "already-delivered";
|
|
395
|
-
store.saveRoleRunDelivery({
|
|
396
|
-
task,
|
|
397
|
-
role,
|
|
398
|
-
run,
|
|
399
|
-
session,
|
|
400
|
-
...(ready.prepared.launchId === undefined
|
|
401
|
-
? {}
|
|
402
|
-
: { launchId: ready.prepared.launchId }),
|
|
403
|
-
now
|
|
404
|
-
});
|
|
405
|
-
results.push({ taskId: task.id, roleName: role.name, runId: run.id, status });
|
|
266
|
+
throw new Error("Managed Provider launch completed without an initial Turn outcome.");
|
|
406
267
|
}
|
|
407
268
|
catch (error) {
|
|
408
269
|
const message = error instanceof Error ? error.message : String(error);
|
|
409
|
-
if (providerSubmissionBegun) {
|
|
410
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
411
|
-
taskId: task.id,
|
|
412
|
-
roleName: role.name,
|
|
413
|
-
runId: run.id,
|
|
414
|
-
attemptId: receiptId,
|
|
415
|
-
status: "delivery-unknown",
|
|
416
|
-
reason: message,
|
|
417
|
-
now
|
|
418
|
-
});
|
|
419
|
-
}
|
|
420
270
|
// Scheduler single-flight backpressure: the Role runtime lifecycle
|
|
421
271
|
// lane was busy when the launch was reserved. The Run stays
|
|
422
272
|
// active-unpushed and its mailbox claim is retained; the next pass
|
|
@@ -953,11 +803,6 @@ function validateRoleSession(role, effective, existing, mode, session) {
|
|
|
953
803
|
if (!compatible) {
|
|
954
804
|
throw new Error(`Ready Role session effective snapshot changed: ${role.taskId}/${role.name}.`);
|
|
955
805
|
}
|
|
956
|
-
if (existing?.nativeSessionId !== undefined
|
|
957
|
-
&& effectiveLaunchSnapshotsCompatibleForTaskSession(existing.effective, effective)
|
|
958
|
-
&& session.nativeSessionId !== existing.nativeSessionId) {
|
|
959
|
-
throw new Error(`Ready Role session changed the fixed native session id: ${role.taskId}/${role.name}.`);
|
|
960
|
-
}
|
|
961
806
|
if (mode === "resume" && session.nativeSessionId !== existing?.nativeSessionId) {
|
|
962
807
|
throw new Error(`Role resume changed the fixed native session id: ${role.taskId}/${role.name}.`);
|
|
963
808
|
}
|
|
@@ -972,13 +817,3 @@ function validateRoleSession(role, effective, existing, mode, session) {
|
|
|
972
817
|
function hasText(value) {
|
|
973
818
|
return typeof value === "string" && value.trim().length > 0;
|
|
974
819
|
}
|
|
975
|
-
/**
|
|
976
|
-
* Fail-closed readiness check for a supported-readiness adapter's first push.
|
|
977
|
-
* When the store cannot answer (no implementation), the push is held rather than
|
|
978
|
-
* proceeding blind — a supported adapter must have a proven provider-ready fold.
|
|
979
|
-
*/
|
|
980
|
-
function providerReadyForPush(store, input) {
|
|
981
|
-
if (store.isRoleGenerationProviderReady === undefined)
|
|
982
|
-
return false;
|
|
983
|
-
return store.isRoleGenerationProviderReady(input);
|
|
984
|
-
}
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
import { createAgentRun } from "../run/agentRun.js";
|
|
2
|
-
import { createRunAssignment
|
|
2
|
+
import { createRunAssignment } from "../context/runContextContract.js";
|
|
3
3
|
import { formatAgentRunReceiptId } from "../task/taskRecordReference.js";
|
|
4
|
-
import { prefixYuiTitleInput } from "../run/runIdentity.js";
|
|
5
|
-
import { resolveTaskRoleSessionTitle } from "../runtime/sessionTitle.js";
|
|
6
4
|
import { effectiveLaunchSnapshotsCompatible, effectiveLaunchSnapshotsCompatibleForTaskSession } from "../executor/effectiveLaunch.js";
|
|
7
5
|
import { hasRuntimeLifecycleWork, RuntimeLifecycleBusyError, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
8
6
|
import { recordLeaderFailure } from "./leaderFailure.js";
|
|
9
7
|
import { isSchedulerTaskWorkspaceReady } from "./ports.js";
|
|
10
8
|
import { RuntimeLaunchError } from "../runtime/ports.js";
|
|
9
|
+
import { roleSessionDispatchModeWithConversationSwitch } from "../runtime/conversationSwitch.js";
|
|
11
10
|
export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
12
11
|
const results = [];
|
|
13
12
|
const wakeups = selection === undefined || selection.full
|
|
@@ -68,7 +67,6 @@ export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
|
68
67
|
let effectiveSession = existingSession;
|
|
69
68
|
let claimed = false;
|
|
70
69
|
let deliveryAttempted = false;
|
|
71
|
-
let providerSubmissionBegun = false;
|
|
72
70
|
let run = null;
|
|
73
71
|
let prepared;
|
|
74
72
|
let preStartFencePersisted = false;
|
|
@@ -125,23 +123,15 @@ export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
|
125
123
|
const resumableSession = hasNativeSession(existingSession)
|
|
126
124
|
&& existingSession.status !== "stopped"
|
|
127
125
|
&& existingSession.status !== "broken";
|
|
128
|
-
//
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
.at(-1);
|
|
138
|
-
const failedResumeWithoutAcceptance = latestRoleRun !== undefined
|
|
139
|
-
&& latestRoleRun.mode === "resume"
|
|
140
|
-
&& latestRoleRun.status === "failed"
|
|
141
|
-
&& latestRoleRun.deliveredAt === undefined;
|
|
142
|
-
const mode = resumableSession && compatibleSession && !failedResumeWithoutAcceptance
|
|
143
|
-
? "resume"
|
|
144
|
-
: "new";
|
|
126
|
+
// A failed or quiet Run is not proof that its Provider Conversation is
|
|
127
|
+
// unusable. Keep resuming the same identity unless an explicit switch
|
|
128
|
+
// request (or exact terminal/missing evidence) authorizes a fresh one.
|
|
129
|
+
const sessionSet = store.getTaskRoleSessionSet?.(task.id, role.name) ?? null;
|
|
130
|
+
const mode = reopenIdentityDrift
|
|
131
|
+
? "new"
|
|
132
|
+
: sessionSet === null
|
|
133
|
+
? resumableSession && compatibleSession ? "resume" : "new"
|
|
134
|
+
: roleSessionDispatchModeWithConversationSwitch(sessionSet, store.listEvents?.(task.id) ?? [], store.getWorkMailbox({ kind: "role", taskId: task.id, roleName: role.name }), role.name, role.effective.agentId, role.effective);
|
|
145
135
|
const runId = store.peekNextAgentRunId(task.id);
|
|
146
136
|
const wakeEnvelope = resolveLeaderWakeEnvelope(store, task.id);
|
|
147
137
|
const contextSnapshot = store.freezeLeaderContextSnapshot?.(task.id, role.name, now);
|
|
@@ -260,19 +250,42 @@ export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
|
260
250
|
now
|
|
261
251
|
});
|
|
262
252
|
if (ready.prepared.turnRejectedDuringLaunch === true) {
|
|
263
|
-
|
|
253
|
+
const persisted = store.saveRoleRunDeliveryFailure({
|
|
264
254
|
taskId: task.id,
|
|
265
255
|
roleName: role.name,
|
|
256
|
+
agentId: run.effective.agentId,
|
|
257
|
+
adapterId: run.effective.adapterId,
|
|
266
258
|
runId: run.id,
|
|
259
|
+
mailboxBatchId: formatAgentRunReceiptId(task.id, run.id),
|
|
260
|
+
...(effectiveSession?.nativeSessionId === undefined
|
|
261
|
+
? {}
|
|
262
|
+
: { nativeSessionId: effectiveSession.nativeSessionId }),
|
|
267
263
|
...(ready.prepared.launchId === undefined
|
|
268
264
|
? {}
|
|
269
|
-
: { launchId: ready.prepared.launchId })
|
|
265
|
+
: { launchId: ready.prepared.launchId }),
|
|
266
|
+
summary: `Provider conclusively rejected the exact initial Leader Run input: ${run.id}.`,
|
|
267
|
+
cleanupRequired: false,
|
|
268
|
+
now
|
|
270
269
|
});
|
|
270
|
+
if (persisted === "failed") {
|
|
271
|
+
delivery.forgetPrepared?.({
|
|
272
|
+
taskId: task.id,
|
|
273
|
+
roleName: role.name,
|
|
274
|
+
runId: run.id,
|
|
275
|
+
...(ready.prepared.launchId === undefined
|
|
276
|
+
? {}
|
|
277
|
+
: { launchId: ready.prepared.launchId })
|
|
278
|
+
});
|
|
279
|
+
}
|
|
271
280
|
results.push({
|
|
272
281
|
taskId: task.id,
|
|
273
282
|
runId: run.id,
|
|
274
|
-
status: "skipped",
|
|
275
|
-
reason: "
|
|
283
|
+
status: persisted === "failed" ? "failed" : "skipped",
|
|
284
|
+
reason: "provider-rejected",
|
|
285
|
+
terminalized: persisted === "failed",
|
|
286
|
+
...(persisted === "failed"
|
|
287
|
+
? {}
|
|
288
|
+
: { error: "Run state changed while recording exact Provider rejection." })
|
|
276
289
|
});
|
|
277
290
|
continue;
|
|
278
291
|
}
|
|
@@ -328,118 +341,11 @@ export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
|
328
341
|
results.push({ taskId: task.id, runId: run.id, status: "dispatched" });
|
|
329
342
|
continue;
|
|
330
343
|
}
|
|
331
|
-
|
|
332
|
-
if (effectiveSession === null || effectiveSession.launchId === undefined
|
|
333
|
-
|| !hasNativeSession(effectiveSession)
|
|
334
|
-
|| store.beginRoleRunProviderTurn?.({
|
|
335
|
-
taskId: task.id,
|
|
336
|
-
roleName: role.name,
|
|
337
|
-
runId: run.id,
|
|
338
|
-
agentId: run.effective.agentId,
|
|
339
|
-
launchId: effectiveSession.launchId,
|
|
340
|
-
nativeSessionId: effectiveSession.nativeSessionId,
|
|
341
|
-
attemptId: receiptId,
|
|
342
|
-
now
|
|
343
|
-
}) !== true) {
|
|
344
|
-
throw new Error("Provider Turn intent could not be durably fenced before delivery.");
|
|
345
|
-
}
|
|
346
|
-
providerSubmissionBegun = true;
|
|
347
|
-
deliveryAttempted = true;
|
|
348
|
-
const outcome = await delivery.sendOnce({
|
|
349
|
-
delivery: ready,
|
|
350
|
-
receiptId,
|
|
351
|
-
text: prefixYuiTitleInput(serializeRunBootstrapEnvelope(run.bootstrapEnvelope), resolveTaskRoleSessionTitle(effectiveSession.title, task, role.name))
|
|
352
|
-
});
|
|
353
|
-
if (outcome === "busy" || outcome === "unavailable") {
|
|
354
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
355
|
-
taskId: task.id,
|
|
356
|
-
roleName: role.name,
|
|
357
|
-
runId: run.id,
|
|
358
|
-
attemptId: receiptId,
|
|
359
|
-
status: "rejected",
|
|
360
|
-
reason: outcome === "busy"
|
|
361
|
-
? "Agent Host was busy before Provider mutation."
|
|
362
|
-
: "Agent Host was unavailable before Provider mutation.",
|
|
363
|
-
now
|
|
364
|
-
});
|
|
365
|
-
providerSubmissionBegun = false;
|
|
366
|
-
results.push({
|
|
367
|
-
taskId: task.id,
|
|
368
|
-
runId: run.id,
|
|
369
|
-
status: "skipped",
|
|
370
|
-
reason: "not-ready"
|
|
371
|
-
});
|
|
372
|
-
continue;
|
|
373
|
-
}
|
|
374
|
-
if (outcome === "rejected") {
|
|
375
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
376
|
-
taskId: task.id,
|
|
377
|
-
roleName: role.name,
|
|
378
|
-
runId: run.id,
|
|
379
|
-
attemptId: receiptId,
|
|
380
|
-
status: "rejected",
|
|
381
|
-
reason: "Provider returned an exact negative acknowledgement.",
|
|
382
|
-
now
|
|
383
|
-
});
|
|
384
|
-
providerSubmissionBegun = false;
|
|
385
|
-
results.push({ taskId: task.id, runId: run.id, status: "skipped", reason: "not-ready" });
|
|
386
|
-
continue;
|
|
387
|
-
}
|
|
388
|
-
if (outcome === "delivery-unknown") {
|
|
389
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
390
|
-
taskId: task.id,
|
|
391
|
-
roleName: role.name,
|
|
392
|
-
runId: run.id,
|
|
393
|
-
attemptId: receiptId,
|
|
394
|
-
status: "delivery-unknown",
|
|
395
|
-
reason: "Provider Turn delivery is ambiguous; automatic retry is fenced.",
|
|
396
|
-
now
|
|
397
|
-
});
|
|
398
|
-
providerSubmissionBegun = false;
|
|
399
|
-
store.saveRoleRunDelivery({
|
|
400
|
-
task,
|
|
401
|
-
role,
|
|
402
|
-
run,
|
|
403
|
-
session: effectiveSession,
|
|
404
|
-
...(ready.prepared.launchId === undefined
|
|
405
|
-
? {}
|
|
406
|
-
: { launchId: ready.prepared.launchId }),
|
|
407
|
-
now
|
|
408
|
-
});
|
|
409
|
-
results.push({
|
|
410
|
-
taskId: task.id,
|
|
411
|
-
runId: run.id,
|
|
412
|
-
status: "skipped",
|
|
413
|
-
reason: "delivery-uncertain"
|
|
414
|
-
});
|
|
415
|
-
continue;
|
|
416
|
-
}
|
|
417
|
-
store.saveRoleRunDelivery({
|
|
418
|
-
task,
|
|
419
|
-
role,
|
|
420
|
-
run,
|
|
421
|
-
session: effectiveSession,
|
|
422
|
-
...(ready.prepared.launchId === undefined
|
|
423
|
-
? {}
|
|
424
|
-
: { launchId: ready.prepared.launchId }),
|
|
425
|
-
now
|
|
426
|
-
});
|
|
427
|
-
results.push({ taskId: task.id, runId: run.id, status: "dispatched" });
|
|
344
|
+
throw new Error("Managed Provider launch completed without an initial Turn outcome.");
|
|
428
345
|
}
|
|
429
346
|
catch (error) {
|
|
430
347
|
const detail = error instanceof Error ? error.message : String(error);
|
|
431
348
|
const message = `Leader dispatch failed: ${detail}`;
|
|
432
|
-
if (providerSubmissionBegun && run !== null) {
|
|
433
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
434
|
-
taskId: task.id,
|
|
435
|
-
roleName: role.name,
|
|
436
|
-
runId: run.id,
|
|
437
|
-
attemptId: formatAgentRunReceiptId(task.id, run.id),
|
|
438
|
-
status: "delivery-unknown",
|
|
439
|
-
reason: detail,
|
|
440
|
-
now
|
|
441
|
-
});
|
|
442
|
-
}
|
|
443
349
|
if (claimed && run !== null) {
|
|
444
350
|
// Scheduler single-flight backpressure: the Role runtime lifecycle
|
|
445
351
|
// lane was busy when the launch was reserved. The claimed Run stays
|
|
@@ -12,6 +12,7 @@ export const DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS = RUNTIME_DIAGNOSTIC_AFTER_
|
|
|
12
12
|
export const RUN_PROGRESS_EVENT = "run.progress";
|
|
13
13
|
export const RUN_STALLED_EVENT = "run.stalled";
|
|
14
14
|
export const RUN_RECOVERED_EVENT = "run.recovered";
|
|
15
|
+
export const RUN_DIAGNOSTIC_FINISHED_EVENT = "runtime.diagnostic-finished";
|
|
15
16
|
/** Structured, non-Message recovery evidence written by an explicit Leader. */
|
|
16
17
|
export const RUN_RECOVERY_REQUESTED_EVENT = "run.recovery-requested";
|
|
17
18
|
export const RUN_RECOVERY_APPLIED_EVENT = "run.recovery-applied";
|
|
@@ -50,10 +51,7 @@ export function projectRoleRunHealth(input) {
|
|
|
50
51
|
const idleMs = evaluation.idleMs;
|
|
51
52
|
const candidateStart = input.deliveredAt ?? input.createdAt;
|
|
52
53
|
const candidateAge = input.now.getTime() - Date.parse(candidateStart);
|
|
53
|
-
const candidate = Number.isFinite(candidateAge)
|
|
54
|
-
&& (input.deliveredAt === undefined
|
|
55
|
-
? candidateAge >= windowMs
|
|
56
|
-
: candidateAge >= diagnosticAfterMs);
|
|
54
|
+
const candidate = Number.isFinite(candidateAge) && candidateAge >= windowMs;
|
|
57
55
|
const providerAcceptance = input.providerAcceptance
|
|
58
56
|
?? (input.deliveredAt === undefined ? "ambiguous" : "accepted");
|
|
59
57
|
const hostLiveness = input.hostLiveness;
|
|
@@ -437,6 +435,8 @@ export function isRoleRunStalled(events, runId) {
|
|
|
437
435
|
if (event.payload.runId !== runId)
|
|
438
436
|
continue;
|
|
439
437
|
if (event.type === RUN_STALLED_EVENT) {
|
|
438
|
+
if (event.payload.status === "diagnostic-only")
|
|
439
|
+
continue;
|
|
440
440
|
if (stalled === undefined
|
|
441
441
|
|| Date.parse(event.createdAt) > Date.parse(stalled.createdAt)) {
|
|
442
442
|
stalled = event;
|
|
@@ -475,8 +475,9 @@ export function isRoleRunStalled(events, runId) {
|
|
|
475
475
|
/**
|
|
476
476
|
* Low-frequency health pass for active Task Role Runs. An unaccepted Run is
|
|
477
477
|
* watched as delivery-stalled after the reasonable delivery window; an
|
|
478
|
-
* accepted Run
|
|
479
|
-
*
|
|
478
|
+
* accepted Run is displayed as checkpoint-overdue after fifteen minutes, but
|
|
479
|
+
* this scheduler performs no runtime inspection until the thirty-minute
|
|
480
|
+
* diagnostic window is due.
|
|
480
481
|
* Leader Runs are only persisted when classification reaches truly-stalled —
|
|
481
482
|
* healthy downstream work, open user input, and recent own progress remain
|
|
482
483
|
* structured waiting/working facts. No branch sends terminal bytes, retries,
|
|
@@ -507,14 +508,43 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
507
508
|
session: store.getRoleSession(task.id, role.name, run.effective.agentId)
|
|
508
509
|
}];
|
|
509
510
|
})));
|
|
510
|
-
const
|
|
511
|
+
const eventsByTask = new Map();
|
|
512
|
+
const diagnosticEvents = (taskId) => {
|
|
513
|
+
const existing = eventsByTask.get(taskId);
|
|
514
|
+
if (existing !== undefined)
|
|
515
|
+
return existing;
|
|
516
|
+
const events = store.listEvents?.(taskId) ?? [];
|
|
517
|
+
eventsByTask.set(taskId, events);
|
|
518
|
+
return events;
|
|
519
|
+
};
|
|
520
|
+
const cadenceCandidates = candidates.filter(({ task, run }) => isStallCandidate(run, now, windowMs, latestRunEventTime(diagnosticEvents(task.id), RUN_DIAGNOSTIC_FINISHED_EVENT, run.id)));
|
|
521
|
+
const stallCandidates = cadenceCandidates.filter(({ task, role, run }) => {
|
|
522
|
+
const progressFacts = foldPortPresent
|
|
523
|
+
? store.getRunProgressFacts?.(task.id, run.id)
|
|
524
|
+
: undefined;
|
|
525
|
+
const progressAt = currentRoleRunProgressAt(store, task.id, role.name, run, foldPortPresent ? undefined : diagnosticEvents(task.id), progressFacts).progressAt;
|
|
526
|
+
return isStallCandidate(run, now, windowMs, latestRunEventTime(diagnosticEvents(task.id), RUN_DIAGNOSTIC_FINISHED_EVENT, run.id), progressAt);
|
|
527
|
+
});
|
|
511
528
|
if (stallCandidates.length === 0)
|
|
512
529
|
return [];
|
|
530
|
+
const diagnosticStartedAt = now.toISOString();
|
|
531
|
+
const finishDiagnostics = (outcome) => {
|
|
532
|
+
for (const { task, role, run } of stallCandidates) {
|
|
533
|
+
store.recordRoleRunDiagnostic?.({
|
|
534
|
+
taskId: task.id,
|
|
535
|
+
roleName: role.name,
|
|
536
|
+
runId: run.id,
|
|
537
|
+
startedAt: diagnosticStartedAt,
|
|
538
|
+
outcome,
|
|
539
|
+
now
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
};
|
|
513
543
|
const stallCandidateKeys = new Set(stallCandidates.map(({ task, role }) => (`${task.id}\0${role.name}`)));
|
|
514
544
|
const observed = new Map();
|
|
515
545
|
// Recent active Runs are known healthy enough for Leader classification from
|
|
516
546
|
// their exact acceptance/creation boundary, but are deliberately not read
|
|
517
|
-
// from tmux or Event history until the
|
|
547
|
+
// from tmux or Event history until the 30-minute diagnostic window.
|
|
518
548
|
for (const candidate of candidates) {
|
|
519
549
|
const key = `${candidate.task.id}\0${candidate.role.name}`;
|
|
520
550
|
if (stallCandidateKeys.has(key))
|
|
@@ -563,7 +593,9 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
563
593
|
}
|
|
564
594
|
catch {
|
|
565
595
|
// Health inspection is advisory. Unknown host state must leave the exact
|
|
566
|
-
// Run/Session fence untouched
|
|
596
|
+
// Run/Session fence untouched. Closing this window schedules the next
|
|
597
|
+
// bounded diagnostic thirty minutes later instead of hot-looping.
|
|
598
|
+
finishDiagnostics("observation-error");
|
|
567
599
|
return [];
|
|
568
600
|
}
|
|
569
601
|
}
|
|
@@ -575,6 +607,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
575
607
|
catch {
|
|
576
608
|
// A malformed/partial provider snapshot is not evidence of a stall.
|
|
577
609
|
// Leave all fences untouched and let the next full pass retry.
|
|
610
|
+
finishDiagnostics("observation-error");
|
|
578
611
|
return [];
|
|
579
612
|
}
|
|
580
613
|
}
|
|
@@ -588,8 +621,10 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
588
621
|
const key = `${candidate.task.id}\0${candidate.role.name}`;
|
|
589
622
|
let live;
|
|
590
623
|
try {
|
|
591
|
-
if (liveStatuses !== undefined && !liveStatuses.has(key))
|
|
624
|
+
if (liveStatuses !== undefined && !liveStatuses.has(key)) {
|
|
625
|
+
finishDiagnostics("observation-error");
|
|
592
626
|
return [];
|
|
627
|
+
}
|
|
593
628
|
live = liveStatuses !== undefined
|
|
594
629
|
? liveStatuses.get(key)
|
|
595
630
|
: byRole === null
|
|
@@ -608,6 +643,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
608
643
|
// Without a complete live snapshot, especially for downstream Runs,
|
|
609
644
|
// Leader classification would be unsafe. Treat the whole advisory pass
|
|
610
645
|
// as unknown instead of escalating a false stall.
|
|
646
|
+
finishDiagnostics("observation-error");
|
|
611
647
|
return [];
|
|
612
648
|
}
|
|
613
649
|
// A missing process is handled by the existing fenced liveness pass. This
|
|
@@ -812,18 +848,18 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
812
848
|
});
|
|
813
849
|
}
|
|
814
850
|
}
|
|
851
|
+
finishDiagnostics("observed");
|
|
815
852
|
return raised;
|
|
816
853
|
}
|
|
817
|
-
function isStallCandidate(run, now, windowMs,
|
|
854
|
+
function isStallCandidate(run, now, windowMs, lastDiagnosticFinishedAt, latestProgressAt) {
|
|
818
855
|
const startAt = run.deliveredAt ?? run.createdAt;
|
|
819
|
-
const
|
|
856
|
+
const baseline = [startAt, lastDiagnosticFinishedAt, latestProgressAt]
|
|
857
|
+
.filter((value) => (value !== undefined && Number.isFinite(Date.parse(value))))
|
|
858
|
+
.reduce((latest, value) => (Date.parse(value) > Date.parse(latest) ? value : latest));
|
|
859
|
+
const ageMs = now.getTime() - Date.parse(baseline);
|
|
820
860
|
if (!Number.isFinite(ageMs))
|
|
821
861
|
return false;
|
|
822
|
-
|
|
823
|
-
// window, but they never enter workflow-stall candidate filtering.
|
|
824
|
-
return run.deliveredAt === undefined
|
|
825
|
-
? ageMs >= windowMs
|
|
826
|
-
: ageMs >= diagnosticAfterMs;
|
|
862
|
+
return ageMs >= windowMs;
|
|
827
863
|
}
|
|
828
864
|
function classifyLeaderStall(store, taskId, observed, now, windowMs) {
|
|
829
865
|
if (store.hasOpenInputRequest(taskId))
|