@zq-silk/yui 0.11.3 → 0.12.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/ARCHITECTURE.md +23 -4
- package/README.md +44 -22
- package/dist/cli/commandCatalog.js +13 -10
- package/dist/cli/interactionPolicy.js +5 -6
- package/dist/cli/updateCommand.js +3 -1
- package/dist/cli/updateOrchestrator.js +173 -28
- package/dist/cli/updatePorts.js +137 -8
- package/dist/cli/upgradeCommand.js +19 -9
- package/dist/cli.js +156 -57
- package/dist/commands/configCommands.js +13 -46
- package/dist/commands/executionAuditCommands.js +3 -2
- package/dist/commands/jobCommands.js +4 -10
- package/dist/commands/taskCommands.js +234 -200
- package/dist/commands/taskCompletionGate.js +29 -54
- package/dist/commands/taskContextCommand.js +61 -13
- package/dist/commands/taskInputCommands.js +8 -5
- package/dist/commands/taskNextActionCommand.js +56 -6
- package/dist/commands/taskOverviewCommand.js +17 -31
- package/dist/commands/taskRoleRuntimeStatus.js +55 -7
- package/dist/commands/taskWorkspaceCommands.js +5 -2
- package/dist/config/configCatalog.js +3 -3
- package/dist/config/yuiConfig.js +3 -7
- package/dist/context/wakeNotification.js +20 -5
- package/dist/controller/agentRuntimeObserver.js +247 -51
- package/dist/controller/clientRuntime.js +39 -3
- package/dist/controller/controller.js +31 -27
- package/dist/controller/fileSchedulerStoreAdapter.js +390 -332
- package/dist/controller/runtime.js +58 -3
- package/dist/controller/runtimeEventInbox.js +17 -7
- package/dist/controller/runtimeEventProcessor.js +12 -19
- package/dist/controller/runtimeHookRunFence.js +19 -4
- package/dist/controller/runtimeObservationHook.js +45 -0
- package/dist/controller/structuredProviderObservation.js +20 -3
- package/dist/core/controllerClient.js +20 -2
- package/dist/core/controllerServer.js +1 -0
- package/dist/execution/resourceBroker.js +5 -4
- package/dist/executor/agentExecutor.js +52 -50
- package/dist/executor/effectiveLaunch.js +8 -48
- package/dist/executor/fileRoleLaunchPlanner.js +129 -45
- package/dist/executor/workspacePreflightClassification.js +23 -2
- package/dist/interaction/operatorPresentation.js +33 -89
- package/dist/lifecycle/exactRunTerminalization.js +69 -5
- package/dist/observability/executionAudit.js +5 -0
- package/dist/observability/orchestrationMetrics.js +8 -3
- package/dist/profile/agentProfile.js +1 -1
- package/dist/release/cliHomeReleaseFence.js +123 -0
- package/dist/release/runtimeRelease.js +20 -0
- package/dist/repository/taskWorkspacePreparer.js +176 -60
- package/dist/resources/sqliteResourceRegistry.js +1 -1
- package/dist/review/deltaRecheck.js +12 -51
- package/dist/review/reviewAcceptance.js +26 -0
- package/dist/review/reviewConfig.js +0 -31
- package/dist/review/reviewDecision.js +113 -0
- package/dist/review/reviewOutcomeClassifier.js +1 -1
- package/dist/review/reviewRound.js +1 -1
- package/dist/review/reviewerAvailability.js +69 -0
- package/dist/run/recoveryProjection.js +45 -6
- package/dist/runtime/agentDriverObservation.js +24 -10
- package/dist/runtime/agentHost.js +159 -85
- package/dist/runtime/builtinAgentDrivers.js +4 -3
- package/dist/runtime/builtinTranscriptObserver.js +301 -64
- package/dist/runtime/builtinTranscriptUsage.js +9 -7
- package/dist/runtime/conversationSwitch.js +277 -0
- package/dist/runtime/index.js +2 -1
- package/dist/runtime/launchBroker.js +12 -0
- package/dist/runtime/processExitOutbox.js +88 -0
- package/dist/runtime/providerRuntimeIdentity.js +29 -1
- package/dist/runtime/runtimeHealthPolicy.js +5 -5
- package/dist/runtime/runtimeObservation.js +28 -0
- package/dist/runtime/runtimeProjection.js +22 -32
- package/dist/runtime/sessionTokenMetrics.js +181 -0
- package/dist/runtime/structuredProviderHost.js +7 -1
- package/dist/runtime/tmuxAdapters.js +4 -1
- package/dist/scheduler/activeRoleRunDelivery.js +34 -199
- package/dist/scheduler/activeTaskProgress.js +7 -6
- package/dist/scheduler/leaderWakeupProcessor.js +42 -138
- package/dist/scheduler/operatorEvent.js +34 -0
- package/dist/scheduler/operatorInputNotificationProcessor.js +54 -94
- package/dist/scheduler/roleRunStall.js +53 -31
- package/dist/scheduler/taskExecutionProjection.js +97 -76
- package/dist/scheduler/taskObservabilityProjection.js +10 -11
- package/dist/storage/migration/productionRegistry.js +379 -0
- package/dist/storage/sqliteSchema.js +66 -23
- package/dist/storage/sqliteStore.js +46 -23
- package/dist/storage/storeRpc.js +0 -1
- package/dist/storage/taskStore.js +15 -30
- package/dist/storage/upgrade/homeClassification.js +52 -0
- package/dist/storage/upgrade/offlineUpgradeInventory.js +145 -7
- package/dist/storage/upgrade/recordVersions.js +3 -2
- package/dist/storage/upgrade/sqliteMigrationTarget.js +30 -8
- package/dist/storage/upgrade/sqliteRecordMigrationTarget.js +23 -11
- package/dist/storage/upgrade/sqliteStateMigration.js +66 -8
- package/dist/storage/upgrade/upgradeOrchestrator.js +333 -12
- package/dist/task/completionReadiness.js +10 -6
- package/dist/task/nextAction.js +82 -58
- package/dist/telemetry/sqliteTelemetryStore.js +1 -1
- package/dist/web/assets/client/components.js +13 -3
- package/dist/web/assets/client/i18n.js +6 -2
- package/dist/web/webSnapshot.js +7 -1
- package/i18n/README.zh-CN.md +2 -2
- package/package.json +1 -1
- package/skills/yui-leader/SKILL.md +60 -20
- package/skills/yui-operator/SKILL.md +13 -4
- package/skills/yui-reviewer/SKILL.md +35 -11
- package/dist/context/sessionContextBudget.js +0 -71
- package/dist/lifecycle/contextBudgetRollover.js +0 -81
- package/dist/lifecycle/taskRoleSessionReset.js +0 -126
- package/dist/scheduler/operatorNotification.js +0 -59
|
@@ -1,14 +1,12 @@
|
|
|
1
1
|
import { createAgentRun } from "../run/agentRun.js";
|
|
2
|
-
import { createRunAssignment
|
|
2
|
+
import { createRunAssignment } from "../context/runContextContract.js";
|
|
3
3
|
import { formatAgentRunReceiptId } from "../task/taskRecordReference.js";
|
|
4
|
-
import {
|
|
5
|
-
import { resolveTaskRoleSessionTitle } from "../runtime/sessionTitle.js";
|
|
6
|
-
import { effectiveLaunchSnapshotsCompatible, effectiveLaunchSnapshotsCompatibleForTaskMain } from "../executor/effectiveLaunch.js";
|
|
4
|
+
import { effectiveLaunchSnapshotsCompatible, effectiveLaunchSnapshotsCompatibleForTaskSession } from "../executor/effectiveLaunch.js";
|
|
7
5
|
import { hasRuntimeLifecycleWork, RuntimeLifecycleBusyError, runtimeLifecycleTarget } from "../runtime/lifecycleReservation.js";
|
|
8
6
|
import { recordLeaderFailure } from "./leaderFailure.js";
|
|
9
|
-
import { createLeaderRecoveryNotification } from "./operatorNotification.js";
|
|
10
7
|
import { isSchedulerTaskWorkspaceReady } from "./ports.js";
|
|
11
8
|
import { RuntimeLaunchError } from "../runtime/ports.js";
|
|
9
|
+
import { roleSessionDispatchModeWithConversationSwitch } from "../runtime/conversationSwitch.js";
|
|
12
10
|
export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
13
11
|
const results = [];
|
|
14
12
|
const wakeups = selection === undefined || selection.full
|
|
@@ -69,7 +67,6 @@ export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
|
69
67
|
let effectiveSession = existingSession;
|
|
70
68
|
let claimed = false;
|
|
71
69
|
let deliveryAttempted = false;
|
|
72
|
-
let providerSubmissionBegun = false;
|
|
73
70
|
let run = null;
|
|
74
71
|
let prepared;
|
|
75
72
|
let preStartFencePersisted = false;
|
|
@@ -114,7 +111,7 @@ export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
|
114
111
|
const compatibleSession = existingSession !== null
|
|
115
112
|
&& (reopening
|
|
116
113
|
? effectiveLaunchSnapshotsCompatible(existingSession.effective, role.effective)
|
|
117
|
-
:
|
|
114
|
+
: effectiveLaunchSnapshotsCompatibleForTaskSession(existingSession.effective, role.effective));
|
|
118
115
|
const reopenIdentityDrift = reopening
|
|
119
116
|
&& hasNativeSession(existingSession)
|
|
120
117
|
&& !compatibleSession;
|
|
@@ -126,23 +123,15 @@ export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
|
126
123
|
const resumableSession = hasNativeSession(existingSession)
|
|
127
124
|
&& existingSession.status !== "stopped"
|
|
128
125
|
&& existingSession.status !== "broken";
|
|
129
|
-
//
|
|
130
|
-
//
|
|
131
|
-
//
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
.at(-1);
|
|
139
|
-
const failedResumeWithoutAcceptance = latestRoleRun !== undefined
|
|
140
|
-
&& latestRoleRun.mode === "resume"
|
|
141
|
-
&& latestRoleRun.status === "failed"
|
|
142
|
-
&& latestRoleRun.deliveredAt === undefined;
|
|
143
|
-
const mode = resumableSession && compatibleSession && !failedResumeWithoutAcceptance
|
|
144
|
-
? "resume"
|
|
145
|
-
: "new";
|
|
126
|
+
// A failed or quiet Run is not proof that its Provider Conversation is
|
|
127
|
+
// unusable. Keep resuming the same identity unless an explicit switch
|
|
128
|
+
// request (or exact terminal/missing evidence) authorizes a fresh one.
|
|
129
|
+
const sessionSet = store.getTaskRoleSessionSet?.(task.id, role.name) ?? null;
|
|
130
|
+
const mode = reopenIdentityDrift
|
|
131
|
+
? "new"
|
|
132
|
+
: sessionSet === null
|
|
133
|
+
? resumableSession && compatibleSession ? "resume" : "new"
|
|
134
|
+
: roleSessionDispatchModeWithConversationSwitch(sessionSet, store.listEvents?.(task.id) ?? [], store.getWorkMailbox({ kind: "role", taskId: task.id, roleName: role.name }), role.name, role.effective.agentId, role.effective);
|
|
146
135
|
const runId = store.peekNextAgentRunId(task.id);
|
|
147
136
|
const wakeEnvelope = resolveLeaderWakeEnvelope(store, task.id);
|
|
148
137
|
const contextSnapshot = store.freezeLeaderContextSnapshot?.(task.id, role.name, now);
|
|
@@ -261,19 +250,42 @@ export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
|
261
250
|
now
|
|
262
251
|
});
|
|
263
252
|
if (ready.prepared.turnRejectedDuringLaunch === true) {
|
|
264
|
-
|
|
253
|
+
const persisted = store.saveRoleRunDeliveryFailure({
|
|
265
254
|
taskId: task.id,
|
|
266
255
|
roleName: role.name,
|
|
256
|
+
agentId: run.effective.agentId,
|
|
257
|
+
adapterId: run.effective.adapterId,
|
|
267
258
|
runId: run.id,
|
|
259
|
+
mailboxBatchId: formatAgentRunReceiptId(task.id, run.id),
|
|
260
|
+
...(effectiveSession?.nativeSessionId === undefined
|
|
261
|
+
? {}
|
|
262
|
+
: { nativeSessionId: effectiveSession.nativeSessionId }),
|
|
268
263
|
...(ready.prepared.launchId === undefined
|
|
269
264
|
? {}
|
|
270
|
-
: { launchId: ready.prepared.launchId })
|
|
265
|
+
: { launchId: ready.prepared.launchId }),
|
|
266
|
+
summary: `Provider conclusively rejected the exact initial Leader Run input: ${run.id}.`,
|
|
267
|
+
cleanupRequired: false,
|
|
268
|
+
now
|
|
271
269
|
});
|
|
270
|
+
if (persisted === "failed") {
|
|
271
|
+
delivery.forgetPrepared?.({
|
|
272
|
+
taskId: task.id,
|
|
273
|
+
roleName: role.name,
|
|
274
|
+
runId: run.id,
|
|
275
|
+
...(ready.prepared.launchId === undefined
|
|
276
|
+
? {}
|
|
277
|
+
: { launchId: ready.prepared.launchId })
|
|
278
|
+
});
|
|
279
|
+
}
|
|
272
280
|
results.push({
|
|
273
281
|
taskId: task.id,
|
|
274
282
|
runId: run.id,
|
|
275
|
-
status: "skipped",
|
|
276
|
-
reason: "
|
|
283
|
+
status: persisted === "failed" ? "failed" : "skipped",
|
|
284
|
+
reason: "provider-rejected",
|
|
285
|
+
terminalized: persisted === "failed",
|
|
286
|
+
...(persisted === "failed"
|
|
287
|
+
? {}
|
|
288
|
+
: { error: "Run state changed while recording exact Provider rejection." })
|
|
277
289
|
});
|
|
278
290
|
continue;
|
|
279
291
|
}
|
|
@@ -329,118 +341,11 @@ export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
|
329
341
|
results.push({ taskId: task.id, runId: run.id, status: "dispatched" });
|
|
330
342
|
continue;
|
|
331
343
|
}
|
|
332
|
-
|
|
333
|
-
if (effectiveSession === null || effectiveSession.launchId === undefined
|
|
334
|
-
|| !hasNativeSession(effectiveSession)
|
|
335
|
-
|| store.beginRoleRunProviderTurn?.({
|
|
336
|
-
taskId: task.id,
|
|
337
|
-
roleName: role.name,
|
|
338
|
-
runId: run.id,
|
|
339
|
-
agentId: run.effective.agentId,
|
|
340
|
-
launchId: effectiveSession.launchId,
|
|
341
|
-
nativeSessionId: effectiveSession.nativeSessionId,
|
|
342
|
-
attemptId: receiptId,
|
|
343
|
-
now
|
|
344
|
-
}) !== true) {
|
|
345
|
-
throw new Error("Provider Turn intent could not be durably fenced before delivery.");
|
|
346
|
-
}
|
|
347
|
-
providerSubmissionBegun = true;
|
|
348
|
-
deliveryAttempted = true;
|
|
349
|
-
const outcome = await delivery.sendOnce({
|
|
350
|
-
delivery: ready,
|
|
351
|
-
receiptId,
|
|
352
|
-
text: prefixYuiTitleInput(serializeRunBootstrapEnvelope(run.bootstrapEnvelope), resolveTaskRoleSessionTitle(effectiveSession.title, task, role.name))
|
|
353
|
-
});
|
|
354
|
-
if (outcome === "busy" || outcome === "unavailable") {
|
|
355
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
356
|
-
taskId: task.id,
|
|
357
|
-
roleName: role.name,
|
|
358
|
-
runId: run.id,
|
|
359
|
-
attemptId: receiptId,
|
|
360
|
-
status: "rejected",
|
|
361
|
-
reason: outcome === "busy"
|
|
362
|
-
? "Agent Host was busy before Provider mutation."
|
|
363
|
-
: "Agent Host was unavailable before Provider mutation.",
|
|
364
|
-
now
|
|
365
|
-
});
|
|
366
|
-
providerSubmissionBegun = false;
|
|
367
|
-
results.push({
|
|
368
|
-
taskId: task.id,
|
|
369
|
-
runId: run.id,
|
|
370
|
-
status: "skipped",
|
|
371
|
-
reason: "not-ready"
|
|
372
|
-
});
|
|
373
|
-
continue;
|
|
374
|
-
}
|
|
375
|
-
if (outcome === "rejected") {
|
|
376
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
377
|
-
taskId: task.id,
|
|
378
|
-
roleName: role.name,
|
|
379
|
-
runId: run.id,
|
|
380
|
-
attemptId: receiptId,
|
|
381
|
-
status: "rejected",
|
|
382
|
-
reason: "Provider returned an exact negative acknowledgement.",
|
|
383
|
-
now
|
|
384
|
-
});
|
|
385
|
-
providerSubmissionBegun = false;
|
|
386
|
-
results.push({ taskId: task.id, runId: run.id, status: "skipped", reason: "not-ready" });
|
|
387
|
-
continue;
|
|
388
|
-
}
|
|
389
|
-
if (outcome === "delivery-unknown") {
|
|
390
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
391
|
-
taskId: task.id,
|
|
392
|
-
roleName: role.name,
|
|
393
|
-
runId: run.id,
|
|
394
|
-
attemptId: receiptId,
|
|
395
|
-
status: "delivery-unknown",
|
|
396
|
-
reason: "Provider Turn delivery is ambiguous; automatic retry is fenced.",
|
|
397
|
-
now
|
|
398
|
-
});
|
|
399
|
-
providerSubmissionBegun = false;
|
|
400
|
-
store.saveRoleRunDelivery({
|
|
401
|
-
task,
|
|
402
|
-
role,
|
|
403
|
-
run,
|
|
404
|
-
session: effectiveSession,
|
|
405
|
-
...(ready.prepared.launchId === undefined
|
|
406
|
-
? {}
|
|
407
|
-
: { launchId: ready.prepared.launchId }),
|
|
408
|
-
now
|
|
409
|
-
});
|
|
410
|
-
results.push({
|
|
411
|
-
taskId: task.id,
|
|
412
|
-
runId: run.id,
|
|
413
|
-
status: "skipped",
|
|
414
|
-
reason: "delivery-uncertain"
|
|
415
|
-
});
|
|
416
|
-
continue;
|
|
417
|
-
}
|
|
418
|
-
store.saveRoleRunDelivery({
|
|
419
|
-
task,
|
|
420
|
-
role,
|
|
421
|
-
run,
|
|
422
|
-
session: effectiveSession,
|
|
423
|
-
...(ready.prepared.launchId === undefined
|
|
424
|
-
? {}
|
|
425
|
-
: { launchId: ready.prepared.launchId }),
|
|
426
|
-
now
|
|
427
|
-
});
|
|
428
|
-
results.push({ taskId: task.id, runId: run.id, status: "dispatched" });
|
|
344
|
+
throw new Error("Managed Provider launch completed without an initial Turn outcome.");
|
|
429
345
|
}
|
|
430
346
|
catch (error) {
|
|
431
347
|
const detail = error instanceof Error ? error.message : String(error);
|
|
432
348
|
const message = `Leader dispatch failed: ${detail}`;
|
|
433
|
-
if (providerSubmissionBegun && run !== null) {
|
|
434
|
-
store.resolveRoleRunProviderSubmission?.({
|
|
435
|
-
taskId: task.id,
|
|
436
|
-
roleName: role.name,
|
|
437
|
-
runId: run.id,
|
|
438
|
-
attemptId: formatAgentRunReceiptId(task.id, run.id),
|
|
439
|
-
status: "delivery-unknown",
|
|
440
|
-
reason: detail,
|
|
441
|
-
now
|
|
442
|
-
});
|
|
443
|
-
}
|
|
444
349
|
if (claimed && run !== null) {
|
|
445
350
|
// Scheduler single-flight backpressure: the Role runtime lifecycle
|
|
446
351
|
// lane was busy when the launch was reserved. The claimed Run stays
|
|
@@ -504,7 +409,6 @@ export async function processLeaderWakeups(store, delivery, now, selection) {
|
|
|
504
409
|
session: effectiveSession,
|
|
505
410
|
claimed: { run, wakeup },
|
|
506
411
|
failure: recordLeaderFailure(task.id, effectiveSession?.nativeSessionId ?? "(unregistered)", message, now, store.getLeaderFailure(task.id)),
|
|
507
|
-
notification: createLeaderRecoveryNotification(task.id, message, now, store.getOperatorNotification(task.id)),
|
|
508
412
|
now
|
|
509
413
|
});
|
|
510
414
|
if (failureResult === "state-changed") {
|
|
@@ -535,7 +439,7 @@ function validateReadySession(role, effective, existing, mode, session) {
|
|
|
535
439
|
throw new Error(`Ready session belongs to another Agent: ${session.agentId}.`);
|
|
536
440
|
}
|
|
537
441
|
const compatible = mode === "resume"
|
|
538
|
-
?
|
|
442
|
+
? effectiveLaunchSnapshotsCompatibleForTaskSession(session.effective, effective)
|
|
539
443
|
: effectiveLaunchSnapshotsCompatible(session.effective, effective);
|
|
540
444
|
if (!compatible) {
|
|
541
445
|
throw new Error("Ready Leader session effective snapshot changed.");
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { enqueueWork } from "../coordination/workMailboxQueue.js";
|
|
2
|
+
import { createTaskEvent } from "../event/taskEvent.js";
|
|
3
|
+
export const LEADER_ATTENTION_REQUIRED_EVENT = "leader.attention-required";
|
|
4
|
+
/** The sole Task-event boundary into the global Operator mailbox. */
|
|
5
|
+
export function enqueueOperatorEvent(store, event, reason, now) {
|
|
6
|
+
enqueueWork(store, { kind: "operator" }, reason, now, [
|
|
7
|
+
{ type: "event", taskId: event.taskId, id: event.id }
|
|
8
|
+
], {
|
|
9
|
+
source: "task-event",
|
|
10
|
+
dedupeKey: `operator-event:${event.taskId}:${event.id}`,
|
|
11
|
+
deliveryMode: "followup",
|
|
12
|
+
lane: "normal"
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
/** Routes one Role fact to its supervisor without exposing Provider details to Operator code. */
|
|
16
|
+
export function routeRoleEvent(store, event, roleName, reason, now) {
|
|
17
|
+
if (roleName === "leader") {
|
|
18
|
+
enqueueOperatorEvent(store, event, reason, now);
|
|
19
|
+
return;
|
|
20
|
+
}
|
|
21
|
+
enqueueWork(store, { kind: "role", taskId: event.taskId, roleName: "leader" }, reason, now, [{ type: "event", taskId: event.taskId, id: event.id }], {
|
|
22
|
+
source: "task-event",
|
|
23
|
+
dedupeKey: `leader-event:${event.taskId}:${event.id}`,
|
|
24
|
+
deliveryMode: "followup",
|
|
25
|
+
lane: "normal"
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
/** Records the semantic boundary used when a Leader can no longer continue. */
|
|
29
|
+
export function recordLeaderAttentionRequired(store, input) {
|
|
30
|
+
const event = createTaskEvent(store.nextEventId(input.taskId), input.taskId, LEADER_ATTENTION_REQUIRED_EVENT, { ...(input.payload ?? {}), reason: input.reason }, input.now);
|
|
31
|
+
store.saveEvent(input.taskId, event);
|
|
32
|
+
enqueueOperatorEvent(store, event, "leader-attention-required", input.now);
|
|
33
|
+
return event;
|
|
34
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createOperatorBatchPresentation } from "../interaction/operatorPresentation.js";
|
|
2
2
|
import { mailboxHasWork, nextPendingBatch } from "../coordination/workMailbox.js";
|
|
3
3
|
export async function processOperatorInputNotifications(store, delivery, selection, now = new Date()) {
|
|
4
4
|
if (selection !== undefined && !selection.full && !selection.operator)
|
|
@@ -19,18 +19,18 @@ export async function processOperatorInputNotifications(store, delivery, selecti
|
|
|
19
19
|
if (claim.status === "empty")
|
|
20
20
|
return [];
|
|
21
21
|
const processing = claim.processing;
|
|
22
|
-
const
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
.
|
|
32
|
-
|
|
33
|
-
|
|
22
|
+
const attentions = deduplicateAttention(processing.batch.refs.flatMap((ref) => {
|
|
23
|
+
if (ref.type === "input") {
|
|
24
|
+
const request = store.getInputRequest(ref.taskId, ref.id);
|
|
25
|
+
return request !== null && request.status === "open"
|
|
26
|
+
? [{ kind: "input", request }]
|
|
27
|
+
: [];
|
|
28
|
+
}
|
|
29
|
+
if (ref.type !== "event" || store.listEvents === undefined)
|
|
30
|
+
return [];
|
|
31
|
+
const event = store.listEvents(ref.taskId).find(({ id }) => id === ref.id);
|
|
32
|
+
return event === undefined ? [] : [{ kind: "event", event }];
|
|
33
|
+
}));
|
|
34
34
|
if (attentions.length === 0) {
|
|
35
35
|
store.completeWorkMailbox(targetMailbox, processing.batchId);
|
|
36
36
|
return [];
|
|
@@ -41,58 +41,35 @@ export async function processOperatorInputNotifications(store, delivery, selecti
|
|
|
41
41
|
store.releaseWorkMailbox(targetMailbox, processing.batchId);
|
|
42
42
|
return attentions.map((attention) => skipped(attention, reason));
|
|
43
43
|
}
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
});
|
|
53
|
-
if (outcome === "unavailable") {
|
|
54
|
-
store.releaseWorkMailbox(targetMailbox, processing.batchId);
|
|
55
|
-
results.push(skipped(attention, "operator-unavailable"));
|
|
56
|
-
results.push(...attentions.slice(index + 1).map((pending) => (skipped(pending, "operator-unavailable"))));
|
|
57
|
-
break;
|
|
58
|
-
}
|
|
59
|
-
else if (outcome === "not-ready") {
|
|
60
|
-
store.releaseWorkMailbox(targetMailbox, processing.batchId);
|
|
61
|
-
results.push(skipped(attention, "operator-not-ready"));
|
|
62
|
-
results.push(...attentions.slice(index + 1).map((pending) => (skipped(pending, "operator-not-ready"))));
|
|
63
|
-
break;
|
|
64
|
-
}
|
|
65
|
-
else {
|
|
66
|
-
results.push({
|
|
67
|
-
...attentionIdentity(attention),
|
|
68
|
-
status: outcome
|
|
69
|
-
});
|
|
70
|
-
if (outcome === "sent") {
|
|
71
|
-
if (index === attentions.length - 1) {
|
|
72
|
-
store.completeWorkMailbox(targetMailbox, processing.batchId);
|
|
73
|
-
}
|
|
74
|
-
else {
|
|
75
|
-
store.releaseWorkMailbox(targetMailbox, processing.batchId);
|
|
76
|
-
}
|
|
77
|
-
results.push(...attentions.slice(index + 1).map((pending) => (skipped(pending, "operator-not-ready"))));
|
|
78
|
-
break;
|
|
79
|
-
}
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
catch (error) {
|
|
44
|
+
try {
|
|
45
|
+
const presentation = createOperatorBatchPresentation(processing.batchId, attentions.map(toPresentationItem));
|
|
46
|
+
const outcome = await delivery.notifyOperatorInputOnce({
|
|
47
|
+
...target,
|
|
48
|
+
receiptId: presentation.receiptId,
|
|
49
|
+
text: presentation.text
|
|
50
|
+
});
|
|
51
|
+
if (outcome === "unavailable" || outcome === "not-ready") {
|
|
83
52
|
store.releaseWorkMailbox(targetMailbox, processing.batchId);
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
status: "failed",
|
|
87
|
-
error: error instanceof Error ? error.message : String(error)
|
|
88
|
-
});
|
|
89
|
-
break;
|
|
53
|
+
const reason = outcome === "unavailable" ? "operator-unavailable" : "operator-not-ready";
|
|
54
|
+
return attentions.map((attention) => skipped(attention, reason));
|
|
90
55
|
}
|
|
91
|
-
|
|
92
|
-
|
|
56
|
+
if (outcome === "sent")
|
|
57
|
+
store.markOperatorTurnStarted(now);
|
|
93
58
|
store.completeWorkMailbox(targetMailbox, processing.batchId);
|
|
59
|
+
return attentions.map((attention) => ({
|
|
60
|
+
...attentionIdentity(attention),
|
|
61
|
+
status: outcome
|
|
62
|
+
}));
|
|
63
|
+
}
|
|
64
|
+
catch (error) {
|
|
65
|
+
store.releaseWorkMailbox(targetMailbox, processing.batchId);
|
|
66
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
67
|
+
return attentions.map((attention) => ({
|
|
68
|
+
...attentionIdentity(attention),
|
|
69
|
+
status: "failed",
|
|
70
|
+
error: detail
|
|
71
|
+
}));
|
|
94
72
|
}
|
|
95
|
-
return results;
|
|
96
73
|
}
|
|
97
74
|
function skipped(attention, reason) {
|
|
98
75
|
return {
|
|
@@ -102,45 +79,28 @@ function skipped(attention, reason) {
|
|
|
102
79
|
};
|
|
103
80
|
}
|
|
104
81
|
function attentionIdentity(attention) {
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
return {
|
|
116
|
-
stallTaskId: attention.notification.taskId,
|
|
117
|
-
taskId: attention.notification.taskId
|
|
82
|
+
return attention.kind === "input"
|
|
83
|
+
? {
|
|
84
|
+
sourceKind: "input",
|
|
85
|
+
sourceId: attention.request.id,
|
|
86
|
+
taskId: attention.request.taskId
|
|
87
|
+
}
|
|
88
|
+
: {
|
|
89
|
+
sourceKind: "event",
|
|
90
|
+
sourceId: attention.event.id,
|
|
91
|
+
taskId: attention.event.taskId
|
|
118
92
|
};
|
|
119
|
-
}
|
|
120
|
-
return {
|
|
121
|
-
terminalTaskId: attention.notification.taskId,
|
|
122
|
-
taskId: attention.notification.taskId
|
|
123
|
-
};
|
|
124
93
|
}
|
|
125
|
-
function
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
if (attention.notification.type === "leader-recovery-failed") {
|
|
130
|
-
return createLeaderRecoveryOperatorPresentation(attention.notification);
|
|
131
|
-
}
|
|
132
|
-
return attention.notification.type === "leader-stalled"
|
|
133
|
-
? createLeaderStallOperatorPresentation(attention.notification)
|
|
134
|
-
: createTaskTerminalOperatorPresentation(attention.notification);
|
|
94
|
+
function toPresentationItem(attention) {
|
|
95
|
+
return attention.kind === "input"
|
|
96
|
+
? { kind: "input-request", request: attention.request }
|
|
97
|
+
: { kind: "task-event", event: attention.event };
|
|
135
98
|
}
|
|
136
99
|
function deduplicateAttention(attentions) {
|
|
137
100
|
const seen = new Set();
|
|
138
101
|
return attentions.filter((attention) => {
|
|
139
|
-
const
|
|
140
|
-
|
|
141
|
-
: attention.notification.type === "leader-stalled"
|
|
142
|
-
? `stall:${attention.notification.taskId}:${attention.notification.runId}:${attention.notification.progressAt}`
|
|
143
|
-
: `recovery:${attention.notification.taskId}:${attention.notification.createdAt}`;
|
|
102
|
+
const identity = attentionIdentity(attention);
|
|
103
|
+
const key = `${identity.sourceKind}:${identity.taskId}:${identity.sourceId}`;
|
|
144
104
|
if (seen.has(key))
|
|
145
105
|
return false;
|
|
146
106
|
seen.add(key);
|
|
@@ -12,6 +12,7 @@ export const DEFAULT_WORKFLOW_STALL_CANDIDATE_AGE_MS = RUNTIME_DIAGNOSTIC_AFTER_
|
|
|
12
12
|
export const RUN_PROGRESS_EVENT = "run.progress";
|
|
13
13
|
export const RUN_STALLED_EVENT = "run.stalled";
|
|
14
14
|
export const RUN_RECOVERED_EVENT = "run.recovered";
|
|
15
|
+
export const RUN_DIAGNOSTIC_FINISHED_EVENT = "runtime.diagnostic-finished";
|
|
15
16
|
/** Structured, non-Message recovery evidence written by an explicit Leader. */
|
|
16
17
|
export const RUN_RECOVERY_REQUESTED_EVENT = "run.recovery-requested";
|
|
17
18
|
export const RUN_RECOVERY_APPLIED_EVENT = "run.recovery-applied";
|
|
@@ -30,20 +31,6 @@ const ACTIVITY_EVENT_TYPES = new Set([
|
|
|
30
31
|
"integration.completed",
|
|
31
32
|
"integration.failed"
|
|
32
33
|
]);
|
|
33
|
-
/**
|
|
34
|
-
* Clear only the Operator projection raised for this exact Leader Run. A
|
|
35
|
-
* later Run (or a recovery notification of another kind) must remain intact;
|
|
36
|
-
* callers use this helper from every supported recovery/terminalization path
|
|
37
|
-
* so attention cleanup cannot drift between CLI and runtime code.
|
|
38
|
-
*/
|
|
39
|
-
export function clearMatchingLeaderStallAttention(store, taskId, runId) {
|
|
40
|
-
const notification = store.getOperatorNotification(taskId);
|
|
41
|
-
if (notification?.type !== "leader-stalled" || notification.runId !== runId) {
|
|
42
|
-
return false;
|
|
43
|
-
}
|
|
44
|
-
store.clearOperatorNotification(taskId);
|
|
45
|
-
return true;
|
|
46
|
-
}
|
|
47
34
|
/**
|
|
48
35
|
* One pure projection used by every Role. Resource activity is retained only
|
|
49
36
|
* as exact-generation diagnostic evidence; it never changes the durable
|
|
@@ -64,10 +51,7 @@ export function projectRoleRunHealth(input) {
|
|
|
64
51
|
const idleMs = evaluation.idleMs;
|
|
65
52
|
const candidateStart = input.deliveredAt ?? input.createdAt;
|
|
66
53
|
const candidateAge = input.now.getTime() - Date.parse(candidateStart);
|
|
67
|
-
const candidate = Number.isFinite(candidateAge)
|
|
68
|
-
&& (input.deliveredAt === undefined
|
|
69
|
-
? candidateAge >= windowMs
|
|
70
|
-
: candidateAge >= diagnosticAfterMs);
|
|
54
|
+
const candidate = Number.isFinite(candidateAge) && candidateAge >= windowMs;
|
|
71
55
|
const providerAcceptance = input.providerAcceptance
|
|
72
56
|
?? (input.deliveredAt === undefined ? "ambiguous" : "accepted");
|
|
73
57
|
const hostLiveness = input.hostLiveness;
|
|
@@ -451,6 +435,8 @@ export function isRoleRunStalled(events, runId) {
|
|
|
451
435
|
if (event.payload.runId !== runId)
|
|
452
436
|
continue;
|
|
453
437
|
if (event.type === RUN_STALLED_EVENT) {
|
|
438
|
+
if (event.payload.status === "diagnostic-only")
|
|
439
|
+
continue;
|
|
454
440
|
if (stalled === undefined
|
|
455
441
|
|| Date.parse(event.createdAt) > Date.parse(stalled.createdAt)) {
|
|
456
442
|
stalled = event;
|
|
@@ -489,8 +475,9 @@ export function isRoleRunStalled(events, runId) {
|
|
|
489
475
|
/**
|
|
490
476
|
* Low-frequency health pass for active Task Role Runs. An unaccepted Run is
|
|
491
477
|
* watched as delivery-stalled after the reasonable delivery window; an
|
|
492
|
-
* accepted Run
|
|
493
|
-
*
|
|
478
|
+
* accepted Run is displayed as checkpoint-overdue after fifteen minutes, but
|
|
479
|
+
* this scheduler performs no runtime inspection until the thirty-minute
|
|
480
|
+
* diagnostic window is due.
|
|
494
481
|
* Leader Runs are only persisted when classification reaches truly-stalled —
|
|
495
482
|
* healthy downstream work, open user input, and recent own progress remain
|
|
496
483
|
* structured waiting/working facts. No branch sends terminal bytes, retries,
|
|
@@ -521,14 +508,43 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
521
508
|
session: store.getRoleSession(task.id, role.name, run.effective.agentId)
|
|
522
509
|
}];
|
|
523
510
|
})));
|
|
524
|
-
const
|
|
511
|
+
const eventsByTask = new Map();
|
|
512
|
+
const diagnosticEvents = (taskId) => {
|
|
513
|
+
const existing = eventsByTask.get(taskId);
|
|
514
|
+
if (existing !== undefined)
|
|
515
|
+
return existing;
|
|
516
|
+
const events = store.listEvents?.(taskId) ?? [];
|
|
517
|
+
eventsByTask.set(taskId, events);
|
|
518
|
+
return events;
|
|
519
|
+
};
|
|
520
|
+
const cadenceCandidates = candidates.filter(({ task, run }) => isStallCandidate(run, now, windowMs, latestRunEventTime(diagnosticEvents(task.id), RUN_DIAGNOSTIC_FINISHED_EVENT, run.id)));
|
|
521
|
+
const stallCandidates = cadenceCandidates.filter(({ task, role, run }) => {
|
|
522
|
+
const progressFacts = foldPortPresent
|
|
523
|
+
? store.getRunProgressFacts?.(task.id, run.id)
|
|
524
|
+
: undefined;
|
|
525
|
+
const progressAt = currentRoleRunProgressAt(store, task.id, role.name, run, foldPortPresent ? undefined : diagnosticEvents(task.id), progressFacts).progressAt;
|
|
526
|
+
return isStallCandidate(run, now, windowMs, latestRunEventTime(diagnosticEvents(task.id), RUN_DIAGNOSTIC_FINISHED_EVENT, run.id), progressAt);
|
|
527
|
+
});
|
|
525
528
|
if (stallCandidates.length === 0)
|
|
526
529
|
return [];
|
|
530
|
+
const diagnosticStartedAt = now.toISOString();
|
|
531
|
+
const finishDiagnostics = (outcome) => {
|
|
532
|
+
for (const { task, role, run } of stallCandidates) {
|
|
533
|
+
store.recordRoleRunDiagnostic?.({
|
|
534
|
+
taskId: task.id,
|
|
535
|
+
roleName: role.name,
|
|
536
|
+
runId: run.id,
|
|
537
|
+
startedAt: diagnosticStartedAt,
|
|
538
|
+
outcome,
|
|
539
|
+
now
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
};
|
|
527
543
|
const stallCandidateKeys = new Set(stallCandidates.map(({ task, role }) => (`${task.id}\0${role.name}`)));
|
|
528
544
|
const observed = new Map();
|
|
529
545
|
// Recent active Runs are known healthy enough for Leader classification from
|
|
530
546
|
// their exact acceptance/creation boundary, but are deliberately not read
|
|
531
|
-
// from tmux or Event history until the
|
|
547
|
+
// from tmux or Event history until the 30-minute diagnostic window.
|
|
532
548
|
for (const candidate of candidates) {
|
|
533
549
|
const key = `${candidate.task.id}\0${candidate.role.name}`;
|
|
534
550
|
if (stallCandidateKeys.has(key))
|
|
@@ -577,7 +593,9 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
577
593
|
}
|
|
578
594
|
catch {
|
|
579
595
|
// Health inspection is advisory. Unknown host state must leave the exact
|
|
580
|
-
// Run/Session fence untouched
|
|
596
|
+
// Run/Session fence untouched. Closing this window schedules the next
|
|
597
|
+
// bounded diagnostic thirty minutes later instead of hot-looping.
|
|
598
|
+
finishDiagnostics("observation-error");
|
|
581
599
|
return [];
|
|
582
600
|
}
|
|
583
601
|
}
|
|
@@ -589,6 +607,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
589
607
|
catch {
|
|
590
608
|
// A malformed/partial provider snapshot is not evidence of a stall.
|
|
591
609
|
// Leave all fences untouched and let the next full pass retry.
|
|
610
|
+
finishDiagnostics("observation-error");
|
|
592
611
|
return [];
|
|
593
612
|
}
|
|
594
613
|
}
|
|
@@ -602,8 +621,10 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
602
621
|
const key = `${candidate.task.id}\0${candidate.role.name}`;
|
|
603
622
|
let live;
|
|
604
623
|
try {
|
|
605
|
-
if (liveStatuses !== undefined && !liveStatuses.has(key))
|
|
624
|
+
if (liveStatuses !== undefined && !liveStatuses.has(key)) {
|
|
625
|
+
finishDiagnostics("observation-error");
|
|
606
626
|
return [];
|
|
627
|
+
}
|
|
607
628
|
live = liveStatuses !== undefined
|
|
608
629
|
? liveStatuses.get(key)
|
|
609
630
|
: byRole === null
|
|
@@ -622,6 +643,7 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
622
643
|
// Without a complete live snapshot, especially for downstream Runs,
|
|
623
644
|
// Leader classification would be unsafe. Treat the whole advisory pass
|
|
624
645
|
// as unknown instead of escalating a false stall.
|
|
646
|
+
finishDiagnostics("observation-error");
|
|
625
647
|
return [];
|
|
626
648
|
}
|
|
627
649
|
// A missing process is handled by the existing fenced liveness pass. This
|
|
@@ -826,18 +848,18 @@ export async function reconcileStalledRoleRuns(store, delivery, now, selection,
|
|
|
826
848
|
});
|
|
827
849
|
}
|
|
828
850
|
}
|
|
851
|
+
finishDiagnostics("observed");
|
|
829
852
|
return raised;
|
|
830
853
|
}
|
|
831
|
-
function isStallCandidate(run, now, windowMs,
|
|
854
|
+
function isStallCandidate(run, now, windowMs, lastDiagnosticFinishedAt, latestProgressAt) {
|
|
832
855
|
const startAt = run.deliveredAt ?? run.createdAt;
|
|
833
|
-
const
|
|
856
|
+
const baseline = [startAt, lastDiagnosticFinishedAt, latestProgressAt]
|
|
857
|
+
.filter((value) => (value !== undefined && Number.isFinite(Date.parse(value))))
|
|
858
|
+
.reduce((latest, value) => (Date.parse(value) > Date.parse(latest) ? value : latest));
|
|
859
|
+
const ageMs = now.getTime() - Date.parse(baseline);
|
|
834
860
|
if (!Number.isFinite(ageMs))
|
|
835
861
|
return false;
|
|
836
|
-
|
|
837
|
-
// window, but they never enter workflow-stall candidate filtering.
|
|
838
|
-
return run.deliveredAt === undefined
|
|
839
|
-
? ageMs >= windowMs
|
|
840
|
-
: ageMs >= diagnosticAfterMs;
|
|
862
|
+
return ageMs >= windowMs;
|
|
841
863
|
}
|
|
842
864
|
function classifyLeaderStall(store, taskId, observed, now, windowMs) {
|
|
843
865
|
if (store.hasOpenInputRequest(taskId))
|