@wjarka/cezarion 0.14.8 → 0.14.9-dev.829
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +76 -2
- package/dist/application-update/bin-recovery.d.ts +5 -0
- package/dist/application-update/bin-recovery.js +96 -0
- package/dist/application-update/bin-recovery.js.map +1 -0
- package/dist/application-update/discovery.d.ts +22 -0
- package/dist/application-update/discovery.js +91 -0
- package/dist/application-update/discovery.js.map +1 -0
- package/dist/application-update/helper.d.ts +57 -0
- package/dist/application-update/helper.js +374 -0
- package/dist/application-update/helper.js.map +1 -0
- package/dist/application-update/launcher.d.ts +8 -0
- package/dist/application-update/launcher.js +53 -0
- package/dist/application-update/launcher.js.map +1 -0
- package/dist/application-update/lock.d.ts +4 -0
- package/dist/application-update/lock.js +70 -0
- package/dist/application-update/lock.js.map +1 -0
- package/dist/application-update/npm-process.d.ts +17 -0
- package/dist/application-update/npm-process.js +133 -0
- package/dist/application-update/npm-process.js.map +1 -0
- package/dist/application-update/service.d.ts +69 -0
- package/dist/application-update/service.js +456 -0
- package/dist/application-update/service.js.map +1 -0
- package/dist/artifacts/cli.d.ts +1 -0
- package/dist/artifacts/cli.js +32 -0
- package/dist/artifacts/cli.js.map +1 -0
- package/dist/artifacts/lifecycle.d.ts +4 -0
- package/dist/artifacts/lifecycle.js +42 -0
- package/dist/artifacts/lifecycle.js.map +1 -0
- package/dist/artifacts/resolve.d.ts +18 -0
- package/dist/artifacts/resolve.js +102 -0
- package/dist/artifacts/resolve.js.map +1 -0
- package/dist/artifacts/store.d.ts +8 -0
- package/dist/artifacts/store.js +323 -0
- package/dist/artifacts/store.js.map +1 -0
- package/dist/ci-wait/mcp.d.ts +3 -0
- package/dist/ci-wait/mcp.js +14 -3
- package/dist/ci-wait/mcp.js.map +1 -1
- package/dist/contract/application-update.d.ts +32 -0
- package/dist/contract/artifacts.d.ts +83 -0
- package/dist/contract/ask-schema.d.ts +32 -0
- package/dist/contract/ask.d.ts +2 -32
- package/dist/contract/conversations.d.ts +165 -0
- package/dist/contract/delegation.d.ts +39 -0
- package/dist/contract/events.d.ts +49 -0
- package/dist/contract/health.d.ts +12 -0
- package/dist/contract/index.d.ts +2 -0
- package/dist/contract/index.js +1374 -1247
- package/dist/contract/runs.d.ts +143 -0
- package/dist/contract/skills.d.ts +28 -0
- package/dist/core/agent-runner.d.ts +33 -4
- package/dist/core/agent-runner.js +5 -0
- package/dist/core/agent-runner.js.map +1 -1
- package/dist/core/claude-cli-runner.d.ts +6 -12
- package/dist/core/claude-cli-runner.js +89 -19
- package/dist/core/claude-cli-runner.js.map +1 -1
- package/dist/core/codex-app-server-runner.d.ts +2 -1
- package/dist/core/codex-app-server-runner.js +327 -92
- package/dist/core/codex-app-server-runner.js.map +1 -1
- package/dist/core/codex-app-server-transport.d.ts +13 -1
- package/dist/core/codex-app-server-transport.js +58 -17
- package/dist/core/codex-app-server-transport.js.map +1 -1
- package/dist/core/cursor-acp-runner.d.ts +4 -1
- package/dist/core/cursor-acp-runner.js +3 -0
- package/dist/core/cursor-acp-runner.js.map +1 -1
- package/dist/core/cursor-provider-error.js +3 -1
- package/dist/core/cursor-provider-error.js.map +1 -1
- package/dist/core/input-submissions.d.ts +14 -0
- package/dist/core/input-submissions.js +26 -0
- package/dist/core/input-submissions.js.map +1 -0
- package/dist/core/opencode-server-runner.d.ts +4 -1
- package/dist/core/opencode-server-runner.js +144 -21
- package/dist/core/opencode-server-runner.js.map +1 -1
- package/dist/core/pi-runner.d.ts +2 -1
- package/dist/core/pi-runner.js +72 -7
- package/dist/core/pi-runner.js.map +1 -1
- package/dist/delegation/cli.js +102 -6
- package/dist/delegation/cli.js.map +1 -1
- package/dist/delegation/conversations.js +13 -5
- package/dist/delegation/conversations.js.map +1 -1
- package/dist/delegation/input.d.ts +2 -1
- package/dist/delegation/input.js +6 -1
- package/dist/delegation/input.js.map +1 -1
- package/dist/delegation/provision.js +5 -3
- package/dist/delegation/provision.js.map +1 -1
- package/dist/delegation/questions.d.ts +19 -0
- package/dist/delegation/questions.js +45 -0
- package/dist/delegation/questions.js.map +1 -0
- package/dist/delegation/results.d.ts +1 -1
- package/dist/delegation/results.js +23 -13
- package/dist/delegation/results.js.map +1 -1
- package/dist/delegation/routes.d.ts +142 -15
- package/dist/delegation/routes.js +4 -1
- package/dist/delegation/routes.js.map +1 -1
- package/dist/delegation/service.d.ts +9 -5
- package/dist/delegation/service.js +106 -7
- package/dist/delegation/service.js.map +1 -1
- package/dist/delegation/transport.d.ts +138 -11
- package/dist/index.js +72 -26
- package/dist/index.js.map +1 -1
- package/dist/open-url.d.ts +2 -0
- package/dist/open-url.js +20 -0
- package/dist/open-url.js.map +1 -0
- package/dist/release/snapshot.d.ts +3 -3
- package/dist/release/snapshot.js +6 -6
- package/dist/release/snapshot.js.map +1 -1
- package/dist/runs/client-request.d.ts +11 -0
- package/dist/runs/client-request.js +26 -0
- package/dist/runs/client-request.js.map +1 -0
- package/dist/runs/delegation-state.d.ts +17 -0
- package/dist/runs/event-history.js +6 -0
- package/dist/runs/event-history.js.map +1 -1
- package/dist/runs/store.d.ts +78 -6
- package/dist/runs/store.js +163 -1
- package/dist/runs/store.js.map +1 -1
- package/dist/server/server.d.ts +3761 -1648
- package/dist/server/server.js +170 -4
- package/dist/server/server.js.map +1 -1
- package/dist/task-cli/cli.d.ts +38 -0
- package/dist/task-cli/cli.js +463 -0
- package/dist/task-cli/cli.js.map +1 -0
- package/dist/task-cli/discovery.d.ts +22 -0
- package/dist/task-cli/discovery.js +95 -0
- package/dist/task-cli/discovery.js.map +1 -0
- package/dist/task-cli/http.d.ts +43 -0
- package/dist/task-cli/http.js +101 -0
- package/dist/task-cli/http.js.map +1 -0
- package/dist/task-cli/projections.d.ts +40 -0
- package/dist/task-cli/projections.js +45 -0
- package/dist/task-cli/projections.js.map +1 -0
- package/dist/task-cli/watch.d.ts +78 -0
- package/dist/task-cli/watch.js +336 -0
- package/dist/task-cli/watch.js.map +1 -0
- package/dist/workflows/run.d.ts +93 -0
- package/dist/workflows/run.js +816 -81
- package/dist/workflows/run.js.map +1 -1
- package/dist/workspace/projects-cli.js +1 -1
- package/package.json +3 -3
- package/scripts/mock-claude.mjs +38 -2
- package/scripts/mock-codex-app-server.mjs +98 -1
- package/scripts/mock-cursor-acp.mjs +3 -0
- package/scripts/mock-opencode-serve.mjs +69 -2
- package/scripts/mock-pi-rpc.mjs +63 -2
- package/web/dist/assets/agent-accounts-JPfZPgKn.js +1 -0
- package/web/dist/assets/centered-state-HwCqQkH3.js +43 -0
- package/web/dist/assets/collapsible-CTymFdL7.js +1 -0
- package/web/dist/assets/commit-list-BZpYGQl2.js +1 -0
- package/web/dist/assets/compare-variants-9VkybaQ7.js +1 -0
- package/web/dist/assets/{diff-tRE59UNb2.js → diff-CZcD4D1r2.js} +2 -2
- package/web/dist/assets/diff-controls-CpcHG3Mm.js +1 -0
- package/web/dist/assets/{diff-stat-Cgnl_hYF.js → diff-stat-Cn7rzy1H.js} +1 -1
- package/web/dist/assets/{diff-view-DN1ort7Q.js → diff-view-C46nHiP2.js} +2 -2
- package/web/dist/assets/dropdown-menu-_YAzCaoQ.js +1 -0
- package/web/dist/assets/git-toolbar-BvKDz6uS.js +1 -0
- package/web/dist/assets/github-CgjJCGrD.js +1 -0
- package/web/dist/assets/{image-preview-C2JGcBlC.js → image-preview-B618kDBH.js} +1 -1
- package/web/dist/assets/index-BJtve0Pm.js +7 -0
- package/web/dist/assets/index-D0WAgmTQ.css +1 -0
- package/web/dist/assets/markdown-CFWZnvMh.js +2 -0
- package/web/dist/assets/pill-QfwkuR50.js +1 -0
- package/web/dist/assets/project-router-CkAA39N5.js +1 -0
- package/web/dist/assets/prompt-templates-Cfna_S0T.js +15 -0
- package/web/dist/assets/repo-git-DgzgJ0cP.js +1 -0
- package/web/dist/assets/{run-diff-CyG0Nciv.js → run-diff-YbCEQn5I.js} +2 -2
- package/web/dist/assets/run-header-DMgIbr1d.js +1 -0
- package/web/dist/assets/skills-BW79kUTd.js +1 -0
- package/web/dist/assets/{tab-link-BWNp3mLY.js → tab-link-MIHvczEm.js} +1 -1
- package/web/dist/assets/task-changes-C96ekt94.js +1 -0
- package/web/dist/assets/task-commits-DBfcMZyG.js +1 -0
- package/web/dist/assets/task-files-BKpH8dKQ.js +2 -0
- package/web/dist/assets/task-thread-Bw_nQuTx.css +1 -0
- package/web/dist/assets/task-thread-Dnq7Ek6W.js +9 -0
- package/web/dist/assets/textarea-D2vAVgYW.js +1 -0
- package/web/dist/assets/thread-loading-NW8Mthco.js +1 -0
- package/web/dist/assets/token-metrics-D188e6_0.js +1 -0
- package/web/dist/assets/utils-DKheTxGD.js +64 -0
- package/web/dist/assets/workflows-BFY1hgp3.js +11 -0
- package/web/dist/assets/{zoomable-image-Z3JDnm0b.js → zoomable-image-YW3Qdrr_.js} +1 -1
- package/web/dist/index.html +14 -14
- package/web/dist/assets/agent-accounts-BZm_2WAv.js +0 -1
- package/web/dist/assets/alert-dialog-B422RI-k.js +0 -43
- package/web/dist/assets/collapsible-Cy_Fmkrd.js +0 -1
- package/web/dist/assets/commit-list-Dcnk7TnI.js +0 -1
- package/web/dist/assets/compare-variants-Ciqwvd_f.js +0 -1
- package/web/dist/assets/diff-controls-BGrXGi7O.js +0 -1
- package/web/dist/assets/dropdown-menu-dwDGT8MM.js +0 -1
- package/web/dist/assets/git-toolbar-UiHnGvE0.js +0 -1
- package/web/dist/assets/github-D-MrkbJ7.js +0 -1
- package/web/dist/assets/ibm-plex-mono-regular-BF3vfe7_.ttf +0 -0
- package/web/dist/assets/index-BszfYvBq.js +0 -7
- package/web/dist/assets/index-CWh4pKP3.css +0 -1
- package/web/dist/assets/markdown-C_IcKVQd.js +0 -2
- package/web/dist/assets/pill-BsbO6G7y.js +0 -1
- package/web/dist/assets/project-router-DbLkll25.js +0 -1
- package/web/dist/assets/prompt-templates-BlTGJB5W.js +0 -15
- package/web/dist/assets/repo-git-Bm3s6Why.js +0 -1
- package/web/dist/assets/run-header-IFyIGCey.js +0 -1
- package/web/dist/assets/skills-C-C4DVFY.js +0 -1
- package/web/dist/assets/task-changes-D2PHtWUW.js +0 -1
- package/web/dist/assets/task-commits-N5u9U0_R.js +0 -1
- package/web/dist/assets/task-files-BS7c7CFm.js +0 -2
- package/web/dist/assets/task-thread-BQFdz1dJ.css +0 -1
- package/web/dist/assets/task-thread-DGM2dZV-.js +0 -9
- package/web/dist/assets/textarea-FjaU370i.js +0 -1
- package/web/dist/assets/thread-loading-77sJ8Yu_.js +0 -1
- package/web/dist/assets/token-metrics-Clxh03bE.js +0 -1
- package/web/dist/assets/utils-PYYcpCFZ.js +0 -64
- package/web/dist/assets/workflows-Czl0ZfXH.js +0 -11
package/dist/workflows/run.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ciWaitRequestSchema, ciWaitResultSchema } from '../contract/index.js';
|
|
2
2
|
import { acquireCiResources } from '../ci-wait/resources.js';
|
|
3
|
+
import { artifactInstructions, provisionArtifactDirectory } from '../artifacts/lifecycle.js';
|
|
3
4
|
import { workerContextHash, workerWorkflowHash, workerStepIdentity, captureWorkerAccount, boundWorkerAccountEnv, WorkerIdentityError } from '../delegation/execution-identity.js';
|
|
4
5
|
import { buildChildEnv } from '../core/agent-env.js';
|
|
5
6
|
import { randomUUID } from 'node:crypto';
|
|
@@ -19,7 +20,8 @@ import { todosPath } from '../todos.js';
|
|
|
19
20
|
// Contract VALUES, like `workspaceUiStateSchema` in workspace/migrations.ts: the attachment
|
|
20
21
|
// vocabulary the routes validate with is the same one the engine stores and re-reads by, so the
|
|
21
22
|
// wire and the disk can never disagree about what counts as an image (#950).
|
|
22
|
-
import { requestWaitRequestSchema, workerWaitRequestSchema, attachmentExtension, delegationStateSchema, pendingHumanAsk, isImageAttachmentName, isImageMediaType, } from '../contract/index.js';
|
|
23
|
+
import { requestWaitRequestSchema, workerWaitRequestSchema, attachmentExtension, delegationStateSchema, pendingHumanAsk, askRequestSchema, isImageAttachmentName, isImageMediaType, } from '../contract/index.js';
|
|
24
|
+
import { inputDeliveryOf } from '../core/agent-runner.js';
|
|
23
25
|
import { discoverSkills } from '../skills.js';
|
|
24
26
|
import { materializeSkillDir } from '../skills-remote.js';
|
|
25
27
|
import { seedAgentConfigLocalLayer } from '../agent-config/seed.js';
|
|
@@ -29,12 +31,13 @@ import { autosaveCommit, createWorktree, resolveBaseRef, worktreeDiff, worktreeS
|
|
|
29
31
|
import { getHeadCommit, getRepoInfo } from '../server/git.js';
|
|
30
32
|
import { ensureOwnedWorkspace, verifyOwnedWorkspace } from '../delegation/workspace.js';
|
|
31
33
|
import { verifyWorkerContext } from '../delegation/context.js';
|
|
32
|
-
import { enqueueAgentInput, agentInputBatch } from '../delegation/input.js';
|
|
34
|
+
import { enqueueAgentInput, agentInputBatch, hasLiveInboxClaim } from '../delegation/input.js';
|
|
33
35
|
import { parseDelegationEffort } from '../delegation/effort.js';
|
|
34
36
|
import { DelegationPolicyError } from '../delegation/policy.js';
|
|
35
37
|
import { reconcileWorkerWait } from '../delegation/wait.js';
|
|
36
38
|
import { reconcileConversationState, projectConversationEvents } from '../delegation/conversations.js';
|
|
37
39
|
import { parentReadiness } from '../delegation/readiness.js';
|
|
40
|
+
import { answersQuestion, openQuestions, questionMessage } from '../delegation/questions.js';
|
|
38
41
|
import { workerOutcome } from '../runs/delegation-state.js';
|
|
39
42
|
import { loadWorkflows } from './load.js';
|
|
40
43
|
import { reclaimWorktrees, rematerializeReclaimedWorktree } from '../runs/retention.js';
|
|
@@ -122,6 +125,8 @@ function stripDoneMarker(text) {
|
|
|
122
125
|
function stripMonitoringMarker(text) {
|
|
123
126
|
return text.replace(/\s*CEZ:MONITORING\s*$/, '');
|
|
124
127
|
}
|
|
128
|
+
/** #505: a worker question routed to its parent is answered there until it falls back. */
|
|
129
|
+
const ROUTED_QUESTION_REFUSAL = 'This question was sent to the parent task; answer it there.';
|
|
125
130
|
/** Emit the v2 `ask.requested` event for a parsed marker (the cockpit renders
|
|
126
131
|
* it as an ask card, #473). Returns the minted request id. */
|
|
127
132
|
function emitAskRequested(sink, ask) {
|
|
@@ -213,6 +218,9 @@ export function repositoryRootLockDisabled(env = process.env) {
|
|
|
213
218
|
}
|
|
214
219
|
const REPOSITORY_ROOT_LOCK_DISABLED_NOTE = 'repository-root lock disabled by CEZ_DISABLE_REPO_LOCK=1 (shared checkout is unsafe)';
|
|
215
220
|
/** Safety cap on autonomous auto-continues per run — stops a stuck agent from nudging forever. */
|
|
221
|
+
/** #505: a real steer is read at the harness's next model step; a quiet boundary this long
|
|
222
|
+
* with input still unread means the harness will not read it without help. */
|
|
223
|
+
export const UNREAD_INPUT_GRACE_MS = 30_000;
|
|
216
224
|
const MAX_AUTO_CONTINUES = 40;
|
|
217
225
|
const AUTONOMOUS_NUDGE = 'Continue working autonomously until the task is fully complete. Do not ask me for confirmation or clarification — make reasonable assumptions and proceed. When everything is done, end the session with your done signal.';
|
|
218
226
|
const MONITORING_WAKE_NUDGE = 'Re-check the downstream work you were monitoring. Continue toward the task goal; emit CEZ:MONITORING again only if it is still pending.';
|
|
@@ -593,8 +601,11 @@ export class RunManager {
|
|
|
593
601
|
workerWakeAdmitted = new Set();
|
|
594
602
|
workerWakeQueuedAt = new Map();
|
|
595
603
|
workerWaitTimers = new Map();
|
|
604
|
+
inboxClaimTimers = new Map();
|
|
596
605
|
reconcilingWorkers = false;
|
|
597
606
|
recovering = false;
|
|
607
|
+
/** Quiet window before unread input is resubmitted or left unconfirmed (#505). */
|
|
608
|
+
unreadInputGraceMs;
|
|
598
609
|
disposed = false;
|
|
599
610
|
onDelegationRun = (run) => {
|
|
600
611
|
if (!this.disposed && !['queued', 'running', 'waiting'].includes(run.status))
|
|
@@ -665,6 +676,7 @@ export class RunManager {
|
|
|
665
676
|
this.store = store;
|
|
666
677
|
this.repoRoot = repoRoot;
|
|
667
678
|
this.dataDir = join(repoRoot, '.ai/cezar');
|
|
679
|
+
this.unreadInputGraceMs = options.unreadInputGraceMs ?? UNREAD_INPUT_GRACE_MS;
|
|
668
680
|
this.semaphore = options.semaphore ?? new WorkspaceSemaphore();
|
|
669
681
|
this.ciResources = acquireCiResources(this.semaphore);
|
|
670
682
|
this.ciSupervisor = this.ciResources.supervisor;
|
|
@@ -711,6 +723,9 @@ export class RunManager {
|
|
|
711
723
|
for (const timer of this.workerWaitTimers.values())
|
|
712
724
|
clearTimeout(timer);
|
|
713
725
|
this.workerWaitTimers.clear();
|
|
726
|
+
for (const timer of this.inboxClaimTimers.values())
|
|
727
|
+
clearTimeout(timer);
|
|
728
|
+
this.inboxClaimTimers.clear();
|
|
714
729
|
this.workerWaiting.clear();
|
|
715
730
|
this.workerWakeAdmitted.clear();
|
|
716
731
|
this.workerWakeQueuedAt.clear();
|
|
@@ -805,6 +820,7 @@ export class RunManager {
|
|
|
805
820
|
* turning into empty command output inside a running agent. */
|
|
806
821
|
agentEnv(runId, generateFollowups = true) {
|
|
807
822
|
return {
|
|
823
|
+
CEZ_ARTIFACTS_DIR: provisionArtifactDirectory(this.dataDir, runId),
|
|
808
824
|
CEZ_HANDOFF_FILE: handoffPath(this.dataDir, runId),
|
|
809
825
|
CEZ_TASK_ID: runId,
|
|
810
826
|
CEZ_TODOS_FILE: generateFollowups ? todosPath(this.dataDir) : '',
|
|
@@ -1190,7 +1206,19 @@ export class RunManager {
|
|
|
1190
1206
|
throw new DelegationPolicyError('invalid_input', error instanceof Error ? error.message : 'Worker execution selection is unavailable');
|
|
1191
1207
|
}
|
|
1192
1208
|
}
|
|
1193
|
-
|
|
1209
|
+
/**
|
|
1210
|
+
* Idempotent start (#504, spec 2026-09-24-cez-task-cli). Lookup and create share one
|
|
1211
|
+
* synchronous path, so the event loop serialises concurrent retries: the first creates, every
|
|
1212
|
+
* later one with the same hash gets that run back, and a different hash is a conflict.
|
|
1213
|
+
*/
|
|
1214
|
+
startRunIdempotent(workflow, input, request) {
|
|
1215
|
+
const existing = this.store.findRunByClientRequestId(request.id);
|
|
1216
|
+
if (existing) {
|
|
1217
|
+
return existing.clientRequestHash === request.hash ? { run: existing, created: false } : { conflict: true };
|
|
1218
|
+
}
|
|
1219
|
+
return { run: this.startRun(workflow, input, undefined, request), created: true };
|
|
1220
|
+
}
|
|
1221
|
+
startRun(workflow, input, group, clientRequest) {
|
|
1194
1222
|
// Sanitize at the manager boundary so CLI runs, workflows, variants, and
|
|
1195
1223
|
// direct callers cannot bypass the HTTP policy.
|
|
1196
1224
|
const effectiveInput = agentModelsLocked(this.repoRoot)
|
|
@@ -1221,6 +1249,8 @@ export class RunManager {
|
|
|
1221
1249
|
worktree: !group && input.worktree === false ? false : undefined,
|
|
1222
1250
|
groupId: group?.groupId,
|
|
1223
1251
|
variant: group?.variant,
|
|
1252
|
+
clientRequestId: clientRequest?.id,
|
|
1253
|
+
clientRequestHash: clientRequest?.hash,
|
|
1224
1254
|
steps: workflow.steps.map((s) => ({ id: s.id, name: s.name ?? s.id, kind: stepKind(s) })),
|
|
1225
1255
|
});
|
|
1226
1256
|
// Persist the full definition so a queued run survives a restart (#367) —
|
|
@@ -1427,6 +1457,14 @@ export class RunManager {
|
|
|
1427
1457
|
// than being dequeued and re-queued (which would churn its position and its record).
|
|
1428
1458
|
const next = this.queue.findIndex((id) => {
|
|
1429
1459
|
const queued = this.store.getRun(id);
|
|
1460
|
+
// An inbox claim may arrive after a wake was queued but before admission.
|
|
1461
|
+
// Preserve its place and slot exemption until ACK/release/expiry re-pumps.
|
|
1462
|
+
if (this.workerWakeQueuedAt.has(id)) {
|
|
1463
|
+
const first = queued?.agentInputs?.find(input => !input.deliveredAt);
|
|
1464
|
+
const wake = queued?.agentInputs?.find(input => input.id === this.workerWait(id)?.wakeId);
|
|
1465
|
+
if ((first && hasLiveInboxClaim(first)) || (wake && hasLiveInboxClaim(wake)))
|
|
1466
|
+
return false;
|
|
1467
|
+
}
|
|
1430
1468
|
return (capacity() || chargedCiWake(id)) && (!queued || this.historyDeletionPending(id) || ((!this.executions.get(id)?.admitted || this.active.has(id)) && !this.executionBlockedByRootFinish(queued) &&
|
|
1431
1469
|
(!anyHold || !accountHeldFor(queued, holds, defaultRunner ?? 'claude'))));
|
|
1432
1470
|
});
|
|
@@ -1473,6 +1511,11 @@ export class RunManager {
|
|
|
1473
1511
|
this.workerWakeAdmitted.add(runId);
|
|
1474
1512
|
this.flushAgentInputs(runId);
|
|
1475
1513
|
}
|
|
1514
|
+
else if (state.pendingHumanAsk && this.routedAsk(runId) && this.workerWaiting.delete(runId)) {
|
|
1515
|
+
// A parent's answer to the worker's routed question (#505).
|
|
1516
|
+
this.workerWakeAdmitted.add(runId);
|
|
1517
|
+
this.answerRoutedQuestion(runId, true);
|
|
1518
|
+
}
|
|
1476
1519
|
continue;
|
|
1477
1520
|
}
|
|
1478
1521
|
if (this.workerWakeQueuedAt.delete(runId))
|
|
@@ -1667,6 +1710,17 @@ export class RunManager {
|
|
|
1667
1710
|
return;
|
|
1668
1711
|
this.recovering = true;
|
|
1669
1712
|
try {
|
|
1713
|
+
for (const run of this.store.listRuns()) {
|
|
1714
|
+
this.store.clearExpiredInboxClaims(run.id, new Date().toISOString());
|
|
1715
|
+
this.armInboxClaimExpiry(run.id);
|
|
1716
|
+
// #505: the session that accepted this input died before the model read it.
|
|
1717
|
+
// Delivery is at-least-once: the replay keeps the input's ID in its text.
|
|
1718
|
+
if (['queued', 'running', 'waiting'].includes(run.status) && !this.isActive(run.id)) {
|
|
1719
|
+
const replayed = this.store.requeueAwaitingReadInputs(run.id);
|
|
1720
|
+
if (replayed.length)
|
|
1721
|
+
this.store.appendEvent(run.id, { type: 'note', message: `replaying ${replayed.length} message${replayed.length === 1 ? '' : 's'} accepted before the restart but not read` });
|
|
1722
|
+
}
|
|
1723
|
+
}
|
|
1670
1724
|
this.reconcileWorkerWaits();
|
|
1671
1725
|
const live = this.store
|
|
1672
1726
|
.listRuns()
|
|
@@ -1743,7 +1797,8 @@ export class RunManager {
|
|
|
1743
1797
|
// runs. A worker reaches here only when it has neither a recoverable ask
|
|
1744
1798
|
// nor a worker wait; preserving that record would strand both it and its
|
|
1745
1799
|
// parent because workers cannot accept an ordinary inactive Continue.
|
|
1746
|
-
if (run.status === 'waiting' && run.delegation?.role !== 'worker'
|
|
1800
|
+
if (run.status === 'waiting' && (run.delegation?.role !== 'worker' ||
|
|
1801
|
+
run.agentInputs?.some(input => !input.deliveredAt && hasLiveInboxClaim(input))))
|
|
1747
1802
|
continue;
|
|
1748
1803
|
if (run.status === 'waiting') {
|
|
1749
1804
|
const finishedAt = new Date().toISOString();
|
|
@@ -1820,7 +1875,15 @@ export class RunManager {
|
|
|
1820
1875
|
finally {
|
|
1821
1876
|
this.recovering = false;
|
|
1822
1877
|
}
|
|
1878
|
+
// A question a crash caught before or after its commit to the parent still gets routed (#505).
|
|
1879
|
+
for (const run of this.store.listRuns())
|
|
1880
|
+
if (run.delegation?.role === 'worker')
|
|
1881
|
+
this.routeWorkerQuestion(run.id);
|
|
1823
1882
|
this.reconcileWorkerWaits();
|
|
1883
|
+
// A parent reply accepted just before the crash still answers its worker (#505).
|
|
1884
|
+
for (const run of this.store.listRuns())
|
|
1885
|
+
if (run.delegation?.role === 'worker')
|
|
1886
|
+
this.answerRoutedQuestion(run.id);
|
|
1824
1887
|
for (const run of this.store.listRuns()) {
|
|
1825
1888
|
if (run.ciWait && !['running', 'waiting', 'queued'].includes(run.status))
|
|
1826
1889
|
this.withdrawCiWait(run.id);
|
|
@@ -2702,6 +2765,9 @@ export class RunManager {
|
|
|
2702
2765
|
return undefined;
|
|
2703
2766
|
if (this.hasPendingHumanAsk(runId) || this.active.get(runId)?.pendingHumanAsk)
|
|
2704
2767
|
return 'Answer the pending human question before finishing.';
|
|
2768
|
+
const questions = this.unansweredQuestions(runId);
|
|
2769
|
+
if (questions.length)
|
|
2770
|
+
return `Answer your workers' questions before finishing: ${questions.map(q => q.workerId).join(', ')}`;
|
|
2705
2771
|
const blockers = this.parentCompletionBlockers(runId);
|
|
2706
2772
|
return blockers.length ? `Workers must finish or be stopped, prove termination, and have their latest results collected before finishing: ${blockers.map(b => `${b.workerId} (${b.reason})`).join(', ')}` : undefined;
|
|
2707
2773
|
}
|
|
@@ -2742,6 +2808,8 @@ export class RunManager {
|
|
|
2742
2808
|
const state = this.active.get(runId);
|
|
2743
2809
|
const blockers = this.parentCompletionBlockers(runId);
|
|
2744
2810
|
const pendingAsk = this.hasPendingHumanAsk(runId) || state?.pendingHumanAsk;
|
|
2811
|
+
// A worker waiting on this parent's answer (#505): waiting on that worker would deadlock.
|
|
2812
|
+
const questions = pendingAsk ? [] : this.unansweredQuestions(runId);
|
|
2745
2813
|
if (!blockers.length && !pendingAsk) {
|
|
2746
2814
|
this.resetParentCompletion(runId);
|
|
2747
2815
|
return false;
|
|
@@ -2754,7 +2822,7 @@ export class RunManager {
|
|
|
2754
2822
|
}
|
|
2755
2823
|
if (parent.delegation?.role !== 'root')
|
|
2756
2824
|
return false;
|
|
2757
|
-
if (parent.delegation.wait && !pendingAsk) {
|
|
2825
|
+
if (parent.delegation.wait && !pendingAsk && !questions.length) {
|
|
2758
2826
|
if (state?.session?.open)
|
|
2759
2827
|
this.parkWorkerWait(runId, state);
|
|
2760
2828
|
else
|
|
@@ -2762,11 +2830,12 @@ export class RunManager {
|
|
|
2762
2830
|
this.store.flush();
|
|
2763
2831
|
return true;
|
|
2764
2832
|
}
|
|
2765
|
-
const message = pendingAsk ? 'Completion blocked: answer the pending human question.' :
|
|
2833
|
+
const message = pendingAsk ? 'Completion blocked: answer the pending human question.' : questions.length
|
|
2834
|
+
? `Completion blocked: answer your workers' questions first: ${questions.map(q => `${q.workerId} (request ${q.messageId})`).join(', ')}.` :
|
|
2766
2835
|
`Completion blocked: finish or explicitly stop outstanding workers, then collect their latest settled results: ${blockers.map(b => `${b.workerId} (${b.reason})`).join(', ')}.`;
|
|
2767
2836
|
this.store.appendEvent(runId, { type: 'note', tone: 'warning', message });
|
|
2768
2837
|
const workerIds = blockers.map(b => b.workerId).filter(id => this.store.getRun(id)?.delegation?.role === 'worker');
|
|
2769
|
-
if (!pendingAsk && !parent.delegation.completion && workerIds.length) {
|
|
2838
|
+
if (!pendingAsk && !questions.length && !parent.delegation.completion && workerIds.length) {
|
|
2770
2839
|
const wait = { id: randomUUID(), mode: 'all', workerIds,
|
|
2771
2840
|
revisions: workerIds.map(workerId => { const d = this.store.getRun(workerId).delegation; return { workerId, revision: d?.role === 'worker' ? d.executionRevision ?? 0 : 0 }; }),
|
|
2772
2841
|
deadline: new Date(Date.now() + 600_000).toISOString(), phase: 'parked', outcomes: [] };
|
|
@@ -2779,7 +2848,7 @@ export class RunManager {
|
|
|
2779
2848
|
this.reconcileWorkerWaits();
|
|
2780
2849
|
}
|
|
2781
2850
|
else {
|
|
2782
|
-
const monitoring = !pendingAsk && !!state?.session?.open && this.hasLiveWorkers(runId);
|
|
2851
|
+
const monitoring = !pendingAsk && !questions.length && !!state?.session?.open && this.hasLiveWorkers(runId);
|
|
2783
2852
|
this.store.commitDelegation([{ id: runId, delegation: { ...parent.delegation, completion: { ...parent.delegation.completion, phase: 'attention' } } }]);
|
|
2784
2853
|
this.store.updateRun(runId, { status: monitoring ? 'running' : 'waiting', activity: monitoring ? 'monitoring' : undefined, finishedAt: undefined });
|
|
2785
2854
|
this.store.flush();
|
|
@@ -2883,6 +2952,90 @@ export class RunManager {
|
|
|
2883
2952
|
this.parkWorkerWait(runId, state);
|
|
2884
2953
|
return registered;
|
|
2885
2954
|
}
|
|
2955
|
+
/** A wait wake is only a pointer. Only provider submission/opening prompts own delivery. */
|
|
2956
|
+
providerOwnsInboxInput(runId, id) {
|
|
2957
|
+
const state = this.active.get(runId);
|
|
2958
|
+
return !!state?.agentInputFlight?.inputIds.includes(id) || state?.openingAgentInputId === id ||
|
|
2959
|
+
this.store.getRun(runId)?.continuationMessage?.agentInputId === id;
|
|
2960
|
+
}
|
|
2961
|
+
canClaimInboxInput(runId, id) {
|
|
2962
|
+
const run = this.store.getRun(runId);
|
|
2963
|
+
const input = run?.agentInputs?.find(entry => entry.id === id);
|
|
2964
|
+
return !this.disposed && !!run && ['queued', 'running', 'waiting'].includes(run.status) &&
|
|
2965
|
+
!this.historyDeletionPending(runId) && !this.executionBlockedByRootFinish(run) && !this.workerExecutionStopped(runId) &&
|
|
2966
|
+
!!input && input.source === 'agent' && !!input.conversation && !input.deliveredAt &&
|
|
2967
|
+
!hasLiveInboxClaim(input) && !this.providerOwnsInboxInput(runId, id);
|
|
2968
|
+
}
|
|
2969
|
+
/** The service supplies authorized, bounded IDs; claim and flight reservations are synchronous. */
|
|
2970
|
+
reserveInboxInputs(runId, generation, ids) {
|
|
2971
|
+
if (!ids.length || ids.some(id => !this.canClaimInboxInput(runId, id)) ||
|
|
2972
|
+
this.store.getRun(runId)?.agentInputs?.some(input => !input.deliveredAt && hasLiveInboxClaim(input)))
|
|
2973
|
+
return undefined;
|
|
2974
|
+
const claim = { receiptId: randomUUID(), generation, expiresAt: new Date(Date.now() + 120_000).toISOString() };
|
|
2975
|
+
this.store.claimInboxInputs(runId, ids, claim);
|
|
2976
|
+
this.armInboxClaimExpiry(runId);
|
|
2977
|
+
return { receiptId: claim.receiptId, expiresAt: claim.expiresAt, inputIds: [...ids] };
|
|
2978
|
+
}
|
|
2979
|
+
acknowledgeInbox(runId, generation, receiptId) {
|
|
2980
|
+
if (this.disposed || this.store.getRun(runId)?.agentInputs?.some(input => input.inboxClaim?.receiptId === receiptId &&
|
|
2981
|
+
!input.deliveredAt && this.providerOwnsInboxInput(runId, input.id)))
|
|
2982
|
+
throw new Error('inbox receipt displaced');
|
|
2983
|
+
const result = this.store.ackInboxInputs(runId, receiptId, generation, new Date().toISOString());
|
|
2984
|
+
this.armInboxClaimExpiry(runId);
|
|
2985
|
+
this.reconcileWorkerWaits();
|
|
2986
|
+
this.deliverConversationInput(runId);
|
|
2987
|
+
this.active.get(runId)?.resumeInboxAutoEnd?.();
|
|
2988
|
+
this.releaseSlot();
|
|
2989
|
+
return result;
|
|
2990
|
+
}
|
|
2991
|
+
releaseInbox(runId, generation, receiptId) {
|
|
2992
|
+
if (this.disposed)
|
|
2993
|
+
throw new Error('inbox manager disposed');
|
|
2994
|
+
const result = this.store.releaseInboxInputs(runId, receiptId, generation);
|
|
2995
|
+
this.armInboxClaimExpiry(runId);
|
|
2996
|
+
this.reconcileWorkerWaits();
|
|
2997
|
+
this.deliverConversationInput(runId);
|
|
2998
|
+
this.releaseSlot();
|
|
2999
|
+
return result;
|
|
3000
|
+
}
|
|
3001
|
+
/** Expiry is an independent wake source, including after the provider's last turn. */
|
|
3002
|
+
armInboxClaimExpiry(runId) {
|
|
3003
|
+
const previous = this.inboxClaimTimers.get(runId);
|
|
3004
|
+
if (previous)
|
|
3005
|
+
clearTimeout(previous);
|
|
3006
|
+
this.inboxClaimTimers.delete(runId);
|
|
3007
|
+
if (this.disposed)
|
|
3008
|
+
return;
|
|
3009
|
+
const deadlines = (this.store.getRun(runId)?.agentInputs ?? []).flatMap(input => input.inboxClaim && !input.inboxClaim.acknowledgedAt ? [Date.parse(input.inboxClaim.expiresAt)] : []);
|
|
3010
|
+
if (!deadlines.length)
|
|
3011
|
+
return;
|
|
3012
|
+
let warned = false;
|
|
3013
|
+
const expire = () => {
|
|
3014
|
+
this.inboxClaimTimers.delete(runId);
|
|
3015
|
+
if (this.disposed || !this.store.getRun(runId))
|
|
3016
|
+
return;
|
|
3017
|
+
try {
|
|
3018
|
+
this.store.clearExpiredInboxClaims(runId, new Date().toISOString());
|
|
3019
|
+
this.reconcileWorkerWaits();
|
|
3020
|
+
this.deliverConversationInput(runId);
|
|
3021
|
+
this.armInboxClaimExpiry(runId);
|
|
3022
|
+
this.releaseSlot();
|
|
3023
|
+
}
|
|
3024
|
+
catch {
|
|
3025
|
+
if (!warned)
|
|
3026
|
+
console.warn('[cez] inbox expiry checkpoint failed; retrying in 1 second');
|
|
3027
|
+
warned = true;
|
|
3028
|
+
if (!this.disposed)
|
|
3029
|
+
arm(1_000);
|
|
3030
|
+
}
|
|
3031
|
+
};
|
|
3032
|
+
const arm = (delay) => {
|
|
3033
|
+
const timer = setTimeout(expire, delay);
|
|
3034
|
+
timer.unref?.();
|
|
3035
|
+
this.inboxClaimTimers.set(runId, timer);
|
|
3036
|
+
};
|
|
3037
|
+
arm(Math.max(0, Math.min(...deadlines) - Date.now()));
|
|
3038
|
+
}
|
|
2886
3039
|
/** Accepted conversation input uses the existing scheduler admission for idle recipients. */
|
|
2887
3040
|
deliverConversationInput(runId) {
|
|
2888
3041
|
if (this.disposed)
|
|
@@ -2890,15 +3043,21 @@ export class RunManager {
|
|
|
2890
3043
|
const run = this.store.getRun(runId);
|
|
2891
3044
|
if (!run?.delegation || run.delegation.role === 'invalid' || this.historyDeletionPending(runId) ||
|
|
2892
3045
|
this.executionBlockedByRootFinish(run) || !['queued', 'running', 'waiting'].includes(run.status) ||
|
|
2893
|
-
this.workerExecutionStopped(runId)
|
|
3046
|
+
this.workerExecutionStopped(runId))
|
|
3047
|
+
return;
|
|
3048
|
+
// A pending question holds every other input; only the parent's reply to a routed one
|
|
3049
|
+
// answers it (#505), and the held inputs follow right behind that answer.
|
|
3050
|
+
if (this.hasPendingHumanAsk(runId)) {
|
|
3051
|
+
this.answerRoutedQuestion(runId);
|
|
2894
3052
|
return;
|
|
3053
|
+
}
|
|
2895
3054
|
// Retain conversation input until the registered CI wait receives scheduler admission.
|
|
2896
3055
|
// Converting this monitor into a worker wake would strand both admission paths.
|
|
2897
3056
|
if (run.ciWait) {
|
|
2898
3057
|
this.queueCiWake(runId);
|
|
2899
3058
|
return;
|
|
2900
3059
|
}
|
|
2901
|
-
const input = run.agentInputs?.find(entry => entry.conversation && !entry.deliveredAt);
|
|
3060
|
+
const input = run.agentInputs?.find(entry => entry.conversation && !entry.deliveredAt && !hasLiveInboxClaim(entry));
|
|
2902
3061
|
if (!input)
|
|
2903
3062
|
return;
|
|
2904
3063
|
const state = this.active.get(runId);
|
|
@@ -2923,6 +3082,231 @@ export class RunManager {
|
|
|
2923
3082
|
else if (!this.recovering)
|
|
2924
3083
|
this.flushAgentInputs(runId);
|
|
2925
3084
|
}
|
|
3085
|
+
/** A worker's pending question goes to its owning parent as a conversation request
|
|
3086
|
+
* (#505). The worker still parks on its ask; only the parent's reply, or a human after
|
|
3087
|
+
* fallback, answers it. Routing is idempotent: the message ID derives from the ask seq. */
|
|
3088
|
+
routeWorkerQuestion(workerId) {
|
|
3089
|
+
const worker = this.store.getRun(workerId);
|
|
3090
|
+
if (worker?.delegation?.role !== 'worker')
|
|
3091
|
+
return;
|
|
3092
|
+
const events = this.store.readEvents(workerId);
|
|
3093
|
+
const ask = pendingHumanAsk(events);
|
|
3094
|
+
if (!ask)
|
|
3095
|
+
return;
|
|
3096
|
+
if (events.some(event => (event.type === 'worker-question-routed' || event.type === 'worker-question-fallback') && event.askSeq === ask.seq))
|
|
3097
|
+
return;
|
|
3098
|
+
const fallback = (reason) => { this.store.appendEvent(workerId, { type: 'worker-question-fallback', askSeq: ask.seq, reason }); };
|
|
3099
|
+
const request = askRequestSchema.safeParse({ questions: ask.questions });
|
|
3100
|
+
if (!request.success) {
|
|
3101
|
+
fallback('the question cannot be carried to the parent');
|
|
3102
|
+
return;
|
|
3103
|
+
}
|
|
3104
|
+
const parentId = worker.delegation.parentRunId;
|
|
3105
|
+
const root = this.store.getRun(parentId);
|
|
3106
|
+
const now = new Date().toISOString();
|
|
3107
|
+
const message = questionMessage({ workerRunId: workerId, parentRunId: parentId, askSeq: ask.seq, request: request.data, now });
|
|
3108
|
+
// Already committed (a crash before the routing event): the question did reach the parent,
|
|
3109
|
+
// whatever its capacity or state is now. Record the routing; fallback sweeps follow.
|
|
3110
|
+
const committed = root?.delegation?.role === 'root' && !!root.delegation.conversation?.messages.some(m => m.id === message.id);
|
|
3111
|
+
if (committed) {
|
|
3112
|
+
this.store.appendEvent(workerId, { type: 'worker-question-routed', askSeq: ask.seq, messageId: message.id, parentRunId: parentId });
|
|
3113
|
+
return;
|
|
3114
|
+
}
|
|
3115
|
+
// Recovery can find a question that never left a worker which has since stopped.
|
|
3116
|
+
if (!['queued', 'running', 'waiting'].includes(worker.status)) {
|
|
3117
|
+
fallback('worker-stopped');
|
|
3118
|
+
return;
|
|
3119
|
+
}
|
|
3120
|
+
if (root?.delegation?.role !== 'root' || !this.parentCanReceive(root)) {
|
|
3121
|
+
fallback('the parent can no longer receive messages');
|
|
3122
|
+
return;
|
|
3123
|
+
}
|
|
3124
|
+
const state = root.delegation.conversation ?? { messages: [], outcomes: [] };
|
|
3125
|
+
// Room in the ledger for the question AND its reply, beside the replies other open questions
|
|
3126
|
+
// reserve. The reply itself is always admitted to this worker's inbox (service.send).
|
|
3127
|
+
if (state.messages.length + openQuestions(state).length + 2 > 1024 || (root.agentInputs ?? []).filter(input => !input.deliveredAt).length >= 32 ||
|
|
3128
|
+
state.messages.filter(m => m.kind === 'request' && m.state === 'accepted' && !state.outcomes.some(o => o.requestId === m.id)).length >= 32) {
|
|
3129
|
+
fallback('the parent conversation is at capacity');
|
|
3130
|
+
return;
|
|
3131
|
+
}
|
|
3132
|
+
const input = { id: message.id, source: 'agent', parentRunId: parentId, text: message.text, createdAt: now,
|
|
3133
|
+
conversation: { senderRunId: workerId, recipientRunId: parentId, kind: 'request' } };
|
|
3134
|
+
this.store.commitConversation(parentId, { ...state, messages: [...state.messages, message] }, { recipientRunId: parentId, input });
|
|
3135
|
+
projectConversationEvents(this.store, this.store.getRun(parentId));
|
|
3136
|
+
this.store.appendEvent(workerId, { type: 'worker-question-routed', askSeq: ask.seq, messageId: message.id, parentRunId: parentId });
|
|
3137
|
+
this.deliverConversationInput(parentId);
|
|
3138
|
+
}
|
|
3139
|
+
/** A human answering a worker question its parent handed back (#505): the reviewing parent
|
|
3140
|
+
* can no longer answer it, so it must not also block the human. */
|
|
3141
|
+
answersFallenBackAsk(runId, opts, deferred) {
|
|
3142
|
+
if (deferred || !(opts.text?.trim() || opts.images?.length))
|
|
3143
|
+
return false;
|
|
3144
|
+
const events = this.store.readEvents(runId);
|
|
3145
|
+
const ask = pendingHumanAsk(events);
|
|
3146
|
+
return !!ask && events.some(event => event.type === 'worker-question-fallback' && event.askSeq === ask.seq);
|
|
3147
|
+
}
|
|
3148
|
+
/** The worker's pending ask when it went to the parent and has not fallen back to a human. */
|
|
3149
|
+
routedAsk(runId) {
|
|
3150
|
+
const run = this.store.getRun(runId);
|
|
3151
|
+
if (run?.delegation?.role !== 'worker')
|
|
3152
|
+
return undefined;
|
|
3153
|
+
const events = this.store.readEvents(runId);
|
|
3154
|
+
const ask = pendingHumanAsk(events);
|
|
3155
|
+
if (!ask || events.some(event => event.type === 'worker-question-fallback' && event.askSeq === ask.seq))
|
|
3156
|
+
return undefined;
|
|
3157
|
+
const routed = events.find(event => event.type === 'worker-question-routed' && event.askSeq === ask.seq);
|
|
3158
|
+
const root = this.store.getRun(run.delegation.parentRunId);
|
|
3159
|
+
const message = routed && root?.delegation?.role === 'root'
|
|
3160
|
+
? root.delegation.conversation?.messages.find(entry => entry.id === routed.messageId) : undefined;
|
|
3161
|
+
return message ? { askSeq: ask.seq, message } : undefined;
|
|
3162
|
+
}
|
|
3163
|
+
/** The parent's reply answers the routed question like a human answer would: through the
|
|
3164
|
+
* live session's native answer seam, or by reopening a session that is gone (#505). */
|
|
3165
|
+
answerRoutedQuestion(runId, admitted = false) {
|
|
3166
|
+
const routed = this.routedAsk(runId);
|
|
3167
|
+
const run = this.store.getRun(runId);
|
|
3168
|
+
const reply = routed && run?.agentInputs?.find(input => !input.deliveredAt && answersQuestion(routed.message, input));
|
|
3169
|
+
// A stopped worker is the human's to continue; a parent answer never revives it.
|
|
3170
|
+
if (!routed || !reply || run?.delegation?.role !== 'worker' || !['queued', 'running', 'waiting'].includes(run.status))
|
|
3171
|
+
return;
|
|
3172
|
+
// Verbatim: a native answer seam parses `Header: label` lines, exactly as a human's.
|
|
3173
|
+
const text = reply.text;
|
|
3174
|
+
const state = this.active.get(runId);
|
|
3175
|
+
if (!state) {
|
|
3176
|
+
// An automated answer is not a human override: the reopened session waits for capacity.
|
|
3177
|
+
if (this.isActive(runId))
|
|
3178
|
+
return;
|
|
3179
|
+
// As on the live path, the answer supersedes a wait registered before the question.
|
|
3180
|
+
if (this.workerWait(runId)) {
|
|
3181
|
+
try {
|
|
3182
|
+
this.withdrawWorkerWait(runId);
|
|
3183
|
+
}
|
|
3184
|
+
catch {
|
|
3185
|
+
return;
|
|
3186
|
+
}
|
|
3187
|
+
}
|
|
3188
|
+
const result = this.continueRun(runId, { text, answerInputId: reply.id }, true);
|
|
3189
|
+
if (!result.ok)
|
|
3190
|
+
this.store.appendEvent(runId, { type: 'note', tone: 'warning', message: `parent answer not delivered: ${result.error}` });
|
|
3191
|
+
return;
|
|
3192
|
+
}
|
|
3193
|
+
if (!state.session?.open || state.cancelled || state.openingAgentInputId === reply.id)
|
|
3194
|
+
return;
|
|
3195
|
+
// Already waiting for admission: only the admission itself may deliver it.
|
|
3196
|
+
if (!admitted && (this.workerWaiting.has(runId) || this.workerWakeQueuedAt.has(runId)))
|
|
3197
|
+
return;
|
|
3198
|
+
if (!admitted && (this.waiting.has(runId) || this.monitoring.has(runId))) {
|
|
3199
|
+
// The parked worker released its slot: the answer rides the ordinary message wake,
|
|
3200
|
+
// which delivers it here once the scheduler admits the worker.
|
|
3201
|
+
const existing = this.workerWait(runId);
|
|
3202
|
+
if (existing?.wakeId !== reply.id) {
|
|
3203
|
+
// An answer supersedes a wait the worker registered before asking, exactly as a
|
|
3204
|
+
// human answer withdraws it; a failed checkpoint leaves both for a later retry.
|
|
3205
|
+
if (existing) {
|
|
3206
|
+
try {
|
|
3207
|
+
this.withdrawWorkerWait(runId);
|
|
3208
|
+
}
|
|
3209
|
+
catch {
|
|
3210
|
+
return;
|
|
3211
|
+
}
|
|
3212
|
+
}
|
|
3213
|
+
const wait = { id: randomUUID(), workerIds: [], outcomes: [],
|
|
3214
|
+
deadline: new Date().toISOString(), phase: 'wake-pending', reason: 'message', wakeId: reply.id };
|
|
3215
|
+
const current = this.store.getRun(runId)?.delegation;
|
|
3216
|
+
if (current?.role !== 'worker')
|
|
3217
|
+
return;
|
|
3218
|
+
this.store.commitDelegation([{ id: runId, delegation: { ...current, wait } }]);
|
|
3219
|
+
}
|
|
3220
|
+
this.clearIdleTimer(state);
|
|
3221
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
3222
|
+
this.waiting.delete(runId);
|
|
3223
|
+
this.monitoring.delete(runId);
|
|
3224
|
+
this.workerWaiting.add(runId);
|
|
3225
|
+
this.queueWorkerWake(runId);
|
|
3226
|
+
return;
|
|
3227
|
+
}
|
|
3228
|
+
const bundle = state.atTurnBoundary === state.session ? this.pendingConversationBatch(runId, state, reply.id) : undefined;
|
|
3229
|
+
if (!state.session.sendMessage([{ type: 'text', text }, ...(bundle ? [{ type: 'text', text: bundle.text }] : [])]))
|
|
3230
|
+
return;
|
|
3231
|
+
this.store.appendEvent(runId, { type: 'human-input-delivered', askSeq: routed.askSeq, source: 'parent' });
|
|
3232
|
+
const delivered = [reply.id, ...(bundle?.inputs.map(input => input.id) ?? [])];
|
|
3233
|
+
this.commitBundledDelivery(runId, state, delivered);
|
|
3234
|
+
// As in submitAgentInput: the wake that admitted this answer is spent, and a stale one
|
|
3235
|
+
// would hold back every later message until the answer's turn ends.
|
|
3236
|
+
if (delivered.includes(this.workerWait(runId)?.wakeId ?? '')) {
|
|
3237
|
+
state.workerWakeTurn = state.session;
|
|
3238
|
+
try {
|
|
3239
|
+
this.withdrawWorkerWait(runId);
|
|
3240
|
+
}
|
|
3241
|
+
catch { /* reconcileWorkerWaits retires it at the turn end */ }
|
|
3242
|
+
}
|
|
3243
|
+
state.pendingHumanAsk = this.hasPendingHumanAsk(runId);
|
|
3244
|
+
if (!state.pendingHumanAsk)
|
|
3245
|
+
this.flushAgentInputs(runId);
|
|
3246
|
+
this.resumeParkedRun(runId, state);
|
|
3247
|
+
}
|
|
3248
|
+
/** A routed question settled without a reply (its worker stopped, a deadline, fallback) can
|
|
3249
|
+
* never be answered by the parent any more: hand it to the human on the worker (#505). */
|
|
3250
|
+
retireSettledQuestions(parentId) {
|
|
3251
|
+
const root = this.store.getRun(parentId);
|
|
3252
|
+
const conversation = root?.delegation?.role === 'root' ? root.delegation.conversation : undefined;
|
|
3253
|
+
if (!conversation)
|
|
3254
|
+
return;
|
|
3255
|
+
for (const message of conversation.messages) {
|
|
3256
|
+
const outcome = message.question && conversation.outcomes.find(entry => entry.requestId === message.id);
|
|
3257
|
+
// A reply still waiting for a live worker answers it; one a stopped worker never took does not.
|
|
3258
|
+
const worker = this.store.getRun(message.senderRunId);
|
|
3259
|
+
const live = !!worker && ['queued', 'running', 'waiting'].includes(worker.status) && !worker.stopping;
|
|
3260
|
+
if (!outcome || (outcome.status === 'replied' && live))
|
|
3261
|
+
continue;
|
|
3262
|
+
const routed = this.routedAsk(message.senderRunId);
|
|
3263
|
+
if (routed?.message.id === message.id) {
|
|
3264
|
+
this.store.appendEvent(message.senderRunId, { type: 'worker-question-fallback', askSeq: routed.askSeq, reason: outcome.status === 'human-fallback' ? `parent-${root.status}` : outcome.status === 'replied' ? 'worker-stopped' : `question-${outcome.status}` });
|
|
3265
|
+
}
|
|
3266
|
+
}
|
|
3267
|
+
}
|
|
3268
|
+
/** Routed questions this parent has not answered yet, by asking worker. */
|
|
3269
|
+
unansweredQuestions(parentId) {
|
|
3270
|
+
return this.store.listRuns().flatMap(run => {
|
|
3271
|
+
if (run.delegation?.role !== 'worker' || run.delegation.parentRunId !== parentId)
|
|
3272
|
+
return [];
|
|
3273
|
+
const routed = this.routedAsk(run.id);
|
|
3274
|
+
const root = this.store.getRun(parentId);
|
|
3275
|
+
const answered = root?.delegation?.role === 'root' && root.delegation.conversation?.outcomes.some(outcome => outcome.requestId === routed?.message.id);
|
|
3276
|
+
return routed && !answered ? [{ workerId: run.id, messageId: routed.message.id }] : [];
|
|
3277
|
+
});
|
|
3278
|
+
}
|
|
3279
|
+
/** The parent can no longer answer: its workers' routed questions go back to the human
|
|
3280
|
+
* (#505). The outcome lands first, so a crash before the worker's event repeats this. */
|
|
3281
|
+
fallbackRoutedQuestions(parentId, reason) {
|
|
3282
|
+
const parent = this.store.getRun(parentId);
|
|
3283
|
+
if (parent?.delegation?.role !== 'root')
|
|
3284
|
+
return;
|
|
3285
|
+
const state = parent.delegation.conversation ?? { messages: [], outcomes: [] };
|
|
3286
|
+
const routed = this.store.listRuns().flatMap(run => {
|
|
3287
|
+
const ask = run.delegation?.role === 'worker' && run.delegation.parentRunId === parentId ? this.routedAsk(run.id) : undefined;
|
|
3288
|
+
// An accepted reply still on its way to a live worker answers it; no human card beside it.
|
|
3289
|
+
const replied = ask && state.outcomes.some(outcome => outcome.requestId === ask.message.id && outcome.status === 'replied');
|
|
3290
|
+
return ask && !(replied && ['queued', 'running', 'waiting'].includes(run.status) && !run.stopping) ? [{ workerId: run.id, ...ask }] : [];
|
|
3291
|
+
});
|
|
3292
|
+
if (!routed.length)
|
|
3293
|
+
return;
|
|
3294
|
+
const observedAt = new Date().toISOString();
|
|
3295
|
+
const missing = routed.filter(entry => !state.outcomes.some(outcome => outcome.requestId === entry.message.id));
|
|
3296
|
+
if (missing.length) {
|
|
3297
|
+
this.store.commitConversation(parentId, { ...state, outcomes: [...state.outcomes,
|
|
3298
|
+
...missing.map(entry => ({ requestId: entry.message.id, status: 'human-fallback', observedAt }))] });
|
|
3299
|
+
projectConversationEvents(this.store, this.store.getRun(parentId));
|
|
3300
|
+
}
|
|
3301
|
+
for (const entry of routed)
|
|
3302
|
+
this.store.appendEvent(entry.workerId, { type: 'worker-question-fallback', askSeq: entry.askSeq, reason });
|
|
3303
|
+
}
|
|
3304
|
+
/** The parent can still read and answer a routed question; answering needs its steer grant. */
|
|
3305
|
+
parentCanReceive(parent) {
|
|
3306
|
+
return parent.delegation?.role === 'root' && parent.delegation.permissions.includes('steer') &&
|
|
3307
|
+
['queued', 'running', 'waiting'].includes(parent.status) && !parent.stopping &&
|
|
3308
|
+
!this.rootFinishRequested(parent.id) && !this.historyDeletionPending(parent.id);
|
|
3309
|
+
}
|
|
2926
3310
|
/** Cancel only the wait; persist its settlement before ordinary wake admission. */
|
|
2927
3311
|
cancelWorkerWait(parentId, waitId) {
|
|
2928
3312
|
const run = this.store.getRun(parentId);
|
|
@@ -2992,6 +3376,7 @@ export class RunManager {
|
|
|
2992
3376
|
if (next && next !== root.delegation.conversation)
|
|
2993
3377
|
this.store.commitConversation(root.id, next);
|
|
2994
3378
|
projectConversationEvents(this.store, this.store.getRun(root.id));
|
|
3379
|
+
this.retireSettledQuestions(root.id);
|
|
2995
3380
|
const key = `conversation:${root.id}`;
|
|
2996
3381
|
const timer = this.workerWaitTimers.get(key);
|
|
2997
3382
|
if (timer) {
|
|
@@ -3037,6 +3422,7 @@ export class RunManager {
|
|
|
3037
3422
|
if (!['queued', 'running', 'waiting'].includes(parent.status)) {
|
|
3038
3423
|
if (!this.recovering) {
|
|
3039
3424
|
this.withdrawWorkerWait(parent.id);
|
|
3425
|
+
this.fallbackRoutedQuestions(parent.id, `parent-${parent.status}`);
|
|
3040
3426
|
for (const child of this.store.listRuns().filter(child => parent.delegation?.role === 'root' && parent.status !== 'review' && child.delegation?.role === 'worker' && child.delegation.parentRunId === parent.id)) {
|
|
3041
3427
|
if (['queued', 'running', 'waiting'].includes(child.status))
|
|
3042
3428
|
this.cancel(child.id);
|
|
@@ -3049,14 +3435,24 @@ export class RunManager {
|
|
|
3049
3435
|
const wait = parent.delegation.wait;
|
|
3050
3436
|
if (!wait)
|
|
3051
3437
|
continue;
|
|
3052
|
-
const delivered =
|
|
3438
|
+
const delivered = parent.agentInputs?.find(input => input.id === wait.wakeId && input.deliveredAt);
|
|
3053
3439
|
if (delivered) {
|
|
3054
3440
|
// Receipt persistence precedes the live status update. After a crash
|
|
3055
3441
|
// this is interrupted execution, not generic waiting-run success.
|
|
3056
|
-
if (this.recovering && parent.status === 'waiting' && !this.hasPendingHumanAsk(parent.id)) {
|
|
3442
|
+
if (this.recovering && !delivered.inboxClaim?.acknowledgedAt && parent.status === 'waiting' && !this.hasPendingHumanAsk(parent.id)) {
|
|
3057
3443
|
this.store.updateRun(parent.id, { status: 'running', activity: undefined });
|
|
3058
3444
|
}
|
|
3445
|
+
const state = this.active.get(parent.id);
|
|
3446
|
+
const inboxParked = delivered.inboxClaim?.acknowledgedAt &&
|
|
3447
|
+
(this.workerWaiting.has(parent.id) || this.workerWakeAdmitted.has(parent.id)) &&
|
|
3448
|
+
state?.session?.open && state.atTurnBoundary === state.session;
|
|
3059
3449
|
this.withdrawWorkerWait(parent.id);
|
|
3450
|
+
// Inbox ACK starts no provider turn. Retain the completed turn's slot
|
|
3451
|
+
// exemption if its registered wait parked while that ACK was in transit.
|
|
3452
|
+
if (inboxParked && state) {
|
|
3453
|
+
this.waiting.add(parent.id);
|
|
3454
|
+
this.armIdleTimer(parent.id, state);
|
|
3455
|
+
}
|
|
3060
3456
|
continue;
|
|
3061
3457
|
}
|
|
3062
3458
|
const now = new Date().toISOString();
|
|
@@ -3098,8 +3494,8 @@ export class RunManager {
|
|
|
3098
3494
|
// already being processed. Their delayed receipts cannot interrupt a wait
|
|
3099
3495
|
// registered by that very turn. Only inputs still awaiting a turn can.
|
|
3100
3496
|
const flight = this.active.get(parent.id)?.agentInputFlight;
|
|
3101
|
-
const pendingInputs = parent.agentInputs?.filter(input => !input.deliveredAt &&
|
|
3102
|
-
input.id !== parent.continuationMessage?.agentInputId && !flight?.inputIds.includes(input.id)) ?? [];
|
|
3497
|
+
const pendingInputs = parent.agentInputs?.filter(input => !input.deliveredAt && !hasLiveInboxClaim(input) &&
|
|
3498
|
+
input.id !== this.active.get(parent.id)?.openingAgentInputId && input.id !== parent.continuationMessage?.agentInputId && !flight?.inputIds.includes(input.id)) ?? [];
|
|
3103
3499
|
const incoming = pendingInputs.find(input => input.conversation &&
|
|
3104
3500
|
!(input.conversation.kind === 'reply' && input.conversation.requestId && wait.requestIds?.includes(input.conversation.requestId)));
|
|
3105
3501
|
if (incoming && next.phase !== 'wake-pending')
|
|
@@ -3107,7 +3503,8 @@ export class RunManager {
|
|
|
3107
3503
|
// A full queue of selected replies still needs admission. Use the last
|
|
3108
3504
|
// already accepted input as the checkpoint rather than adding a 33rd input.
|
|
3109
3505
|
// Its content is preserved; delivery includes the wait receipt separately.
|
|
3110
|
-
if (next.phase === 'wake-pending' && pendingInputs.length
|
|
3506
|
+
if (next.phase === 'wake-pending' && pendingInputs.length &&
|
|
3507
|
+
(parent.agentInputs ?? []).filter(input => !input.deliveredAt).length >= 32 &&
|
|
3111
3508
|
!parent.agentInputs?.some(input => input.id === next.wakeId)) {
|
|
3112
3509
|
next = { ...next, wakeId: pendingInputs.at(-1).id };
|
|
3113
3510
|
}
|
|
@@ -3172,7 +3569,7 @@ export class RunManager {
|
|
|
3172
3569
|
!['queued', 'running', 'waiting'].includes(run.status))
|
|
3173
3570
|
return;
|
|
3174
3571
|
const existing = run.agentInputs?.find(input => input.id === wait.wakeId);
|
|
3175
|
-
if (existing?.deliveredAt)
|
|
3572
|
+
if (existing?.deliveredAt || (existing && hasLiveInboxClaim(existing)))
|
|
3176
3573
|
return;
|
|
3177
3574
|
// 32 summaries, including worst-case JSON escaping, must fit the ordinary
|
|
3178
3575
|
// input bound. Full 4k summaries remain in durable outcomes and events.
|
|
@@ -3192,6 +3589,10 @@ export class RunManager {
|
|
|
3192
3589
|
return;
|
|
3193
3590
|
}
|
|
3194
3591
|
else if (!existing) {
|
|
3592
|
+
// A full inbox snapshot can temporarily own every reply. ACK/expiry will
|
|
3593
|
+
// reconcile again when a lifecycle input can fit or adopt an unread reply.
|
|
3594
|
+
if ((run.agentInputs ?? []).filter(input => !input.deliveredAt).length >= 32)
|
|
3595
|
+
return;
|
|
3195
3596
|
this.store.commitAgentInputs(parentId, enqueueAgentInput(run, input));
|
|
3196
3597
|
this.store.appendEvent(parentId, { type: 'agent-input', input });
|
|
3197
3598
|
}
|
|
@@ -3202,7 +3603,9 @@ export class RunManager {
|
|
|
3202
3603
|
if (this.recovering)
|
|
3203
3604
|
return;
|
|
3204
3605
|
const state = this.active.get(parentId);
|
|
3205
|
-
|
|
3606
|
+
// A parent's reply to a routed question (#505) is the one message a pending ask admits.
|
|
3607
|
+
const parentAnswer = wait.reason === 'message' && this.routedAsk(parentId) !== undefined;
|
|
3608
|
+
if ((!parentAnswer && (this.hasPendingHumanAsk(parentId) || state?.pendingHumanAsk)) || this.workerWakeQueuedAt.has(parentId))
|
|
3206
3609
|
return;
|
|
3207
3610
|
if (state) {
|
|
3208
3611
|
// Expiring a registered intent NEVER exempts a still-executing turn.
|
|
@@ -3292,7 +3695,227 @@ export class RunManager {
|
|
|
3292
3695
|
const run = this.store.getRun(runId);
|
|
3293
3696
|
return !!run && ['queued', 'running', 'waiting'].includes(run.status) &&
|
|
3294
3697
|
!(run.delegation?.role === 'worker' && run.delegation.destroy) &&
|
|
3295
|
-
!!run.agentInputs?.some(input => !input.deliveredAt);
|
|
3698
|
+
(!!run.agentInputs?.some(input => !input.deliveredAt) || this.harnessOwesInput(this.active.get(runId)));
|
|
3699
|
+
}
|
|
3700
|
+
/** Only the current session's observation writes consumedAt (#505). An ID reported
|
|
3701
|
+
* before its acceptance checkpoint lands is applied when that checkpoint commits. */
|
|
3702
|
+
handleAgentInputConsumed(runId, state, session, ids) {
|
|
3703
|
+
if (!session || this.active.get(runId) !== state || state.session !== session)
|
|
3704
|
+
return;
|
|
3705
|
+
const inFlight = state.agentInputFlight?.inputIds ?? [];
|
|
3706
|
+
for (const id of ids)
|
|
3707
|
+
if (inFlight.includes(id))
|
|
3708
|
+
state.consumedBeforeAck?.add(id);
|
|
3709
|
+
const read = ids.filter(id => !inFlight.includes(id));
|
|
3710
|
+
for (const id of read)
|
|
3711
|
+
state.unreadInputIds?.delete(id);
|
|
3712
|
+
if (!this.harnessOwesInput(state))
|
|
3713
|
+
this.clearUnreadInputTimer(state);
|
|
3714
|
+
try {
|
|
3715
|
+
if (!read.length)
|
|
3716
|
+
return;
|
|
3717
|
+
this.store.commitAgentInputsConsumed(runId, read, new Date().toISOString());
|
|
3718
|
+
this.reconcileWorkerWaits(); // projects the consumed receipt
|
|
3719
|
+
}
|
|
3720
|
+
catch (error) {
|
|
3721
|
+
console.warn(`[cez] agent input consumption checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3722
|
+
}
|
|
3723
|
+
}
|
|
3724
|
+
/** #505: input the harness accepted but will never read returns to the queue, so
|
|
3725
|
+
* DONE, auto-end and park see queued work. Out of a turn (an OpenCode lost wake)
|
|
3726
|
+
* it is submitted right away; at a turn-end the handler's own flush submits it. */
|
|
3727
|
+
retireUnreadInputs(runId, state, ids, outOfTurn) {
|
|
3728
|
+
if (!ids.length || state.cancelled || this.active.get(runId) !== state)
|
|
3729
|
+
return;
|
|
3730
|
+
// Durable first: a failed checkpoint must never let this boundary finish (DONE, park)
|
|
3731
|
+
// over input that is still marked delivered but was never read.
|
|
3732
|
+
try {
|
|
3733
|
+
this.store.requeueUnconsumedAgentInputs(runId, ids);
|
|
3734
|
+
}
|
|
3735
|
+
catch (error) {
|
|
3736
|
+
if (state.agentInputError)
|
|
3737
|
+
return;
|
|
3738
|
+
state.agentInputError = `agent input requeue checkpoint failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
3739
|
+
try {
|
|
3740
|
+
state.session?.interrupt();
|
|
3741
|
+
}
|
|
3742
|
+
finally {
|
|
3743
|
+
try {
|
|
3744
|
+
this.store.appendEvent(runId, { type: 'error', message: state.agentInputError });
|
|
3745
|
+
}
|
|
3746
|
+
catch {
|
|
3747
|
+
console.error(`[cez] ${state.agentInputError}`);
|
|
3748
|
+
}
|
|
3749
|
+
}
|
|
3750
|
+
return;
|
|
3751
|
+
}
|
|
3752
|
+
for (const id of ids)
|
|
3753
|
+
state.unreadInputIds?.delete(id);
|
|
3754
|
+
// Reported before its own acknowledgement landed: that acknowledgement must not
|
|
3755
|
+
// mark it delivered again.
|
|
3756
|
+
for (const id of ids)
|
|
3757
|
+
if (state.agentInputFlight?.inputIds.includes(id))
|
|
3758
|
+
(state.retiredInFlight ??= new Set()).add(id);
|
|
3759
|
+
this.store.appendEvent(runId, { type: 'note', message: `resubmitting ${ids.length} message${ids.length === 1 ? '' : 's'} the agent did not read` });
|
|
3760
|
+
if (outOfTurn)
|
|
3761
|
+
this.flushAgentInputs(runId);
|
|
3762
|
+
}
|
|
3763
|
+
/** #505: the harness started the opening turn, so the opening input was accepted.
|
|
3764
|
+
* Its replay-prompt retirement still waits for that turn's successful end. */
|
|
3765
|
+
recordOpeningAccepted(runId, state) {
|
|
3766
|
+
const id = state.openingAgentInputId;
|
|
3767
|
+
if (!id || state.cancelled || this.active.get(runId) !== state)
|
|
3768
|
+
return;
|
|
3769
|
+
const inputs = this.store.getRun(runId)?.agentInputs ?? [];
|
|
3770
|
+
if (!inputs.some(input => input.id === id && !input.deliveredAt))
|
|
3771
|
+
return;
|
|
3772
|
+
const deliveredAt = new Date().toISOString();
|
|
3773
|
+
try {
|
|
3774
|
+
this.store.commitAgentInputs(runId, inputs.map(input => input.id === id ? { ...input, deliveredAt } : input));
|
|
3775
|
+
}
|
|
3776
|
+
catch (error) {
|
|
3777
|
+
console.warn(`[cez] opening input acceptance checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3778
|
+
}
|
|
3779
|
+
}
|
|
3780
|
+
/** The conversation messages the next submission would carry, if nothing but
|
|
3781
|
+
* conversation input leads the queue (lifecycle input stays a barrier). */
|
|
3782
|
+
pendingConversationBatch(runId, state, answerId) {
|
|
3783
|
+
const run = this.store.getRun(runId);
|
|
3784
|
+
if (!run || run.ciWait || state.agentInputFlight || this.workerWait(runId))
|
|
3785
|
+
return undefined;
|
|
3786
|
+
const batch = agentInputBatch((run.agentInputs ?? []).filter(input => input.id !== state.openingAgentInputId && input.id !== answerId), input => this.formatAgentInput(runId, input));
|
|
3787
|
+
return batch?.inputs.every(input => input.conversation) ? batch : undefined;
|
|
3788
|
+
}
|
|
3789
|
+
/** Bundled messages rode the answer's own write: delivered now, read when that turn ends. */
|
|
3790
|
+
commitBundledDelivery(runId, state, ids) {
|
|
3791
|
+
const deliveredAt = new Date().toISOString();
|
|
3792
|
+
const awaiting = state.inputDelivery?.consumption === 'observable' ? { awaitingRead: true } : {};
|
|
3793
|
+
try {
|
|
3794
|
+
this.store.commitAgentInputs(runId, (this.store.getRun(runId)?.agentInputs ?? []).map(input => ids.includes(input.id) && !input.deliveredAt ? { ...input, deliveredAt, ...awaiting } : input));
|
|
3795
|
+
state.bundledInputIds = [...(state.bundledInputIds ?? []), ...ids];
|
|
3796
|
+
}
|
|
3797
|
+
catch (error) {
|
|
3798
|
+
console.warn(`[cez] bundled delivery checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3799
|
+
}
|
|
3800
|
+
}
|
|
3801
|
+
/** The turn that carried bundled messages has ended: the model read them. */
|
|
3802
|
+
readBundledInputs(runId, state) {
|
|
3803
|
+
const ids = state.bundledInputIds ?? [];
|
|
3804
|
+
state.bundledInputIds = undefined;
|
|
3805
|
+
if (!ids.length || state.cancelled)
|
|
3806
|
+
return;
|
|
3807
|
+
try {
|
|
3808
|
+
this.store.commitAgentInputsConsumed(runId, ids, new Date().toISOString());
|
|
3809
|
+
this.reconcileWorkerWaits();
|
|
3810
|
+
}
|
|
3811
|
+
catch (error) {
|
|
3812
|
+
console.warn(`[cez] bundled read checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3813
|
+
}
|
|
3814
|
+
}
|
|
3815
|
+
/** #505: the session closed without reporting it read (failure, stop, exit): the input
|
|
3816
|
+
* goes back to the queue so a later session delivers it; never shown as delivered. */
|
|
3817
|
+
requeueUnreadAtClose(runId, state, session) {
|
|
3818
|
+
this.clearUnreadInputTimer(state);
|
|
3819
|
+
// Everything this session accepted but never saw read: steered input, messages bundled
|
|
3820
|
+
// behind a human answer, and an opening instruction whose turn never succeeded.
|
|
3821
|
+
const opening = state.openingAgentInputId && this.store.getRun(runId)?.agentInputs?.some(input => input.id === state.openingAgentInputId && input.deliveredAt && !input.consumedAt) ? [state.openingAgentInputId] : [];
|
|
3822
|
+
const ids = [...new Set([...(state.unreadInputIds ?? []), ...(state.bundledInputIds ?? []), ...opening])];
|
|
3823
|
+
if (!session || !ids.length || this.active.get(runId) !== state || state.session !== session)
|
|
3824
|
+
return;
|
|
3825
|
+
state.unreadInputIds?.clear();
|
|
3826
|
+
state.bundledInputIds = undefined;
|
|
3827
|
+
try {
|
|
3828
|
+
this.store.requeueUnconsumedAgentInputs(runId, ids);
|
|
3829
|
+
}
|
|
3830
|
+
catch (error) {
|
|
3831
|
+
console.warn(`[cez] agent input requeue failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3832
|
+
}
|
|
3833
|
+
}
|
|
3834
|
+
/** An idle, open boundary with nothing left to deliver: DONE closes, otherwise the
|
|
3835
|
+
* run parks as waiting or monitoring. Shared by acknowledgements that land after a
|
|
3836
|
+
* turn ended and by the unread-input grace timer (#505). */
|
|
3837
|
+
settleIdleBoundary(runId, state, session) {
|
|
3838
|
+
if (state.atTurnBoundary !== session || state.pendingHumanAsk || this.workerWait(runId) || this.store.getRun(runId)?.ciWait || this.hasQueuedAgentInputs(runId))
|
|
3839
|
+
return;
|
|
3840
|
+
if (state.doneAtBoundary === session) {
|
|
3841
|
+
if (this.deferParentCompletion(runId))
|
|
3842
|
+
return;
|
|
3843
|
+
this.store.appendEvent(runId, { type: 'lifecycle', message: 'goal achieved — session closed' });
|
|
3844
|
+
appendHandoffHeartbeat(this.dataDir, runId, 'turn complete — goal achieved, session closed');
|
|
3845
|
+
session.end();
|
|
3846
|
+
}
|
|
3847
|
+
else if (state.parkAfterAck?.session === session && !this.waiting.has(runId) && !this.monitoring.has(runId)) {
|
|
3848
|
+
// A wake turn can finish before its HTTP/RPC acknowledgement. Retiring
|
|
3849
|
+
// that wait must release its completed turn, not invent another turn.
|
|
3850
|
+
if ((state.parkAfterAck.monitoring || this.hasLiveWorkers(runId)) && !this.parentCompletionAttention(runId)) {
|
|
3851
|
+
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
3852
|
+
if (state.currentStepId)
|
|
3853
|
+
this.store.updateStep(runId, state.currentStepId, { status: 'running' });
|
|
3854
|
+
this.monitoring.add(runId);
|
|
3855
|
+
this.clearIdleTimer(state);
|
|
3856
|
+
this.armMonitoringWakeTimer(runId, state);
|
|
3857
|
+
}
|
|
3858
|
+
else {
|
|
3859
|
+
this.store.updateRun(runId, { status: 'waiting', activity: undefined });
|
|
3860
|
+
if (state.currentStepId)
|
|
3861
|
+
this.store.updateStep(runId, state.currentStepId, { status: 'waiting' });
|
|
3862
|
+
this.store.flush(); // Match ordinary turn-end durability before releasing capacity.
|
|
3863
|
+
this.armIdleTimer(runId, state);
|
|
3864
|
+
}
|
|
3865
|
+
this.waiting.add(runId);
|
|
3866
|
+
this.releaseSlot();
|
|
3867
|
+
}
|
|
3868
|
+
}
|
|
3869
|
+
/** #505 liveness bound: a harness that accepted input and then neither read it nor ran
|
|
3870
|
+
* another turn must not hold the run and its slot forever. After a quiet grace window the
|
|
3871
|
+
* input is resubmitted once; after a second it is left delivered but unconfirmed. */
|
|
3872
|
+
armUnreadInputTimer(runId, state) {
|
|
3873
|
+
this.clearUnreadInputTimer(state);
|
|
3874
|
+
const session = state.session;
|
|
3875
|
+
if (!session?.open || !this.harnessOwesInput(state) || state.agentInputFlight)
|
|
3876
|
+
return;
|
|
3877
|
+
const timer = setTimeout(() => {
|
|
3878
|
+
state.unreadInputTimer = undefined;
|
|
3879
|
+
if (this.disposed || this.active.get(runId) !== state || state.session !== session || !session.open || state.cancelled ||
|
|
3880
|
+
state.agentInputFlight || !this.harnessOwesInput(state))
|
|
3881
|
+
return;
|
|
3882
|
+
const ids = [...state.unreadInputIds];
|
|
3883
|
+
const retried = ids.filter(id => state.unreadRetried?.has(id));
|
|
3884
|
+
const fresh = ids.filter(id => !state.unreadRetried?.has(id));
|
|
3885
|
+
if (retried.length) {
|
|
3886
|
+
for (const id of retried)
|
|
3887
|
+
state.unreadInputIds?.delete(id);
|
|
3888
|
+
try {
|
|
3889
|
+
this.store.commitAgentInputsUnconfirmed(runId, retried);
|
|
3890
|
+
}
|
|
3891
|
+
catch { /* receipt stays awaiting; a restart replays it */ }
|
|
3892
|
+
this.store.appendEvent(runId, { type: 'note', message: `delivered ${retried.length} message${retried.length === 1 ? '' : 's'}; the agent did not confirm reading ${retried.length === 1 ? 'it' : 'them'}` });
|
|
3893
|
+
}
|
|
3894
|
+
if (fresh.length) {
|
|
3895
|
+
state.unreadRetried ??= new Set();
|
|
3896
|
+
for (const id of fresh)
|
|
3897
|
+
state.unreadRetried.add(id);
|
|
3898
|
+
this.retireUnreadInputs(runId, state, fresh, true);
|
|
3899
|
+
}
|
|
3900
|
+
// No content since the last turn ended: the harness is idle, whatever turn it announced.
|
|
3901
|
+
if (!this.harnessOwesInput(state) && !state.agentInputFlight) {
|
|
3902
|
+
state.atTurnBoundary = session;
|
|
3903
|
+
if (state.parkAfterAck?.session !== session)
|
|
3904
|
+
state.parkAfterAck = { session, monitoring: false };
|
|
3905
|
+
this.settleIdleBoundary(runId, state, session);
|
|
3906
|
+
}
|
|
3907
|
+
}, this.unreadInputGraceMs);
|
|
3908
|
+
timer.unref?.();
|
|
3909
|
+
state.unreadInputTimer = timer;
|
|
3910
|
+
}
|
|
3911
|
+
clearUnreadInputTimer(state) {
|
|
3912
|
+
if (state.unreadInputTimer)
|
|
3913
|
+
clearTimeout(state.unreadInputTimer);
|
|
3914
|
+
state.unreadInputTimer = undefined;
|
|
3915
|
+
}
|
|
3916
|
+
/** #505: the harness accepted input this session has not reported reading yet. */
|
|
3917
|
+
harnessOwesInput(state) {
|
|
3918
|
+
return !!state?.unreadInputIds?.size;
|
|
3296
3919
|
}
|
|
3297
3920
|
/** A callback from an old/replaced/disposed session carries no authority. */
|
|
3298
3921
|
handleAgentInputReady(runId, state, session) {
|
|
@@ -3328,7 +3951,7 @@ export class RunManager {
|
|
|
3328
3951
|
};
|
|
3329
3952
|
let acknowledgement;
|
|
3330
3953
|
try {
|
|
3331
|
-
acknowledgement = session.sendAgentMessage(content);
|
|
3954
|
+
acknowledgement = session.sendAgentMessage(content, inputIds);
|
|
3332
3955
|
}
|
|
3333
3956
|
catch (error) {
|
|
3334
3957
|
fail(error, false);
|
|
@@ -3361,8 +3984,25 @@ export class RunManager {
|
|
|
3361
3984
|
}
|
|
3362
3985
|
else {
|
|
3363
3986
|
const deliveredAt = new Date().toISOString();
|
|
3364
|
-
|
|
3365
|
-
|
|
3987
|
+
// An observable harness still owes a read: mark it so a restart replays it (#505).
|
|
3988
|
+
const awaiting = state.inputDelivery?.consumption === 'observable' ? { awaitingRead: true } : {};
|
|
3989
|
+
// Input the harness reported unread while this very POST was pending stays queued:
|
|
3990
|
+
// the flush after this acknowledgement resubmits it (#505 review).
|
|
3991
|
+
const accepted = inputIds.filter(id => !state.retiredInFlight?.has(id));
|
|
3992
|
+
for (const id of inputIds)
|
|
3993
|
+
state.retiredInFlight?.delete(id);
|
|
3994
|
+
this.store.commitAgentInputs(runId, queue.map(input => accepted.includes(input.id) && !input.deliveredAt
|
|
3995
|
+
? { ...input, deliveredAt, ...awaiting } : input));
|
|
3996
|
+
if (state.inputDelivery?.consumption === 'observable') {
|
|
3997
|
+
const early = accepted.filter(id => state.consumedBeforeAck?.has(id));
|
|
3998
|
+
for (const id of accepted)
|
|
3999
|
+
if (!early.includes(id))
|
|
4000
|
+
state.unreadInputIds?.add(id);
|
|
4001
|
+
for (const id of early)
|
|
4002
|
+
state.consumedBeforeAck?.delete(id);
|
|
4003
|
+
if (early.length)
|
|
4004
|
+
this.store.commitAgentInputsConsumed(runId, early, deliveredAt);
|
|
4005
|
+
}
|
|
3366
4006
|
}
|
|
3367
4007
|
if (inputIds.includes(this.workerWait(runId)?.wakeId ?? ''))
|
|
3368
4008
|
this.withdrawWorkerWait(runId);
|
|
@@ -3379,38 +4019,13 @@ export class RunManager {
|
|
|
3379
4019
|
state.agentInputFlight = undefined;
|
|
3380
4020
|
if (!authorized || !session.open || state.agentSessionError || state.agentInputError)
|
|
3381
4021
|
return;
|
|
4022
|
+
// A liveness resubmission gets its own quiet window, and so does input acknowledged
|
|
4023
|
+
// after its turn already ended: no later turn-end would arm one (#505 review).
|
|
4024
|
+
if (inputIds.some(id => state.unreadRetried?.has(id)) || state.atTurnBoundary === session)
|
|
4025
|
+
this.armUnreadInputTimer(runId, state);
|
|
3382
4026
|
if (this.flushAgentInputs(runId))
|
|
3383
4027
|
return;
|
|
3384
|
-
|
|
3385
|
-
return;
|
|
3386
|
-
if (state.doneAtBoundary === session) {
|
|
3387
|
-
if (this.deferParentCompletion(runId))
|
|
3388
|
-
return;
|
|
3389
|
-
this.store.appendEvent(runId, { type: 'lifecycle', message: 'goal achieved — session closed' });
|
|
3390
|
-
appendHandoffHeartbeat(this.dataDir, runId, 'turn complete — goal achieved, session closed');
|
|
3391
|
-
session.end();
|
|
3392
|
-
}
|
|
3393
|
-
else if (state.parkAfterAck?.session === session && !this.waiting.has(runId) && !this.monitoring.has(runId)) {
|
|
3394
|
-
// A wake turn can finish before its HTTP/RPC acknowledgement. Retiring
|
|
3395
|
-
// that wait must release its completed turn, not invent another turn.
|
|
3396
|
-
if ((state.parkAfterAck.monitoring || this.hasLiveWorkers(runId)) && !this.parentCompletionAttention(runId)) {
|
|
3397
|
-
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
3398
|
-
if (state.currentStepId)
|
|
3399
|
-
this.store.updateStep(runId, state.currentStepId, { status: 'running' });
|
|
3400
|
-
this.monitoring.add(runId);
|
|
3401
|
-
this.clearIdleTimer(state);
|
|
3402
|
-
this.armMonitoringWakeTimer(runId, state);
|
|
3403
|
-
}
|
|
3404
|
-
else {
|
|
3405
|
-
this.store.updateRun(runId, { status: 'waiting', activity: undefined });
|
|
3406
|
-
if (state.currentStepId)
|
|
3407
|
-
this.store.updateStep(runId, state.currentStepId, { status: 'waiting' });
|
|
3408
|
-
this.store.flush(); // Match ordinary turn-end durability before releasing capacity.
|
|
3409
|
-
this.armIdleTimer(runId, state);
|
|
3410
|
-
}
|
|
3411
|
-
this.waiting.add(runId);
|
|
3412
|
-
this.releaseSlot();
|
|
3413
|
-
}
|
|
4028
|
+
this.settleIdleBoundary(runId, state, session);
|
|
3414
4029
|
}).catch(error => {
|
|
3415
4030
|
// Bookkeeping failures cannot become unhandled promise rejections.
|
|
3416
4031
|
if (state.agentInputFlight === undefined)
|
|
@@ -3445,10 +4060,9 @@ export class RunManager {
|
|
|
3445
4060
|
return false;
|
|
3446
4061
|
if (state.pendingHumanAsk || this.hasUnansweredHumanAsk(runId))
|
|
3447
4062
|
return false;
|
|
3448
|
-
// The opening prompt already carries
|
|
3449
|
-
//
|
|
3450
|
-
|
|
3451
|
-
return false;
|
|
4063
|
+
// The opening prompt already carries its own input; later input steers the
|
|
4064
|
+
// opening turn instead of waiting for it to end (#505).
|
|
4065
|
+
const queue = (run.agentInputs ?? []).filter(input => input.id !== state.openingAgentInputId);
|
|
3452
4066
|
if (run.ciWait) {
|
|
3453
4067
|
if (!this.ciWakeAdmitted.has(runId) || state.atTurnBoundary !== state.session)
|
|
3454
4068
|
return false;
|
|
@@ -3456,7 +4070,7 @@ export class RunManager {
|
|
|
3456
4070
|
const input = run.agentInputs?.find(entry => entry.id === run.ciWait?.wakeId && !entry.deliveredAt);
|
|
3457
4071
|
return !!input && this.submitAgentInput(runId, state, [{ type: 'text', text: this.formatAgentInput(runId, input) }], [input.id]);
|
|
3458
4072
|
}
|
|
3459
|
-
const batch = agentInputBatch(
|
|
4073
|
+
const batch = agentInputBatch(queue, input => this.formatAgentInput(runId, input));
|
|
3460
4074
|
return !!batch && this.submitAgentInput(runId, state, [{ type: 'text', text: batch.text }], batch.inputs.map(input => input.id));
|
|
3461
4075
|
}
|
|
3462
4076
|
formatAgentInput(runId, input) {
|
|
@@ -3553,6 +4167,10 @@ export class RunManager {
|
|
|
3553
4167
|
return false;
|
|
3554
4168
|
if (!userAuthored && run.ciWait)
|
|
3555
4169
|
return false;
|
|
4170
|
+
if (userAuthored && this.routedAsk(runId)) {
|
|
4171
|
+
this.store.appendEvent(runId, { type: 'note', tone: 'warning', message: ROUTED_QUESTION_REFUSAL });
|
|
4172
|
+
return false;
|
|
4173
|
+
}
|
|
3556
4174
|
const text = content
|
|
3557
4175
|
.filter((b) => b.type === 'text')
|
|
3558
4176
|
.map((b) => b.text)
|
|
@@ -3584,7 +4202,11 @@ export class RunManager {
|
|
|
3584
4202
|
? [...expanded, pastedAttachmentsNote(persisted)]
|
|
3585
4203
|
: expanded.length ? expanded : [{ type: 'text', text: 'The user attached a document, but it could not be saved. Ask them to attach it again before discussing its contents.' }];
|
|
3586
4204
|
const answeringAskSeq = userAuthored ? this.pendingHumanAskSeq(runId) : undefined;
|
|
3587
|
-
|
|
4205
|
+
// #505: an answer that opens a new turn (a CEZ:ASK answer at a boundary) carries the
|
|
4206
|
+
// held conversation messages in the same submission, after the answer.
|
|
4207
|
+
const bundle = userAuthored && answeringAskSeq !== undefined && state.atTurnBoundary === state.session
|
|
4208
|
+
? this.pendingConversationBatch(runId, state) : undefined;
|
|
4209
|
+
const delivered = userAuthored ? state.session.sendMessage(bundle ? [...deliverable, { type: 'text', text: bundle.text }] : deliverable)
|
|
3588
4210
|
: this.submitAgentInput(runId, state, deliverable);
|
|
3589
4211
|
if (delivered) {
|
|
3590
4212
|
if (userAuthored) {
|
|
@@ -3594,6 +4216,12 @@ export class RunManager {
|
|
|
3594
4216
|
this.store.appendEvent(runId, { type: 'human-input-delivered', askSeq: answeringAskSeq });
|
|
3595
4217
|
}
|
|
3596
4218
|
state.pendingHumanAsk = this.hasPendingHumanAsk(runId);
|
|
4219
|
+
if (bundle)
|
|
4220
|
+
this.commitBundledDelivery(runId, state, bundle.inputs.map(input => input.id));
|
|
4221
|
+
// A native answer continues the turn, and a bundle can leave messages that did not
|
|
4222
|
+
// fit its batch: either way the rest steer right behind the answer.
|
|
4223
|
+
if (answeringAskSeq !== undefined && !state.pendingHumanAsk)
|
|
4224
|
+
this.flushAgentInputs(runId);
|
|
3597
4225
|
}
|
|
3598
4226
|
this.resumeParkedRun(runId, state);
|
|
3599
4227
|
}
|
|
@@ -3715,7 +4343,7 @@ export class RunManager {
|
|
|
3715
4343
|
return { ok: false, error: 'run is still stopping' };
|
|
3716
4344
|
if (this.historyDeletionPending(runId))
|
|
3717
4345
|
return { ok: false, error: 'Parent history deletion is pending; retry deletion' };
|
|
3718
|
-
if (run.delegation?.role === 'worker' && this.store.getRun(run.delegation.parentRunId)?.status === 'review') {
|
|
4346
|
+
if (run.delegation?.role === 'worker' && this.store.getRun(run.delegation.parentRunId)?.status === 'review' && !opts.answerInputId && !this.answersFallenBackAsk(runId, opts, deferForCapacity)) {
|
|
3719
4347
|
return { ok: false, error: 'Continue the reviewing parent before continuing its worker' };
|
|
3720
4348
|
}
|
|
3721
4349
|
// Held against the identity of the step this continuation extends (#452), which for a mixed
|
|
@@ -3750,8 +4378,10 @@ export class RunManager {
|
|
|
3750
4378
|
return { ok: false, error: 'worker cannot continue' };
|
|
3751
4379
|
if (this.executionBlockedByRootFinish(run))
|
|
3752
4380
|
return { ok: false, error: 'parent finish is pending' };
|
|
4381
|
+
if (pendingHumanAsk && !deferForCapacity && !opts.answerInputId && this.routedAsk(runId))
|
|
4382
|
+
return { ok: false, error: ROUTED_QUESTION_REFUSAL };
|
|
3753
4383
|
const answers = deferForCapacity ? [run.continuationMessage, ...(run.queuedMessages ?? [])] : [opts];
|
|
3754
|
-
if (pendingHumanAsk && (conversation || (deferForCapacity && run.continuationMessage?.origin !== 'human') ||
|
|
4384
|
+
if (pendingHumanAsk && !opts.answerInputId && (conversation || (deferForCapacity && run.continuationMessage?.origin !== 'human') ||
|
|
3755
4385
|
!answers.some(answer => answer?.text?.trim() || answer?.images?.length))) {
|
|
3756
4386
|
return { ok: false, error: 'pending human question requires an explicit answer' };
|
|
3757
4387
|
}
|
|
@@ -3899,7 +4529,7 @@ export class RunManager {
|
|
|
3899
4529
|
const continuationStep = { id: stepId, name: 'Continue', kind: 'agent', synthetic: 'continuation' };
|
|
3900
4530
|
if (run.delegation?.role !== 'worker')
|
|
3901
4531
|
this.store.addStep(runId, continuationStep);
|
|
3902
|
-
const recovered = deferForCapacity && !conversation ? run.continuationMessage : undefined;
|
|
4532
|
+
const recovered = deferForCapacity && !conversation && !opts.answerInputId ? run.continuationMessage : undefined;
|
|
3903
4533
|
const message = recovered ?? this.toQueuedMessage(runId, [
|
|
3904
4534
|
...(opts.text?.trim() ? [{ type: 'text', text: opts.text.trim() }] : []),
|
|
3905
4535
|
...(opts.images ?? []),
|
|
@@ -3909,8 +4539,9 @@ export class RunManager {
|
|
|
3909
4539
|
// and the original prompt/URLs must reach disk before the first startup await.
|
|
3910
4540
|
const acceptedPatch = {
|
|
3911
4541
|
continuationMessage: { ...message, id: stepId, text: prompt,
|
|
3912
|
-
...(conversation ? { agentInputId: conversation.input.id } : {}),
|
|
3913
|
-
|
|
4542
|
+
...(conversation ? { agentInputId: conversation.input.id } : opts.answerInputId ? { agentInputId: opts.answerInputId } : {}),
|
|
4543
|
+
// A parent's answer (#505) answers the ask exactly like a human's.
|
|
4544
|
+
origin: recovered ? recovered.origin : deferForCapacity && !opts.answerInputId ? 'lifecycle' : 'human' },
|
|
3914
4545
|
status: deferForCapacity ? 'queued' : 'running',
|
|
3915
4546
|
error: undefined,
|
|
3916
4547
|
finishedAt: undefined,
|
|
@@ -4087,12 +4718,15 @@ export class RunManager {
|
|
|
4087
4718
|
const openingInput = record?.agentInputs?.find(input => input.id === checkpoint?.agentInputId);
|
|
4088
4719
|
// Hydration has already included later human updates and task edits. Append
|
|
4089
4720
|
// the current obligation outcome without replacing that composed prompt.
|
|
4090
|
-
|
|
4721
|
+
// A parent's answer to a routed question (#505) is shown as its conversation reply, not
|
|
4722
|
+
// as a human message, and its own reply outcome is not news to the worker.
|
|
4723
|
+
const parentAnswer = openingInput?.conversation?.kind === 'reply' && this.routedAsk(runId) !== undefined;
|
|
4724
|
+
if (openingInput && !parentAnswer)
|
|
4091
4725
|
prompt += this.agentInputOutcomeText(openingInput);
|
|
4092
4726
|
const saved = this.readPersistedAttachments(runId, checkpoint?.id === stepId ? checkpoint.images ?? [] : [], 'continuation');
|
|
4093
4727
|
const openingImages = [...(images.length ? contentBlocksOf(images) : saved.blocks), ...persistedImages];
|
|
4094
4728
|
const attachments = [...saved.attachments, ...persistedAttachments];
|
|
4095
|
-
if (checkpoint?.origin !== 'lifecycle')
|
|
4729
|
+
if (checkpoint?.origin !== 'lifecycle' && !parentAnswer)
|
|
4096
4730
|
this.store.appendEvent(runId, {
|
|
4097
4731
|
type: 'user-message',
|
|
4098
4732
|
stepId,
|
|
@@ -4122,12 +4756,19 @@ export class RunManager {
|
|
|
4122
4756
|
return;
|
|
4123
4757
|
}
|
|
4124
4758
|
if (event.type === 'text') {
|
|
4759
|
+
this.clearUnreadInputTimer(state); // real content: the harness is working
|
|
4125
4760
|
turnText = appendTurnText(turnText, event.text);
|
|
4126
4761
|
const text = stripAskMarker(stripTaskMarkers(stripMonitoringMarker(stripDoneMarker(event.text))), false);
|
|
4127
4762
|
if (text)
|
|
4128
4763
|
this.store.appendEvent(runId, { type: 'text', text, stepId });
|
|
4129
4764
|
return;
|
|
4130
4765
|
}
|
|
4766
|
+
if (event.type === 'input-unconsumed') {
|
|
4767
|
+
this.retireUnreadInputs(runId, state, event.inputIds, true);
|
|
4768
|
+
return;
|
|
4769
|
+
}
|
|
4770
|
+
if (event.type === 'tool-call' || event.type === 'tool-result')
|
|
4771
|
+
this.clearUnreadInputTimer(state);
|
|
4131
4772
|
this.store.appendEvent(runId, { ...event, stepId });
|
|
4132
4773
|
if (event.type === 'error') {
|
|
4133
4774
|
// Preserve the initiating fault: interrupt may fail an in-flight HTTP request.
|
|
@@ -4151,6 +4792,8 @@ export class RunManager {
|
|
|
4151
4792
|
if (isClaudeScheduleWakeup(event, backend))
|
|
4152
4793
|
sawClaudeScheduleWakeup = true;
|
|
4153
4794
|
if (event.type === 'turn-end') {
|
|
4795
|
+
this.retireUnreadInputs(runId, state, event.unconsumedInputIds ?? [], false);
|
|
4796
|
+
this.readBundledInputs(runId, state);
|
|
4154
4797
|
state.workerWakeTurn = undefined;
|
|
4155
4798
|
state.ciWakeTurn = undefined;
|
|
4156
4799
|
state.atTurnBoundary = state.session;
|
|
@@ -4162,7 +4805,13 @@ export class RunManager {
|
|
|
4162
4805
|
const id = state.openingAgentInputId;
|
|
4163
4806
|
const deliveredAt = new Date().toISOString();
|
|
4164
4807
|
try {
|
|
4165
|
-
|
|
4808
|
+
// The successful opening turn handled its instruction: delivered and read (#505).
|
|
4809
|
+
this.store.commitAgentInputs(runId, (this.store.getRun(runId)?.agentInputs ?? []).map(input => {
|
|
4810
|
+
if (input.id !== id)
|
|
4811
|
+
return input;
|
|
4812
|
+
const { awaitingRead: _awaiting, ...handled } = input;
|
|
4813
|
+
return { ...handled, deliveredAt: input.deliveredAt ?? deliveredAt, consumedAt: input.consumedAt ?? deliveredAt };
|
|
4814
|
+
}), id);
|
|
4166
4815
|
}
|
|
4167
4816
|
catch (error) {
|
|
4168
4817
|
state.agentInputError = `agent input delivery checkpoint failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
@@ -4183,7 +4832,7 @@ export class RunManager {
|
|
|
4183
4832
|
this.reconcileWorkerWaits();
|
|
4184
4833
|
}
|
|
4185
4834
|
if (openingAnswerAskSeq !== undefined && state.session?.open) {
|
|
4186
|
-
this.store.appendEvent(runId, { type: 'human-input-delivered', askSeq: openingAnswerAskSeq });
|
|
4835
|
+
this.store.appendEvent(runId, { type: 'human-input-delivered', askSeq: openingAnswerAskSeq, ...(parentAnswer ? { source: 'parent' } : {}) });
|
|
4187
4836
|
openingAnswerAskSeq = undefined;
|
|
4188
4837
|
state.openingAnswerAskSeq = undefined;
|
|
4189
4838
|
}
|
|
@@ -4221,7 +4870,11 @@ export class RunManager {
|
|
|
4221
4870
|
const workerWaitParked = ciWaitParked || completionBlocked || (!!sessionOpen && this.parkWorkerWait(runId, state));
|
|
4222
4871
|
state.doneAtBoundary = done ? state.session : undefined;
|
|
4223
4872
|
state.parkAfterAck = sessionOpen && state.session ? { session: state.session, monitoring: !!monitoring } : undefined;
|
|
4224
|
-
|
|
4873
|
+
// #505: input the harness accepted but has not read yet runs as its next turn.
|
|
4874
|
+
// An in-flight submission is settled by its acknowledgement (parkAfterAck), not here.
|
|
4875
|
+
const agentInputDelivered = !ask && !workerWaitParked && (this.flushAgentInputs(runId) || this.harnessOwesInput(state) || !!state.agentInputFlight);
|
|
4876
|
+
if (!ask && !workerWaitParked)
|
|
4877
|
+
this.armUnreadInputTimer(runId, state);
|
|
4225
4878
|
if (state.agentInputError)
|
|
4226
4879
|
return;
|
|
4227
4880
|
if (done && !workerWaitParked && !state.pendingHumanAsk && !agentInputDelivered && !state.agentInputFlight && !this.hasQueuedAgentInputs(runId)) {
|
|
@@ -4257,8 +4910,10 @@ export class RunManager {
|
|
|
4257
4910
|
// `ScheduleWakeup` → non-attention `running`/`activity:'monitoring'`
|
|
4258
4911
|
// (#490, #46). Both free the slot; only genuine user waits keep the
|
|
4259
4912
|
// idle timer. The autonomous nudge above still wins over either.
|
|
4260
|
-
if (ask)
|
|
4913
|
+
if (ask) {
|
|
4261
4914
|
emitAskRequested(sink, ask);
|
|
4915
|
+
this.routeWorkerQuestion(runId);
|
|
4916
|
+
}
|
|
4262
4917
|
if (monitoring) {
|
|
4263
4918
|
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
4264
4919
|
this.store.updateStep(runId, stepId, { status: 'running' });
|
|
@@ -4395,11 +5050,14 @@ export class RunManager {
|
|
|
4395
5050
|
failBeforeSpawn(err.message);
|
|
4396
5051
|
return;
|
|
4397
5052
|
}
|
|
4398
|
-
|
|
5053
|
+
const settlePrelaunchCancellation = () => {
|
|
4399
5054
|
this.store.updateStep(runId, stepId, { status: 'cancelled' });
|
|
4400
5055
|
this.store.updateRun(runId, { status: 'cancelled', finishedAt: new Date().toISOString(), currentStepId: undefined });
|
|
4401
5056
|
this.clearAutosaveTimer(state);
|
|
4402
5057
|
this.dropActive(runId);
|
|
5058
|
+
};
|
|
5059
|
+
if (state.cancelled) {
|
|
5060
|
+
settlePrelaunchCancellation();
|
|
4403
5061
|
return;
|
|
4404
5062
|
}
|
|
4405
5063
|
if (this.holdContinuationForRootFinish(runId, { stepId, sessionId, backend, prompt, images }, state))
|
|
@@ -4424,12 +5082,36 @@ export class RunManager {
|
|
|
4424
5082
|
}
|
|
4425
5083
|
const delegation = this.provisionSession(runId, state);
|
|
4426
5084
|
const ciTools = await this.provisionCiSession(runId, state);
|
|
5085
|
+
// Stop/Finish can arrive while the local tool controller is starting.
|
|
5086
|
+
const launchRun = this.store.getRun(runId);
|
|
5087
|
+
if (state.cancelled || this.workerExecutionStopped(runId) || this.active.get(runId) !== state ||
|
|
5088
|
+
(launchRun && this.executionBlockedByRootFinish(launchRun))) {
|
|
5089
|
+
ciTools?.revoke();
|
|
5090
|
+
if (state.revokeCiTools === ciTools?.revoke)
|
|
5091
|
+
state.revokeCiTools = undefined;
|
|
5092
|
+
delegation?.revoke();
|
|
5093
|
+
if (state.revokeDelegation === delegation?.revoke)
|
|
5094
|
+
state.revokeDelegation = undefined;
|
|
5095
|
+
if (this.active.get(runId) !== state)
|
|
5096
|
+
return;
|
|
5097
|
+
if (state.cancelled || this.workerExecutionStopped(runId)) {
|
|
5098
|
+
state.cancelled = true;
|
|
5099
|
+
settlePrelaunchCancellation();
|
|
5100
|
+
}
|
|
5101
|
+
else {
|
|
5102
|
+
this.holdContinuationForRootFinish(runId, { stepId, sessionId, backend, prompt, images }, state);
|
|
5103
|
+
}
|
|
5104
|
+
return;
|
|
5105
|
+
}
|
|
5106
|
+
state.inputDelivery = inputDeliveryOf(runner);
|
|
5107
|
+
state.unreadInputIds = new Set();
|
|
5108
|
+
state.consumedBeforeAck = new Set();
|
|
4427
5109
|
try {
|
|
4428
5110
|
session = runner.startSession({
|
|
4429
5111
|
// The Continue step is a fresh agent session on the same run — the
|
|
4430
5112
|
// run's extra system prompt (already resolved at execute time and
|
|
4431
5113
|
// echoed on the record) rides along with the handoff contract.
|
|
4432
|
-
systemPrompt: composeSystemPrompt(record?.systemPrompt, delegation?.instructions, generateFollowups ? HANDOFF_INSTRUCTIONS : HANDOFF_ONLY_INSTRUCTIONS),
|
|
5114
|
+
systemPrompt: composeSystemPrompt(record?.systemPrompt, delegation?.instructions, artifactInstructions(continueProfile.env.CEZ_ARTIFACTS_DIR), generateFollowups ? HANDOFF_INSTRUCTIONS : HANDOFF_ONLY_INSTRUCTIONS),
|
|
4433
5115
|
userPrompt: attachments.length
|
|
4434
5116
|
? `${openingPrompt}\n\n${pastedAttachmentsText(attachments)}`
|
|
4435
5117
|
: openingPrompt,
|
|
@@ -4449,9 +5131,12 @@ export class RunManager {
|
|
|
4449
5131
|
}, onEvent, {
|
|
4450
5132
|
onUiEvent: (event) => {
|
|
4451
5133
|
completedAssistantText = appendCompletedAssistantText(completedAssistantText, event);
|
|
5134
|
+
if (event.type === 'turn.started')
|
|
5135
|
+
this.recordOpeningAccepted(runId, state);
|
|
4452
5136
|
this.handleRunnerUiEvent(runId, state, sink, event);
|
|
4453
5137
|
},
|
|
4454
5138
|
onAgentInputReady: () => this.handleAgentInputReady(runId, state, session),
|
|
5139
|
+
onAgentInputConsumed: (ids) => this.handleAgentInputConsumed(runId, state, session, ids),
|
|
4455
5140
|
});
|
|
4456
5141
|
}
|
|
4457
5142
|
catch (error) {
|
|
@@ -4473,7 +5158,7 @@ export class RunManager {
|
|
|
4473
5158
|
if (session.pid !== undefined)
|
|
4474
5159
|
registerRunProcess(runId, session.pid);
|
|
4475
5160
|
setupComplete = true;
|
|
4476
|
-
const result = await session.result.finally(() => state.agentInputFlight?.settled);
|
|
5161
|
+
const result = await session.result.finally(() => state.agentInputFlight?.settled).finally(() => this.requeueUnreadAtClose(runId, state, session));
|
|
4477
5162
|
if (this.preserveRunAfterDisposal(runId, state))
|
|
4478
5163
|
return;
|
|
4479
5164
|
this.persistWorkerAssistantResult(runId, stepId, result.text);
|
|
@@ -4862,6 +5547,11 @@ export class RunManager {
|
|
|
4862
5547
|
startImages = undefined;
|
|
4863
5548
|
startAttachments = [];
|
|
4864
5549
|
checkFailure = null;
|
|
5550
|
+
// Stale setup cannot finalize a replacement owner. Disposal also clears
|
|
5551
|
+
// active, but must still honor a previously accepted Stop/Finish intent.
|
|
5552
|
+
if (this.active.get(runId) !== state &&
|
|
5553
|
+
!(this.disposed && (state.cancelled || state.finishRequested)))
|
|
5554
|
+
return;
|
|
4865
5555
|
if (state.cancelled || this.preserveRunAfterDisposal(runId, state))
|
|
4866
5556
|
break;
|
|
4867
5557
|
const current = this.store.getRun(runId);
|
|
@@ -5052,12 +5742,19 @@ export class RunManager {
|
|
|
5052
5742
|
return;
|
|
5053
5743
|
}
|
|
5054
5744
|
if (event.type === 'text') {
|
|
5745
|
+
this.clearUnreadInputTimer(state); // real content: the harness is working
|
|
5055
5746
|
turnText = appendTurnText(turnText, event.text);
|
|
5056
5747
|
const text = stripAskMarker(stripTaskMarkers(stripMonitoringMarker(stripDoneMarker(event.text))), false);
|
|
5057
5748
|
if (text)
|
|
5058
5749
|
emit({ type: 'text', text, stepId: step.id });
|
|
5059
5750
|
return;
|
|
5060
5751
|
}
|
|
5752
|
+
if (event.type === 'input-unconsumed') {
|
|
5753
|
+
this.retireUnreadInputs(runId, state, event.inputIds, true);
|
|
5754
|
+
return;
|
|
5755
|
+
}
|
|
5756
|
+
if (event.type === 'tool-call' || event.type === 'tool-result')
|
|
5757
|
+
this.clearUnreadInputTimer(state);
|
|
5061
5758
|
emit({ ...event, stepId: step.id });
|
|
5062
5759
|
if (event.type === 'error') {
|
|
5063
5760
|
// Preserve the initiating fault: interrupt may fail an in-flight HTTP request.
|
|
@@ -5082,6 +5779,8 @@ export class RunManager {
|
|
|
5082
5779
|
if (isClaudeScheduleWakeup(event, backend))
|
|
5083
5780
|
sawClaudeScheduleWakeup = true;
|
|
5084
5781
|
if (event.type === 'turn-end') {
|
|
5782
|
+
this.retireUnreadInputs(runId, state, event.unconsumedInputIds ?? [], false);
|
|
5783
|
+
this.readBundledInputs(runId, state);
|
|
5085
5784
|
state.workerWakeTurn = undefined;
|
|
5086
5785
|
state.ciWakeTurn = undefined;
|
|
5087
5786
|
state.atTurnBoundary = state.session;
|
|
@@ -5117,7 +5816,11 @@ export class RunManager {
|
|
|
5117
5816
|
const workerWaitParked = ciWaitParked || completionBlocked || (!!sessionOpen && this.parkWorkerWait(runId, state));
|
|
5118
5817
|
state.doneAtBoundary = done ? state.session : undefined;
|
|
5119
5818
|
state.parkAfterAck = interactive && sessionOpen && state.session ? { session: state.session, monitoring: !!monitoring } : undefined;
|
|
5120
|
-
|
|
5819
|
+
// #505: input the harness accepted but has not read yet runs as its next turn.
|
|
5820
|
+
// An in-flight submission is settled by its acknowledgement (parkAfterAck), not here.
|
|
5821
|
+
const agentInputDelivered = !ask && !workerWaitParked && (this.flushAgentInputs(runId) || this.harnessOwesInput(state) || !!state.agentInputFlight);
|
|
5822
|
+
if (!ask && !workerWaitParked)
|
|
5823
|
+
this.armUnreadInputTimer(runId, state);
|
|
5121
5824
|
if (state.agentInputError)
|
|
5122
5825
|
return;
|
|
5123
5826
|
if (done && !workerWaitParked && !state.pendingHumanAsk && !agentInputDelivered && !state.agentInputFlight && !this.hasQueuedAgentInputs(runId)) {
|
|
@@ -5136,8 +5839,10 @@ export class RunManager {
|
|
|
5136
5839
|
// still working through `CEZ:MONITORING` or Claude's native
|
|
5137
5840
|
// `ScheduleWakeup`. Monitoring parks as `running` with no user-wait
|
|
5138
5841
|
// idle timer, so the cockpit stays non-attention (#490, #46).
|
|
5139
|
-
if (ask)
|
|
5842
|
+
if (ask) {
|
|
5140
5843
|
emitAskRequested(sink, ask);
|
|
5844
|
+
this.routeWorkerQuestion(runId);
|
|
5845
|
+
}
|
|
5141
5846
|
if (monitoring) {
|
|
5142
5847
|
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
5143
5848
|
this.store.updateStep(runId, step.id, { status: 'running' });
|
|
@@ -5232,19 +5937,38 @@ export class RunManager {
|
|
|
5232
5937
|
const runner = createRunner(stepBackend);
|
|
5233
5938
|
state.agentSessionError = undefined;
|
|
5234
5939
|
let session;
|
|
5940
|
+
const shouldAutoEnd = () => this.active.get(runId) !== state || state.session !== session ||
|
|
5941
|
+
(!this.workerWait(runId) && !this.store.getRun(runId)?.ciWait && !this.parentCompletionPending(runId) &&
|
|
5942
|
+
state.workerWakeTurn !== session && state.ciWakeTurn !== session && !state.agentInputFlight &&
|
|
5943
|
+
!this.hasQueuedAgentInputs(runId));
|
|
5235
5944
|
state.currentStepId = step.id;
|
|
5236
5945
|
this.beginUsageInvocation(runId, state, step.id);
|
|
5237
5946
|
const delegation = this.provisionSession(runId, state);
|
|
5238
5947
|
const ciTools = await this.provisionCiSession(runId, state);
|
|
5239
|
-
|
|
5240
|
-
|
|
5948
|
+
// Revalidate after provisioning, then let execute settle cancellation or
|
|
5949
|
+
// requeue a Finish-held step through its existing no-session paths.
|
|
5950
|
+
const launchRun = this.store.getRun(runId);
|
|
5951
|
+
if (state.cancelled || this.workerExecutionStopped(runId) || this.active.get(runId) !== state ||
|
|
5952
|
+
(launchRun && this.executionBlockedByRootFinish(launchRun))) {
|
|
5953
|
+
ciTools?.revoke();
|
|
5954
|
+
if (state.revokeCiTools === ciTools?.revoke)
|
|
5955
|
+
state.revokeCiTools = undefined;
|
|
5956
|
+
delegation?.revoke();
|
|
5957
|
+
if (state.revokeDelegation === delegation?.revoke)
|
|
5958
|
+
state.revokeDelegation = undefined;
|
|
5959
|
+
if (this.workerExecutionStopped(runId))
|
|
5241
5960
|
state.cancelled = true;
|
|
5242
|
-
|
|
5243
|
-
|
|
5961
|
+
state.currentStepId = undefined;
|
|
5962
|
+
return null;
|
|
5963
|
+
}
|
|
5964
|
+
state.inputDelivery = inputDeliveryOf(runner);
|
|
5965
|
+
state.unreadInputIds = new Set();
|
|
5966
|
+
state.consumedBeforeAck = new Set();
|
|
5967
|
+
try {
|
|
5244
5968
|
session = runner.startSession({
|
|
5245
5969
|
// Skill body, then the run's extra prompt (POST override or config
|
|
5246
5970
|
// default), then the handoff/todos contract — every agent step.
|
|
5247
|
-
systemPrompt: composeSystemPrompt(systemPrompt, extraSystemPrompt, delegation?.instructions, followupsEnabled() && input.generateFollowups !== false
|
|
5971
|
+
systemPrompt: composeSystemPrompt(systemPrompt, extraSystemPrompt, delegation?.instructions, artifactInstructions(stepProfile.env.CEZ_ARTIFACTS_DIR), followupsEnabled() && input.generateFollowups !== false
|
|
5248
5972
|
? HANDOFF_INSTRUCTIONS
|
|
5249
5973
|
: HANDOFF_ONLY_INSTRUCTIONS),
|
|
5250
5974
|
userPrompt,
|
|
@@ -5264,13 +5988,14 @@ export class RunManager {
|
|
|
5264
5988
|
timeoutMs: interactive ? 0 : undefined,
|
|
5265
5989
|
}, onEvent, {
|
|
5266
5990
|
autoEndAfterFirstTurn: !interactive,
|
|
5267
|
-
|
|
5268
|
-
|
|
5991
|
+
// Unread inbox claims must survive this window so release/expiry can replay them.
|
|
5992
|
+
shouldAutoEnd,
|
|
5269
5993
|
onUiEvent: (event) => {
|
|
5270
5994
|
completedAssistantText = appendCompletedAssistantText(completedAssistantText, event);
|
|
5271
5995
|
this.handleRunnerUiEvent(runId, state, sink, event);
|
|
5272
5996
|
},
|
|
5273
5997
|
onAgentInputReady: () => this.handleAgentInputReady(runId, state, session),
|
|
5998
|
+
onAgentInputConsumed: (ids) => this.handleAgentInputConsumed(runId, state, session, ids),
|
|
5274
5999
|
});
|
|
5275
6000
|
}
|
|
5276
6001
|
catch (err) {
|
|
@@ -5283,6 +6008,14 @@ export class RunManager {
|
|
|
5283
6008
|
}
|
|
5284
6009
|
const sessionClosed = this.trackWorkerSessionResult(runId, session);
|
|
5285
6010
|
state.session = session;
|
|
6011
|
+
state.resumeInboxAutoEnd = interactive ? undefined : () => {
|
|
6012
|
+
// ACK may consume the last input after the runner's one-shot close timer fired.
|
|
6013
|
+
// Release/expiry instead submit a provider turn through normal admission.
|
|
6014
|
+
if (this.active.get(runId) === state && state.session === session && session?.open &&
|
|
6015
|
+
state.atTurnBoundary === session && !state.cancelled && !state.pendingHumanAsk &&
|
|
6016
|
+
!this.hasPendingHumanAsk(runId) && shouldAutoEnd())
|
|
6017
|
+
session.end();
|
|
6018
|
+
};
|
|
5286
6019
|
state.sessionEverOpened = true;
|
|
5287
6020
|
state.currentStepId = step.id;
|
|
5288
6021
|
state.interrupt = () => session?.interrupt();
|
|
@@ -5293,7 +6026,7 @@ export class RunManager {
|
|
|
5293
6026
|
if (session.pid !== undefined)
|
|
5294
6027
|
registerRunProcess(runId, session.pid);
|
|
5295
6028
|
setupComplete = true;
|
|
5296
|
-
const result = await session.result.finally(() => state.agentInputFlight?.settled);
|
|
6029
|
+
const result = await session.result.finally(() => state.agentInputFlight?.settled).finally(() => this.requeueUnreadAtClose(runId, state, session));
|
|
5297
6030
|
if (this.isDisposedDelegatedRun(runId))
|
|
5298
6031
|
return null;
|
|
5299
6032
|
this.persistWorkerAssistantResult(runId, step.id, result.text);
|
|
@@ -5341,6 +6074,7 @@ export class RunManager {
|
|
|
5341
6074
|
if (at >= 0)
|
|
5342
6075
|
this.queue.splice(at, 1);
|
|
5343
6076
|
}
|
|
6077
|
+
state.resumeInboxAutoEnd = undefined;
|
|
5344
6078
|
state.workerWakeTurn = undefined;
|
|
5345
6079
|
state.ciWakeTurn = undefined;
|
|
5346
6080
|
state.atTurnBoundary = undefined;
|
|
@@ -5418,6 +6152,7 @@ export class RunManager {
|
|
|
5418
6152
|
if (state.currentStepId)
|
|
5419
6153
|
this.store.updateStep(runId, state.currentStepId, { status: 'waiting' });
|
|
5420
6154
|
this.releaseSlot();
|
|
6155
|
+
this.routeWorkerQuestion(runId);
|
|
5421
6156
|
return;
|
|
5422
6157
|
}
|
|
5423
6158
|
// Pi can resume autonomously when an async subagent completes — the mapper
|