@wjarka/cezarion 0.14.8 → 0.14.9-dev.1009
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +131 -2
- package/dist/application-update/bin-recovery.d.ts +5 -0
- package/dist/application-update/bin-recovery.js +96 -0
- package/dist/application-update/bin-recovery.js.map +1 -0
- package/dist/application-update/discovery.d.ts +22 -0
- package/dist/application-update/discovery.js +91 -0
- package/dist/application-update/discovery.js.map +1 -0
- package/dist/application-update/helper.d.ts +57 -0
- package/dist/application-update/helper.js +374 -0
- package/dist/application-update/helper.js.map +1 -0
- package/dist/application-update/launcher.d.ts +8 -0
- package/dist/application-update/launcher.js +53 -0
- package/dist/application-update/launcher.js.map +1 -0
- package/dist/application-update/lock.d.ts +4 -0
- package/dist/application-update/lock.js +70 -0
- package/dist/application-update/lock.js.map +1 -0
- package/dist/application-update/npm-process.d.ts +17 -0
- package/dist/application-update/npm-process.js +133 -0
- package/dist/application-update/npm-process.js.map +1 -0
- package/dist/application-update/service.d.ts +69 -0
- package/dist/application-update/service.js +456 -0
- package/dist/application-update/service.js.map +1 -0
- package/dist/artifacts/cli.d.ts +1 -0
- package/dist/artifacts/cli.js +32 -0
- package/dist/artifacts/cli.js.map +1 -0
- package/dist/artifacts/lifecycle.d.ts +4 -0
- package/dist/artifacts/lifecycle.js +42 -0
- package/dist/artifacts/lifecycle.js.map +1 -0
- package/dist/artifacts/resolve.d.ts +18 -0
- package/dist/artifacts/resolve.js +102 -0
- package/dist/artifacts/resolve.js.map +1 -0
- package/dist/artifacts/store.d.ts +8 -0
- package/dist/artifacts/store.js +323 -0
- package/dist/artifacts/store.js.map +1 -0
- package/dist/automations/scheduler.d.ts +6 -0
- package/dist/automations/scheduler.js +45 -17
- package/dist/automations/scheduler.js.map +1 -1
- package/dist/automations/store.d.ts +12 -4
- package/dist/automations/store.js +199 -22
- package/dist/automations/store.js.map +1 -1
- package/dist/automations/types.d.ts +2 -0
- package/dist/automations/types.js +1 -0
- package/dist/automations/types.js.map +1 -1
- package/dist/ci-wait/mcp.d.ts +3 -0
- package/dist/ci-wait/mcp.js +14 -3
- package/dist/ci-wait/mcp.js.map +1 -1
- package/dist/contract/application-update.d.ts +32 -0
- package/dist/contract/artifacts.d.ts +83 -0
- package/dist/contract/ask-schema.d.ts +32 -0
- package/dist/contract/ask.d.ts +2 -32
- package/dist/contract/automations.d.ts +6 -0
- package/dist/contract/conversations.d.ts +165 -0
- package/dist/contract/delegation.d.ts +68 -0
- package/dist/contract/discovery.d.ts +134 -0
- package/dist/contract/events.d.ts +49 -0
- package/dist/contract/health.d.ts +12 -0
- package/dist/contract/index.d.ts +3 -0
- package/dist/contract/index.js +1654 -1454
- package/dist/contract/projects.d.ts +40 -0
- package/dist/contract/runs.d.ts +186 -0
- package/dist/contract/skills.d.ts +33 -0
- package/dist/core/agent-runner.d.ts +33 -4
- package/dist/core/agent-runner.js +5 -0
- package/dist/core/agent-runner.js.map +1 -1
- package/dist/core/claude-cli-runner.d.ts +6 -12
- package/dist/core/claude-cli-runner.js +90 -20
- package/dist/core/claude-cli-runner.js.map +1 -1
- package/dist/core/codex-app-server-runner.d.ts +2 -1
- package/dist/core/codex-app-server-runner.js +327 -92
- package/dist/core/codex-app-server-runner.js.map +1 -1
- package/dist/core/codex-app-server-transport.d.ts +13 -1
- package/dist/core/codex-app-server-transport.js +58 -17
- package/dist/core/codex-app-server-transport.js.map +1 -1
- package/dist/core/cursor-acp-runner.d.ts +6 -1
- package/dist/core/cursor-acp-runner.js +162 -26
- package/dist/core/cursor-acp-runner.js.map +1 -1
- package/dist/core/cursor-provider-error.d.ts +3 -0
- package/dist/core/cursor-provider-error.js +16 -1
- package/dist/core/cursor-provider-error.js.map +1 -1
- package/dist/core/input-submissions.d.ts +14 -0
- package/dist/core/input-submissions.js +26 -0
- package/dist/core/input-submissions.js.map +1 -0
- package/dist/core/opencode-server-runner.d.ts +4 -1
- package/dist/core/opencode-server-runner.js +156 -26
- package/dist/core/opencode-server-runner.js.map +1 -1
- package/dist/core/pi-runner.d.ts +2 -1
- package/dist/core/pi-runner.js +88 -10
- package/dist/core/pi-runner.js.map +1 -1
- package/dist/core/provider-auth.d.ts +99 -1
- package/dist/core/provider-auth.js +203 -2
- package/dist/core/provider-auth.js.map +1 -1
- package/dist/delegation/cli.d.ts +1 -0
- package/dist/delegation/cli.js +103 -7
- package/dist/delegation/cli.js.map +1 -1
- package/dist/delegation/conversations.js +13 -5
- package/dist/delegation/conversations.js.map +1 -1
- package/dist/delegation/input.d.ts +2 -1
- package/dist/delegation/input.js +6 -1
- package/dist/delegation/input.js.map +1 -1
- package/dist/delegation/policy.d.ts +3 -2
- package/dist/delegation/policy.js +3 -1
- package/dist/delegation/policy.js.map +1 -1
- package/dist/delegation/process-liveness.d.ts +66 -0
- package/dist/delegation/process-liveness.js +215 -0
- package/dist/delegation/process-liveness.js.map +1 -0
- package/dist/delegation/provision.d.ts +1 -1
- package/dist/delegation/provision.js +10 -5
- package/dist/delegation/provision.js.map +1 -1
- package/dist/delegation/questions.d.ts +19 -0
- package/dist/delegation/questions.js +45 -0
- package/dist/delegation/questions.js.map +1 -0
- package/dist/delegation/results.d.ts +1 -1
- package/dist/delegation/results.js +33 -19
- package/dist/delegation/results.js.map +1 -1
- package/dist/delegation/routes.d.ts +185 -11
- package/dist/delegation/routes.js +6 -2
- package/dist/delegation/routes.js.map +1 -1
- package/dist/delegation/service.d.ts +23 -5
- package/dist/delegation/service.js +240 -17
- package/dist/delegation/service.js.map +1 -1
- package/dist/delegation/transport.d.ts +178 -15
- package/dist/delegation/workspace.d.ts +1 -1
- package/dist/delegation/workspace.js +13 -6
- package/dist/delegation/workspace.js.map +1 -1
- package/dist/discovery/catalog.d.ts +14 -0
- package/dist/discovery/catalog.js +31 -0
- package/dist/discovery/catalog.js.map +1 -0
- package/dist/discovery/cli.d.ts +8 -0
- package/dist/discovery/cli.js +104 -0
- package/dist/discovery/cli.js.map +1 -0
- package/dist/git-worktree.d.ts +3 -1
- package/dist/git-worktree.js +26 -4
- package/dist/git-worktree.js.map +1 -1
- package/dist/index.js +93 -26
- package/dist/index.js.map +1 -1
- package/dist/open-url.d.ts +2 -0
- package/dist/open-url.js +20 -0
- package/dist/open-url.js.map +1 -0
- package/dist/release/snapshot.d.ts +3 -3
- package/dist/release/snapshot.js +6 -6
- package/dist/release/snapshot.js.map +1 -1
- package/dist/runs/agent-tmpdir.d.ts +2 -0
- package/dist/runs/agent-tmpdir.js +5 -2
- package/dist/runs/agent-tmpdir.js.map +1 -1
- package/dist/runs/client-request.d.ts +11 -0
- package/dist/runs/client-request.js +26 -0
- package/dist/runs/client-request.js.map +1 -0
- package/dist/runs/delegation-state.d.ts +17 -0
- package/dist/runs/event-history.js +6 -0
- package/dist/runs/event-history.js.map +1 -1
- package/dist/runs/retention.d.ts +7 -2
- package/dist/runs/retention.js +39 -7
- package/dist/runs/retention.js.map +1 -1
- package/dist/runs/store.d.ts +100 -6
- package/dist/runs/store.js +245 -2
- package/dist/runs/store.js.map +1 -1
- package/dist/runs/webhook.d.ts +125 -0
- package/dist/runs/webhook.js +256 -0
- package/dist/runs/webhook.js.map +1 -0
- package/dist/server/project-context.d.ts +2 -0
- package/dist/server/project-context.js +1 -0
- package/dist/server/project-context.js.map +1 -1
- package/dist/server/provider-auth-runtime.d.ts +30 -4
- package/dist/server/provider-auth-runtime.js +71 -6
- package/dist/server/provider-auth-runtime.js.map +1 -1
- package/dist/server/server.d.ts +8226 -4860
- package/dist/server/server.js +311 -15
- package/dist/server/server.js.map +1 -1
- package/dist/server-install/platforms/ubuntu-vps.d.ts +5 -1
- package/dist/server-install/platforms/ubuntu-vps.js +160 -18
- package/dist/server-install/platforms/ubuntu-vps.js.map +1 -1
- package/dist/server-install/steps.d.ts +4 -2
- package/dist/server-install/steps.js +5 -4
- package/dist/server-install/steps.js.map +1 -1
- package/dist/skills-remote.d.ts +30 -5
- package/dist/skills-remote.js +122 -32
- package/dist/skills-remote.js.map +1 -1
- package/dist/skills.js +15 -6
- package/dist/skills.js.map +1 -1
- package/dist/task-cli/cli.d.ts +38 -0
- package/dist/task-cli/cli.js +580 -0
- package/dist/task-cli/cli.js.map +1 -0
- package/dist/task-cli/discovery.d.ts +22 -0
- package/dist/task-cli/discovery.js +96 -0
- package/dist/task-cli/discovery.js.map +1 -0
- package/dist/task-cli/http.d.ts +46 -0
- package/dist/task-cli/http.js +101 -0
- package/dist/task-cli/http.js.map +1 -0
- package/dist/task-cli/projections.d.ts +40 -0
- package/dist/task-cli/projections.js +45 -0
- package/dist/task-cli/projections.js.map +1 -0
- package/dist/task-cli/watch.d.ts +78 -0
- package/dist/task-cli/watch.js +336 -0
- package/dist/task-cli/watch.js.map +1 -0
- package/dist/workflows/run.d.ts +160 -2
- package/dist/workflows/run.js +1141 -108
- package/dist/workflows/run.js.map +1 -1
- package/dist/workspace/agent-profiles.d.ts +22 -0
- package/dist/workspace/agent-profiles.js +15 -0
- package/dist/workspace/agent-profiles.js.map +1 -1
- package/dist/workspace/config.d.ts +9 -0
- package/dist/workspace/config.js +13 -1
- package/dist/workspace/config.js.map +1 -1
- package/dist/workspace/projects-cli.js +1 -1
- package/dist/workspace/projects.d.ts +24 -1
- package/dist/workspace/projects.js +14 -1
- package/dist/workspace/projects.js.map +1 -1
- package/package.json +5 -5
- package/scripts/mock-claude.mjs +64 -2
- package/scripts/mock-codex-app-server.mjs +124 -1
- package/scripts/mock-cursor-acp.mjs +54 -3
- package/scripts/mock-cursor-hang-stdin-late.mjs +35 -0
- package/scripts/mock-cursor-hang-stdin.mjs +30 -0
- package/scripts/mock-opencode-serve.mjs +102 -2
- package/scripts/mock-pi-rpc.mjs +82 -3
- package/web/dist/assets/agent-accounts-CerclGiO.js +1 -0
- package/web/dist/assets/centered-state-DH8qtCz-.js +43 -0
- package/web/dist/assets/collapsible-De2G0xqQ.js +1 -0
- package/web/dist/assets/commit-list-s6JPY3qv.js +1 -0
- package/web/dist/assets/compare-variants-BQu_t-_g.js +1 -0
- package/web/dist/assets/{diff-tRE59UNb2.js → diff-8M5HlDxq2.js} +2 -2
- package/web/dist/assets/diff-controls-B6ok0TtQ.js +1 -0
- package/web/dist/assets/{diff-stat-Cgnl_hYF.js → diff-stat-D1AODN2a.js} +1 -1
- package/web/dist/assets/{diff-view-DN1ort7Q.js → diff-view-Bs5Gh7UF.js} +2 -2
- package/web/dist/assets/dropdown-menu-7KGQMoFj.js +1 -0
- package/web/dist/assets/git-toolbar-DqbapMXr.js +1 -0
- package/web/dist/assets/github-DCkk6sBm.js +1 -0
- package/web/dist/assets/{image-preview-C2JGcBlC.js → image-preview-jzHoEyLM.js} +1 -1
- package/web/dist/assets/index-Bnva-ttR.js +7 -0
- package/web/dist/assets/index-D5NkwqY7.css +1 -0
- package/web/dist/assets/markdown-DvbymqXz.js +2 -0
- package/web/dist/assets/pill-BL6ugwXo.js +1 -0
- package/web/dist/assets/project-router-ChC5Rb-Y.js +1 -0
- package/web/dist/assets/prompt-templates-j8qlK7c-.js +15 -0
- package/web/dist/assets/repo-git-_h1NjOVt.js +1 -0
- package/web/dist/assets/{run-diff-CyG0Nciv.js → run-diff-BuATgZpx.js} +2 -2
- package/web/dist/assets/run-header-DuAWswYM.js +1 -0
- package/web/dist/assets/skills-C-fq5BrZ.js +1 -0
- package/web/dist/assets/{tab-link-BWNp3mLY.js → tab-link-BmGKj9Rt.js} +1 -1
- package/web/dist/assets/task-changes-CHa6JWOS.js +1 -0
- package/web/dist/assets/task-commits-BqCqyWm1.js +1 -0
- package/web/dist/assets/task-files-BSA0l-YP.js +2 -0
- package/web/dist/assets/task-thread-Bw_nQuTx.css +1 -0
- package/web/dist/assets/task-thread-Q7Nj33DH.js +9 -0
- package/web/dist/assets/textarea-ByZ-txS7.js +1 -0
- package/web/dist/assets/thread-loading-DHT9p5o5.js +1 -0
- package/web/dist/assets/token-metrics-k4XoovDS.js +1 -0
- package/web/dist/assets/utils-BTFBhZ55.js +64 -0
- package/web/dist/assets/workflows-HJOd3lVi.js +11 -0
- package/web/dist/assets/{zoomable-image-Z3JDnm0b.js → zoomable-image-Ba6y6Wyl.js} +1 -1
- package/web/dist/index.html +14 -14
- package/web/dist/assets/agent-accounts-BZm_2WAv.js +0 -1
- package/web/dist/assets/alert-dialog-B422RI-k.js +0 -43
- package/web/dist/assets/collapsible-Cy_Fmkrd.js +0 -1
- package/web/dist/assets/commit-list-Dcnk7TnI.js +0 -1
- package/web/dist/assets/compare-variants-Ciqwvd_f.js +0 -1
- package/web/dist/assets/diff-controls-BGrXGi7O.js +0 -1
- package/web/dist/assets/dropdown-menu-dwDGT8MM.js +0 -1
- package/web/dist/assets/git-toolbar-UiHnGvE0.js +0 -1
- package/web/dist/assets/github-D-MrkbJ7.js +0 -1
- package/web/dist/assets/ibm-plex-mono-regular-BF3vfe7_.ttf +0 -0
- package/web/dist/assets/index-BszfYvBq.js +0 -7
- package/web/dist/assets/index-CWh4pKP3.css +0 -1
- package/web/dist/assets/markdown-C_IcKVQd.js +0 -2
- package/web/dist/assets/pill-BsbO6G7y.js +0 -1
- package/web/dist/assets/project-router-DbLkll25.js +0 -1
- package/web/dist/assets/prompt-templates-BlTGJB5W.js +0 -15
- package/web/dist/assets/repo-git-Bm3s6Why.js +0 -1
- package/web/dist/assets/run-header-IFyIGCey.js +0 -1
- package/web/dist/assets/skills-C-C4DVFY.js +0 -1
- package/web/dist/assets/task-changes-D2PHtWUW.js +0 -1
- package/web/dist/assets/task-commits-N5u9U0_R.js +0 -1
- package/web/dist/assets/task-files-BS7c7CFm.js +0 -2
- package/web/dist/assets/task-thread-BQFdz1dJ.css +0 -1
- package/web/dist/assets/task-thread-DGM2dZV-.js +0 -9
- package/web/dist/assets/textarea-FjaU370i.js +0 -1
- package/web/dist/assets/thread-loading-77sJ8Yu_.js +0 -1
- package/web/dist/assets/token-metrics-Clxh03bE.js +0 -1
- package/web/dist/assets/utils-PYYcpCFZ.js +0 -64
- package/web/dist/assets/workflows-Czl0ZfXH.js +0 -11
package/dist/workflows/run.js
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ciWaitRequestSchema, ciWaitResultSchema } from '../contract/index.js';
|
|
2
2
|
import { acquireCiResources } from '../ci-wait/resources.js';
|
|
3
|
+
import { artifactInstructions, provisionArtifactDirectory } from '../artifacts/lifecycle.js';
|
|
3
4
|
import { workerContextHash, workerWorkflowHash, workerStepIdentity, captureWorkerAccount, boundWorkerAccountEnv, WorkerIdentityError } from '../delegation/execution-identity.js';
|
|
4
5
|
import { buildChildEnv } from '../core/agent-env.js';
|
|
5
6
|
import { randomUUID } from 'node:crypto';
|
|
@@ -8,6 +9,7 @@ import { existsSync, mkdirSync, readFileSync, readdirSync, rmSync, writeFileSync
|
|
|
8
9
|
import { dirname, join } from 'node:path';
|
|
9
10
|
import { parseAskMarkerResult, stripAskMarker, } from '../core/ask.js';
|
|
10
11
|
import { hasRegisteredRunProcess, onUsage, registerRunProcess, unregisterRunProcess } from '../core/process-usage.js';
|
|
12
|
+
import { inspectGeneration, isCurrentProcess, processStartToken, recordedProcessLive } from '../delegation/process-liveness.js';
|
|
11
13
|
import { parseUsageLimit } from '../core/usage-limit.js';
|
|
12
14
|
import { createRunner } from '../core/runner-factory.js';
|
|
13
15
|
import { modelConflictsWithRunner } from '../core/model-presets.js';
|
|
@@ -19,7 +21,8 @@ import { todosPath } from '../todos.js';
|
|
|
19
21
|
// Contract VALUES, like `workspaceUiStateSchema` in workspace/migrations.ts: the attachment
|
|
20
22
|
// vocabulary the routes validate with is the same one the engine stores and re-reads by, so the
|
|
21
23
|
// wire and the disk can never disagree about what counts as an image (#950).
|
|
22
|
-
import { requestWaitRequestSchema, workerWaitRequestSchema, attachmentExtension, delegationStateSchema, pendingHumanAsk, isImageAttachmentName, isImageMediaType, } from '../contract/index.js';
|
|
24
|
+
import { requestWaitRequestSchema, workerWaitRequestSchema, attachmentExtension, delegationStateSchema, pendingHumanAsk, askRequestSchema, isImageAttachmentName, isImageMediaType, } from '../contract/index.js';
|
|
25
|
+
import { inputDeliveryOf } from '../core/agent-runner.js';
|
|
23
26
|
import { discoverSkills } from '../skills.js';
|
|
24
27
|
import { materializeSkillDir } from '../skills-remote.js';
|
|
25
28
|
import { seedAgentConfigLocalLayer } from '../agent-config/seed.js';
|
|
@@ -29,16 +32,17 @@ import { autosaveCommit, createWorktree, resolveBaseRef, worktreeDiff, worktreeS
|
|
|
29
32
|
import { getHeadCommit, getRepoInfo } from '../server/git.js';
|
|
30
33
|
import { ensureOwnedWorkspace, verifyOwnedWorkspace } from '../delegation/workspace.js';
|
|
31
34
|
import { verifyWorkerContext } from '../delegation/context.js';
|
|
32
|
-
import { enqueueAgentInput, agentInputBatch } from '../delegation/input.js';
|
|
35
|
+
import { enqueueAgentInput, agentInputBatch, hasLiveInboxClaim } from '../delegation/input.js';
|
|
33
36
|
import { parseDelegationEffort } from '../delegation/effort.js';
|
|
34
37
|
import { DelegationPolicyError } from '../delegation/policy.js';
|
|
35
38
|
import { reconcileWorkerWait } from '../delegation/wait.js';
|
|
36
39
|
import { reconcileConversationState, projectConversationEvents } from '../delegation/conversations.js';
|
|
37
40
|
import { parentReadiness } from '../delegation/readiness.js';
|
|
41
|
+
import { answersQuestion, openQuestions, questionMessage } from '../delegation/questions.js';
|
|
38
42
|
import { workerOutcome } from '../runs/delegation-state.js';
|
|
39
43
|
import { loadWorkflows } from './load.js';
|
|
40
44
|
import { reclaimWorktrees, rematerializeReclaimedWorktree } from '../runs/retention.js';
|
|
41
|
-
import { AgentTempDirError, agentTmpEnv, removeAgentTmpDir, sweepAgentTmpDirs, } from '../runs/agent-tmpdir.js';
|
|
45
|
+
import { AgentTempDirError, agentTmpEnv, agentTmpDirLocations, removeAgentTmpDir, sweepAgentTmpDirs, } from '../runs/agent-tmpdir.js';
|
|
42
46
|
import { extractTaskRefs, refineTaskRefs, titleRefNumber } from '../runs/task-refs.js';
|
|
43
47
|
import { parseTaskMarkers, stripTaskMarkers } from '../runs/task-markers.js';
|
|
44
48
|
import { autoNamingActive, generateRunName, liveTitleUpdatesEnabled, postValidateTitle } from '../runs/auto-name.js';
|
|
@@ -122,6 +126,8 @@ function stripDoneMarker(text) {
|
|
|
122
126
|
function stripMonitoringMarker(text) {
|
|
123
127
|
return text.replace(/\s*CEZ:MONITORING\s*$/, '');
|
|
124
128
|
}
|
|
129
|
+
/** #505: a worker question routed to its parent is answered there until it falls back. */
|
|
130
|
+
const ROUTED_QUESTION_REFUSAL = 'This question was sent to the parent task; answer it there.';
|
|
125
131
|
/** Emit the v2 `ask.requested` event for a parsed marker (the cockpit renders
|
|
126
132
|
* it as an ask card, #473). Returns the minted request id. */
|
|
127
133
|
function emitAskRequested(sink, ask) {
|
|
@@ -213,6 +219,9 @@ export function repositoryRootLockDisabled(env = process.env) {
|
|
|
213
219
|
}
|
|
214
220
|
const REPOSITORY_ROOT_LOCK_DISABLED_NOTE = 'repository-root lock disabled by CEZ_DISABLE_REPO_LOCK=1 (shared checkout is unsafe)';
|
|
215
221
|
/** Safety cap on autonomous auto-continues per run — stops a stuck agent from nudging forever. */
|
|
222
|
+
/** #505: a real steer is read at the harness's next model step; a quiet boundary this long
|
|
223
|
+
* with input still unread means the harness will not read it without help. */
|
|
224
|
+
export const UNREAD_INPUT_GRACE_MS = 30_000;
|
|
216
225
|
const MAX_AUTO_CONTINUES = 40;
|
|
217
226
|
const AUTONOMOUS_NUDGE = 'Continue working autonomously until the task is fully complete. Do not ask me for confirmation or clarification — make reasonable assumptions and proceed. When everything is done, end the session with your done signal.';
|
|
218
227
|
const MONITORING_WAKE_NUDGE = 'Re-check the downstream work you were monitoring. Continue toward the task goal; emit CEZ:MONITORING again only if it is still pending.';
|
|
@@ -418,6 +427,12 @@ export function isUntouchedCancelledRun(run) {
|
|
|
418
427
|
return run.status === 'cancelled' && !run.startedAt && !!run.workflowDef &&
|
|
419
428
|
run.steps.every(step => step.status === 'pending' && !step.startedAt && !step.sessionId);
|
|
420
429
|
}
|
|
430
|
+
/** #469: how long a non-gone orphan probe stands before the next scan. */
|
|
431
|
+
const ORPHAN_PROBE_CACHE_MS = 2_000;
|
|
432
|
+
/** #469: admission refused because a process of the previous generation still runs. */
|
|
433
|
+
class WorkerOrphanAliveError extends Error {
|
|
434
|
+
}
|
|
435
|
+
const admissionError = (error, fallback) => error instanceof WorkerOrphanAliveError ? error.message : fallback;
|
|
421
436
|
/**
|
|
422
437
|
* The mini workflow engine: executes a `WorkflowDef` against a repo, one step
|
|
423
438
|
* at a time, persisting every event to the RunStore (which the SSE endpoints
|
|
@@ -447,6 +462,13 @@ export class RunManager {
|
|
|
447
462
|
beginWorkerExecution(runId, admitted = true) {
|
|
448
463
|
if (this.store.getRun(runId)?.delegation?.role !== 'worker')
|
|
449
464
|
return;
|
|
465
|
+
// #469: one site for Continue, --resume, parent replies and queued revival after a crash.
|
|
466
|
+
// Fresh: admission is one-shot, so a cached "alive" must not refuse an orphan that has since died.
|
|
467
|
+
if (!this.executions.has(runId))
|
|
468
|
+
this.settleOrphanedWorkerExecution(runId, { admitting: true, fresh: true });
|
|
469
|
+
const refusal = this.executions.has(runId) ? undefined : this.orphanAdmissionRefusal(runId);
|
|
470
|
+
if (refusal)
|
|
471
|
+
throw new WorkerOrphanAliveError(refusal);
|
|
450
472
|
const existing = this.executions.get(runId);
|
|
451
473
|
if (existing) {
|
|
452
474
|
const proof = this.store.readWorkerExecution(runId);
|
|
@@ -461,6 +483,9 @@ export class RunManager {
|
|
|
461
483
|
const promise = new Promise(done => { resolve = done; });
|
|
462
484
|
this.executions.set(runId, { generation, admitted, promise, resolve, sessions: new Set(), turns: new Set(), deliveries: new Set() });
|
|
463
485
|
this.finalizedWorkers.delete(runId);
|
|
486
|
+
// #469: an earlier generation's destroy blocker never describes this one.
|
|
487
|
+
this.orphanBlockers.delete(runId);
|
|
488
|
+
this.reportedOrphanBlockers.delete(runId);
|
|
464
489
|
this.stoppedWorkers.delete(runId);
|
|
465
490
|
}
|
|
466
491
|
async finishWorkerExecution(runId) {
|
|
@@ -510,6 +535,196 @@ export class RunManager {
|
|
|
510
535
|
}
|
|
511
536
|
catch { /* Finalized in this process; preserve exact generation for an explicit retry. */ }
|
|
512
537
|
}
|
|
538
|
+
/** #469: the generation a dead controller left `starting`, if this manager may judge it. A
|
|
539
|
+
* generation this process controls is never judged here: the in-memory execution map owns it,
|
|
540
|
+
* and a disposed manager's still-running sessions look like orphans. `busy` is transient (this
|
|
541
|
+
* manager holds the run); `unknown` is a present record that proves nothing. */
|
|
542
|
+
orphanState(runId, admitting = false) {
|
|
543
|
+
const run = this.store.getRun(runId);
|
|
544
|
+
if (run?.delegation?.role !== 'worker' || this.disposed)
|
|
545
|
+
return { state: 'none' };
|
|
546
|
+
if (this.executions.has(runId) || this.active.has(runId) || this.starting.has(runId) || (!admitting && this.queue.includes(runId)))
|
|
547
|
+
return { state: 'busy' };
|
|
548
|
+
const proof = this.store.readWorkerExecution(runId);
|
|
549
|
+
if (proof?.phase !== 'starting')
|
|
550
|
+
return { state: 'none' };
|
|
551
|
+
const record = this.store.readWorkerProcesses(runId, proof.generation);
|
|
552
|
+
// A present record that proves nothing (unreadable, malformed, another generation) blocks both
|
|
553
|
+
// finalization and reaping; only an absent one is legacy, scan-only evidence.
|
|
554
|
+
if (record === 'unknown')
|
|
555
|
+
return { state: 'unknown' };
|
|
556
|
+
if (record !== 'absent' && isCurrentProcess(record.controller))
|
|
557
|
+
return { state: 'none' };
|
|
558
|
+
// Finalization deletes the scratch too, so a process working there keeps the generation alive.
|
|
559
|
+
return { state: 'orphan', generation: proof.generation, ...(record === 'absent' ? {} : { record }),
|
|
560
|
+
paths: [run.delegation.workspace.path, ...agentTmpDirLocations(this.dataDir, runId)],
|
|
561
|
+
// No process of this worker can predate its record (1 s slack for tick rounding).
|
|
562
|
+
...(Number.isFinite(Date.parse(run.createdAt)) ? { since: Date.parse(run.createdAt) - 1_000 } : {}) };
|
|
563
|
+
}
|
|
564
|
+
orphanedWorkerGeneration(runId, admitting = false) {
|
|
565
|
+
const orphan = this.orphanState(runId, admitting);
|
|
566
|
+
return orphan.state === 'orphan' ? orphan : undefined;
|
|
567
|
+
}
|
|
568
|
+
/** Completes a crashed generation's proof once every one of its processes is proven gone (#469).
|
|
569
|
+
* Sync and signal-free; recovery, resume, delivery, collect and destroy then take their normal paths.
|
|
570
|
+
* A non-gone probe is cached briefly: on darwin the scan is a synchronous `lsof` on the event loop. */
|
|
571
|
+
settleOrphanedWorkerExecution(runId, opts = {}) {
|
|
572
|
+
const orphan = this.orphanedWorkerGeneration(runId, opts.admitting);
|
|
573
|
+
if (!orphan)
|
|
574
|
+
return false;
|
|
575
|
+
const cached = this.orphanProbes.get(runId);
|
|
576
|
+
if (!opts.fresh && cached?.generation === orphan.generation && Date.now() - cached.at < ORPHAN_PROBE_CACHE_MS)
|
|
577
|
+
return false;
|
|
578
|
+
const probe = inspectGeneration(orphan);
|
|
579
|
+
if (probe.liveness !== 'gone') {
|
|
580
|
+
this.orphanProbes.set(runId, { generation: orphan.generation, at: Date.now(), probe });
|
|
581
|
+
return false;
|
|
582
|
+
}
|
|
583
|
+
// A stale `alive` must not outlive a `gone` probe, even when the commit below fails.
|
|
584
|
+
this.orphanProbes.delete(runId);
|
|
585
|
+
try {
|
|
586
|
+
if (!this.store.commitWorkerExecutionComplete(runId, orphan.generation))
|
|
587
|
+
return false;
|
|
588
|
+
}
|
|
589
|
+
catch {
|
|
590
|
+
return false;
|
|
591
|
+
}
|
|
592
|
+
this.orphanProbes.delete(runId);
|
|
593
|
+
this.orphanBlockers.delete(runId);
|
|
594
|
+
this.reportedOrphanBlockers.delete(runId);
|
|
595
|
+
this.clearOrphanReprobe(runId);
|
|
596
|
+
this.store.appendEvent(runId, { type: 'lifecycle', message: "the interrupted worker's processes are gone; its execution was finalized" });
|
|
597
|
+
removeAgentTmpDir(this.dataDir, runId);
|
|
598
|
+
return true;
|
|
599
|
+
}
|
|
600
|
+
orphanProbes = new Map();
|
|
601
|
+
orphanReprobes = new Map();
|
|
602
|
+
orphanBlockers = new Map();
|
|
603
|
+
reportedOrphanBlockers = new Map();
|
|
604
|
+
// Private and overridable so tests need not wait out production cadence.
|
|
605
|
+
orphanReprobeMs = 15_000;
|
|
606
|
+
orphanReprobeLimitMs = 15 * 60_000;
|
|
607
|
+
orphanReprobeSlowMs = 60_000;
|
|
608
|
+
orphanTermGraceMs = 10_000;
|
|
609
|
+
/** #469: a survivor that dies after recovery has no other wake source (no exit callback for a
|
|
610
|
+
* process another cezar spawned). Unref'd; finalization's `run` event then lets worker waits and
|
|
611
|
+
* outcomes observe it. Fast for the first window, then slow but uncapped, so a long-lived
|
|
612
|
+
* survivor is still noticed when it exits. A tick while this manager holds the run is skipped;
|
|
613
|
+
* only terminal reasons (proof settled, generation changed, run gone, unknown record, dispose)
|
|
614
|
+
* stop it. */
|
|
615
|
+
armOrphanReprobe(runId) {
|
|
616
|
+
const armed = this.orphanedWorkerGeneration(runId);
|
|
617
|
+
if (this.disposed || this.orphanReprobes.has(runId) || !armed)
|
|
618
|
+
return;
|
|
619
|
+
const slowAfter = Date.now() + this.orphanReprobeLimitMs;
|
|
620
|
+
const schedule = () => {
|
|
621
|
+
const timer = setTimeout(() => {
|
|
622
|
+
const orphan = this.orphanState(runId);
|
|
623
|
+
if (orphan.state === 'busy')
|
|
624
|
+
return schedule();
|
|
625
|
+
if (orphan.state !== 'orphan' || orphan.generation !== armed.generation || this.settleOrphanedWorkerExecution(runId))
|
|
626
|
+
this.clearOrphanReprobe(runId);
|
|
627
|
+
else
|
|
628
|
+
schedule();
|
|
629
|
+
}, Date.now() < slowAfter ? this.orphanReprobeMs : this.orphanReprobeSlowMs);
|
|
630
|
+
timer.unref?.();
|
|
631
|
+
this.orphanReprobes.set(runId, timer);
|
|
632
|
+
};
|
|
633
|
+
schedule();
|
|
634
|
+
}
|
|
635
|
+
clearOrphanReprobe(runId) {
|
|
636
|
+
const timer = this.orphanReprobes.get(runId);
|
|
637
|
+
if (timer)
|
|
638
|
+
clearTimeout(timer);
|
|
639
|
+
this.orphanReprobes.delete(runId);
|
|
640
|
+
}
|
|
641
|
+
/** Why the last destroy could not prove termination (#469), and whether that differs from the
|
|
642
|
+
* reason last reported, so a retry loop appends one lifecycle event per distinct blocker set. */
|
|
643
|
+
takeWorkerTerminationBlocker(runId) {
|
|
644
|
+
const blocker = this.orphanBlockers.get(runId);
|
|
645
|
+
if (!blocker)
|
|
646
|
+
return undefined;
|
|
647
|
+
const key = JSON.stringify(blocker);
|
|
648
|
+
const changed = this.reportedOrphanBlockers.get(runId) !== key;
|
|
649
|
+
this.reportedOrphanBlockers.set(runId, key);
|
|
650
|
+
return { blocker, changed };
|
|
651
|
+
}
|
|
652
|
+
/** Continue over a live orphan names what still runs; other refusals keep the generic text. */
|
|
653
|
+
orphanAdmissionRefusal(runId) {
|
|
654
|
+
const orphan = this.orphanedWorkerGeneration(runId, true);
|
|
655
|
+
const probe = orphan && this.orphanProbes.get(runId);
|
|
656
|
+
if (!probe || probe.generation !== orphan.generation)
|
|
657
|
+
return undefined;
|
|
658
|
+
if (probe.probe.controller !== undefined)
|
|
659
|
+
return `the worker is still controlled by a live cezar (pid ${probe.probe.controller})`;
|
|
660
|
+
const pid = probe.probe.pids[0];
|
|
661
|
+
return pid === undefined ? undefined : `a process of the previous execution is still running (pid ${pid})`;
|
|
662
|
+
}
|
|
663
|
+
/** Destroy only (#469): signal the recorded, token-verified survivors of a generation whose
|
|
664
|
+
* controller is proven dead, then finalize on `gone`. Scan-only processes are never signalled. */
|
|
665
|
+
async reapOrphanedWorker(runId, timeoutMs) {
|
|
666
|
+
const deadline = Date.now() + Math.min(30_000, Math.max(0, Number.isFinite(timeoutMs) ? timeoutMs : 0));
|
|
667
|
+
this.orphanBlockers.delete(runId);
|
|
668
|
+
const initial = this.orphanState(runId);
|
|
669
|
+
if (initial.state === 'unknown') {
|
|
670
|
+
this.orphanBlockers.set(runId, { kind: 'unreadable' });
|
|
671
|
+
return;
|
|
672
|
+
}
|
|
673
|
+
const orphan = this.orphanedWorkerGeneration(runId);
|
|
674
|
+
if (!orphan || this.settleOrphanedWorkerExecution(runId, { fresh: true }))
|
|
675
|
+
return;
|
|
676
|
+
const same = () => this.orphanedWorkerGeneration(runId)?.generation === orphan.generation;
|
|
677
|
+
// >= 500 ms: every probe may be a synchronous darwin `lsof`.
|
|
678
|
+
const pause = () => new Promise(resolve => setTimeout(resolve, Math.max(0, Math.min(500, deadline - Date.now()))));
|
|
679
|
+
// A live controller is another cezar's: nothing to signal or wait for. Otherwise signal only
|
|
680
|
+
// recorded, token-verified survivors (none for a legacy record-less generation), then wait
|
|
681
|
+
// out every other holder until the deadline; the scan's processes are never signalled.
|
|
682
|
+
if (!orphan.record || !recordedProcessLive(orphan.record.controller)) {
|
|
683
|
+
if (orphan.record) {
|
|
684
|
+
const targets = orphan.record.processes.filter(entry => entry.startToken !== undefined && recordedProcessLive(entry));
|
|
685
|
+
const signal = (name) => {
|
|
686
|
+
for (const entry of targets) {
|
|
687
|
+
if (!same())
|
|
688
|
+
return;
|
|
689
|
+
// Re-verified immediately before every signal, exactly: a reused PID is never touched.
|
|
690
|
+
if (processStartToken(entry.pid) === entry.startToken)
|
|
691
|
+
try {
|
|
692
|
+
process.kill(entry.pid, name);
|
|
693
|
+
}
|
|
694
|
+
catch { /* already gone */ }
|
|
695
|
+
}
|
|
696
|
+
};
|
|
697
|
+
signal('SIGTERM');
|
|
698
|
+
const killAt = Math.min(deadline, Date.now() + this.orphanTermGraceMs);
|
|
699
|
+
while (Date.now() < killAt && targets.some(recordedProcessLive))
|
|
700
|
+
await pause();
|
|
701
|
+
if (targets.some(recordedProcessLive))
|
|
702
|
+
signal('SIGKILL');
|
|
703
|
+
}
|
|
704
|
+
while (!this.settleOrphanedWorkerExecution(runId, { fresh: true }) && Date.now() < deadline && same())
|
|
705
|
+
await pause();
|
|
706
|
+
}
|
|
707
|
+
if (!same())
|
|
708
|
+
return;
|
|
709
|
+
const probe = this.orphanProbes.get(runId)?.probe;
|
|
710
|
+
if (probe?.controller !== undefined)
|
|
711
|
+
this.orphanBlockers.set(runId, { kind: 'controller', pid: probe.controller });
|
|
712
|
+
else if (probe?.pids.length)
|
|
713
|
+
this.orphanBlockers.set(runId, { kind: 'processes', pids: probe.pids });
|
|
714
|
+
}
|
|
715
|
+
/** Best effort: a failed write leaves the working-directory scan as this process's evidence. */
|
|
716
|
+
recordWorkerProcess(runId, pid) {
|
|
717
|
+
const generation = this.executions.get(runId)?.generation;
|
|
718
|
+
if (!generation)
|
|
719
|
+
return;
|
|
720
|
+
try {
|
|
721
|
+
if (!this.store.appendWorkerProcess(runId, generation, pid))
|
|
722
|
+
console.warn(`[cez] worker ${runId} process record unavailable; relying on the working-directory scan`);
|
|
723
|
+
}
|
|
724
|
+
catch {
|
|
725
|
+
console.warn(`[cez] worker ${runId} process record write failed; relying on the working-directory scan`);
|
|
726
|
+
}
|
|
727
|
+
}
|
|
513
728
|
trackTurnEnd(runId, text) {
|
|
514
729
|
const task = this.recordTurnEnd(runId, text);
|
|
515
730
|
const turns = this.executions.get(runId)?.turns;
|
|
@@ -524,12 +739,16 @@ export class RunManager {
|
|
|
524
739
|
this.cancel(runId);
|
|
525
740
|
return { workerId: runId, state: !this.executions.has(runId) && this.store.readWorkerExecution(runId)?.phase === 'complete' ? 'terminated' : 'stopping' };
|
|
526
741
|
}
|
|
527
|
-
async awaitRunTermination(runId, timeoutMs) {
|
|
742
|
+
async awaitRunTermination(runId, timeoutMs, opts = {}) {
|
|
528
743
|
const execution = this.executions.get(runId);
|
|
529
744
|
if (!execution) {
|
|
530
745
|
const generation = this.finalizedWorkers.get(runId);
|
|
531
746
|
if (!this.disposed && generation)
|
|
532
747
|
this.persistWorkerCompletion(runId, generation);
|
|
748
|
+
else if (!generation && opts.reapOrphans)
|
|
749
|
+
await this.reapOrphanedWorker(runId, timeoutMs);
|
|
750
|
+
else if (!generation)
|
|
751
|
+
this.settleOrphanedWorkerExecution(runId);
|
|
533
752
|
return this.store.readWorkerExecution(runId)?.phase === 'complete';
|
|
534
753
|
}
|
|
535
754
|
if (this.disposed)
|
|
@@ -593,8 +812,11 @@ export class RunManager {
|
|
|
593
812
|
workerWakeAdmitted = new Set();
|
|
594
813
|
workerWakeQueuedAt = new Map();
|
|
595
814
|
workerWaitTimers = new Map();
|
|
815
|
+
inboxClaimTimers = new Map();
|
|
596
816
|
reconcilingWorkers = false;
|
|
597
817
|
recovering = false;
|
|
818
|
+
/** Quiet window before unread input is resubmitted or left unconfirmed (#505). */
|
|
819
|
+
unreadInputGraceMs;
|
|
598
820
|
disposed = false;
|
|
599
821
|
onDelegationRun = (run) => {
|
|
600
822
|
if (!this.disposed && !['queued', 'running', 'waiting'].includes(run.status))
|
|
@@ -665,6 +887,7 @@ export class RunManager {
|
|
|
665
887
|
this.store = store;
|
|
666
888
|
this.repoRoot = repoRoot;
|
|
667
889
|
this.dataDir = join(repoRoot, '.ai/cezar');
|
|
890
|
+
this.unreadInputGraceMs = options.unreadInputGraceMs ?? UNREAD_INPUT_GRACE_MS;
|
|
668
891
|
this.semaphore = options.semaphore ?? new WorkspaceSemaphore();
|
|
669
892
|
this.ciResources = acquireCiResources(this.semaphore);
|
|
670
893
|
this.ciSupervisor = this.ciResources.supervisor;
|
|
@@ -705,12 +928,20 @@ export class RunManager {
|
|
|
705
928
|
this.ciResources.release();
|
|
706
929
|
this.delegationProvisioner = undefined;
|
|
707
930
|
this.finalizedWorkers.clear();
|
|
931
|
+
for (const runId of [...this.orphanReprobes.keys()])
|
|
932
|
+
this.clearOrphanReprobe(runId);
|
|
933
|
+
this.orphanProbes.clear();
|
|
934
|
+
this.orphanBlockers.clear();
|
|
935
|
+
this.reportedOrphanBlockers.clear();
|
|
708
936
|
for (const settle of this.terminationWaiters)
|
|
709
937
|
settle();
|
|
710
938
|
this.store.off('run', this.onDelegationRun);
|
|
711
939
|
for (const timer of this.workerWaitTimers.values())
|
|
712
940
|
clearTimeout(timer);
|
|
713
941
|
this.workerWaitTimers.clear();
|
|
942
|
+
for (const timer of this.inboxClaimTimers.values())
|
|
943
|
+
clearTimeout(timer);
|
|
944
|
+
this.inboxClaimTimers.clear();
|
|
714
945
|
this.workerWaiting.clear();
|
|
715
946
|
this.workerWakeAdmitted.clear();
|
|
716
947
|
this.workerWakeQueuedAt.clear();
|
|
@@ -805,6 +1036,7 @@ export class RunManager {
|
|
|
805
1036
|
* turning into empty command output inside a running agent. */
|
|
806
1037
|
agentEnv(runId, generateFollowups = true) {
|
|
807
1038
|
return {
|
|
1039
|
+
CEZ_ARTIFACTS_DIR: provisionArtifactDirectory(this.dataDir, runId),
|
|
808
1040
|
CEZ_HANDOFF_FILE: handoffPath(this.dataDir, runId),
|
|
809
1041
|
CEZ_TASK_ID: runId,
|
|
810
1042
|
CEZ_TODOS_FILE: generateFollowups ? todosPath(this.dataDir) : '',
|
|
@@ -1155,7 +1387,7 @@ export class RunManager {
|
|
|
1155
1387
|
* does not know is refused rather than degraded to the default, because a worker silently
|
|
1156
1388
|
* running on another login is a billing boundary crossed without anyone choosing it.
|
|
1157
1389
|
*/
|
|
1158
|
-
async selectDelegationExecutionSettings(runId, selection) {
|
|
1390
|
+
async selectDelegationExecutionSettings(runId, selection, validateModel) {
|
|
1159
1391
|
const parent = this.delegationExecutionSettings(runId);
|
|
1160
1392
|
const runner = selection.backend ?? parent.runner;
|
|
1161
1393
|
const sameBackend = runner === parent.runner;
|
|
@@ -1180,17 +1412,34 @@ export class RunManager {
|
|
|
1180
1412
|
env = { ...env, ...boundWorkerAccountEnv(accountBinding, env) };
|
|
1181
1413
|
const native = await readAgentModelSettings(runner, this.repoRoot, env);
|
|
1182
1414
|
const chosen = selection.model ?? (sameBackend ? parent.model : locked ? undefined : native.model);
|
|
1415
|
+
const normalized = normalizeModelForBackend(runner, chosen, { configuredProvider: native.provider });
|
|
1416
|
+
const settings = { ...parent, runner, model: normalized?.backendModel, modelIdentity: normalized ? formatModelIdentity(normalized.identity) : undefined, effort: effortPin ?? (sameBackend ? parent.effort : undefined),
|
|
1417
|
+
agentProfile: accountBinding.profileId, accountBinding };
|
|
1418
|
+
// A populated catalog gives a useful choice list before the legacy preset fallback rejects.
|
|
1419
|
+
await validateModel?.(settings);
|
|
1183
1420
|
if (chosen && modelConflictsWithRunner(chosen, runner))
|
|
1184
1421
|
throw new Error('Model is incompatible with the selected backend');
|
|
1185
|
-
|
|
1186
|
-
return { ...parent, runner, model, effort: effortPin ?? (sameBackend ? parent.effort : undefined),
|
|
1187
|
-
agentProfile: accountBinding.profileId, accountBinding };
|
|
1422
|
+
return settings;
|
|
1188
1423
|
}
|
|
1189
1424
|
catch (error) {
|
|
1425
|
+
if (error instanceof DelegationPolicyError)
|
|
1426
|
+
throw error;
|
|
1190
1427
|
throw new DelegationPolicyError('invalid_input', error instanceof Error ? error.message : 'Worker execution selection is unavailable');
|
|
1191
1428
|
}
|
|
1192
1429
|
}
|
|
1193
|
-
|
|
1430
|
+
/**
|
|
1431
|
+
* Idempotent start (#504, spec 2026-09-24-cez-task-cli). Lookup and create share one
|
|
1432
|
+
* synchronous path, so the event loop serialises concurrent retries: the first creates, every
|
|
1433
|
+
* later one with the same hash gets that run back, and a different hash is a conflict.
|
|
1434
|
+
*/
|
|
1435
|
+
startRunIdempotent(workflow, input, request) {
|
|
1436
|
+
const existing = this.store.findRunByClientRequestId(request.id);
|
|
1437
|
+
if (existing) {
|
|
1438
|
+
return existing.clientRequestHash === request.hash ? { run: existing, created: false } : { conflict: true };
|
|
1439
|
+
}
|
|
1440
|
+
return { run: this.startRun(workflow, input, undefined, request), created: true };
|
|
1441
|
+
}
|
|
1442
|
+
startRun(workflow, input, group, clientRequest) {
|
|
1194
1443
|
// Sanitize at the manager boundary so CLI runs, workflows, variants, and
|
|
1195
1444
|
// direct callers cannot bypass the HTTP policy.
|
|
1196
1445
|
const effectiveInput = agentModelsLocked(this.repoRoot)
|
|
@@ -1216,11 +1465,14 @@ export class RunManager {
|
|
|
1216
1465
|
// auto-nudge reads `input.autonomous` (`execute`), but the record is the
|
|
1217
1466
|
// only source those after-the-fact consumers have.
|
|
1218
1467
|
autonomous: input.autonomous === true,
|
|
1468
|
+
notify: input.notify === true,
|
|
1219
1469
|
// Persist the explicit opt-out so queued-run restart recovery and the
|
|
1220
1470
|
// session Git routes can distinguish it from a removed isolated worktree.
|
|
1221
1471
|
worktree: !group && input.worktree === false ? false : undefined,
|
|
1222
1472
|
groupId: group?.groupId,
|
|
1223
1473
|
variant: group?.variant,
|
|
1474
|
+
clientRequestId: clientRequest?.id,
|
|
1475
|
+
clientRequestHash: clientRequest?.hash,
|
|
1224
1476
|
steps: workflow.steps.map((s) => ({ id: s.id, name: s.name ?? s.id, kind: stepKind(s) })),
|
|
1225
1477
|
});
|
|
1226
1478
|
// Persist the full definition so a queued run survives a restart (#367) —
|
|
@@ -1427,6 +1679,14 @@ export class RunManager {
|
|
|
1427
1679
|
// than being dequeued and re-queued (which would churn its position and its record).
|
|
1428
1680
|
const next = this.queue.findIndex((id) => {
|
|
1429
1681
|
const queued = this.store.getRun(id);
|
|
1682
|
+
// An inbox claim may arrive after a wake was queued but before admission.
|
|
1683
|
+
// Preserve its place and slot exemption until ACK/release/expiry re-pumps.
|
|
1684
|
+
if (this.workerWakeQueuedAt.has(id)) {
|
|
1685
|
+
const first = queued?.agentInputs?.find(input => !input.deliveredAt);
|
|
1686
|
+
const wake = queued?.agentInputs?.find(input => input.id === this.workerWait(id)?.wakeId);
|
|
1687
|
+
if ((first && hasLiveInboxClaim(first)) || (wake && hasLiveInboxClaim(wake)))
|
|
1688
|
+
return false;
|
|
1689
|
+
}
|
|
1430
1690
|
return (capacity() || chargedCiWake(id)) && (!queued || this.historyDeletionPending(id) || ((!this.executions.get(id)?.admitted || this.active.has(id)) && !this.executionBlockedByRootFinish(queued) &&
|
|
1431
1691
|
(!anyHold || !accountHeldFor(queued, holds, defaultRunner ?? 'claude'))));
|
|
1432
1692
|
});
|
|
@@ -1441,11 +1701,11 @@ export class RunManager {
|
|
|
1441
1701
|
try {
|
|
1442
1702
|
this.beginWorkerExecution(candidate);
|
|
1443
1703
|
}
|
|
1444
|
-
catch {
|
|
1704
|
+
catch (error) {
|
|
1445
1705
|
this.queue.splice(next, 1);
|
|
1446
1706
|
this.pendingJobs.delete(candidate);
|
|
1447
1707
|
this.pendingContinuations.delete(candidate);
|
|
1448
|
-
this.store.updateRun(candidate, { status: 'failed', error: 'worker execution checkpoint unavailable', finishedAt: new Date().toISOString() });
|
|
1708
|
+
this.store.updateRun(candidate, { status: 'failed', error: admissionError(error, 'worker execution checkpoint unavailable'), finishedAt: new Date().toISOString() });
|
|
1449
1709
|
continue;
|
|
1450
1710
|
}
|
|
1451
1711
|
}
|
|
@@ -1473,6 +1733,11 @@ export class RunManager {
|
|
|
1473
1733
|
this.workerWakeAdmitted.add(runId);
|
|
1474
1734
|
this.flushAgentInputs(runId);
|
|
1475
1735
|
}
|
|
1736
|
+
else if (state.pendingHumanAsk && this.routedAsk(runId) && this.workerWaiting.delete(runId)) {
|
|
1737
|
+
// A parent's answer to the worker's routed question (#505).
|
|
1738
|
+
this.workerWakeAdmitted.add(runId);
|
|
1739
|
+
this.answerRoutedQuestion(runId, true);
|
|
1740
|
+
}
|
|
1476
1741
|
continue;
|
|
1477
1742
|
}
|
|
1478
1743
|
if (this.workerWakeQueuedAt.delete(runId))
|
|
@@ -1579,7 +1844,7 @@ export class RunManager {
|
|
|
1579
1844
|
const sameAccount = agentProfile === undefined || agentProfile === sessionAccount;
|
|
1580
1845
|
this.pendingContinuations.set(run.id, {
|
|
1581
1846
|
stepId: queuedContinuation.id,
|
|
1582
|
-
sessionId: sessionBackend === backend && sameAccount ?
|
|
1847
|
+
sessionId: sessionBackend === backend && sameAccount ? this.resumableSessionId(run.id, sessionStep) : undefined,
|
|
1583
1848
|
backend,
|
|
1584
1849
|
prompt: run.continuationMessage?.text ?? RESTART_CONTINUATION_PROMPT,
|
|
1585
1850
|
images: [],
|
|
@@ -1667,6 +1932,22 @@ export class RunManager {
|
|
|
1667
1932
|
return;
|
|
1668
1933
|
this.recovering = true;
|
|
1669
1934
|
try {
|
|
1935
|
+
for (const run of this.store.listRuns()) {
|
|
1936
|
+
this.store.clearExpiredInboxClaims(run.id, new Date().toISOString());
|
|
1937
|
+
this.armInboxClaimExpiry(run.id);
|
|
1938
|
+
// #505: the session that accepted this input died before the model read it.
|
|
1939
|
+
// Delivery is at-least-once: the replay keeps the input's ID in its text.
|
|
1940
|
+
if (['queued', 'running', 'waiting'].includes(run.status) && !this.isActive(run.id)) {
|
|
1941
|
+
const replayed = this.store.requeueAwaitingReadInputs(run.id);
|
|
1942
|
+
if (replayed.length)
|
|
1943
|
+
this.store.appendEvent(run.id, { type: 'note', message: `replaying ${replayed.length} message${replayed.length === 1 ? '' : 's'} accepted before the restart but not read` });
|
|
1944
|
+
}
|
|
1945
|
+
}
|
|
1946
|
+
// #469: a generation whose processes died with the old controller is finalized first, so it
|
|
1947
|
+
// re-launches through the ordinary Continue path and its scratch is not retained.
|
|
1948
|
+
for (const run of this.store.listRuns())
|
|
1949
|
+
if (run.delegation?.role === 'worker' && !this.settleOrphanedWorkerExecution(run.id))
|
|
1950
|
+
this.armOrphanReprobe(run.id);
|
|
1670
1951
|
this.reconcileWorkerWaits();
|
|
1671
1952
|
const live = this.store
|
|
1672
1953
|
.listRuns()
|
|
@@ -1743,7 +2024,8 @@ export class RunManager {
|
|
|
1743
2024
|
// runs. A worker reaches here only when it has neither a recoverable ask
|
|
1744
2025
|
// nor a worker wait; preserving that record would strand both it and its
|
|
1745
2026
|
// parent because workers cannot accept an ordinary inactive Continue.
|
|
1746
|
-
if (run.status === 'waiting' && run.delegation?.role !== 'worker'
|
|
2027
|
+
if (run.status === 'waiting' && (run.delegation?.role !== 'worker' ||
|
|
2028
|
+
run.agentInputs?.some(input => !input.deliveredAt && hasLiveInboxClaim(input))))
|
|
1747
2029
|
continue;
|
|
1748
2030
|
if (run.status === 'waiting') {
|
|
1749
2031
|
const finishedAt = new Date().toISOString();
|
|
@@ -1820,7 +2102,15 @@ export class RunManager {
|
|
|
1820
2102
|
finally {
|
|
1821
2103
|
this.recovering = false;
|
|
1822
2104
|
}
|
|
2105
|
+
// A question a crash caught before or after its commit to the parent still gets routed (#505).
|
|
2106
|
+
for (const run of this.store.listRuns())
|
|
2107
|
+
if (run.delegation?.role === 'worker')
|
|
2108
|
+
this.routeWorkerQuestion(run.id);
|
|
1823
2109
|
this.reconcileWorkerWaits();
|
|
2110
|
+
// A parent reply accepted just before the crash still answers its worker (#505).
|
|
2111
|
+
for (const run of this.store.listRuns())
|
|
2112
|
+
if (run.delegation?.role === 'worker')
|
|
2113
|
+
this.answerRoutedQuestion(run.id);
|
|
1824
2114
|
for (const run of this.store.listRuns()) {
|
|
1825
2115
|
if (run.ciWait && !['running', 'waiting', 'queued'].includes(run.status))
|
|
1826
2116
|
this.withdrawCiWait(run.id);
|
|
@@ -2702,6 +2992,9 @@ export class RunManager {
|
|
|
2702
2992
|
return undefined;
|
|
2703
2993
|
if (this.hasPendingHumanAsk(runId) || this.active.get(runId)?.pendingHumanAsk)
|
|
2704
2994
|
return 'Answer the pending human question before finishing.';
|
|
2995
|
+
const questions = this.unansweredQuestions(runId);
|
|
2996
|
+
if (questions.length)
|
|
2997
|
+
return `Answer your workers' questions before finishing: ${questions.map(q => q.workerId).join(', ')}`;
|
|
2705
2998
|
const blockers = this.parentCompletionBlockers(runId);
|
|
2706
2999
|
return blockers.length ? `Workers must finish or be stopped, prove termination, and have their latest results collected before finishing: ${blockers.map(b => `${b.workerId} (${b.reason})`).join(', ')}` : undefined;
|
|
2707
3000
|
}
|
|
@@ -2742,6 +3035,8 @@ export class RunManager {
|
|
|
2742
3035
|
const state = this.active.get(runId);
|
|
2743
3036
|
const blockers = this.parentCompletionBlockers(runId);
|
|
2744
3037
|
const pendingAsk = this.hasPendingHumanAsk(runId) || state?.pendingHumanAsk;
|
|
3038
|
+
// A worker waiting on this parent's answer (#505): waiting on that worker would deadlock.
|
|
3039
|
+
const questions = pendingAsk ? [] : this.unansweredQuestions(runId);
|
|
2745
3040
|
if (!blockers.length && !pendingAsk) {
|
|
2746
3041
|
this.resetParentCompletion(runId);
|
|
2747
3042
|
return false;
|
|
@@ -2754,7 +3049,7 @@ export class RunManager {
|
|
|
2754
3049
|
}
|
|
2755
3050
|
if (parent.delegation?.role !== 'root')
|
|
2756
3051
|
return false;
|
|
2757
|
-
if (parent.delegation.wait && !pendingAsk) {
|
|
3052
|
+
if (parent.delegation.wait && !pendingAsk && !questions.length) {
|
|
2758
3053
|
if (state?.session?.open)
|
|
2759
3054
|
this.parkWorkerWait(runId, state);
|
|
2760
3055
|
else
|
|
@@ -2762,11 +3057,12 @@ export class RunManager {
|
|
|
2762
3057
|
this.store.flush();
|
|
2763
3058
|
return true;
|
|
2764
3059
|
}
|
|
2765
|
-
const message = pendingAsk ? 'Completion blocked: answer the pending human question.' :
|
|
3060
|
+
const message = pendingAsk ? 'Completion blocked: answer the pending human question.' : questions.length
|
|
3061
|
+
? `Completion blocked: answer your workers' questions first: ${questions.map(q => `${q.workerId} (request ${q.messageId})`).join(', ')}.` :
|
|
2766
3062
|
`Completion blocked: finish or explicitly stop outstanding workers, then collect their latest settled results: ${blockers.map(b => `${b.workerId} (${b.reason})`).join(', ')}.`;
|
|
2767
3063
|
this.store.appendEvent(runId, { type: 'note', tone: 'warning', message });
|
|
2768
3064
|
const workerIds = blockers.map(b => b.workerId).filter(id => this.store.getRun(id)?.delegation?.role === 'worker');
|
|
2769
|
-
if (!pendingAsk && !parent.delegation.completion && workerIds.length) {
|
|
3065
|
+
if (!pendingAsk && !questions.length && !parent.delegation.completion && workerIds.length) {
|
|
2770
3066
|
const wait = { id: randomUUID(), mode: 'all', workerIds,
|
|
2771
3067
|
revisions: workerIds.map(workerId => { const d = this.store.getRun(workerId).delegation; return { workerId, revision: d?.role === 'worker' ? d.executionRevision ?? 0 : 0 }; }),
|
|
2772
3068
|
deadline: new Date(Date.now() + 600_000).toISOString(), phase: 'parked', outcomes: [] };
|
|
@@ -2779,7 +3075,7 @@ export class RunManager {
|
|
|
2779
3075
|
this.reconcileWorkerWaits();
|
|
2780
3076
|
}
|
|
2781
3077
|
else {
|
|
2782
|
-
const monitoring = !pendingAsk && !!state?.session?.open && this.hasLiveWorkers(runId);
|
|
3078
|
+
const monitoring = !pendingAsk && !questions.length && !!state?.session?.open && this.hasLiveWorkers(runId);
|
|
2783
3079
|
this.store.commitDelegation([{ id: runId, delegation: { ...parent.delegation, completion: { ...parent.delegation.completion, phase: 'attention' } } }]);
|
|
2784
3080
|
this.store.updateRun(runId, { status: monitoring ? 'running' : 'waiting', activity: monitoring ? 'monitoring' : undefined, finishedAt: undefined });
|
|
2785
3081
|
this.store.flush();
|
|
@@ -2883,6 +3179,90 @@ export class RunManager {
|
|
|
2883
3179
|
this.parkWorkerWait(runId, state);
|
|
2884
3180
|
return registered;
|
|
2885
3181
|
}
|
|
3182
|
+
/** A wait wake is only a pointer. Only provider submission/opening prompts own delivery. */
|
|
3183
|
+
providerOwnsInboxInput(runId, id) {
|
|
3184
|
+
const state = this.active.get(runId);
|
|
3185
|
+
return !!state?.agentInputFlight?.inputIds.includes(id) || state?.openingAgentInputId === id ||
|
|
3186
|
+
this.store.getRun(runId)?.continuationMessage?.agentInputId === id;
|
|
3187
|
+
}
|
|
3188
|
+
canClaimInboxInput(runId, id) {
|
|
3189
|
+
const run = this.store.getRun(runId);
|
|
3190
|
+
const input = run?.agentInputs?.find(entry => entry.id === id);
|
|
3191
|
+
return !this.disposed && !!run && ['queued', 'running', 'waiting'].includes(run.status) &&
|
|
3192
|
+
!this.historyDeletionPending(runId) && !this.executionBlockedByRootFinish(run) && !this.workerExecutionStopped(runId) &&
|
|
3193
|
+
!!input && input.source === 'agent' && !!input.conversation && !input.deliveredAt &&
|
|
3194
|
+
!hasLiveInboxClaim(input) && !this.providerOwnsInboxInput(runId, id);
|
|
3195
|
+
}
|
|
3196
|
+
/** The service supplies authorized, bounded IDs; claim and flight reservations are synchronous. */
|
|
3197
|
+
reserveInboxInputs(runId, generation, ids) {
|
|
3198
|
+
if (!ids.length || ids.some(id => !this.canClaimInboxInput(runId, id)) ||
|
|
3199
|
+
this.store.getRun(runId)?.agentInputs?.some(input => !input.deliveredAt && hasLiveInboxClaim(input)))
|
|
3200
|
+
return undefined;
|
|
3201
|
+
const claim = { receiptId: randomUUID(), generation, expiresAt: new Date(Date.now() + 120_000).toISOString() };
|
|
3202
|
+
this.store.claimInboxInputs(runId, ids, claim);
|
|
3203
|
+
this.armInboxClaimExpiry(runId);
|
|
3204
|
+
return { receiptId: claim.receiptId, expiresAt: claim.expiresAt, inputIds: [...ids] };
|
|
3205
|
+
}
|
|
3206
|
+
acknowledgeInbox(runId, generation, receiptId) {
|
|
3207
|
+
if (this.disposed || this.store.getRun(runId)?.agentInputs?.some(input => input.inboxClaim?.receiptId === receiptId &&
|
|
3208
|
+
!input.deliveredAt && this.providerOwnsInboxInput(runId, input.id)))
|
|
3209
|
+
throw new Error('inbox receipt displaced');
|
|
3210
|
+
const result = this.store.ackInboxInputs(runId, receiptId, generation, new Date().toISOString());
|
|
3211
|
+
this.armInboxClaimExpiry(runId);
|
|
3212
|
+
this.reconcileWorkerWaits();
|
|
3213
|
+
this.deliverConversationInput(runId);
|
|
3214
|
+
this.active.get(runId)?.resumeInboxAutoEnd?.();
|
|
3215
|
+
this.releaseSlot();
|
|
3216
|
+
return result;
|
|
3217
|
+
}
|
|
3218
|
+
releaseInbox(runId, generation, receiptId) {
|
|
3219
|
+
if (this.disposed)
|
|
3220
|
+
throw new Error('inbox manager disposed');
|
|
3221
|
+
const result = this.store.releaseInboxInputs(runId, receiptId, generation);
|
|
3222
|
+
this.armInboxClaimExpiry(runId);
|
|
3223
|
+
this.reconcileWorkerWaits();
|
|
3224
|
+
this.deliverConversationInput(runId);
|
|
3225
|
+
this.releaseSlot();
|
|
3226
|
+
return result;
|
|
3227
|
+
}
|
|
3228
|
+
/** Expiry is an independent wake source, including after the provider's last turn. */
|
|
3229
|
+
armInboxClaimExpiry(runId) {
|
|
3230
|
+
const previous = this.inboxClaimTimers.get(runId);
|
|
3231
|
+
if (previous)
|
|
3232
|
+
clearTimeout(previous);
|
|
3233
|
+
this.inboxClaimTimers.delete(runId);
|
|
3234
|
+
if (this.disposed)
|
|
3235
|
+
return;
|
|
3236
|
+
const deadlines = (this.store.getRun(runId)?.agentInputs ?? []).flatMap(input => input.inboxClaim && !input.inboxClaim.acknowledgedAt ? [Date.parse(input.inboxClaim.expiresAt)] : []);
|
|
3237
|
+
if (!deadlines.length)
|
|
3238
|
+
return;
|
|
3239
|
+
let warned = false;
|
|
3240
|
+
const expire = () => {
|
|
3241
|
+
this.inboxClaimTimers.delete(runId);
|
|
3242
|
+
if (this.disposed || !this.store.getRun(runId))
|
|
3243
|
+
return;
|
|
3244
|
+
try {
|
|
3245
|
+
this.store.clearExpiredInboxClaims(runId, new Date().toISOString());
|
|
3246
|
+
this.reconcileWorkerWaits();
|
|
3247
|
+
this.deliverConversationInput(runId);
|
|
3248
|
+
this.armInboxClaimExpiry(runId);
|
|
3249
|
+
this.releaseSlot();
|
|
3250
|
+
}
|
|
3251
|
+
catch {
|
|
3252
|
+
if (!warned)
|
|
3253
|
+
console.warn('[cez] inbox expiry checkpoint failed; retrying in 1 second');
|
|
3254
|
+
warned = true;
|
|
3255
|
+
if (!this.disposed)
|
|
3256
|
+
arm(1_000);
|
|
3257
|
+
}
|
|
3258
|
+
};
|
|
3259
|
+
const arm = (delay) => {
|
|
3260
|
+
const timer = setTimeout(expire, delay);
|
|
3261
|
+
timer.unref?.();
|
|
3262
|
+
this.inboxClaimTimers.set(runId, timer);
|
|
3263
|
+
};
|
|
3264
|
+
arm(Math.max(0, Math.min(...deadlines) - Date.now()));
|
|
3265
|
+
}
|
|
2886
3266
|
/** Accepted conversation input uses the existing scheduler admission for idle recipients. */
|
|
2887
3267
|
deliverConversationInput(runId) {
|
|
2888
3268
|
if (this.disposed)
|
|
@@ -2890,15 +3270,21 @@ export class RunManager {
|
|
|
2890
3270
|
const run = this.store.getRun(runId);
|
|
2891
3271
|
if (!run?.delegation || run.delegation.role === 'invalid' || this.historyDeletionPending(runId) ||
|
|
2892
3272
|
this.executionBlockedByRootFinish(run) || !['queued', 'running', 'waiting'].includes(run.status) ||
|
|
2893
|
-
this.workerExecutionStopped(runId)
|
|
3273
|
+
this.workerExecutionStopped(runId))
|
|
3274
|
+
return;
|
|
3275
|
+
// A pending question holds every other input; only the parent's reply to a routed one
|
|
3276
|
+
// answers it (#505), and the held inputs follow right behind that answer.
|
|
3277
|
+
if (this.hasPendingHumanAsk(runId)) {
|
|
3278
|
+
this.answerRoutedQuestion(runId);
|
|
2894
3279
|
return;
|
|
3280
|
+
}
|
|
2895
3281
|
// Retain conversation input until the registered CI wait receives scheduler admission.
|
|
2896
3282
|
// Converting this monitor into a worker wake would strand both admission paths.
|
|
2897
3283
|
if (run.ciWait) {
|
|
2898
3284
|
this.queueCiWake(runId);
|
|
2899
3285
|
return;
|
|
2900
3286
|
}
|
|
2901
|
-
const input = run.agentInputs?.find(entry => entry.conversation && !entry.deliveredAt);
|
|
3287
|
+
const input = run.agentInputs?.find(entry => entry.conversation && !entry.deliveredAt && !hasLiveInboxClaim(entry));
|
|
2902
3288
|
if (!input)
|
|
2903
3289
|
return;
|
|
2904
3290
|
const state = this.active.get(runId);
|
|
@@ -2923,6 +3309,231 @@ export class RunManager {
|
|
|
2923
3309
|
else if (!this.recovering)
|
|
2924
3310
|
this.flushAgentInputs(runId);
|
|
2925
3311
|
}
|
|
3312
|
+
/** A worker's pending question goes to its owning parent as a conversation request
|
|
3313
|
+
* (#505). The worker still parks on its ask; only the parent's reply, or a human after
|
|
3314
|
+
* fallback, answers it. Routing is idempotent: the message ID derives from the ask seq. */
|
|
3315
|
+
routeWorkerQuestion(workerId) {
|
|
3316
|
+
const worker = this.store.getRun(workerId);
|
|
3317
|
+
if (worker?.delegation?.role !== 'worker')
|
|
3318
|
+
return;
|
|
3319
|
+
const events = this.store.readEvents(workerId);
|
|
3320
|
+
const ask = pendingHumanAsk(events);
|
|
3321
|
+
if (!ask)
|
|
3322
|
+
return;
|
|
3323
|
+
if (events.some(event => (event.type === 'worker-question-routed' || event.type === 'worker-question-fallback') && event.askSeq === ask.seq))
|
|
3324
|
+
return;
|
|
3325
|
+
const fallback = (reason) => { this.store.appendEvent(workerId, { type: 'worker-question-fallback', askSeq: ask.seq, reason }); };
|
|
3326
|
+
const request = askRequestSchema.safeParse({ questions: ask.questions });
|
|
3327
|
+
if (!request.success) {
|
|
3328
|
+
fallback('the question cannot be carried to the parent');
|
|
3329
|
+
return;
|
|
3330
|
+
}
|
|
3331
|
+
const parentId = worker.delegation.parentRunId;
|
|
3332
|
+
const root = this.store.getRun(parentId);
|
|
3333
|
+
const now = new Date().toISOString();
|
|
3334
|
+
const message = questionMessage({ workerRunId: workerId, parentRunId: parentId, askSeq: ask.seq, request: request.data, now });
|
|
3335
|
+
// Already committed (a crash before the routing event): the question did reach the parent,
|
|
3336
|
+
// whatever its capacity or state is now. Record the routing; fallback sweeps follow.
|
|
3337
|
+
const committed = root?.delegation?.role === 'root' && !!root.delegation.conversation?.messages.some(m => m.id === message.id);
|
|
3338
|
+
if (committed) {
|
|
3339
|
+
this.store.appendEvent(workerId, { type: 'worker-question-routed', askSeq: ask.seq, messageId: message.id, parentRunId: parentId });
|
|
3340
|
+
return;
|
|
3341
|
+
}
|
|
3342
|
+
// Recovery can find a question that never left a worker which has since stopped.
|
|
3343
|
+
if (!['queued', 'running', 'waiting'].includes(worker.status)) {
|
|
3344
|
+
fallback('worker-stopped');
|
|
3345
|
+
return;
|
|
3346
|
+
}
|
|
3347
|
+
if (root?.delegation?.role !== 'root' || !this.parentCanReceive(root)) {
|
|
3348
|
+
fallback('the parent can no longer receive messages');
|
|
3349
|
+
return;
|
|
3350
|
+
}
|
|
3351
|
+
const state = root.delegation.conversation ?? { messages: [], outcomes: [] };
|
|
3352
|
+
// Room in the ledger for the question AND its reply, beside the replies other open questions
|
|
3353
|
+
// reserve. The reply itself is always admitted to this worker's inbox (service.send).
|
|
3354
|
+
if (state.messages.length + openQuestions(state).length + 2 > 1024 || (root.agentInputs ?? []).filter(input => !input.deliveredAt).length >= 32 ||
|
|
3355
|
+
state.messages.filter(m => m.kind === 'request' && m.state === 'accepted' && !state.outcomes.some(o => o.requestId === m.id)).length >= 32) {
|
|
3356
|
+
fallback('the parent conversation is at capacity');
|
|
3357
|
+
return;
|
|
3358
|
+
}
|
|
3359
|
+
const input = { id: message.id, source: 'agent', parentRunId: parentId, text: message.text, createdAt: now,
|
|
3360
|
+
conversation: { senderRunId: workerId, recipientRunId: parentId, kind: 'request' } };
|
|
3361
|
+
this.store.commitConversation(parentId, { ...state, messages: [...state.messages, message] }, { recipientRunId: parentId, input });
|
|
3362
|
+
projectConversationEvents(this.store, this.store.getRun(parentId));
|
|
3363
|
+
this.store.appendEvent(workerId, { type: 'worker-question-routed', askSeq: ask.seq, messageId: message.id, parentRunId: parentId });
|
|
3364
|
+
this.deliverConversationInput(parentId);
|
|
3365
|
+
}
|
|
3366
|
+
/** A human answering a worker question its parent handed back (#505): the reviewing parent
|
|
3367
|
+
* can no longer answer it, so it must not also block the human. */
|
|
3368
|
+
answersFallenBackAsk(runId, opts, deferred) {
|
|
3369
|
+
if (deferred || !(opts.text?.trim() || opts.images?.length))
|
|
3370
|
+
return false;
|
|
3371
|
+
const events = this.store.readEvents(runId);
|
|
3372
|
+
const ask = pendingHumanAsk(events);
|
|
3373
|
+
return !!ask && events.some(event => event.type === 'worker-question-fallback' && event.askSeq === ask.seq);
|
|
3374
|
+
}
|
|
3375
|
+
/** The worker's pending ask when it went to the parent and has not fallen back to a human. */
|
|
3376
|
+
routedAsk(runId) {
|
|
3377
|
+
const run = this.store.getRun(runId);
|
|
3378
|
+
if (run?.delegation?.role !== 'worker')
|
|
3379
|
+
return undefined;
|
|
3380
|
+
const events = this.store.readEvents(runId);
|
|
3381
|
+
const ask = pendingHumanAsk(events);
|
|
3382
|
+
if (!ask || events.some(event => event.type === 'worker-question-fallback' && event.askSeq === ask.seq))
|
|
3383
|
+
return undefined;
|
|
3384
|
+
const routed = events.find(event => event.type === 'worker-question-routed' && event.askSeq === ask.seq);
|
|
3385
|
+
const root = this.store.getRun(run.delegation.parentRunId);
|
|
3386
|
+
const message = routed && root?.delegation?.role === 'root'
|
|
3387
|
+
? root.delegation.conversation?.messages.find(entry => entry.id === routed.messageId) : undefined;
|
|
3388
|
+
return message ? { askSeq: ask.seq, message } : undefined;
|
|
3389
|
+
}
|
|
3390
|
+
/** The parent's reply answers the routed question like a human answer would: through the
|
|
3391
|
+
* live session's native answer seam, or by reopening a session that is gone (#505). */
|
|
3392
|
+
answerRoutedQuestion(runId, admitted = false) {
|
|
3393
|
+
const routed = this.routedAsk(runId);
|
|
3394
|
+
const run = this.store.getRun(runId);
|
|
3395
|
+
const reply = routed && run?.agentInputs?.find(input => !input.deliveredAt && answersQuestion(routed.message, input));
|
|
3396
|
+
// A stopped worker is the human's to continue; a parent answer never revives it.
|
|
3397
|
+
if (!routed || !reply || run?.delegation?.role !== 'worker' || !['queued', 'running', 'waiting'].includes(run.status))
|
|
3398
|
+
return;
|
|
3399
|
+
// Verbatim: a native answer seam parses `Header: label` lines, exactly as a human's.
|
|
3400
|
+
const text = reply.text;
|
|
3401
|
+
const state = this.active.get(runId);
|
|
3402
|
+
if (!state) {
|
|
3403
|
+
// An automated answer is not a human override: the reopened session waits for capacity.
|
|
3404
|
+
if (this.isActive(runId))
|
|
3405
|
+
return;
|
|
3406
|
+
// As on the live path, the answer supersedes a wait registered before the question.
|
|
3407
|
+
if (this.workerWait(runId)) {
|
|
3408
|
+
try {
|
|
3409
|
+
this.withdrawWorkerWait(runId);
|
|
3410
|
+
}
|
|
3411
|
+
catch {
|
|
3412
|
+
return;
|
|
3413
|
+
}
|
|
3414
|
+
}
|
|
3415
|
+
const result = this.continueRun(runId, { text, answerInputId: reply.id }, true);
|
|
3416
|
+
if (!result.ok)
|
|
3417
|
+
this.store.appendEvent(runId, { type: 'note', tone: 'warning', message: `parent answer not delivered: ${result.error}` });
|
|
3418
|
+
return;
|
|
3419
|
+
}
|
|
3420
|
+
if (!state.session?.open || state.cancelled || state.openingAgentInputId === reply.id)
|
|
3421
|
+
return;
|
|
3422
|
+
// Already waiting for admission: only the admission itself may deliver it.
|
|
3423
|
+
if (!admitted && (this.workerWaiting.has(runId) || this.workerWakeQueuedAt.has(runId)))
|
|
3424
|
+
return;
|
|
3425
|
+
if (!admitted && (this.waiting.has(runId) || this.monitoring.has(runId))) {
|
|
3426
|
+
// The parked worker released its slot: the answer rides the ordinary message wake,
|
|
3427
|
+
// which delivers it here once the scheduler admits the worker.
|
|
3428
|
+
const existing = this.workerWait(runId);
|
|
3429
|
+
if (existing?.wakeId !== reply.id) {
|
|
3430
|
+
// An answer supersedes a wait the worker registered before asking, exactly as a
|
|
3431
|
+
// human answer withdraws it; a failed checkpoint leaves both for a later retry.
|
|
3432
|
+
if (existing) {
|
|
3433
|
+
try {
|
|
3434
|
+
this.withdrawWorkerWait(runId);
|
|
3435
|
+
}
|
|
3436
|
+
catch {
|
|
3437
|
+
return;
|
|
3438
|
+
}
|
|
3439
|
+
}
|
|
3440
|
+
const wait = { id: randomUUID(), workerIds: [], outcomes: [],
|
|
3441
|
+
deadline: new Date().toISOString(), phase: 'wake-pending', reason: 'message', wakeId: reply.id };
|
|
3442
|
+
const current = this.store.getRun(runId)?.delegation;
|
|
3443
|
+
if (current?.role !== 'worker')
|
|
3444
|
+
return;
|
|
3445
|
+
this.store.commitDelegation([{ id: runId, delegation: { ...current, wait } }]);
|
|
3446
|
+
}
|
|
3447
|
+
this.clearIdleTimer(state);
|
|
3448
|
+
this.clearMonitoringWakeTimer(state, runId);
|
|
3449
|
+
this.waiting.delete(runId);
|
|
3450
|
+
this.monitoring.delete(runId);
|
|
3451
|
+
this.workerWaiting.add(runId);
|
|
3452
|
+
this.queueWorkerWake(runId);
|
|
3453
|
+
return;
|
|
3454
|
+
}
|
|
3455
|
+
const bundle = state.atTurnBoundary === state.session ? this.pendingConversationBatch(runId, state, reply.id) : undefined;
|
|
3456
|
+
if (!state.session.sendMessage([{ type: 'text', text }, ...(bundle ? [{ type: 'text', text: bundle.text }] : [])]))
|
|
3457
|
+
return;
|
|
3458
|
+
this.store.appendEvent(runId, { type: 'human-input-delivered', askSeq: routed.askSeq, source: 'parent' });
|
|
3459
|
+
const delivered = [reply.id, ...(bundle?.inputs.map(input => input.id) ?? [])];
|
|
3460
|
+
this.commitBundledDelivery(runId, state, delivered);
|
|
3461
|
+
// As in submitAgentInput: the wake that admitted this answer is spent, and a stale one
|
|
3462
|
+
// would hold back every later message until the answer's turn ends.
|
|
3463
|
+
if (delivered.includes(this.workerWait(runId)?.wakeId ?? '')) {
|
|
3464
|
+
state.workerWakeTurn = state.session;
|
|
3465
|
+
try {
|
|
3466
|
+
this.withdrawWorkerWait(runId);
|
|
3467
|
+
}
|
|
3468
|
+
catch { /* reconcileWorkerWaits retires it at the turn end */ }
|
|
3469
|
+
}
|
|
3470
|
+
state.pendingHumanAsk = this.hasPendingHumanAsk(runId);
|
|
3471
|
+
if (!state.pendingHumanAsk)
|
|
3472
|
+
this.flushAgentInputs(runId);
|
|
3473
|
+
this.resumeParkedRun(runId, state);
|
|
3474
|
+
}
|
|
3475
|
+
/** A routed question settled without a reply (its worker stopped, a deadline, fallback) can
|
|
3476
|
+
* never be answered by the parent any more: hand it to the human on the worker (#505). */
|
|
3477
|
+
retireSettledQuestions(parentId) {
|
|
3478
|
+
const root = this.store.getRun(parentId);
|
|
3479
|
+
const conversation = root?.delegation?.role === 'root' ? root.delegation.conversation : undefined;
|
|
3480
|
+
if (!conversation)
|
|
3481
|
+
return;
|
|
3482
|
+
for (const message of conversation.messages) {
|
|
3483
|
+
const outcome = message.question && conversation.outcomes.find(entry => entry.requestId === message.id);
|
|
3484
|
+
// A reply still waiting for a live worker answers it; one a stopped worker never took does not.
|
|
3485
|
+
const worker = this.store.getRun(message.senderRunId);
|
|
3486
|
+
const live = !!worker && ['queued', 'running', 'waiting'].includes(worker.status) && !worker.stopping;
|
|
3487
|
+
if (!outcome || (outcome.status === 'replied' && live))
|
|
3488
|
+
continue;
|
|
3489
|
+
const routed = this.routedAsk(message.senderRunId);
|
|
3490
|
+
if (routed?.message.id === message.id) {
|
|
3491
|
+
this.store.appendEvent(message.senderRunId, { type: 'worker-question-fallback', askSeq: routed.askSeq, reason: outcome.status === 'human-fallback' ? `parent-${root.status}` : outcome.status === 'replied' ? 'worker-stopped' : `question-${outcome.status}` });
|
|
3492
|
+
}
|
|
3493
|
+
}
|
|
3494
|
+
}
|
|
3495
|
+
/** Routed questions this parent has not answered yet, by asking worker. */
|
|
3496
|
+
unansweredQuestions(parentId) {
|
|
3497
|
+
return this.store.listRuns().flatMap(run => {
|
|
3498
|
+
if (run.delegation?.role !== 'worker' || run.delegation.parentRunId !== parentId)
|
|
3499
|
+
return [];
|
|
3500
|
+
const routed = this.routedAsk(run.id);
|
|
3501
|
+
const root = this.store.getRun(parentId);
|
|
3502
|
+
const answered = root?.delegation?.role === 'root' && root.delegation.conversation?.outcomes.some(outcome => outcome.requestId === routed?.message.id);
|
|
3503
|
+
return routed && !answered ? [{ workerId: run.id, messageId: routed.message.id }] : [];
|
|
3504
|
+
});
|
|
3505
|
+
}
|
|
3506
|
+
/** The parent can no longer answer: its workers' routed questions go back to the human
|
|
3507
|
+
* (#505). The outcome lands first, so a crash before the worker's event repeats this. */
|
|
3508
|
+
fallbackRoutedQuestions(parentId, reason) {
|
|
3509
|
+
const parent = this.store.getRun(parentId);
|
|
3510
|
+
if (parent?.delegation?.role !== 'root')
|
|
3511
|
+
return;
|
|
3512
|
+
const state = parent.delegation.conversation ?? { messages: [], outcomes: [] };
|
|
3513
|
+
const routed = this.store.listRuns().flatMap(run => {
|
|
3514
|
+
const ask = run.delegation?.role === 'worker' && run.delegation.parentRunId === parentId ? this.routedAsk(run.id) : undefined;
|
|
3515
|
+
// An accepted reply still on its way to a live worker answers it; no human card beside it.
|
|
3516
|
+
const replied = ask && state.outcomes.some(outcome => outcome.requestId === ask.message.id && outcome.status === 'replied');
|
|
3517
|
+
return ask && !(replied && ['queued', 'running', 'waiting'].includes(run.status) && !run.stopping) ? [{ workerId: run.id, ...ask }] : [];
|
|
3518
|
+
});
|
|
3519
|
+
if (!routed.length)
|
|
3520
|
+
return;
|
|
3521
|
+
const observedAt = new Date().toISOString();
|
|
3522
|
+
const missing = routed.filter(entry => !state.outcomes.some(outcome => outcome.requestId === entry.message.id));
|
|
3523
|
+
if (missing.length) {
|
|
3524
|
+
this.store.commitConversation(parentId, { ...state, outcomes: [...state.outcomes,
|
|
3525
|
+
...missing.map(entry => ({ requestId: entry.message.id, status: 'human-fallback', observedAt }))] });
|
|
3526
|
+
projectConversationEvents(this.store, this.store.getRun(parentId));
|
|
3527
|
+
}
|
|
3528
|
+
for (const entry of routed)
|
|
3529
|
+
this.store.appendEvent(entry.workerId, { type: 'worker-question-fallback', askSeq: entry.askSeq, reason });
|
|
3530
|
+
}
|
|
3531
|
+
/** The parent can still read and answer a routed question; answering needs its steer grant. */
|
|
3532
|
+
parentCanReceive(parent) {
|
|
3533
|
+
return parent.delegation?.role === 'root' && parent.delegation.permissions.includes('steer') &&
|
|
3534
|
+
['queued', 'running', 'waiting'].includes(parent.status) && !parent.stopping &&
|
|
3535
|
+
!this.rootFinishRequested(parent.id) && !this.historyDeletionPending(parent.id);
|
|
3536
|
+
}
|
|
2926
3537
|
/** Cancel only the wait; persist its settlement before ordinary wake admission. */
|
|
2927
3538
|
cancelWorkerWait(parentId, waitId) {
|
|
2928
3539
|
const run = this.store.getRun(parentId);
|
|
@@ -2992,6 +3603,7 @@ export class RunManager {
|
|
|
2992
3603
|
if (next && next !== root.delegation.conversation)
|
|
2993
3604
|
this.store.commitConversation(root.id, next);
|
|
2994
3605
|
projectConversationEvents(this.store, this.store.getRun(root.id));
|
|
3606
|
+
this.retireSettledQuestions(root.id);
|
|
2995
3607
|
const key = `conversation:${root.id}`;
|
|
2996
3608
|
const timer = this.workerWaitTimers.get(key);
|
|
2997
3609
|
if (timer) {
|
|
@@ -3037,6 +3649,7 @@ export class RunManager {
|
|
|
3037
3649
|
if (!['queued', 'running', 'waiting'].includes(parent.status)) {
|
|
3038
3650
|
if (!this.recovering) {
|
|
3039
3651
|
this.withdrawWorkerWait(parent.id);
|
|
3652
|
+
this.fallbackRoutedQuestions(parent.id, `parent-${parent.status}`);
|
|
3040
3653
|
for (const child of this.store.listRuns().filter(child => parent.delegation?.role === 'root' && parent.status !== 'review' && child.delegation?.role === 'worker' && child.delegation.parentRunId === parent.id)) {
|
|
3041
3654
|
if (['queued', 'running', 'waiting'].includes(child.status))
|
|
3042
3655
|
this.cancel(child.id);
|
|
@@ -3049,14 +3662,24 @@ export class RunManager {
|
|
|
3049
3662
|
const wait = parent.delegation.wait;
|
|
3050
3663
|
if (!wait)
|
|
3051
3664
|
continue;
|
|
3052
|
-
const delivered =
|
|
3665
|
+
const delivered = parent.agentInputs?.find(input => input.id === wait.wakeId && input.deliveredAt);
|
|
3053
3666
|
if (delivered) {
|
|
3054
3667
|
// Receipt persistence precedes the live status update. After a crash
|
|
3055
3668
|
// this is interrupted execution, not generic waiting-run success.
|
|
3056
|
-
if (this.recovering && parent.status === 'waiting' && !this.hasPendingHumanAsk(parent.id)) {
|
|
3669
|
+
if (this.recovering && !delivered.inboxClaim?.acknowledgedAt && parent.status === 'waiting' && !this.hasPendingHumanAsk(parent.id)) {
|
|
3057
3670
|
this.store.updateRun(parent.id, { status: 'running', activity: undefined });
|
|
3058
3671
|
}
|
|
3672
|
+
const state = this.active.get(parent.id);
|
|
3673
|
+
const inboxParked = delivered.inboxClaim?.acknowledgedAt &&
|
|
3674
|
+
(this.workerWaiting.has(parent.id) || this.workerWakeAdmitted.has(parent.id)) &&
|
|
3675
|
+
state?.session?.open && state.atTurnBoundary === state.session;
|
|
3059
3676
|
this.withdrawWorkerWait(parent.id);
|
|
3677
|
+
// Inbox ACK starts no provider turn. Retain the completed turn's slot
|
|
3678
|
+
// exemption if its registered wait parked while that ACK was in transit.
|
|
3679
|
+
if (inboxParked && state) {
|
|
3680
|
+
this.waiting.add(parent.id);
|
|
3681
|
+
this.armIdleTimer(parent.id, state);
|
|
3682
|
+
}
|
|
3060
3683
|
continue;
|
|
3061
3684
|
}
|
|
3062
3685
|
const now = new Date().toISOString();
|
|
@@ -3098,8 +3721,8 @@ export class RunManager {
|
|
|
3098
3721
|
// already being processed. Their delayed receipts cannot interrupt a wait
|
|
3099
3722
|
// registered by that very turn. Only inputs still awaiting a turn can.
|
|
3100
3723
|
const flight = this.active.get(parent.id)?.agentInputFlight;
|
|
3101
|
-
const pendingInputs = parent.agentInputs?.filter(input => !input.deliveredAt &&
|
|
3102
|
-
input.id !== parent.continuationMessage?.agentInputId && !flight?.inputIds.includes(input.id)) ?? [];
|
|
3724
|
+
const pendingInputs = parent.agentInputs?.filter(input => !input.deliveredAt && !hasLiveInboxClaim(input) &&
|
|
3725
|
+
input.id !== this.active.get(parent.id)?.openingAgentInputId && input.id !== parent.continuationMessage?.agentInputId && !flight?.inputIds.includes(input.id)) ?? [];
|
|
3103
3726
|
const incoming = pendingInputs.find(input => input.conversation &&
|
|
3104
3727
|
!(input.conversation.kind === 'reply' && input.conversation.requestId && wait.requestIds?.includes(input.conversation.requestId)));
|
|
3105
3728
|
if (incoming && next.phase !== 'wake-pending')
|
|
@@ -3107,7 +3730,8 @@ export class RunManager {
|
|
|
3107
3730
|
// A full queue of selected replies still needs admission. Use the last
|
|
3108
3731
|
// already accepted input as the checkpoint rather than adding a 33rd input.
|
|
3109
3732
|
// Its content is preserved; delivery includes the wait receipt separately.
|
|
3110
|
-
if (next.phase === 'wake-pending' && pendingInputs.length
|
|
3733
|
+
if (next.phase === 'wake-pending' && pendingInputs.length &&
|
|
3734
|
+
(parent.agentInputs ?? []).filter(input => !input.deliveredAt).length >= 32 &&
|
|
3111
3735
|
!parent.agentInputs?.some(input => input.id === next.wakeId)) {
|
|
3112
3736
|
next = { ...next, wakeId: pendingInputs.at(-1).id };
|
|
3113
3737
|
}
|
|
@@ -3172,7 +3796,7 @@ export class RunManager {
|
|
|
3172
3796
|
!['queued', 'running', 'waiting'].includes(run.status))
|
|
3173
3797
|
return;
|
|
3174
3798
|
const existing = run.agentInputs?.find(input => input.id === wait.wakeId);
|
|
3175
|
-
if (existing?.deliveredAt)
|
|
3799
|
+
if (existing?.deliveredAt || (existing && hasLiveInboxClaim(existing)))
|
|
3176
3800
|
return;
|
|
3177
3801
|
// 32 summaries, including worst-case JSON escaping, must fit the ordinary
|
|
3178
3802
|
// input bound. Full 4k summaries remain in durable outcomes and events.
|
|
@@ -3192,6 +3816,10 @@ export class RunManager {
|
|
|
3192
3816
|
return;
|
|
3193
3817
|
}
|
|
3194
3818
|
else if (!existing) {
|
|
3819
|
+
// A full inbox snapshot can temporarily own every reply. ACK/expiry will
|
|
3820
|
+
// reconcile again when a lifecycle input can fit or adopt an unread reply.
|
|
3821
|
+
if ((run.agentInputs ?? []).filter(input => !input.deliveredAt).length >= 32)
|
|
3822
|
+
return;
|
|
3195
3823
|
this.store.commitAgentInputs(parentId, enqueueAgentInput(run, input));
|
|
3196
3824
|
this.store.appendEvent(parentId, { type: 'agent-input', input });
|
|
3197
3825
|
}
|
|
@@ -3202,7 +3830,9 @@ export class RunManager {
|
|
|
3202
3830
|
if (this.recovering)
|
|
3203
3831
|
return;
|
|
3204
3832
|
const state = this.active.get(parentId);
|
|
3205
|
-
|
|
3833
|
+
// A parent's reply to a routed question (#505) is the one message a pending ask admits.
|
|
3834
|
+
const parentAnswer = wait.reason === 'message' && this.routedAsk(parentId) !== undefined;
|
|
3835
|
+
if ((!parentAnswer && (this.hasPendingHumanAsk(parentId) || state?.pendingHumanAsk)) || this.workerWakeQueuedAt.has(parentId))
|
|
3206
3836
|
return;
|
|
3207
3837
|
if (state) {
|
|
3208
3838
|
// Expiring a registered intent NEVER exempts a still-executing turn.
|
|
@@ -3292,7 +3922,227 @@ export class RunManager {
|
|
|
3292
3922
|
const run = this.store.getRun(runId);
|
|
3293
3923
|
return !!run && ['queued', 'running', 'waiting'].includes(run.status) &&
|
|
3294
3924
|
!(run.delegation?.role === 'worker' && run.delegation.destroy) &&
|
|
3295
|
-
!!run.agentInputs?.some(input => !input.deliveredAt);
|
|
3925
|
+
(!!run.agentInputs?.some(input => !input.deliveredAt) || this.harnessOwesInput(this.active.get(runId)));
|
|
3926
|
+
}
|
|
3927
|
+
/** Only the current session's observation writes consumedAt (#505). An ID reported
|
|
3928
|
+
* before its acceptance checkpoint lands is applied when that checkpoint commits. */
|
|
3929
|
+
handleAgentInputConsumed(runId, state, session, ids) {
|
|
3930
|
+
if (!session || this.active.get(runId) !== state || state.session !== session)
|
|
3931
|
+
return;
|
|
3932
|
+
const inFlight = state.agentInputFlight?.inputIds ?? [];
|
|
3933
|
+
for (const id of ids)
|
|
3934
|
+
if (inFlight.includes(id))
|
|
3935
|
+
state.consumedBeforeAck?.add(id);
|
|
3936
|
+
const read = ids.filter(id => !inFlight.includes(id));
|
|
3937
|
+
for (const id of read)
|
|
3938
|
+
state.unreadInputIds?.delete(id);
|
|
3939
|
+
if (!this.harnessOwesInput(state))
|
|
3940
|
+
this.clearUnreadInputTimer(state);
|
|
3941
|
+
try {
|
|
3942
|
+
if (!read.length)
|
|
3943
|
+
return;
|
|
3944
|
+
this.store.commitAgentInputsConsumed(runId, read, new Date().toISOString());
|
|
3945
|
+
this.reconcileWorkerWaits(); // projects the consumed receipt
|
|
3946
|
+
}
|
|
3947
|
+
catch (error) {
|
|
3948
|
+
console.warn(`[cez] agent input consumption checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3949
|
+
}
|
|
3950
|
+
}
|
|
3951
|
+
/** #505: input the harness accepted but will never read returns to the queue, so
|
|
3952
|
+
* DONE, auto-end and park see queued work. Out of a turn (an OpenCode lost wake)
|
|
3953
|
+
* it is submitted right away; at a turn-end the handler's own flush submits it. */
|
|
3954
|
+
retireUnreadInputs(runId, state, ids, outOfTurn) {
|
|
3955
|
+
if (!ids.length || state.cancelled || this.active.get(runId) !== state)
|
|
3956
|
+
return;
|
|
3957
|
+
// Durable first: a failed checkpoint must never let this boundary finish (DONE, park)
|
|
3958
|
+
// over input that is still marked delivered but was never read.
|
|
3959
|
+
try {
|
|
3960
|
+
this.store.requeueUnconsumedAgentInputs(runId, ids);
|
|
3961
|
+
}
|
|
3962
|
+
catch (error) {
|
|
3963
|
+
if (state.agentInputError)
|
|
3964
|
+
return;
|
|
3965
|
+
state.agentInputError = `agent input requeue checkpoint failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
3966
|
+
try {
|
|
3967
|
+
state.session?.interrupt();
|
|
3968
|
+
}
|
|
3969
|
+
finally {
|
|
3970
|
+
try {
|
|
3971
|
+
this.store.appendEvent(runId, { type: 'error', message: state.agentInputError });
|
|
3972
|
+
}
|
|
3973
|
+
catch {
|
|
3974
|
+
console.error(`[cez] ${state.agentInputError}`);
|
|
3975
|
+
}
|
|
3976
|
+
}
|
|
3977
|
+
return;
|
|
3978
|
+
}
|
|
3979
|
+
for (const id of ids)
|
|
3980
|
+
state.unreadInputIds?.delete(id);
|
|
3981
|
+
// Reported before its own acknowledgement landed: that acknowledgement must not
|
|
3982
|
+
// mark it delivered again.
|
|
3983
|
+
for (const id of ids)
|
|
3984
|
+
if (state.agentInputFlight?.inputIds.includes(id))
|
|
3985
|
+
(state.retiredInFlight ??= new Set()).add(id);
|
|
3986
|
+
this.store.appendEvent(runId, { type: 'note', message: `resubmitting ${ids.length} message${ids.length === 1 ? '' : 's'} the agent did not read` });
|
|
3987
|
+
if (outOfTurn)
|
|
3988
|
+
this.flushAgentInputs(runId);
|
|
3989
|
+
}
|
|
3990
|
+
/** #505: the harness started the opening turn, so the opening input was accepted.
|
|
3991
|
+
* Its replay-prompt retirement still waits for that turn's successful end. */
|
|
3992
|
+
recordOpeningAccepted(runId, state) {
|
|
3993
|
+
const id = state.openingAgentInputId;
|
|
3994
|
+
if (!id || state.cancelled || this.active.get(runId) !== state)
|
|
3995
|
+
return;
|
|
3996
|
+
const inputs = this.store.getRun(runId)?.agentInputs ?? [];
|
|
3997
|
+
if (!inputs.some(input => input.id === id && !input.deliveredAt))
|
|
3998
|
+
return;
|
|
3999
|
+
const deliveredAt = new Date().toISOString();
|
|
4000
|
+
try {
|
|
4001
|
+
this.store.commitAgentInputs(runId, inputs.map(input => input.id === id ? { ...input, deliveredAt } : input));
|
|
4002
|
+
}
|
|
4003
|
+
catch (error) {
|
|
4004
|
+
console.warn(`[cez] opening input acceptance checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
4005
|
+
}
|
|
4006
|
+
}
|
|
4007
|
+
/** The conversation messages the next submission would carry, if nothing but
|
|
4008
|
+
* conversation input leads the queue (lifecycle input stays a barrier). */
|
|
4009
|
+
pendingConversationBatch(runId, state, answerId) {
|
|
4010
|
+
const run = this.store.getRun(runId);
|
|
4011
|
+
if (!run || run.ciWait || state.agentInputFlight || this.workerWait(runId))
|
|
4012
|
+
return undefined;
|
|
4013
|
+
const batch = agentInputBatch((run.agentInputs ?? []).filter(input => input.id !== state.openingAgentInputId && input.id !== answerId), input => this.formatAgentInput(runId, input));
|
|
4014
|
+
return batch?.inputs.every(input => input.conversation) ? batch : undefined;
|
|
4015
|
+
}
|
|
4016
|
+
/** Bundled messages rode the answer's own write: delivered now, read when that turn ends. */
|
|
4017
|
+
commitBundledDelivery(runId, state, ids) {
|
|
4018
|
+
const deliveredAt = new Date().toISOString();
|
|
4019
|
+
const awaiting = state.inputDelivery?.consumption === 'observable' ? { awaitingRead: true } : {};
|
|
4020
|
+
try {
|
|
4021
|
+
this.store.commitAgentInputs(runId, (this.store.getRun(runId)?.agentInputs ?? []).map(input => ids.includes(input.id) && !input.deliveredAt ? { ...input, deliveredAt, ...awaiting } : input));
|
|
4022
|
+
state.bundledInputIds = [...(state.bundledInputIds ?? []), ...ids];
|
|
4023
|
+
}
|
|
4024
|
+
catch (error) {
|
|
4025
|
+
console.warn(`[cez] bundled delivery checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
4026
|
+
}
|
|
4027
|
+
}
|
|
4028
|
+
/** The turn that carried bundled messages has ended: the model read them. */
|
|
4029
|
+
readBundledInputs(runId, state) {
|
|
4030
|
+
const ids = state.bundledInputIds ?? [];
|
|
4031
|
+
state.bundledInputIds = undefined;
|
|
4032
|
+
if (!ids.length || state.cancelled)
|
|
4033
|
+
return;
|
|
4034
|
+
try {
|
|
4035
|
+
this.store.commitAgentInputsConsumed(runId, ids, new Date().toISOString());
|
|
4036
|
+
this.reconcileWorkerWaits();
|
|
4037
|
+
}
|
|
4038
|
+
catch (error) {
|
|
4039
|
+
console.warn(`[cez] bundled read checkpoint failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
4040
|
+
}
|
|
4041
|
+
}
|
|
4042
|
+
/** #505: the session closed without reporting it read (failure, stop, exit): the input
|
|
4043
|
+
* goes back to the queue so a later session delivers it; never shown as delivered. */
|
|
4044
|
+
requeueUnreadAtClose(runId, state, session) {
|
|
4045
|
+
this.clearUnreadInputTimer(state);
|
|
4046
|
+
// Everything this session accepted but never saw read: steered input, messages bundled
|
|
4047
|
+
// behind a human answer, and an opening instruction whose turn never succeeded.
|
|
4048
|
+
const opening = state.openingAgentInputId && this.store.getRun(runId)?.agentInputs?.some(input => input.id === state.openingAgentInputId && input.deliveredAt && !input.consumedAt) ? [state.openingAgentInputId] : [];
|
|
4049
|
+
const ids = [...new Set([...(state.unreadInputIds ?? []), ...(state.bundledInputIds ?? []), ...opening])];
|
|
4050
|
+
if (!session || !ids.length || this.active.get(runId) !== state || state.session !== session)
|
|
4051
|
+
return;
|
|
4052
|
+
state.unreadInputIds?.clear();
|
|
4053
|
+
state.bundledInputIds = undefined;
|
|
4054
|
+
try {
|
|
4055
|
+
this.store.requeueUnconsumedAgentInputs(runId, ids);
|
|
4056
|
+
}
|
|
4057
|
+
catch (error) {
|
|
4058
|
+
console.warn(`[cez] agent input requeue failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
4059
|
+
}
|
|
4060
|
+
}
|
|
4061
|
+
/** An idle, open boundary with nothing left to deliver: DONE closes, otherwise the
|
|
4062
|
+
* run parks as waiting or monitoring. Shared by acknowledgements that land after a
|
|
4063
|
+
* turn ended and by the unread-input grace timer (#505). */
|
|
4064
|
+
settleIdleBoundary(runId, state, session) {
|
|
4065
|
+
if (state.atTurnBoundary !== session || state.pendingHumanAsk || this.workerWait(runId) || this.store.getRun(runId)?.ciWait || this.hasQueuedAgentInputs(runId))
|
|
4066
|
+
return;
|
|
4067
|
+
if (state.doneAtBoundary === session) {
|
|
4068
|
+
if (this.deferParentCompletion(runId))
|
|
4069
|
+
return;
|
|
4070
|
+
this.store.appendEvent(runId, { type: 'lifecycle', message: 'goal achieved — session closed' });
|
|
4071
|
+
appendHandoffHeartbeat(this.dataDir, runId, 'turn complete — goal achieved, session closed');
|
|
4072
|
+
session.end();
|
|
4073
|
+
}
|
|
4074
|
+
else if (state.parkAfterAck?.session === session && !this.waiting.has(runId) && !this.monitoring.has(runId)) {
|
|
4075
|
+
// A wake turn can finish before its HTTP/RPC acknowledgement. Retiring
|
|
4076
|
+
// that wait must release its completed turn, not invent another turn.
|
|
4077
|
+
if ((state.parkAfterAck.monitoring || this.hasLiveWorkers(runId)) && !this.parentCompletionAttention(runId)) {
|
|
4078
|
+
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
4079
|
+
if (state.currentStepId)
|
|
4080
|
+
this.store.updateStep(runId, state.currentStepId, { status: 'running' });
|
|
4081
|
+
this.monitoring.add(runId);
|
|
4082
|
+
this.clearIdleTimer(state);
|
|
4083
|
+
this.armMonitoringWakeTimer(runId, state);
|
|
4084
|
+
}
|
|
4085
|
+
else {
|
|
4086
|
+
this.store.updateRun(runId, { status: 'waiting', activity: undefined });
|
|
4087
|
+
if (state.currentStepId)
|
|
4088
|
+
this.store.updateStep(runId, state.currentStepId, { status: 'waiting' });
|
|
4089
|
+
this.store.flush(); // Match ordinary turn-end durability before releasing capacity.
|
|
4090
|
+
this.armIdleTimer(runId, state);
|
|
4091
|
+
}
|
|
4092
|
+
this.waiting.add(runId);
|
|
4093
|
+
this.releaseSlot();
|
|
4094
|
+
}
|
|
4095
|
+
}
|
|
4096
|
+
/** #505 liveness bound: a harness that accepted input and then neither read it nor ran
|
|
4097
|
+
* another turn must not hold the run and its slot forever. After a quiet grace window the
|
|
4098
|
+
* input is resubmitted once; after a second it is left delivered but unconfirmed. */
|
|
4099
|
+
armUnreadInputTimer(runId, state) {
|
|
4100
|
+
this.clearUnreadInputTimer(state);
|
|
4101
|
+
const session = state.session;
|
|
4102
|
+
if (!session?.open || !this.harnessOwesInput(state) || state.agentInputFlight)
|
|
4103
|
+
return;
|
|
4104
|
+
const timer = setTimeout(() => {
|
|
4105
|
+
state.unreadInputTimer = undefined;
|
|
4106
|
+
if (this.disposed || this.active.get(runId) !== state || state.session !== session || !session.open || state.cancelled ||
|
|
4107
|
+
state.agentInputFlight || !this.harnessOwesInput(state))
|
|
4108
|
+
return;
|
|
4109
|
+
const ids = [...state.unreadInputIds];
|
|
4110
|
+
const retried = ids.filter(id => state.unreadRetried?.has(id));
|
|
4111
|
+
const fresh = ids.filter(id => !state.unreadRetried?.has(id));
|
|
4112
|
+
if (retried.length) {
|
|
4113
|
+
for (const id of retried)
|
|
4114
|
+
state.unreadInputIds?.delete(id);
|
|
4115
|
+
try {
|
|
4116
|
+
this.store.commitAgentInputsUnconfirmed(runId, retried);
|
|
4117
|
+
}
|
|
4118
|
+
catch { /* receipt stays awaiting; a restart replays it */ }
|
|
4119
|
+
this.store.appendEvent(runId, { type: 'note', message: `delivered ${retried.length} message${retried.length === 1 ? '' : 's'}; the agent did not confirm reading ${retried.length === 1 ? 'it' : 'them'}` });
|
|
4120
|
+
}
|
|
4121
|
+
if (fresh.length) {
|
|
4122
|
+
state.unreadRetried ??= new Set();
|
|
4123
|
+
for (const id of fresh)
|
|
4124
|
+
state.unreadRetried.add(id);
|
|
4125
|
+
this.retireUnreadInputs(runId, state, fresh, true);
|
|
4126
|
+
}
|
|
4127
|
+
// No content since the last turn ended: the harness is idle, whatever turn it announced.
|
|
4128
|
+
if (!this.harnessOwesInput(state) && !state.agentInputFlight) {
|
|
4129
|
+
state.atTurnBoundary = session;
|
|
4130
|
+
if (state.parkAfterAck?.session !== session)
|
|
4131
|
+
state.parkAfterAck = { session, monitoring: false };
|
|
4132
|
+
this.settleIdleBoundary(runId, state, session);
|
|
4133
|
+
}
|
|
4134
|
+
}, this.unreadInputGraceMs);
|
|
4135
|
+
timer.unref?.();
|
|
4136
|
+
state.unreadInputTimer = timer;
|
|
4137
|
+
}
|
|
4138
|
+
clearUnreadInputTimer(state) {
|
|
4139
|
+
if (state.unreadInputTimer)
|
|
4140
|
+
clearTimeout(state.unreadInputTimer);
|
|
4141
|
+
state.unreadInputTimer = undefined;
|
|
4142
|
+
}
|
|
4143
|
+
/** #505: the harness accepted input this session has not reported reading yet. */
|
|
4144
|
+
harnessOwesInput(state) {
|
|
4145
|
+
return !!state?.unreadInputIds?.size;
|
|
3296
4146
|
}
|
|
3297
4147
|
/** A callback from an old/replaced/disposed session carries no authority. */
|
|
3298
4148
|
handleAgentInputReady(runId, state, session) {
|
|
@@ -3328,7 +4178,7 @@ export class RunManager {
|
|
|
3328
4178
|
};
|
|
3329
4179
|
let acknowledgement;
|
|
3330
4180
|
try {
|
|
3331
|
-
acknowledgement = session.sendAgentMessage(content);
|
|
4181
|
+
acknowledgement = session.sendAgentMessage(content, inputIds);
|
|
3332
4182
|
}
|
|
3333
4183
|
catch (error) {
|
|
3334
4184
|
fail(error, false);
|
|
@@ -3361,8 +4211,25 @@ export class RunManager {
|
|
|
3361
4211
|
}
|
|
3362
4212
|
else {
|
|
3363
4213
|
const deliveredAt = new Date().toISOString();
|
|
3364
|
-
|
|
3365
|
-
|
|
4214
|
+
// An observable harness still owes a read: mark it so a restart replays it (#505).
|
|
4215
|
+
const awaiting = state.inputDelivery?.consumption === 'observable' ? { awaitingRead: true } : {};
|
|
4216
|
+
// Input the harness reported unread while this very POST was pending stays queued:
|
|
4217
|
+
// the flush after this acknowledgement resubmits it (#505 review).
|
|
4218
|
+
const accepted = inputIds.filter(id => !state.retiredInFlight?.has(id));
|
|
4219
|
+
for (const id of inputIds)
|
|
4220
|
+
state.retiredInFlight?.delete(id);
|
|
4221
|
+
this.store.commitAgentInputs(runId, queue.map(input => accepted.includes(input.id) && !input.deliveredAt
|
|
4222
|
+
? { ...input, deliveredAt, ...awaiting } : input));
|
|
4223
|
+
if (state.inputDelivery?.consumption === 'observable') {
|
|
4224
|
+
const early = accepted.filter(id => state.consumedBeforeAck?.has(id));
|
|
4225
|
+
for (const id of accepted)
|
|
4226
|
+
if (!early.includes(id))
|
|
4227
|
+
state.unreadInputIds?.add(id);
|
|
4228
|
+
for (const id of early)
|
|
4229
|
+
state.consumedBeforeAck?.delete(id);
|
|
4230
|
+
if (early.length)
|
|
4231
|
+
this.store.commitAgentInputsConsumed(runId, early, deliveredAt);
|
|
4232
|
+
}
|
|
3366
4233
|
}
|
|
3367
4234
|
if (inputIds.includes(this.workerWait(runId)?.wakeId ?? ''))
|
|
3368
4235
|
this.withdrawWorkerWait(runId);
|
|
@@ -3379,38 +4246,13 @@ export class RunManager {
|
|
|
3379
4246
|
state.agentInputFlight = undefined;
|
|
3380
4247
|
if (!authorized || !session.open || state.agentSessionError || state.agentInputError)
|
|
3381
4248
|
return;
|
|
4249
|
+
// A liveness resubmission gets its own quiet window, and so does input acknowledged
|
|
4250
|
+
// after its turn already ended: no later turn-end would arm one (#505 review).
|
|
4251
|
+
if (inputIds.some(id => state.unreadRetried?.has(id)) || state.atTurnBoundary === session)
|
|
4252
|
+
this.armUnreadInputTimer(runId, state);
|
|
3382
4253
|
if (this.flushAgentInputs(runId))
|
|
3383
4254
|
return;
|
|
3384
|
-
|
|
3385
|
-
return;
|
|
3386
|
-
if (state.doneAtBoundary === session) {
|
|
3387
|
-
if (this.deferParentCompletion(runId))
|
|
3388
|
-
return;
|
|
3389
|
-
this.store.appendEvent(runId, { type: 'lifecycle', message: 'goal achieved — session closed' });
|
|
3390
|
-
appendHandoffHeartbeat(this.dataDir, runId, 'turn complete — goal achieved, session closed');
|
|
3391
|
-
session.end();
|
|
3392
|
-
}
|
|
3393
|
-
else if (state.parkAfterAck?.session === session && !this.waiting.has(runId) && !this.monitoring.has(runId)) {
|
|
3394
|
-
// A wake turn can finish before its HTTP/RPC acknowledgement. Retiring
|
|
3395
|
-
// that wait must release its completed turn, not invent another turn.
|
|
3396
|
-
if ((state.parkAfterAck.monitoring || this.hasLiveWorkers(runId)) && !this.parentCompletionAttention(runId)) {
|
|
3397
|
-
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
3398
|
-
if (state.currentStepId)
|
|
3399
|
-
this.store.updateStep(runId, state.currentStepId, { status: 'running' });
|
|
3400
|
-
this.monitoring.add(runId);
|
|
3401
|
-
this.clearIdleTimer(state);
|
|
3402
|
-
this.armMonitoringWakeTimer(runId, state);
|
|
3403
|
-
}
|
|
3404
|
-
else {
|
|
3405
|
-
this.store.updateRun(runId, { status: 'waiting', activity: undefined });
|
|
3406
|
-
if (state.currentStepId)
|
|
3407
|
-
this.store.updateStep(runId, state.currentStepId, { status: 'waiting' });
|
|
3408
|
-
this.store.flush(); // Match ordinary turn-end durability before releasing capacity.
|
|
3409
|
-
this.armIdleTimer(runId, state);
|
|
3410
|
-
}
|
|
3411
|
-
this.waiting.add(runId);
|
|
3412
|
-
this.releaseSlot();
|
|
3413
|
-
}
|
|
4255
|
+
this.settleIdleBoundary(runId, state, session);
|
|
3414
4256
|
}).catch(error => {
|
|
3415
4257
|
// Bookkeeping failures cannot become unhandled promise rejections.
|
|
3416
4258
|
if (state.agentInputFlight === undefined)
|
|
@@ -3445,10 +4287,9 @@ export class RunManager {
|
|
|
3445
4287
|
return false;
|
|
3446
4288
|
if (state.pendingHumanAsk || this.hasUnansweredHumanAsk(runId))
|
|
3447
4289
|
return false;
|
|
3448
|
-
// The opening prompt already carries
|
|
3449
|
-
//
|
|
3450
|
-
|
|
3451
|
-
return false;
|
|
4290
|
+
// The opening prompt already carries its own input; later input steers the
|
|
4291
|
+
// opening turn instead of waiting for it to end (#505).
|
|
4292
|
+
const queue = (run.agentInputs ?? []).filter(input => input.id !== state.openingAgentInputId);
|
|
3452
4293
|
if (run.ciWait) {
|
|
3453
4294
|
if (!this.ciWakeAdmitted.has(runId) || state.atTurnBoundary !== state.session)
|
|
3454
4295
|
return false;
|
|
@@ -3456,7 +4297,7 @@ export class RunManager {
|
|
|
3456
4297
|
const input = run.agentInputs?.find(entry => entry.id === run.ciWait?.wakeId && !entry.deliveredAt);
|
|
3457
4298
|
return !!input && this.submitAgentInput(runId, state, [{ type: 'text', text: this.formatAgentInput(runId, input) }], [input.id]);
|
|
3458
4299
|
}
|
|
3459
|
-
const batch = agentInputBatch(
|
|
4300
|
+
const batch = agentInputBatch(queue, input => this.formatAgentInput(runId, input));
|
|
3460
4301
|
return !!batch && this.submitAgentInput(runId, state, [{ type: 'text', text: batch.text }], batch.inputs.map(input => input.id));
|
|
3461
4302
|
}
|
|
3462
4303
|
formatAgentInput(runId, input) {
|
|
@@ -3553,6 +4394,10 @@ export class RunManager {
|
|
|
3553
4394
|
return false;
|
|
3554
4395
|
if (!userAuthored && run.ciWait)
|
|
3555
4396
|
return false;
|
|
4397
|
+
if (userAuthored && this.routedAsk(runId)) {
|
|
4398
|
+
this.store.appendEvent(runId, { type: 'note', tone: 'warning', message: ROUTED_QUESTION_REFUSAL });
|
|
4399
|
+
return false;
|
|
4400
|
+
}
|
|
3556
4401
|
const text = content
|
|
3557
4402
|
.filter((b) => b.type === 'text')
|
|
3558
4403
|
.map((b) => b.text)
|
|
@@ -3584,7 +4429,11 @@ export class RunManager {
|
|
|
3584
4429
|
? [...expanded, pastedAttachmentsNote(persisted)]
|
|
3585
4430
|
: expanded.length ? expanded : [{ type: 'text', text: 'The user attached a document, but it could not be saved. Ask them to attach it again before discussing its contents.' }];
|
|
3586
4431
|
const answeringAskSeq = userAuthored ? this.pendingHumanAskSeq(runId) : undefined;
|
|
3587
|
-
|
|
4432
|
+
// #505: an answer that opens a new turn (a CEZ:ASK answer at a boundary) carries the
|
|
4433
|
+
// held conversation messages in the same submission, after the answer.
|
|
4434
|
+
const bundle = userAuthored && answeringAskSeq !== undefined && state.atTurnBoundary === state.session
|
|
4435
|
+
? this.pendingConversationBatch(runId, state) : undefined;
|
|
4436
|
+
const delivered = userAuthored ? state.session.sendMessage(bundle ? [...deliverable, { type: 'text', text: bundle.text }] : deliverable)
|
|
3588
4437
|
: this.submitAgentInput(runId, state, deliverable);
|
|
3589
4438
|
if (delivered) {
|
|
3590
4439
|
if (userAuthored) {
|
|
@@ -3594,6 +4443,12 @@ export class RunManager {
|
|
|
3594
4443
|
this.store.appendEvent(runId, { type: 'human-input-delivered', askSeq: answeringAskSeq });
|
|
3595
4444
|
}
|
|
3596
4445
|
state.pendingHumanAsk = this.hasPendingHumanAsk(runId);
|
|
4446
|
+
if (bundle)
|
|
4447
|
+
this.commitBundledDelivery(runId, state, bundle.inputs.map(input => input.id));
|
|
4448
|
+
// A native answer continues the turn, and a bundle can leave messages that did not
|
|
4449
|
+
// fit its batch: either way the rest steer right behind the answer.
|
|
4450
|
+
if (answeringAskSeq !== undefined && !state.pendingHumanAsk)
|
|
4451
|
+
this.flushAgentInputs(runId);
|
|
3597
4452
|
}
|
|
3598
4453
|
this.resumeParkedRun(runId, state);
|
|
3599
4454
|
}
|
|
@@ -3690,6 +4545,20 @@ export class RunManager {
|
|
|
3690
4545
|
}
|
|
3691
4546
|
return -1;
|
|
3692
4547
|
}
|
|
4548
|
+
/**
|
|
4549
|
+
* Session ids Cursor will accept on `session/load`. A launch UUID written
|
|
4550
|
+
* before `session.started` is only for the step rail (#583).
|
|
4551
|
+
*/
|
|
4552
|
+
resumableSessionId(runId, sessionStep) {
|
|
4553
|
+
if (!sessionStep?.sessionId)
|
|
4554
|
+
return undefined;
|
|
4555
|
+
if ((sessionStep.backend ?? this.store.getRun(runId)?.runner) === 'cursor') {
|
|
4556
|
+
const confirmed = this.store.readEvents(runId).some((event) => event.type === 'session' && event.sessionId === sessionStep.sessionId);
|
|
4557
|
+
if (!confirmed)
|
|
4558
|
+
return undefined;
|
|
4559
|
+
}
|
|
4560
|
+
return sessionStep.sessionId;
|
|
4561
|
+
}
|
|
3693
4562
|
/**
|
|
3694
4563
|
* "Continue" (spec 003): reopen a finished run's claude session in-process
|
|
3695
4564
|
* (`claude --resume <sessionId>`) as a new synthetic step. The session then
|
|
@@ -3715,7 +4584,7 @@ export class RunManager {
|
|
|
3715
4584
|
return { ok: false, error: 'run is still stopping' };
|
|
3716
4585
|
if (this.historyDeletionPending(runId))
|
|
3717
4586
|
return { ok: false, error: 'Parent history deletion is pending; retry deletion' };
|
|
3718
|
-
if (run.delegation?.role === 'worker' && this.store.getRun(run.delegation.parentRunId)?.status === 'review') {
|
|
4587
|
+
if (run.delegation?.role === 'worker' && this.store.getRun(run.delegation.parentRunId)?.status === 'review' && !opts.answerInputId && !this.answersFallenBackAsk(runId, opts, deferForCapacity)) {
|
|
3719
4588
|
return { ok: false, error: 'Continue the reviewing parent before continuing its worker' };
|
|
3720
4589
|
}
|
|
3721
4590
|
// Held against the identity of the step this continuation extends (#452), which for a mixed
|
|
@@ -3750,8 +4619,10 @@ export class RunManager {
|
|
|
3750
4619
|
return { ok: false, error: 'worker cannot continue' };
|
|
3751
4620
|
if (this.executionBlockedByRootFinish(run))
|
|
3752
4621
|
return { ok: false, error: 'parent finish is pending' };
|
|
4622
|
+
if (pendingHumanAsk && !deferForCapacity && !opts.answerInputId && this.routedAsk(runId))
|
|
4623
|
+
return { ok: false, error: ROUTED_QUESTION_REFUSAL };
|
|
3753
4624
|
const answers = deferForCapacity ? [run.continuationMessage, ...(run.queuedMessages ?? [])] : [opts];
|
|
3754
|
-
if (pendingHumanAsk && (conversation || (deferForCapacity && run.continuationMessage?.origin !== 'human') ||
|
|
4625
|
+
if (pendingHumanAsk && !opts.answerInputId && (conversation || (deferForCapacity && run.continuationMessage?.origin !== 'human') ||
|
|
3755
4626
|
!answers.some(answer => answer?.text?.trim() || answer?.images?.length))) {
|
|
3756
4627
|
return { ok: false, error: 'pending human question requires an explicit answer' };
|
|
3757
4628
|
}
|
|
@@ -3788,6 +4659,7 @@ export class RunManager {
|
|
|
3788
4659
|
// account is what decides whether the session can be reattached (#452).
|
|
3789
4660
|
const accountSwitched = (opts.agentProfile ?? resumeIdentity?.account.profileId ?? run.agentProfile ?? sessionAccount) !== sessionAccount;
|
|
3790
4661
|
const resume = sessionBackend === targetRunner && !accountSwitched;
|
|
4662
|
+
const resumeSessionId = resume ? this.resumableSessionId(runId, sessionStep) : undefined;
|
|
3791
4663
|
// Follow-up runner/model/account override (#401, spec 2026-07-29-agent-profiles): the composer
|
|
3792
4664
|
// lets the user pick which backend, model and login handle this continuation — the same flat
|
|
3793
4665
|
// pill the /new composer offers. Omitted → the run's current backend/model/account is kept
|
|
@@ -3857,8 +4729,8 @@ export class RunManager {
|
|
|
3857
4729
|
} };
|
|
3858
4730
|
this.beginWorkerExecution(runId, false);
|
|
3859
4731
|
}
|
|
3860
|
-
catch {
|
|
3861
|
-
return { ok: false, error: 'original execution checkpoint unavailable' };
|
|
4732
|
+
catch (error) {
|
|
4733
|
+
return { ok: false, error: admissionError(error, 'original execution checkpoint unavailable') };
|
|
3862
4734
|
}
|
|
3863
4735
|
const content = [
|
|
3864
4736
|
...(opts.text?.trim() ? [{ type: 'text', text: opts.text.trim() }] : []), ...(opts.images ?? []),
|
|
@@ -3879,8 +4751,8 @@ export class RunManager {
|
|
|
3879
4751
|
try {
|
|
3880
4752
|
this.beginWorkerExecution(runId, !deferForCapacity);
|
|
3881
4753
|
}
|
|
3882
|
-
catch {
|
|
3883
|
-
return { ok: false, error: 'worker execution checkpoint unavailable' };
|
|
4754
|
+
catch (error) {
|
|
4755
|
+
return { ok: false, error: admissionError(error, 'worker execution checkpoint unavailable') };
|
|
3884
4756
|
}
|
|
3885
4757
|
// Everything that could refuse this continuation has now passed, so a pending usage-limit
|
|
3886
4758
|
// resume is superseded either way: this IS that resume (it re-stamps its own counter), or a
|
|
@@ -3899,7 +4771,7 @@ export class RunManager {
|
|
|
3899
4771
|
const continuationStep = { id: stepId, name: 'Continue', kind: 'agent', synthetic: 'continuation' };
|
|
3900
4772
|
if (run.delegation?.role !== 'worker')
|
|
3901
4773
|
this.store.addStep(runId, continuationStep);
|
|
3902
|
-
const recovered = deferForCapacity && !conversation ? run.continuationMessage : undefined;
|
|
4774
|
+
const recovered = deferForCapacity && !conversation && !opts.answerInputId ? run.continuationMessage : undefined;
|
|
3903
4775
|
const message = recovered ?? this.toQueuedMessage(runId, [
|
|
3904
4776
|
...(opts.text?.trim() ? [{ type: 'text', text: opts.text.trim() }] : []),
|
|
3905
4777
|
...(opts.images ?? []),
|
|
@@ -3909,8 +4781,9 @@ export class RunManager {
|
|
|
3909
4781
|
// and the original prompt/URLs must reach disk before the first startup await.
|
|
3910
4782
|
const acceptedPatch = {
|
|
3911
4783
|
continuationMessage: { ...message, id: stepId, text: prompt,
|
|
3912
|
-
...(conversation ? { agentInputId: conversation.input.id } : {}),
|
|
3913
|
-
|
|
4784
|
+
...(conversation ? { agentInputId: conversation.input.id } : opts.answerInputId ? { agentInputId: opts.answerInputId } : {}),
|
|
4785
|
+
// A parent's answer (#505) answers the ask exactly like a human's.
|
|
4786
|
+
origin: recovered ? recovered.origin : deferForCapacity && !opts.answerInputId ? 'lifecycle' : 'human' },
|
|
3914
4787
|
status: deferForCapacity ? 'queued' : 'running',
|
|
3915
4788
|
error: undefined,
|
|
3916
4789
|
finishedAt: undefined,
|
|
@@ -3939,7 +4812,7 @@ export class RunManager {
|
|
|
3939
4812
|
if (deferForCapacity) {
|
|
3940
4813
|
this.pendingContinuations.set(runId, {
|
|
3941
4814
|
stepId,
|
|
3942
|
-
sessionId:
|
|
4815
|
+
sessionId: resumeSessionId,
|
|
3943
4816
|
backend: targetRunner,
|
|
3944
4817
|
prompt,
|
|
3945
4818
|
images,
|
|
@@ -3957,7 +4830,7 @@ export class RunManager {
|
|
|
3957
4830
|
return { ok: true };
|
|
3958
4831
|
}
|
|
3959
4832
|
this.starting.add(runId);
|
|
3960
|
-
void this.runContinuation(runId, stepId,
|
|
4833
|
+
void this.runContinuation(runId, stepId, resumeSessionId, targetRunner, prompt, images).catch((err) => {
|
|
3961
4834
|
const message = err instanceof Error ? err.message : String(err);
|
|
3962
4835
|
this.store.updateRun(runId, {
|
|
3963
4836
|
status: 'failed',
|
|
@@ -4087,12 +4960,15 @@ export class RunManager {
|
|
|
4087
4960
|
const openingInput = record?.agentInputs?.find(input => input.id === checkpoint?.agentInputId);
|
|
4088
4961
|
// Hydration has already included later human updates and task edits. Append
|
|
4089
4962
|
// the current obligation outcome without replacing that composed prompt.
|
|
4090
|
-
|
|
4963
|
+
// A parent's answer to a routed question (#505) is shown as its conversation reply, not
|
|
4964
|
+
// as a human message, and its own reply outcome is not news to the worker.
|
|
4965
|
+
const parentAnswer = openingInput?.conversation?.kind === 'reply' && this.routedAsk(runId) !== undefined;
|
|
4966
|
+
if (openingInput && !parentAnswer)
|
|
4091
4967
|
prompt += this.agentInputOutcomeText(openingInput);
|
|
4092
4968
|
const saved = this.readPersistedAttachments(runId, checkpoint?.id === stepId ? checkpoint.images ?? [] : [], 'continuation');
|
|
4093
4969
|
const openingImages = [...(images.length ? contentBlocksOf(images) : saved.blocks), ...persistedImages];
|
|
4094
4970
|
const attachments = [...saved.attachments, ...persistedAttachments];
|
|
4095
|
-
if (checkpoint?.origin !== 'lifecycle')
|
|
4971
|
+
if (checkpoint?.origin !== 'lifecycle' && !parentAnswer)
|
|
4096
4972
|
this.store.appendEvent(runId, {
|
|
4097
4973
|
type: 'user-message',
|
|
4098
4974
|
stepId,
|
|
@@ -4103,7 +4979,6 @@ export class RunManager {
|
|
|
4103
4979
|
let openingAnswerAskSeq = checkpoint?.origin === 'human' ? this.pendingHumanAskSeq(runId) : undefined;
|
|
4104
4980
|
state.openingAnswerAskSeq = openingAnswerAskSeq;
|
|
4105
4981
|
state.pendingHumanAsk = openingAnswerAskSeq === undefined && this.hasPendingHumanAsk(runId);
|
|
4106
|
-
let stepCost = 0;
|
|
4107
4982
|
let turnText = '';
|
|
4108
4983
|
let completedAssistantText = '';
|
|
4109
4984
|
let sawClaudeScheduleWakeup = false;
|
|
@@ -4122,13 +4997,22 @@ export class RunManager {
|
|
|
4122
4997
|
return;
|
|
4123
4998
|
}
|
|
4124
4999
|
if (event.type === 'text') {
|
|
5000
|
+
this.clearUnreadInputTimer(state); // real content: the harness is working
|
|
4125
5001
|
turnText = appendTurnText(turnText, event.text);
|
|
4126
5002
|
const text = stripAskMarker(stripTaskMarkers(stripMonitoringMarker(stripDoneMarker(event.text))), false);
|
|
4127
5003
|
if (text)
|
|
4128
5004
|
this.store.appendEvent(runId, { type: 'text', text, stepId });
|
|
4129
5005
|
return;
|
|
4130
5006
|
}
|
|
5007
|
+
if (event.type === 'input-unconsumed') {
|
|
5008
|
+
this.retireUnreadInputs(runId, state, event.inputIds, true);
|
|
5009
|
+
return;
|
|
5010
|
+
}
|
|
5011
|
+
if (event.type === 'tool-call' || event.type === 'tool-result')
|
|
5012
|
+
this.clearUnreadInputTimer(state);
|
|
4131
5013
|
this.store.appendEvent(runId, { ...event, stepId });
|
|
5014
|
+
if (event.type === 'cost')
|
|
5015
|
+
this.recordReportedCost(runId, stepId, state, backend, event.usd);
|
|
4132
5016
|
if (event.type === 'error') {
|
|
4133
5017
|
// Preserve the initiating fault: interrupt may fail an in-flight HTTP request.
|
|
4134
5018
|
sessionError ??= state.agentInputError ?? event.message;
|
|
@@ -4144,13 +5028,11 @@ export class RunManager {
|
|
|
4144
5028
|
if (event.type === 'token-usage') {
|
|
4145
5029
|
this.store.updateStep(runId, stepId, { tokensUsed: event.tokensUsed });
|
|
4146
5030
|
}
|
|
4147
|
-
if (event.type === 'cost') {
|
|
4148
|
-
stepCost += event.usd;
|
|
4149
|
-
this.store.updateStep(runId, stepId, { costUsd: stepCost });
|
|
4150
|
-
}
|
|
4151
5031
|
if (isClaudeScheduleWakeup(event, backend))
|
|
4152
5032
|
sawClaudeScheduleWakeup = true;
|
|
4153
5033
|
if (event.type === 'turn-end') {
|
|
5034
|
+
this.retireUnreadInputs(runId, state, event.unconsumedInputIds ?? [], false);
|
|
5035
|
+
this.readBundledInputs(runId, state);
|
|
4154
5036
|
state.workerWakeTurn = undefined;
|
|
4155
5037
|
state.ciWakeTurn = undefined;
|
|
4156
5038
|
state.atTurnBoundary = state.session;
|
|
@@ -4162,7 +5044,13 @@ export class RunManager {
|
|
|
4162
5044
|
const id = state.openingAgentInputId;
|
|
4163
5045
|
const deliveredAt = new Date().toISOString();
|
|
4164
5046
|
try {
|
|
4165
|
-
|
|
5047
|
+
// The successful opening turn handled its instruction: delivered and read (#505).
|
|
5048
|
+
this.store.commitAgentInputs(runId, (this.store.getRun(runId)?.agentInputs ?? []).map(input => {
|
|
5049
|
+
if (input.id !== id)
|
|
5050
|
+
return input;
|
|
5051
|
+
const { awaitingRead: _awaiting, ...handled } = input;
|
|
5052
|
+
return { ...handled, deliveredAt: input.deliveredAt ?? deliveredAt, consumedAt: input.consumedAt ?? deliveredAt };
|
|
5053
|
+
}), id);
|
|
4166
5054
|
}
|
|
4167
5055
|
catch (error) {
|
|
4168
5056
|
state.agentInputError = `agent input delivery checkpoint failed: ${error instanceof Error ? error.message : String(error)}`;
|
|
@@ -4183,7 +5071,7 @@ export class RunManager {
|
|
|
4183
5071
|
this.reconcileWorkerWaits();
|
|
4184
5072
|
}
|
|
4185
5073
|
if (openingAnswerAskSeq !== undefined && state.session?.open) {
|
|
4186
|
-
this.store.appendEvent(runId, { type: 'human-input-delivered', askSeq: openingAnswerAskSeq });
|
|
5074
|
+
this.store.appendEvent(runId, { type: 'human-input-delivered', askSeq: openingAnswerAskSeq, ...(parentAnswer ? { source: 'parent' } : {}) });
|
|
4187
5075
|
openingAnswerAskSeq = undefined;
|
|
4188
5076
|
state.openingAnswerAskSeq = undefined;
|
|
4189
5077
|
}
|
|
@@ -4221,7 +5109,11 @@ export class RunManager {
|
|
|
4221
5109
|
const workerWaitParked = ciWaitParked || completionBlocked || (!!sessionOpen && this.parkWorkerWait(runId, state));
|
|
4222
5110
|
state.doneAtBoundary = done ? state.session : undefined;
|
|
4223
5111
|
state.parkAfterAck = sessionOpen && state.session ? { session: state.session, monitoring: !!monitoring } : undefined;
|
|
4224
|
-
|
|
5112
|
+
// #505: input the harness accepted but has not read yet runs as its next turn.
|
|
5113
|
+
// An in-flight submission is settled by its acknowledgement (parkAfterAck), not here.
|
|
5114
|
+
const agentInputDelivered = !ask && !workerWaitParked && (this.flushAgentInputs(runId) || this.harnessOwesInput(state) || !!state.agentInputFlight);
|
|
5115
|
+
if (!ask && !workerWaitParked)
|
|
5116
|
+
this.armUnreadInputTimer(runId, state);
|
|
4225
5117
|
if (state.agentInputError)
|
|
4226
5118
|
return;
|
|
4227
5119
|
if (done && !workerWaitParked && !state.pendingHumanAsk && !agentInputDelivered && !state.agentInputFlight && !this.hasQueuedAgentInputs(runId)) {
|
|
@@ -4257,8 +5149,10 @@ export class RunManager {
|
|
|
4257
5149
|
// `ScheduleWakeup` → non-attention `running`/`activity:'monitoring'`
|
|
4258
5150
|
// (#490, #46). Both free the slot; only genuine user waits keep the
|
|
4259
5151
|
// idle timer. The autonomous nudge above still wins over either.
|
|
4260
|
-
if (ask)
|
|
5152
|
+
if (ask) {
|
|
4261
5153
|
emitAskRequested(sink, ask);
|
|
5154
|
+
this.routeWorkerQuestion(runId);
|
|
5155
|
+
}
|
|
4262
5156
|
if (monitoring) {
|
|
4263
5157
|
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
4264
5158
|
this.store.updateStep(runId, stepId, { status: 'running' });
|
|
@@ -4395,11 +5289,14 @@ export class RunManager {
|
|
|
4395
5289
|
failBeforeSpawn(err.message);
|
|
4396
5290
|
return;
|
|
4397
5291
|
}
|
|
4398
|
-
|
|
5292
|
+
const settlePrelaunchCancellation = () => {
|
|
4399
5293
|
this.store.updateStep(runId, stepId, { status: 'cancelled' });
|
|
4400
5294
|
this.store.updateRun(runId, { status: 'cancelled', finishedAt: new Date().toISOString(), currentStepId: undefined });
|
|
4401
5295
|
this.clearAutosaveTimer(state);
|
|
4402
5296
|
this.dropActive(runId);
|
|
5297
|
+
};
|
|
5298
|
+
if (state.cancelled) {
|
|
5299
|
+
settlePrelaunchCancellation();
|
|
4403
5300
|
return;
|
|
4404
5301
|
}
|
|
4405
5302
|
if (this.holdContinuationForRootFinish(runId, { stepId, sessionId, backend, prompt, images }, state))
|
|
@@ -4424,12 +5321,36 @@ export class RunManager {
|
|
|
4424
5321
|
}
|
|
4425
5322
|
const delegation = this.provisionSession(runId, state);
|
|
4426
5323
|
const ciTools = await this.provisionCiSession(runId, state);
|
|
5324
|
+
// Stop/Finish can arrive while the local tool controller is starting.
|
|
5325
|
+
const launchRun = this.store.getRun(runId);
|
|
5326
|
+
if (state.cancelled || this.workerExecutionStopped(runId) || this.active.get(runId) !== state ||
|
|
5327
|
+
(launchRun && this.executionBlockedByRootFinish(launchRun))) {
|
|
5328
|
+
ciTools?.revoke();
|
|
5329
|
+
if (state.revokeCiTools === ciTools?.revoke)
|
|
5330
|
+
state.revokeCiTools = undefined;
|
|
5331
|
+
delegation?.revoke();
|
|
5332
|
+
if (state.revokeDelegation === delegation?.revoke)
|
|
5333
|
+
state.revokeDelegation = undefined;
|
|
5334
|
+
if (this.active.get(runId) !== state)
|
|
5335
|
+
return;
|
|
5336
|
+
if (state.cancelled || this.workerExecutionStopped(runId)) {
|
|
5337
|
+
state.cancelled = true;
|
|
5338
|
+
settlePrelaunchCancellation();
|
|
5339
|
+
}
|
|
5340
|
+
else {
|
|
5341
|
+
this.holdContinuationForRootFinish(runId, { stepId, sessionId, backend, prompt, images }, state);
|
|
5342
|
+
}
|
|
5343
|
+
return;
|
|
5344
|
+
}
|
|
5345
|
+
state.inputDelivery = inputDeliveryOf(runner);
|
|
5346
|
+
state.unreadInputIds = new Set();
|
|
5347
|
+
state.consumedBeforeAck = new Set();
|
|
4427
5348
|
try {
|
|
4428
5349
|
session = runner.startSession({
|
|
4429
5350
|
// The Continue step is a fresh agent session on the same run — the
|
|
4430
5351
|
// run's extra system prompt (already resolved at execute time and
|
|
4431
5352
|
// echoed on the record) rides along with the handoff contract.
|
|
4432
|
-
systemPrompt: composeSystemPrompt(record?.systemPrompt, delegation?.instructions, generateFollowups ? HANDOFF_INSTRUCTIONS : HANDOFF_ONLY_INSTRUCTIONS),
|
|
5353
|
+
systemPrompt: composeSystemPrompt(record?.systemPrompt, delegation?.instructions, artifactInstructions(continueProfile.env.CEZ_ARTIFACTS_DIR), generateFollowups ? HANDOFF_INSTRUCTIONS : HANDOFF_ONLY_INSTRUCTIONS),
|
|
4433
5354
|
userPrompt: attachments.length
|
|
4434
5355
|
? `${openingPrompt}\n\n${pastedAttachmentsText(attachments)}`
|
|
4435
5356
|
: openingPrompt,
|
|
@@ -4449,9 +5370,12 @@ export class RunManager {
|
|
|
4449
5370
|
}, onEvent, {
|
|
4450
5371
|
onUiEvent: (event) => {
|
|
4451
5372
|
completedAssistantText = appendCompletedAssistantText(completedAssistantText, event);
|
|
5373
|
+
if (event.type === 'turn.started')
|
|
5374
|
+
this.recordOpeningAccepted(runId, state);
|
|
4452
5375
|
this.handleRunnerUiEvent(runId, state, sink, event);
|
|
4453
5376
|
},
|
|
4454
5377
|
onAgentInputReady: () => this.handleAgentInputReady(runId, state, session),
|
|
5378
|
+
onAgentInputConsumed: (ids) => this.handleAgentInputConsumed(runId, state, session, ids),
|
|
4455
5379
|
});
|
|
4456
5380
|
}
|
|
4457
5381
|
catch (error) {
|
|
@@ -4470,10 +5394,12 @@ export class RunManager {
|
|
|
4470
5394
|
try {
|
|
4471
5395
|
this.flushDeferred(runId);
|
|
4472
5396
|
this.flushAgentInputs(runId);
|
|
4473
|
-
if (session.pid !== undefined)
|
|
5397
|
+
if (session.pid !== undefined) {
|
|
4474
5398
|
registerRunProcess(runId, session.pid);
|
|
5399
|
+
this.recordWorkerProcess(runId, session.pid);
|
|
5400
|
+
}
|
|
4475
5401
|
setupComplete = true;
|
|
4476
|
-
const result = await session.result.finally(() => state.agentInputFlight?.settled);
|
|
5402
|
+
const result = await session.result.finally(() => state.agentInputFlight?.settled).finally(() => this.requeueUnreadAtClose(runId, state, session));
|
|
4477
5403
|
if (this.preserveRunAfterDisposal(runId, state))
|
|
4478
5404
|
return;
|
|
4479
5405
|
this.persistWorkerAssistantResult(runId, stepId, result.text);
|
|
@@ -4862,6 +5788,11 @@ export class RunManager {
|
|
|
4862
5788
|
startImages = undefined;
|
|
4863
5789
|
startAttachments = [];
|
|
4864
5790
|
checkFailure = null;
|
|
5791
|
+
// Stale setup cannot finalize a replacement owner. Disposal also clears
|
|
5792
|
+
// active, but must still honor a previously accepted Stop/Finish intent.
|
|
5793
|
+
if (this.active.get(runId) !== state &&
|
|
5794
|
+
!(this.disposed && (state.cancelled || state.finishRequested)))
|
|
5795
|
+
return;
|
|
4865
5796
|
if (state.cancelled || this.preserveRunAfterDisposal(runId, state))
|
|
4866
5797
|
break;
|
|
4867
5798
|
const current = this.store.getRun(runId);
|
|
@@ -5033,7 +5964,6 @@ export class RunManager {
|
|
|
5033
5964
|
this.store.updateStep(runId, step.id, { sessionId, backend });
|
|
5034
5965
|
const stepRecord = this.store.getRun(runId)?.steps.find((s) => s.id === step.id);
|
|
5035
5966
|
const startTokens = stepRecord?.tokensUsed ?? 0;
|
|
5036
|
-
let stepCost = stepRecord?.costUsd ?? 0;
|
|
5037
5967
|
let turnText = '';
|
|
5038
5968
|
let completedAssistantText = '';
|
|
5039
5969
|
let sawClaudeScheduleWakeup = false;
|
|
@@ -5052,13 +5982,22 @@ export class RunManager {
|
|
|
5052
5982
|
return;
|
|
5053
5983
|
}
|
|
5054
5984
|
if (event.type === 'text') {
|
|
5985
|
+
this.clearUnreadInputTimer(state); // real content: the harness is working
|
|
5055
5986
|
turnText = appendTurnText(turnText, event.text);
|
|
5056
5987
|
const text = stripAskMarker(stripTaskMarkers(stripMonitoringMarker(stripDoneMarker(event.text))), false);
|
|
5057
5988
|
if (text)
|
|
5058
5989
|
emit({ type: 'text', text, stepId: step.id });
|
|
5059
5990
|
return;
|
|
5060
5991
|
}
|
|
5992
|
+
if (event.type === 'input-unconsumed') {
|
|
5993
|
+
this.retireUnreadInputs(runId, state, event.inputIds, true);
|
|
5994
|
+
return;
|
|
5995
|
+
}
|
|
5996
|
+
if (event.type === 'tool-call' || event.type === 'tool-result')
|
|
5997
|
+
this.clearUnreadInputTimer(state);
|
|
5061
5998
|
emit({ ...event, stepId: step.id });
|
|
5999
|
+
if (event.type === 'cost')
|
|
6000
|
+
this.recordReportedCost(runId, step.id, state, backend, event.usd);
|
|
5062
6001
|
if (event.type === 'error') {
|
|
5063
6002
|
// Preserve the initiating fault: interrupt may fail an in-flight HTTP request.
|
|
5064
6003
|
sessionError ??= state.agentInputError ?? event.message;
|
|
@@ -5075,13 +6014,11 @@ export class RunManager {
|
|
|
5075
6014
|
if (event.type === 'token-usage') {
|
|
5076
6015
|
this.store.updateStep(runId, step.id, { tokensUsed: startTokens + event.tokensUsed });
|
|
5077
6016
|
}
|
|
5078
|
-
if (event.type === 'cost') {
|
|
5079
|
-
stepCost += event.usd;
|
|
5080
|
-
this.store.updateStep(runId, step.id, { costUsd: stepCost });
|
|
5081
|
-
}
|
|
5082
6017
|
if (isClaudeScheduleWakeup(event, backend))
|
|
5083
6018
|
sawClaudeScheduleWakeup = true;
|
|
5084
6019
|
if (event.type === 'turn-end') {
|
|
6020
|
+
this.retireUnreadInputs(runId, state, event.unconsumedInputIds ?? [], false);
|
|
6021
|
+
this.readBundledInputs(runId, state);
|
|
5085
6022
|
state.workerWakeTurn = undefined;
|
|
5086
6023
|
state.ciWakeTurn = undefined;
|
|
5087
6024
|
state.atTurnBoundary = state.session;
|
|
@@ -5117,7 +6054,11 @@ export class RunManager {
|
|
|
5117
6054
|
const workerWaitParked = ciWaitParked || completionBlocked || (!!sessionOpen && this.parkWorkerWait(runId, state));
|
|
5118
6055
|
state.doneAtBoundary = done ? state.session : undefined;
|
|
5119
6056
|
state.parkAfterAck = interactive && sessionOpen && state.session ? { session: state.session, monitoring: !!monitoring } : undefined;
|
|
5120
|
-
|
|
6057
|
+
// #505: input the harness accepted but has not read yet runs as its next turn.
|
|
6058
|
+
// An in-flight submission is settled by its acknowledgement (parkAfterAck), not here.
|
|
6059
|
+
const agentInputDelivered = !ask && !workerWaitParked && (this.flushAgentInputs(runId) || this.harnessOwesInput(state) || !!state.agentInputFlight);
|
|
6060
|
+
if (!ask && !workerWaitParked)
|
|
6061
|
+
this.armUnreadInputTimer(runId, state);
|
|
5121
6062
|
if (state.agentInputError)
|
|
5122
6063
|
return;
|
|
5123
6064
|
if (done && !workerWaitParked && !state.pendingHumanAsk && !agentInputDelivered && !state.agentInputFlight && !this.hasQueuedAgentInputs(runId)) {
|
|
@@ -5136,8 +6077,10 @@ export class RunManager {
|
|
|
5136
6077
|
// still working through `CEZ:MONITORING` or Claude's native
|
|
5137
6078
|
// `ScheduleWakeup`. Monitoring parks as `running` with no user-wait
|
|
5138
6079
|
// idle timer, so the cockpit stays non-attention (#490, #46).
|
|
5139
|
-
if (ask)
|
|
6080
|
+
if (ask) {
|
|
5140
6081
|
emitAskRequested(sink, ask);
|
|
6082
|
+
this.routeWorkerQuestion(runId);
|
|
6083
|
+
}
|
|
5141
6084
|
if (monitoring) {
|
|
5142
6085
|
this.store.updateRun(runId, { status: 'running', activity: 'monitoring' });
|
|
5143
6086
|
this.store.updateStep(runId, step.id, { status: 'running' });
|
|
@@ -5232,19 +6175,38 @@ export class RunManager {
|
|
|
5232
6175
|
const runner = createRunner(stepBackend);
|
|
5233
6176
|
state.agentSessionError = undefined;
|
|
5234
6177
|
let session;
|
|
6178
|
+
const shouldAutoEnd = () => this.active.get(runId) !== state || state.session !== session ||
|
|
6179
|
+
(!this.workerWait(runId) && !this.store.getRun(runId)?.ciWait && !this.parentCompletionPending(runId) &&
|
|
6180
|
+
state.workerWakeTurn !== session && state.ciWakeTurn !== session && !state.agentInputFlight &&
|
|
6181
|
+
!this.hasQueuedAgentInputs(runId));
|
|
5235
6182
|
state.currentStepId = step.id;
|
|
5236
6183
|
this.beginUsageInvocation(runId, state, step.id);
|
|
5237
6184
|
const delegation = this.provisionSession(runId, state);
|
|
5238
6185
|
const ciTools = await this.provisionCiSession(runId, state);
|
|
5239
|
-
|
|
5240
|
-
|
|
6186
|
+
// Revalidate after provisioning, then let execute settle cancellation or
|
|
6187
|
+
// requeue a Finish-held step through its existing no-session paths.
|
|
6188
|
+
const launchRun = this.store.getRun(runId);
|
|
6189
|
+
if (state.cancelled || this.workerExecutionStopped(runId) || this.active.get(runId) !== state ||
|
|
6190
|
+
(launchRun && this.executionBlockedByRootFinish(launchRun))) {
|
|
6191
|
+
ciTools?.revoke();
|
|
6192
|
+
if (state.revokeCiTools === ciTools?.revoke)
|
|
6193
|
+
state.revokeCiTools = undefined;
|
|
6194
|
+
delegation?.revoke();
|
|
6195
|
+
if (state.revokeDelegation === delegation?.revoke)
|
|
6196
|
+
state.revokeDelegation = undefined;
|
|
6197
|
+
if (this.workerExecutionStopped(runId))
|
|
5241
6198
|
state.cancelled = true;
|
|
5242
|
-
|
|
5243
|
-
|
|
6199
|
+
state.currentStepId = undefined;
|
|
6200
|
+
return null;
|
|
6201
|
+
}
|
|
6202
|
+
state.inputDelivery = inputDeliveryOf(runner);
|
|
6203
|
+
state.unreadInputIds = new Set();
|
|
6204
|
+
state.consumedBeforeAck = new Set();
|
|
6205
|
+
try {
|
|
5244
6206
|
session = runner.startSession({
|
|
5245
6207
|
// Skill body, then the run's extra prompt (POST override or config
|
|
5246
6208
|
// default), then the handoff/todos contract — every agent step.
|
|
5247
|
-
systemPrompt: composeSystemPrompt(systemPrompt, extraSystemPrompt, delegation?.instructions, followupsEnabled() && input.generateFollowups !== false
|
|
6209
|
+
systemPrompt: composeSystemPrompt(systemPrompt, extraSystemPrompt, delegation?.instructions, artifactInstructions(stepProfile.env.CEZ_ARTIFACTS_DIR), followupsEnabled() && input.generateFollowups !== false
|
|
5248
6210
|
? HANDOFF_INSTRUCTIONS
|
|
5249
6211
|
: HANDOFF_ONLY_INSTRUCTIONS),
|
|
5250
6212
|
userPrompt,
|
|
@@ -5264,13 +6226,14 @@ export class RunManager {
|
|
|
5264
6226
|
timeoutMs: interactive ? 0 : undefined,
|
|
5265
6227
|
}, onEvent, {
|
|
5266
6228
|
autoEndAfterFirstTurn: !interactive,
|
|
5267
|
-
|
|
5268
|
-
|
|
6229
|
+
// Unread inbox claims must survive this window so release/expiry can replay them.
|
|
6230
|
+
shouldAutoEnd,
|
|
5269
6231
|
onUiEvent: (event) => {
|
|
5270
6232
|
completedAssistantText = appendCompletedAssistantText(completedAssistantText, event);
|
|
5271
6233
|
this.handleRunnerUiEvent(runId, state, sink, event);
|
|
5272
6234
|
},
|
|
5273
6235
|
onAgentInputReady: () => this.handleAgentInputReady(runId, state, session),
|
|
6236
|
+
onAgentInputConsumed: (ids) => this.handleAgentInputConsumed(runId, state, session, ids),
|
|
5274
6237
|
});
|
|
5275
6238
|
}
|
|
5276
6239
|
catch (err) {
|
|
@@ -5283,6 +6246,14 @@ export class RunManager {
|
|
|
5283
6246
|
}
|
|
5284
6247
|
const sessionClosed = this.trackWorkerSessionResult(runId, session);
|
|
5285
6248
|
state.session = session;
|
|
6249
|
+
state.resumeInboxAutoEnd = interactive ? undefined : () => {
|
|
6250
|
+
// ACK may consume the last input after the runner's one-shot close timer fired.
|
|
6251
|
+
// Release/expiry instead submit a provider turn through normal admission.
|
|
6252
|
+
if (this.active.get(runId) === state && state.session === session && session?.open &&
|
|
6253
|
+
state.atTurnBoundary === session && !state.cancelled && !state.pendingHumanAsk &&
|
|
6254
|
+
!this.hasPendingHumanAsk(runId) && shouldAutoEnd())
|
|
6255
|
+
session.end();
|
|
6256
|
+
};
|
|
5286
6257
|
state.sessionEverOpened = true;
|
|
5287
6258
|
state.currentStepId = step.id;
|
|
5288
6259
|
state.interrupt = () => session?.interrupt();
|
|
@@ -5290,10 +6261,12 @@ export class RunManager {
|
|
|
5290
6261
|
try {
|
|
5291
6262
|
this.flushDeferred(runId);
|
|
5292
6263
|
this.flushAgentInputs(runId);
|
|
5293
|
-
if (session.pid !== undefined)
|
|
6264
|
+
if (session.pid !== undefined) {
|
|
5294
6265
|
registerRunProcess(runId, session.pid);
|
|
6266
|
+
this.recordWorkerProcess(runId, session.pid);
|
|
6267
|
+
}
|
|
5295
6268
|
setupComplete = true;
|
|
5296
|
-
const result = await session.result.finally(() => state.agentInputFlight?.settled);
|
|
6269
|
+
const result = await session.result.finally(() => state.agentInputFlight?.settled).finally(() => this.requeueUnreadAtClose(runId, state, session));
|
|
5297
6270
|
if (this.isDisposedDelegatedRun(runId))
|
|
5298
6271
|
return null;
|
|
5299
6272
|
this.persistWorkerAssistantResult(runId, step.id, result.text);
|
|
@@ -5341,6 +6314,7 @@ export class RunManager {
|
|
|
5341
6314
|
if (at >= 0)
|
|
5342
6315
|
this.queue.splice(at, 1);
|
|
5343
6316
|
}
|
|
6317
|
+
state.resumeInboxAutoEnd = undefined;
|
|
5344
6318
|
state.workerWakeTurn = undefined;
|
|
5345
6319
|
state.ciWakeTurn = undefined;
|
|
5346
6320
|
state.atTurnBoundary = undefined;
|
|
@@ -5418,6 +6392,7 @@ export class RunManager {
|
|
|
5418
6392
|
if (state.currentStepId)
|
|
5419
6393
|
this.store.updateStep(runId, state.currentStepId, { status: 'waiting' });
|
|
5420
6394
|
this.releaseSlot();
|
|
6395
|
+
this.routeWorkerQuestion(runId);
|
|
5421
6396
|
return;
|
|
5422
6397
|
}
|
|
5423
6398
|
// Pi can resume autonomously when an async subagent completes — the mapper
|
|
@@ -5446,6 +6421,64 @@ export class RunManager {
|
|
|
5446
6421
|
recordedTurns: new Set(),
|
|
5447
6422
|
};
|
|
5448
6423
|
}
|
|
6424
|
+
/** Claude's `total_cost_usd` is cumulative for the conversation, including
|
|
6425
|
+
* turns in Continue steps. Other runners emit per-message cost deltas. */
|
|
6426
|
+
recordReportedCost(runId, stepId, state, backend, reportedUsd) {
|
|
6427
|
+
if (!Number.isFinite(reportedUsd) || reportedUsd < 0)
|
|
6428
|
+
return;
|
|
6429
|
+
const run = this.store.getRun(runId);
|
|
6430
|
+
const step = run?.steps.find((candidate) => candidate.id === stepId);
|
|
6431
|
+
if (!run || !step)
|
|
6432
|
+
return;
|
|
6433
|
+
let delta = reportedUsd;
|
|
6434
|
+
if (backend === 'claude') {
|
|
6435
|
+
const sessionKey = step.sessionId ?? step.id;
|
|
6436
|
+
state.reportedClaudeCost ??= new Map();
|
|
6437
|
+
let previous = state.reportedClaudeCost.get(sessionKey);
|
|
6438
|
+
if (previous === undefined) {
|
|
6439
|
+
// The current cost event was appended immediately before this call. Rebuild
|
|
6440
|
+
// the checkpoint from raw reports, since older records summed cumulative
|
|
6441
|
+
// reports into costUsd and cannot be used as the baseline on Continue.
|
|
6442
|
+
const events = this.store.readEvents(runId);
|
|
6443
|
+
let currentReportIndex = -1;
|
|
6444
|
+
for (let index = events.length - 1; index >= 0; index--) {
|
|
6445
|
+
const event = events[index];
|
|
6446
|
+
if (event.type === 'cost' && event.stepId === stepId && event.usd === reportedUsd) {
|
|
6447
|
+
currentReportIndex = index;
|
|
6448
|
+
break;
|
|
6449
|
+
}
|
|
6450
|
+
}
|
|
6451
|
+
const sessionStepIds = new Set(run.steps.filter((candidate) => (candidate.backend === 'claude' || candidate.backend === undefined) && (step.sessionId
|
|
6452
|
+
? candidate.sessionId === step.sessionId
|
|
6453
|
+
: candidate.id === step.id)).map((candidate) => candidate.id));
|
|
6454
|
+
previous = 0;
|
|
6455
|
+
const priorCostByStep = new Map();
|
|
6456
|
+
for (let index = 0; index < events.length; index++) {
|
|
6457
|
+
if (index === currentReportIndex)
|
|
6458
|
+
continue;
|
|
6459
|
+
const event = events[index];
|
|
6460
|
+
if (event.type === 'cost' && event.stepId && sessionStepIds.has(event.stepId) &&
|
|
6461
|
+
typeof event.usd === 'number' && Number.isFinite(event.usd) && event.usd >= 0) {
|
|
6462
|
+
priorCostByStep.set(event.stepId, (priorCostByStep.get(event.stepId) ?? 0) + Math.max(0, event.usd - previous));
|
|
6463
|
+
previous = Math.max(previous, event.usd);
|
|
6464
|
+
}
|
|
6465
|
+
}
|
|
6466
|
+
// The event log is appended before the step total. A process exit between
|
|
6467
|
+
// those writes leaves a report whose USD never reached the run record.
|
|
6468
|
+
// Restore missing amounts, while preserving older inflated totals rather
|
|
6469
|
+
// than silently migrating historical run state.
|
|
6470
|
+
for (const [priorStepId, expectedCost] of priorCostByStep) {
|
|
6471
|
+
const priorStep = run.steps.find((candidate) => candidate.id === priorStepId);
|
|
6472
|
+
if (priorStep && (priorStep.costUsd === undefined || priorStep.costUsd < expectedCost)) {
|
|
6473
|
+
this.store.updateStep(runId, priorStepId, { costUsd: expectedCost });
|
|
6474
|
+
}
|
|
6475
|
+
}
|
|
6476
|
+
}
|
|
6477
|
+
delta = Math.max(0, reportedUsd - previous);
|
|
6478
|
+
state.reportedClaudeCost.set(sessionKey, Math.max(previous, reportedUsd));
|
|
6479
|
+
}
|
|
6480
|
+
this.store.updateStep(runId, stepId, { costUsd: (step.costUsd ?? 0) + delta });
|
|
6481
|
+
}
|
|
5449
6482
|
/** Fold backend-neutral completed-turn usage into the current step exactly
|
|
5450
6483
|
* once. Invocation/turn counters are written before the event reaches the
|
|
5451
6484
|
* NDJSON sink so crashes cannot preserve a falsely complete subtotal. */
|