@qodeca/xezar 0.13.1 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -15
- package/dist/agent-config/account-identity.d.ts +2 -8
- package/dist/agent-config/account-identity.js +54 -10
- package/dist/agent-config/account-identity.js.map +1 -1
- package/dist/agent-config/catalog.d.ts +49 -7
- package/dist/agent-config/catalog.js +107 -0
- package/dist/agent-config/catalog.js.map +1 -1
- package/dist/agent-config/model-settings/pi.d.ts +12 -6
- package/dist/agent-config/model-settings/pi.js +12 -6
- package/dist/agent-config/model-settings/pi.js.map +1 -1
- package/dist/agent-config/seed.js +10 -0
- package/dist/agent-config/seed.js.map +1 -1
- package/dist/contract/github.d.ts +15 -0
- package/dist/contract/index.d.ts +9 -0
- package/dist/contract/index.js +1527 -895
- package/dist/contract/mcp-api-reference.d.ts +174 -0
- package/dist/contract/mcp-audit.d.ts +103 -0
- package/dist/contract/mcp-discovery.d.ts +257 -0
- package/dist/contract/mcp-event-catalog.d.ts +121 -0
- package/dist/contract/mcp-event-catalog.test.d.ts +1 -0
- package/dist/contract/mcp-idempotency.d.ts +200 -0
- package/dist/contract/mcp-journal.d.ts +268 -0
- package/dist/contract/mcp-leader.d.ts +88 -0
- package/dist/contract/mcp-ownership.d.ts +78 -0
- package/dist/contract/mcp-versioning.d.ts +139 -0
- package/dist/contract/runs.d.ts +81 -0
- package/dist/core/agent-profiles.d.ts +15 -4
- package/dist/core/agent-profiles.js +22 -8
- package/dist/core/agent-profiles.js.map +1 -1
- package/dist/core/opencode-server-runner.js +71 -11
- package/dist/core/opencode-server-runner.js.map +1 -1
- package/dist/core/secret-redaction.d.ts +7 -1
- package/dist/core/secret-redaction.js +44 -10
- package/dist/core/secret-redaction.js.map +1 -1
- package/dist/index.js +138 -27
- package/dist/index.js.map +1 -1
- package/dist/machine-identity.d.ts +66 -0
- package/dist/machine-identity.js +96 -0
- package/dist/machine-identity.js.map +1 -0
- package/dist/mcp/adapters/claude-code.d.ts +41 -0
- package/dist/mcp/adapters/claude-code.js +40 -0
- package/dist/mcp/adapters/claude-code.js.map +1 -0
- package/dist/mcp/adapters/codex.d.ts +140 -0
- package/dist/mcp/adapters/codex.js +323 -0
- package/dist/mcp/adapters/codex.js.map +1 -0
- package/dist/mcp/adapters/opencode.d.ts +148 -0
- package/dist/mcp/adapters/opencode.js +528 -0
- package/dist/mcp/adapters/opencode.js.map +1 -0
- package/dist/mcp/adapters/pi-link.d.ts +102 -0
- package/dist/mcp/adapters/pi-link.js +235 -0
- package/dist/mcp/adapters/pi-link.js.map +1 -0
- package/dist/mcp/adapters/pi.d.ts +188 -0
- package/dist/mcp/adapters/pi.js +418 -0
- package/dist/mcp/adapters/pi.js.map +1 -0
- package/dist/mcp/api-reference.d.ts +26 -0
- package/dist/mcp/api-reference.js +133 -0
- package/dist/mcp/api-reference.js.map +1 -0
- package/dist/mcp/audit-trail.d.ts +162 -0
- package/dist/mcp/audit-trail.js +306 -0
- package/dist/mcp/audit-trail.js.map +1 -0
- package/dist/mcp/bridge.d.ts +81 -0
- package/dist/mcp/bridge.js +407 -0
- package/dist/mcp/bridge.js.map +1 -0
- package/dist/mcp/connection-file.d.ts +49 -0
- package/dist/mcp/connection-file.js +63 -0
- package/dist/mcp/connection-file.js.map +1 -0
- package/dist/mcp/echo-guard.d.ts +94 -0
- package/dist/mcp/echo-guard.js +160 -0
- package/dist/mcp/echo-guard.js.map +1 -0
- package/dist/mcp/event-catalog.d.ts +120 -0
- package/dist/mcp/event-catalog.js +306 -0
- package/dist/mcp/event-catalog.js.map +1 -0
- package/dist/mcp/event-controller.d.ts +230 -0
- package/dist/mcp/event-controller.js +596 -0
- package/dist/mcp/event-controller.js.map +1 -0
- package/dist/mcp/event-journal.d.ts +57 -0
- package/dist/mcp/event-journal.js +423 -0
- package/dist/mcp/event-journal.js.map +1 -0
- package/dist/mcp/index.d.ts +63 -0
- package/dist/mcp/index.js +386 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/mcp/ipc.d.ts +202 -0
- package/dist/mcp/ipc.js +206 -0
- package/dist/mcp/ipc.js.map +1 -0
- package/dist/mcp/leader-delivery.d.ts +119 -0
- package/dist/mcp/leader-delivery.js +450 -0
- package/dist/mcp/leader-delivery.js.map +1 -0
- package/dist/mcp/operation-receipts.d.ts +210 -0
- package/dist/mcp/operation-receipts.js +723 -0
- package/dist/mcp/operation-receipts.js.map +1 -0
- package/dist/mcp/project-catalogs.d.ts +23 -0
- package/dist/mcp/project-catalogs.js +25 -0
- package/dist/mcp/project-catalogs.js.map +1 -0
- package/dist/mcp/project-leaders.d.ts +26 -0
- package/dist/mcp/project-leaders.js +17 -0
- package/dist/mcp/project-leaders.js.map +1 -0
- package/dist/mcp/protocol.d.ts +50 -0
- package/dist/mcp/protocol.js +47 -0
- package/dist/mcp/protocol.js.map +1 -0
- package/dist/mcp/reconnect.d.ts +297 -0
- package/dist/mcp/reconnect.js +465 -0
- package/dist/mcp/reconnect.js.map +1 -0
- package/dist/mcp/resource-ownership.d.ts +228 -0
- package/dist/mcp/resource-ownership.js +390 -0
- package/dist/mcp/resource-ownership.js.map +1 -0
- package/dist/mcp/service-adapter.d.ts +16488 -0
- package/dist/mcp/service-adapter.js +148 -0
- package/dist/mcp/service-adapter.js.map +1 -0
- package/dist/mcp/service.d.ts +83 -0
- package/dist/mcp/service.js +233 -0
- package/dist/mcp/service.js.map +1 -0
- package/dist/mcp/session-binding.d.ts +105 -0
- package/dist/mcp/session-binding.js +163 -0
- package/dist/mcp/session-binding.js.map +1 -0
- package/dist/mcp/stale-write.d.ts +145 -0
- package/dist/mcp/stale-write.js +219 -0
- package/dist/mcp/stale-write.js.map +1 -0
- package/dist/mcp/tool.d.ts +48 -0
- package/dist/mcp/tool.js +24 -0
- package/dist/mcp/tool.js.map +1 -0
- package/dist/mcp/tools/discovery.d.ts +40 -0
- package/dist/mcp/tools/discovery.js +232 -0
- package/dist/mcp/tools/discovery.js.map +1 -0
- package/dist/mcp/tools/execution-control.d.ts +171 -0
- package/dist/mcp/tools/execution-control.js +513 -0
- package/dist/mcp/tools/execution-control.js.map +1 -0
- package/dist/mcp/tools/handoff-git.d.ts +26 -0
- package/dist/mcp/tools/handoff-git.js +544 -0
- package/dist/mcp/tools/handoff-git.js.map +1 -0
- package/dist/mcp/tools/index.d.ts +2 -0
- package/dist/mcp/tools/index.js +23 -0
- package/dist/mcp/tools/index.js.map +1 -0
- package/dist/mcp/tools/leader-events.d.ts +46 -0
- package/dist/mcp/tools/leader-events.js +126 -0
- package/dist/mcp/tools/leader-events.js.map +1 -0
- package/dist/mcp/tools/local-handoff.d.ts +102 -0
- package/dist/mcp/tools/local-handoff.js +306 -0
- package/dist/mcp/tools/local-handoff.js.map +1 -0
- package/dist/mcp/tools/project-config.d.ts +340 -0
- package/dist/mcp/tools/project-config.js +1075 -0
- package/dist/mcp/tools/project-config.js.map +1 -0
- package/dist/mcp/tools/results-evidence.d.ts +112 -0
- package/dist/mcp/tools/results-evidence.js +1045 -0
- package/dist/mcp/tools/results-evidence.js.map +1 -0
- package/dist/mcp/tools/task-create.d.ts +70 -0
- package/dist/mcp/tools/task-create.js +473 -0
- package/dist/mcp/tools/task-create.js.map +1 -0
- package/dist/mcp/tools/task-reads.d.ts +102 -0
- package/dist/mcp/tools/task-reads.js +689 -0
- package/dist/mcp/tools/task-reads.js.map +1 -0
- package/dist/mcp/tools/work-organisation.d.ts +2 -0
- package/dist/mcp/tools/work-organisation.js +502 -0
- package/dist/mcp/tools/work-organisation.js.map +1 -0
- package/dist/paths.d.ts +17 -7
- package/dist/paths.js +18 -8
- package/dist/paths.js.map +1 -1
- package/dist/runs/project-writer.js +76 -11
- package/dist/runs/project-writer.js.map +1 -1
- package/dist/runs/retention.d.ts +24 -1
- package/dist/runs/retention.js +62 -1
- package/dist/runs/retention.js.map +1 -1
- package/dist/server/forge/github.js +44 -0
- package/dist/server/forge/github.js.map +1 -1
- package/dist/server/forge/types.d.ts +16 -0
- package/dist/server/project-context.d.ts +81 -8
- package/dist/server/project-context.js +143 -35
- package/dist/server/project-context.js.map +1 -1
- package/dist/server/server.d.ts +1050 -53
- package/dist/server/server.js +326 -120
- package/dist/server/server.js.map +1 -1
- package/dist/server/validators.d.ts +26 -6
- package/dist/server/validators.js +17 -0
- package/dist/server/validators.js.map +1 -1
- package/dist/skills-remote.js +64 -6
- package/dist/skills-remote.js.map +1 -1
- package/dist/workflows/run.d.ts +277 -12
- package/dist/workflows/run.js +554 -59
- package/dist/workflows/run.js.map +1 -1
- package/dist/workspace/agent-profiles.d.ts +30 -3
- package/dist/workspace/agent-profiles.js +39 -4
- package/dist/workspace/agent-profiles.js.map +1 -1
- package/dist/workspace/config.js +25 -4
- package/dist/workspace/config.js.map +1 -1
- package/dist/workspace/project-owner.d.ts +210 -0
- package/dist/workspace/project-owner.js +518 -0
- package/dist/workspace/project-owner.js.map +1 -0
- package/package.json +5 -3
- package/scripts/pi-leader-extension.ts +420 -0
- package/scripts/sync-readme.mjs +83 -2
- package/web/dist/assets/{alert-dialog-BVsVcbFy.js → alert-dialog-D6rP6d39.js} +1 -1
- package/web/dist/assets/arrow-down-DMOfFIzW.js +1 -0
- package/web/dist/assets/arrow-left-C4Ec2BUg.js +1 -0
- package/web/dist/assets/{centered-state-Djgly6Ni.js → centered-state-B0ZqrPaa.js} +12 -12
- package/web/dist/assets/chevron-right-mHJ-crvn.js +1 -0
- package/web/dist/assets/{collapsible-DZ4D-DHG.js → collapsible-DchEHgUM.js} +1 -1
- package/web/dist/assets/{commit-list-B-z1JL9G.js → commit-list-BpMim1Rv.js} +1 -1
- package/web/dist/assets/{compare-variants-D62m4j3B.js → compare-variants-D5ItQbAq.js} +1 -1
- package/web/dist/assets/{diff-BKew9rQX.js → diff-HqqZ_WDR.js} +2 -2
- package/web/dist/assets/{diff-stat-D0mc5V6Z.js → diff-stat-NsU-3ecr.js} +1 -1
- package/web/dist/assets/{diff-view-xpIcuLIC.js → diff-view-B7c11Xux.js} +1 -1
- package/web/dist/assets/{dropdown-menu-CEQziH7X.js → dropdown-menu-CtTpnyod.js} +1 -1
- package/web/dist/assets/{editable-title-BCX3T2BY.js → editable-title-BXtuGQry.js} +1 -1
- package/web/dist/assets/{ellipsis-vertical-ByVdDc4z.js → ellipsis-vertical-MHtGZSH6.js} +1 -1
- package/web/dist/assets/{file-xBSB5hXO.js → file-DKLw5SOb.js} +1 -1
- package/web/dist/assets/{folder-CldgbZ5A.js → folder-Drw6-ufH.js} +1 -1
- package/web/dist/assets/{git-toolbar-DJANqg7Y.js → git-toolbar-BQH8bORV.js} +1 -1
- package/web/dist/assets/github-ynnpKFCJ.js +1 -0
- package/web/dist/assets/{image-preview-Cv5mZr0n.js → image-preview-DLhMN_R9.js} +1 -1
- package/web/dist/assets/index-BEsrNdXt.js +29 -0
- package/web/dist/assets/index-Da8DOBau.css +2 -0
- package/web/dist/assets/{markdown-CgEEE2ND.js → markdown-q365eN6Y.js} +1 -1
- package/web/dist/assets/{new-task-form-1r2bgsdD.js → new-task-form-CW-hNIWQ.js} +1 -1
- package/web/dist/assets/{pill-q-x22_ej.js → pill-B_TFulXa.js} +1 -1
- package/web/dist/assets/{project-router-VDN4jQM8.js → project-router-CLPc-ImZ.js} +1 -1
- package/web/dist/assets/{prompt-templates-BmZtCe-L.js → prompt-templates-CLc4RxyO.js} +1 -1
- package/web/dist/assets/{repo-git-btG0sdbO.js → repo-git-Is7R6-Tj.js} +1 -1
- package/web/dist/assets/{run-diff-vguEMJzh.js → run-diff-DHMvDlVo.js} +2 -2
- package/web/dist/assets/run-header-DvZJbQ8I.js +1 -0
- package/web/dist/assets/{search-x-QmhaAbg5.js → search-x-Ce2MMetu.js} +1 -1
- package/web/dist/assets/{skill-empty-hint-BrzUwByZ.js → skill-empty-hint-C_xCSvzB.js} +1 -1
- package/web/dist/assets/skills-CivqD4si.js +1 -0
- package/web/dist/assets/{sparkles-CP8QDOgX.js → sparkles-BCqPoj5g.js} +1 -1
- package/web/dist/assets/{square-terminal-CGypshSD.js → square-terminal-ikj5g-oK.js} +1 -1
- package/web/dist/assets/{tab-link-D4lFzzI1.js → tab-link-VsEjFV1Y.js} +1 -1
- package/web/dist/assets/{task-changes-BtZId14z.js → task-changes-CH-E81-_.js} +1 -1
- package/web/dist/assets/{task-commits-BoeyJXrv.js → task-commits-DqfkMFxp.js} +1 -1
- package/web/dist/assets/{task-files-DIy2oUnj.js → task-files-D_nkHZ_a.js} +2 -2
- package/web/dist/assets/task-thread-bxQompxJ.js +9 -0
- package/web/dist/assets/{textarea-CyBCyKIk.js → textarea-xajuW1L_.js} +1 -1
- package/web/dist/assets/{thread-loading-Cuq2jchC.js → thread-loading-C3JQZbIE.js} +1 -1
- package/web/dist/assets/{trash-2-B6Dp3ZNs.js → trash-2-hJrBhpUD.js} +1 -1
- package/web/dist/assets/{triangle-alert-xBya8FUp.js → triangle-alert-D1u82y1a.js} +1 -1
- package/web/dist/assets/{upload-D6MXEB9p.js → upload-Bn6FBZ59.js} +1 -1
- package/web/dist/assets/{use-desktop-CPBwzQ7M.js → use-desktop-DOIMfHfY.js} +1 -1
- package/web/dist/assets/{use-submit-shortcut-JTpNzU-u.js → use-submit-shortcut-Dj4DHTdE.js} +1 -1
- package/web/dist/assets/utils-YwBpOoRN.js +64 -0
- package/web/dist/assets/{workflows-DG115scU.js → workflows-CTaAKz2m.js} +3 -3
- package/web/dist/assets/{zoomable-image-GTYwLtkS.js → zoomable-image-C1-6P3tg.js} +1 -1
- package/web/dist/index.html +23 -23
- package/web/dist/assets/arrow-down-CYgAlZtW.js +0 -1
- package/web/dist/assets/arrow-left-BOqE0a3k.js +0 -1
- package/web/dist/assets/chevron-right-Cc9BWnbN.js +0 -1
- package/web/dist/assets/github-D6vK0jFn.js +0 -1
- package/web/dist/assets/index-CkmR6h7s.css +0 -2
- package/web/dist/assets/index-DVDqF4bM.js +0 -9
- package/web/dist/assets/refresh-cw-DcNCj2nu.js +0 -1
- package/web/dist/assets/run-header-DM7MiKj9.js +0 -1
- package/web/dist/assets/skills-DMfxQtrD.js +0 -1
- package/web/dist/assets/task-thread-CbMPmrWq.js +0 -9
- package/web/dist/assets/utils--z9hRtl3.js +0 -64
package/dist/workflows/run.js
CHANGED
|
@@ -68,6 +68,35 @@ const DONE_MARKER_RE = /XEZ:DONE\s*$/;
|
|
|
68
68
|
* backends can't split the marker across text events.
|
|
69
69
|
*/
|
|
70
70
|
const MONITORING_MARKER_RE = /XEZ:MONITORING\s*$/;
|
|
71
|
+
/**
|
|
72
|
+
* Why a NON-FINAL agent step may not be marked `done` (#317), or null when it may.
|
|
73
|
+
*
|
|
74
|
+
* Such a step runs one turn and its session closes by itself, so it has no way to wait for an
|
|
75
|
+
* answer — and until #317 nothing decided it was done except "the session ended without an
|
|
76
|
+
* error". Run b86c6066's implement step ended on an `XEZ:ASK` design question, was marked done,
|
|
77
|
+
* and readiness, gates and the seal all ran on a branch with no work in it.
|
|
78
|
+
*
|
|
79
|
+
* The decision now belongs to the agent contract every step is given (`HANDOFF_INSTRUCTIONS`):
|
|
80
|
+
* `XEZ:DONE` means "my goal is achieved", and a turn that ends without it is "waiting on the
|
|
81
|
+
* user". A turn that ends on anything else — a question, the monitoring marker, or plain prose —
|
|
82
|
+
* stops the workflow here instead of carrying on without an answer. Deliberately fail-closed on
|
|
83
|
+
* plain prose: that is the case no agent-side marker can report, and the one the BLOCKED file
|
|
84
|
+
* could not catch either. `turnText` is null when the session ended without finishing a turn.
|
|
85
|
+
*/
|
|
86
|
+
export function unfinishedStepReason(turnText) {
|
|
87
|
+
const text = turnText?.trimEnd() ?? '';
|
|
88
|
+
if (DONE_MARKER_RE.test(text))
|
|
89
|
+
return null;
|
|
90
|
+
const next = 'A step before the last one cannot wait for an answer, so the workflow stopped here instead of ' +
|
|
91
|
+
'carrying on without one. Read the step\'s last message, then Continue this task to answer it.';
|
|
92
|
+
if (parseAskMarkerResult(text).kind !== 'none') {
|
|
93
|
+
return `the agent ended its turn on a question (XEZ:ASK) instead of finishing the step. ${next}`;
|
|
94
|
+
}
|
|
95
|
+
if (MONITORING_MARKER_RE.test(text)) {
|
|
96
|
+
return `the agent ended its turn still waiting on its own work (XEZ:MONITORING) instead of finishing the step. ${next}`;
|
|
97
|
+
}
|
|
98
|
+
return `the agent ended its turn without the XEZ:DONE completion marker, so nothing says the step's work is finished — it may be waiting on a question. ${next}`;
|
|
99
|
+
}
|
|
71
100
|
/**
|
|
72
101
|
* Preserve boundaries between complete assistant text blocks while a turn is
|
|
73
102
|
* accumulated for marker parsing. The runners join these same v1 blocks with
|
|
@@ -194,6 +223,17 @@ export const MAX_AUTO_RESUMES = 12;
|
|
|
194
223
|
* note instead of fired, so the only tasks a sweep can revive are ones someone is still waiting on.
|
|
195
224
|
*/
|
|
196
225
|
export const AUTO_RESUME_MISSED_WINDOW_MS = 24 * 60 * 60_000;
|
|
226
|
+
/**
|
|
227
|
+
* How long a resumed turn must stay live before it counts as proof that the limit lifted (#285).
|
|
228
|
+
*
|
|
229
|
+
* The in-flight hold exists for the window where a resume is TESTING the account: a limit that
|
|
230
|
+
* has not lifted refuses the turn at its first API call, and the measured doomed run lives about
|
|
231
|
+
* 200 ms. A turn still running minutes later got past that call, so holding every other task on
|
|
232
|
+
* the account until it COMPLETES — which a long turn may not do for hours — is the hold outliving
|
|
233
|
+
* its purpose. Not the first streamed event: a refused Claude turn still streams its session init
|
|
234
|
+
* and a synthetic assistant frame before the error, so "it said something" proves nothing.
|
|
235
|
+
*/
|
|
236
|
+
export const AUTO_RESUME_PROOF_MS = 2 * 60_000;
|
|
197
237
|
/**
|
|
198
238
|
* How often the queue checks that it is not wedged.
|
|
199
239
|
*
|
|
@@ -249,6 +289,18 @@ export function runAccountKey(run, fallbackRunner) {
|
|
|
249
289
|
function resumeInFlight(run) {
|
|
250
290
|
return (run.autoResumeAttempts !== undefined && (run.status === 'queued' || run.status === 'running'));
|
|
251
291
|
}
|
|
292
|
+
/**
|
|
293
|
+
* Has this resume's turn stayed live long enough to prove the window open (#285)? Read off the
|
|
294
|
+
* durable record — the running step's own `startedAt` — so a restart answers exactly as the
|
|
295
|
+
* process that started the turn would have. A resume still `queued` has proven nothing.
|
|
296
|
+
*/
|
|
297
|
+
function resumeProven(run, now, proofMs) {
|
|
298
|
+
if (run.status !== 'running' || !run.currentStepId)
|
|
299
|
+
return false;
|
|
300
|
+
const startedAt = run.steps.find((step) => step.id === run.currentStepId)?.startedAt;
|
|
301
|
+
const at = startedAt ? Date.parse(startedAt) : Number.NaN;
|
|
302
|
+
return Number.isFinite(at) && now - at >= proofMs;
|
|
303
|
+
}
|
|
252
304
|
const AUTO_RESUME_PROMPT = 'The provider usage limit that interrupted this task has reset. Read the handoff file (XEZ_HANDOFF_FILE) to recover context, then continue the task from where you left off.';
|
|
253
305
|
/**
|
|
254
306
|
* The wake instant as a human reads it — local, to the SECOND, with the zone named. The
|
|
@@ -417,6 +469,12 @@ export class RunManager {
|
|
|
417
469
|
* from the record rather than losing the wait. Runs here are `failed` and therefore NOT in
|
|
418
470
|
* `active`, which is why the timer cannot live on an `ActiveRun` like the monitoring one. */
|
|
419
471
|
autoResumeTimers = new Map();
|
|
472
|
+
/** Wake-ups for the instant a live resume's proof window closes (#285). The hold lifts by
|
|
473
|
+
* derivation, and a derived release is not an event, so these are only the pump that notices;
|
|
474
|
+
* the hold itself stays on the records. */
|
|
475
|
+
resumeProofTimers = new Set();
|
|
476
|
+
/** `AUTO_RESUME_PROOF_MS`, overridable only so a test need not wait two minutes. */
|
|
477
|
+
resumeProofMs;
|
|
420
478
|
pumping = false;
|
|
421
479
|
/** A pump that arrived while one was in flight — replayed by `pump()`'s own
|
|
422
480
|
* loop so a slot freed mid-sweep is never a lost wakeup. */
|
|
@@ -438,15 +496,77 @@ export class RunManager {
|
|
|
438
496
|
offUsage;
|
|
439
497
|
/** The stalled-queue watchdog (see `rescueStalledQueue`). */
|
|
440
498
|
queueWatchdog;
|
|
441
|
-
/**
|
|
442
|
-
*
|
|
443
|
-
*
|
|
444
|
-
*
|
|
445
|
-
|
|
499
|
+
/**
|
|
500
|
+
* Every background WRITE this manager started on its own initiative and has not settled yet.
|
|
501
|
+
* Two producers, one set: the queue-watchdog rescue sweeps (#125) and the terminal-transition
|
|
502
|
+
* worktree retention sweeps (#200). Neither is cancellable and both cross an await before they
|
|
503
|
+
* touch disk — a rescue appends NDJSON after `reviveWorkflow`, a retention sweep spawns
|
|
504
|
+
* `git worktree remove` after `resolveWorktreeRetention` — so `clearInterval` and the
|
|
505
|
+
* `disposed` flag between them still leave work running.
|
|
506
|
+
*
|
|
507
|
+
* dispose() settles this set, which is what makes its promise mean "provably finished writing"
|
|
508
|
+
* before the caller removes the data root. Retention was the half that was NOT tracked, and
|
|
509
|
+
* the harm differs by caller — be precise about which, because only one of them deletes
|
|
510
|
+
* anything. In a TEST teardown a disposed manager could still be spawning git inside a
|
|
511
|
+
* directory the case was already `rmSync`-ing, which is the reported ENOTEMPTY. In a LIVE
|
|
512
|
+
* server nothing removes the project directory at all: the sweep instead stamps
|
|
513
|
+
* `worktreeReclaimedAt` onto a record, scheduling a debounced `runs.json` write from a store
|
|
514
|
+
* whose lifecycle has ended — so a re-add inside that window can see the stale in-memory index
|
|
515
|
+
* overwrite the fresh one. Every producer therefore enrols through a wrapper
|
|
516
|
+
* (`rescueStalledQueue`, `enforceRetention`), so nothing can start a background write teardown
|
|
517
|
+
* cannot see.
|
|
518
|
+
*
|
|
519
|
+
* The live-server half needed the OTHER end fixed too, and the promise is worthless without it:
|
|
520
|
+
* `ProjectContexts.dispose` used to drop this promise on the floor (`teardown()` was
|
|
521
|
+
* synchronous), so the guarantee held only for callers that awaited — which was tests. It now
|
|
522
|
+
* awaits the manager before `store.flush()` + `removeAllListeners()`, and `DELETE /projects/:id`
|
|
523
|
+
* awaits that in turn.
|
|
524
|
+
*/
|
|
525
|
+
writesInFlight = new Set();
|
|
526
|
+
/**
|
|
527
|
+
* The run BODIES — `execute()` / `runContinuation()` — that have started and not settled yet,
|
|
528
|
+
* plus the turn-end bookkeeping (`recordTurnEnd`) that a body fires and does not await.
|
|
529
|
+
*
|
|
530
|
+
* Deliberately NOT `writesInFlight`. dispose() is documented as "the manager makes no further
|
|
531
|
+
* moves ON ITS OWN" and is explicitly not a run-stopper, so awaiting a live agent session in
|
|
532
|
+
* its promise would silently change what every existing caller gets. `quiesce()` is the opt-in
|
|
533
|
+
* that cancels first and then awaits this set; dispose()'s semantics are untouched.
|
|
534
|
+
*/
|
|
535
|
+
runsInFlight = new Set();
|
|
536
|
+
/**
|
|
537
|
+
* Cancellations that arrived while a run was still MATERIALIZING — dequeued into `starting`,
|
|
538
|
+
* or floated by the direct Continue path — and had nowhere to land.
|
|
539
|
+
*
|
|
540
|
+
* `cancel()` stops an active run by writing `state.cancelled` and calling `state.interrupt()`,
|
|
541
|
+
* and neither exists until the body has built its `ActiveRun`. Before this set, `cancel()` for
|
|
542
|
+
* such a run simply returned false and delivered NOTHING, so the run went on to spawn an agent
|
|
543
|
+
* turn that no one had asked for and that `quiesce()` then waited out — up to the runner's
|
|
544
|
+
* 30-minute default (#200). The body consumes the request through `adoptActive`, at the first
|
|
545
|
+
* instant a cancellation has somewhere to go.
|
|
546
|
+
*
|
|
547
|
+
* `dropActive` clears the entry too, because a body can leave without ever reaching
|
|
548
|
+
* `adoptActive` (the continuation whose isolation is gone), and a stale entry would cancel the
|
|
549
|
+
* NEXT run to reuse that id — which, for a Continue, is the same id again.
|
|
550
|
+
*/
|
|
551
|
+
cancelRequested = new Set();
|
|
446
552
|
/** Set by dispose(): this manager makes no further moves. Re-checked at every await boundary a
|
|
447
553
|
* rescue crosses, because a sweep that was already past its first check when dispose() landed
|
|
448
554
|
* would otherwise write an event — and re-populate the queue dispose() had just emptied. */
|
|
449
555
|
disposed = false;
|
|
556
|
+
/**
|
|
557
|
+
* Set by `quiesce()` for the length of its drain: the scheduler starts nothing new.
|
|
558
|
+
*
|
|
559
|
+
* `disposed` cannot cover this window, because `quiesce()` disposes LAST — the whole point is
|
|
560
|
+
* to stop the runs while `cancel()` can still see them. During the drain the manager is
|
|
561
|
+
* otherwise fully live, and every settling run pumps the entire workspace on its way out
|
|
562
|
+
* (`dropActive` → `releaseSlot` → `semaphore.release()`, which awaits `pump()` on every
|
|
563
|
+
* registered participant, this one included). A pump reconciles auto-resumes from the RECORDS,
|
|
564
|
+
* and a record whose `autoResumeAt` has already passed arms at zero delay — so the drain could
|
|
565
|
+
* fire a resume, spawn a fresh agent turn into the very repo root the caller is about to
|
|
566
|
+
* delete, and then dutifully wait for it. Bounded by `MAX_AUTO_RESUMES`, and the exact opposite
|
|
567
|
+
* of what the method promises.
|
|
568
|
+
*/
|
|
569
|
+
quiescing = false;
|
|
450
570
|
/** Set by the watchdog for exactly one sweep: ignore the usage-limit hold and make progress. */
|
|
451
571
|
forceNextPump = false;
|
|
452
572
|
/** Runs the watchdog started despite the hold. The spawn-time gate (`requeueWhileHeld`) would
|
|
@@ -464,6 +584,7 @@ export class RunManager {
|
|
|
464
584
|
this.repoRoot = repoRoot;
|
|
465
585
|
this.dataDir = store.dataDir;
|
|
466
586
|
this.semaphore = options.semaphore ?? new WorkspaceSemaphore();
|
|
587
|
+
this.resumeProofMs = options.resumeProofMs ?? AUTO_RESUME_PROOF_MS;
|
|
467
588
|
this.offSemaphore = this.semaphore.register({
|
|
468
589
|
busySlots: () => this.busySlots(),
|
|
469
590
|
pump: () => this.pump(),
|
|
@@ -473,7 +594,12 @@ export class RunManager {
|
|
|
473
594
|
// Memory guard (#memory-guard): the shared process-tree sampler already ticks ~every 2 s for
|
|
474
595
|
// the runs table; piggyback on it to enforce the per-task memory ceiling.
|
|
475
596
|
this.offUsage = onUsage((snapshot) => void this.enforceMemoryLimit(snapshot));
|
|
476
|
-
|
|
597
|
+
// `.catch()` for the same reason `enforceRetention` carries one: a floated promise that
|
|
598
|
+
// rejects is a process-level unhandled rejection, and this one CAN reject —
|
|
599
|
+
// `sweepStalledQueue` → `reviveQueuedRun` → `reviveWorkflow` → `loadWorkflows` reads the
|
|
600
|
+
// project's YAML, and `rescueStalledQueue` re-raises through its try/finally. The interval
|
|
601
|
+
// is the only caller that does not await; tests await it and still see the throw.
|
|
602
|
+
this.queueWatchdog = setInterval(() => void this.rescueStalledQueue().catch(() => undefined), QUEUE_WATCHDOG_MS);
|
|
477
603
|
this.queueWatchdog.unref?.();
|
|
478
604
|
}
|
|
479
605
|
/**
|
|
@@ -487,13 +613,32 @@ export class RunManager {
|
|
|
487
613
|
* dispose only guarantees the manager makes no further moves on its own.
|
|
488
614
|
*
|
|
489
615
|
* Every side effect below is SYNCHRONOUS, so a caller that ignores the return value behaves
|
|
490
|
-
* exactly as it did before #125. The returned promise settles the
|
|
491
|
-
* synchronously: a queue-watchdog rescue
|
|
492
|
-
*
|
|
616
|
+
* exactly as it did before #125. The returned promise settles the things dispose cannot do
|
|
617
|
+
* synchronously: the background writes already in flight — a queue-watchdog rescue (#125) and
|
|
618
|
+
* a worktree retention sweep (#200), both tracked in `writesInFlight`. `clearInterval` stops
|
|
619
|
+
* the next tick, never the running one — the same discipline `AGENTS.md` records for the e2e
|
|
493
620
|
* fixture servers, where `kill()` only delivers the signal and the helper awaits the exit.
|
|
494
621
|
* Await it whenever the data root is about to be removed; a test that deletes its temp
|
|
495
622
|
* directory without awaiting is the ENOENT in #125.
|
|
496
623
|
*
|
|
624
|
+
* It does NOT wait for running runs, and that is the point of `quiesce()`: dispose clears
|
|
625
|
+
* `active`/`starting`/`queue` without stopping anything, so after it returns a leaked run is
|
|
626
|
+
* invisible to `cancel()` and can no longer be stopped at all. Reach for `quiesce()` whenever
|
|
627
|
+
* the runs are yours to end; reach for `dispose()` when they are not.
|
|
628
|
+
*
|
|
629
|
+
* ## The states a disposed manager leaves behind, and how they end
|
|
630
|
+
*
|
|
631
|
+
* - A run that was `active` keeps running to its own terminal status and writes it to the
|
|
632
|
+
* record; only the manager's registries forgot it. `cancel()` answers false for it from here
|
|
633
|
+
* on, which is the leak `quiesce()` exists to avoid.
|
|
634
|
+
* - A run that was `queued` — or one `startRun()` accepts AFTER dispose, which still writes a
|
|
635
|
+
* `queued` record and still floats a `pump()` — never starts in this process: `pump()` bails
|
|
636
|
+
* on `disposed`, so nothing dequeues it. Its only exit is the next process: `recover()`
|
|
637
|
+
* re-queues every `queued` record at boot, and `rescueStalledQueue` re-adopts one the engine
|
|
638
|
+
* is holding no work item for. That is deliberate — a disposed manager's project has been
|
|
639
|
+
* removed from the registry, and starting its work would be the surprise — but it does mean
|
|
640
|
+
* the record outlives the process as `queued` rather than `cancelled`.
|
|
641
|
+
*
|
|
497
642
|
* It never rejects: a sweep that fails still belongs to whoever started it (the watchdog floats
|
|
498
643
|
* it exactly as before), and teardown must not become a second place that error surfaces.
|
|
499
644
|
*/
|
|
@@ -512,6 +657,9 @@ export class RunManager {
|
|
|
512
657
|
for (const timer of this.autoResumeTimers.values())
|
|
513
658
|
clearTimeout(timer);
|
|
514
659
|
this.autoResumeTimers.clear();
|
|
660
|
+
for (const timer of this.resumeProofTimers)
|
|
661
|
+
clearTimeout(timer);
|
|
662
|
+
this.resumeProofTimers.clear();
|
|
515
663
|
this.active.clear();
|
|
516
664
|
this.waiting.clear();
|
|
517
665
|
this.starting.clear();
|
|
@@ -520,7 +668,113 @@ export class RunManager {
|
|
|
520
668
|
this.pendingContinuations.clear();
|
|
521
669
|
this.memoryPausing.clear();
|
|
522
670
|
this.lastNamerKey.clear();
|
|
523
|
-
|
|
671
|
+
this.cancelRequested.clear();
|
|
672
|
+
return Promise.allSettled([...this.writesInFlight]).then(() => undefined);
|
|
673
|
+
}
|
|
674
|
+
/**
|
|
675
|
+
* Stop every run this manager owns, then dispose — "the data root is about to disappear".
|
|
676
|
+
*
|
|
677
|
+
* dispose() alone cannot deliver that: it is not a run-stopper by design, and because it
|
|
678
|
+
* clears `active`/`starting`/`queue` first, a run left behind is afterwards invisible to
|
|
679
|
+
* `cancel()` (which looks only in `queue` then `active`) and can never be stopped. The order
|
|
680
|
+
* here is the whole fix — cancel, drain, dispose — and it is the order four test files were
|
|
681
|
+
* already hand-rolling three different ways.
|
|
682
|
+
*
|
|
683
|
+
* ## Why the cancel is INSIDE the loop, and why termination needs `quiescing`
|
|
684
|
+
*
|
|
685
|
+
* Cancelling once up front is not enough, because the population is not fixed while the drain
|
|
686
|
+
* runs. `starting` promotes to `active` (the `ActiveRun` a `cancel()` needs did not exist a
|
|
687
|
+
* moment ago), a queued continuation crosses `rematerializeReclaimedWorktree` before it
|
|
688
|
+
* registers anywhere at all, and every settling body pumps the whole workspace on its way out.
|
|
689
|
+
* So each pass re-asks `activeRunIds()` and re-issues `cancel()`; `cancelRequested` carries a
|
|
690
|
+
* cancellation into the one window where neither registry can hold it.
|
|
691
|
+
*
|
|
692
|
+
* Termination then rests on `quiescing`, not on "the queue is empty". The manager has three
|
|
693
|
+
* self-starting entry points — `pump()`, `fireAutoResume()` and the queue watchdog's
|
|
694
|
+
* `rescueStalledQueue()` — and all three bail while it is set, so no pass can enrol a run the
|
|
695
|
+
* previous pass did not cancel. Ask "who fires this?" of any new one: the watchdog is an
|
|
696
|
+
* unref'd 60 s interval, so it is invisible in a short drain and lands squarely in a long one
|
|
697
|
+
* (an agent session that will not close is bounded only by the runner's 30-minute default),
|
|
698
|
+
* where it would re-adopt queued records — NDJSON, `pendingJobs`, a `queue` push — inside the
|
|
699
|
+
* window this method exists to make quiet.
|
|
700
|
+
*
|
|
701
|
+
* What a pass CAN still enrol is bounded bookkeeping: a body fires one `recordTurnEnd` per turn
|
|
702
|
+
* it finishes, and a cancelled run finishes no further turns (`autoContinueTurn` refuses a
|
|
703
|
+
* cancelled state). Each iteration therefore awaits a strictly older generation of work than the
|
|
704
|
+
* last, and the generations are finite.
|
|
705
|
+
*
|
|
706
|
+
* An external caller CAN still enrol during the drain — `startRun()` and `continueRun()` are
|
|
707
|
+
* not gated by `quiescing`, because refusing a user's request is the HTTP layer's decision and
|
|
708
|
+
* not a teardown helper's. The loop absorbs one: the next pass finds it, cancels it and waits
|
|
709
|
+
* for it, which is exactly what the "an agent turn enrolled after the drain took its snapshot"
|
|
710
|
+
* case pins. What is NOT bounded is an unbroken STREAM of them — a caller that keeps starting
|
|
711
|
+
* work into a project it is tearing down gets what it asked for.
|
|
712
|
+
*
|
|
713
|
+
* ## Each pass delivers its cancel ONCE — the bodies are what make that enough
|
|
714
|
+
*
|
|
715
|
+
* Re-issuing `cancel()` per pass reaches runs whose REGISTRY membership changed; it does not
|
|
716
|
+
* reach a run that was already in `active` when the pass ran, because the next re-issue waits
|
|
717
|
+
* on the same `Promise.allSettled` the stuck body is holding open. So the guarantee has to live
|
|
718
|
+
* in the body: every point at which a run body can park must consume a cancellation that
|
|
719
|
+
* arrived while it was elsewhere. Those points are `adoptActive` (the `starting` window),
|
|
720
|
+
* `acquireRepoRoot` (the lease wait, which races its own abort), the step loop's
|
|
721
|
+
* `if (state.cancelled) break`, `publishSession` (the gap between the loop's check and a live
|
|
722
|
+
* `state.interrupt`, which is what #199 hung on), and `state.interrupt()` itself once a session
|
|
723
|
+
* is up. Adding a new `await` inside a run body means asking which of those covers it.
|
|
724
|
+
*
|
|
725
|
+
* ## If a tracked promise never settles
|
|
726
|
+
*
|
|
727
|
+
* `quiesce()` has no deadline of its own and stays pending — the caller's timeout is the only
|
|
728
|
+
* bound. That is deliberate. Cancellation is delivered, an agent step's own wall clock still
|
|
729
|
+
* applies, and a session that refuses to close is a real writer: returning early would hand the
|
|
730
|
+
* caller a promise that says "nothing is writing any more" while something is, which is the
|
|
731
|
+
* exact lie this method exists to remove. An invented deadline would only move the ENOTEMPTY a
|
|
732
|
+
* few hundred milliseconds later — and, in the #199 hang, would have shipped a teardown that
|
|
733
|
+
* deleted a repository out from under a live agent CLI instead of failing loudly at 90 s.
|
|
734
|
+
*
|
|
735
|
+
* Never rejects, the same contract as dispose(): teardown must not become a second place a run's
|
|
736
|
+
* error surfaces.
|
|
737
|
+
*/
|
|
738
|
+
async quiesce() {
|
|
739
|
+
this.quiescing = true;
|
|
740
|
+
try {
|
|
741
|
+
// Drain to a FIXPOINT, not one snapshot: a settling body enrols its own turn-end
|
|
742
|
+
// bookkeeping (`recordTurnEnd`, which spawns git in the worktree), and `Promise.allSettled`
|
|
743
|
+
// freezes the set at the instant it is called, so a single pass would miss precisely the
|
|
744
|
+
// promises that outlive the body.
|
|
745
|
+
while (this.runsInFlight.size > 0) {
|
|
746
|
+
for (const runId of this.activeRunIds())
|
|
747
|
+
this.cancel(runId);
|
|
748
|
+
await Promise.allSettled([...this.runsInFlight]);
|
|
749
|
+
}
|
|
750
|
+
// Nothing is running any more, but the queue can still hold records — a run cancelled
|
|
751
|
+
// before it ever started leaves none, a `startRun()` that raced the drain does. Cancel what
|
|
752
|
+
// is left so it stops as `cancelled` rather than being silently forgotten by dispose().
|
|
753
|
+
for (const runId of this.activeRunIds())
|
|
754
|
+
this.cancel(runId);
|
|
755
|
+
}
|
|
756
|
+
finally {
|
|
757
|
+
// Synchronously before `dispose()` sets `disposed`, so there is no instant in which the
|
|
758
|
+
// manager is neither quiescing nor disposed and a pump could slip through.
|
|
759
|
+
this.quiescing = false;
|
|
760
|
+
}
|
|
761
|
+
return this.dispose();
|
|
762
|
+
}
|
|
763
|
+
/**
|
|
764
|
+
* Enrol a run body in `runsInFlight` so `quiesce()` can wait for it.
|
|
765
|
+
*
|
|
766
|
+
* The body arrives with its own `.catch(…)` already attached — the failure policy stays at the
|
|
767
|
+
* call site, where it can name the run — and the extra `.catch` here only keeps the tracked
|
|
768
|
+
* copy from ever rejecting, so a handler that itself throws cannot turn teardown into an
|
|
769
|
+
* unhandled rejection.
|
|
770
|
+
*/
|
|
771
|
+
trackRun(body) {
|
|
772
|
+
const tracked = body
|
|
773
|
+
.catch(() => undefined)
|
|
774
|
+
.finally(() => {
|
|
775
|
+
this.runsInFlight.delete(tracked);
|
|
776
|
+
});
|
|
777
|
+
this.runsInFlight.add(tracked);
|
|
524
778
|
}
|
|
525
779
|
/**
|
|
526
780
|
* Pause any active run whose whole process tree exceeds the WORKSPACE
|
|
@@ -757,6 +1011,18 @@ export class RunManager {
|
|
|
757
1011
|
* root (spec 006 degradation rule), which is always the tighter bound.
|
|
758
1012
|
*/
|
|
759
1013
|
async pump() {
|
|
1014
|
+
// A pump floated by something that started before teardown must not run afterwards: it spawns
|
|
1015
|
+
// `git rev-parse` inside `repoRoot` (`getRepoInfo`) and `reconcileAutoResumes` re-arms from
|
|
1016
|
+
// the RECORDS, which would repopulate the very timer map dispose() had just cleared (#200).
|
|
1017
|
+
// Nothing is lost by bailing — dispose empties the queue, so there is nothing left to start.
|
|
1018
|
+
//
|
|
1019
|
+
// `quiescing` is the same bail one phase earlier, and it is what makes `quiesce()` terminate:
|
|
1020
|
+
// during its drain every settling run pumps this manager (`dropActive` → `releaseSlot` →
|
|
1021
|
+
// `semaphore.release()`, which awaits `pump()` on every participant), and a pump that re-armed
|
|
1022
|
+
// a past-due `autoResumeAt` would start a fresh agent turn into a repo root the caller is
|
|
1023
|
+
// about to delete — while `quiesce()` waited for it.
|
|
1024
|
+
if (this.disposed || this.quiescing)
|
|
1025
|
+
return;
|
|
760
1026
|
this.reconcileMonitoringWakeTimers();
|
|
761
1027
|
this.reconcileAutoResumes();
|
|
762
1028
|
// A pump requested while one is in flight can't just be dropped: the
|
|
@@ -772,6 +1038,13 @@ export class RunManager {
|
|
|
772
1038
|
do {
|
|
773
1039
|
this.pumpAgain = false;
|
|
774
1040
|
const repo = await getRepoInfo(this.repoRoot);
|
|
1041
|
+
// Re-read after every await this loop crosses, not just at the top: teardown can land in
|
|
1042
|
+
// any of these gaps, and a pump that was already past the head check would go on to spawn
|
|
1043
|
+
// `git rev-parse` and dequeue work into a data root its owner has finished with — the
|
|
1044
|
+
// same class `reclaimRetiredWorktrees` re-checks for, one severity lower because a pump's
|
|
1045
|
+
// own reads are harmless and only what it STARTS is not.
|
|
1046
|
+
if (this.disposed || this.quiescing)
|
|
1047
|
+
return;
|
|
775
1048
|
const maxParallel = this.semaphore.maxParallel();
|
|
776
1049
|
// Per-project ceiling (spec 2026-07-22-per-project-concurrency): this
|
|
777
1050
|
// project never runs more than its own configured `maxParallel`; absent
|
|
@@ -805,6 +1078,8 @@ export class RunManager {
|
|
|
805
1078
|
// Only pay for the config read when something is actually held: a queued record may name
|
|
806
1079
|
// no runner, and then the account it would use is the configured default.
|
|
807
1080
|
const defaultRunner = anyHold ? (await loadConfig(this.repoRoot)).defaultRunner : undefined;
|
|
1081
|
+
if (this.disposed || this.quiescing)
|
|
1082
|
+
return;
|
|
808
1083
|
while (this.queue.length > 0 && capacity()) {
|
|
809
1084
|
// FIFO among the runs that CAN start; a held one keeps its place in the queue rather
|
|
810
1085
|
// than being dequeued and re-queued (which would churn its position and its record).
|
|
@@ -832,7 +1107,7 @@ export class RunManager {
|
|
|
832
1107
|
this.starting.add(runId);
|
|
833
1108
|
if (continuation) {
|
|
834
1109
|
const hydrated = this.hydrateQueuedContinuation(runId, continuation);
|
|
835
|
-
|
|
1110
|
+
this.trackRun(this.runContinuation(runId, hydrated.stepId, hydrated.sessionId, hydrated.backend, hydrated.prompt, hydrated.images, hydrated.persistedImages, hydrated.persistedAttachments).catch((err) => {
|
|
836
1111
|
const message = err instanceof Error ? err.message : String(err);
|
|
837
1112
|
this.store.updateRun(runId, {
|
|
838
1113
|
status: 'failed',
|
|
@@ -841,7 +1116,7 @@ export class RunManager {
|
|
|
841
1116
|
});
|
|
842
1117
|
this.starting.delete(runId);
|
|
843
1118
|
this.dropActive(runId);
|
|
844
|
-
});
|
|
1119
|
+
}));
|
|
845
1120
|
continue;
|
|
846
1121
|
}
|
|
847
1122
|
if (!job)
|
|
@@ -851,7 +1126,7 @@ export class RunManager {
|
|
|
851
1126
|
// in the same synchronous tick as the `pendingJobs.delete` above, so no
|
|
852
1127
|
// handler can observe a half-dequeued run.
|
|
853
1128
|
const input = this.hydrateQueuedInput(runId, job.input);
|
|
854
|
-
|
|
1129
|
+
this.trackRun(this.execute(runId, job.workflow, input).catch((err) => {
|
|
855
1130
|
const message = err instanceof Error ? err.message : String(err);
|
|
856
1131
|
this.store.updateRun(runId, {
|
|
857
1132
|
status: 'failed',
|
|
@@ -865,7 +1140,7 @@ export class RunManager {
|
|
|
865
1140
|
}
|
|
866
1141
|
this.starting.delete(runId);
|
|
867
1142
|
this.dropActive(runId);
|
|
868
|
-
});
|
|
1143
|
+
}));
|
|
869
1144
|
}
|
|
870
1145
|
} while (this.pumpAgain);
|
|
871
1146
|
}
|
|
@@ -1057,6 +1332,10 @@ export class RunManager {
|
|
|
1057
1332
|
this.memoryPausing.delete(runId);
|
|
1058
1333
|
this.lastNamerKey.delete(runId);
|
|
1059
1334
|
this.forceStarted.delete(runId);
|
|
1335
|
+
// A body can leave without ever reaching `adoptActive` — the continuation whose isolation is
|
|
1336
|
+
// gone bails between the two — and an unconsumed request would then cancel the NEXT run to
|
|
1337
|
+
// carry this id, which for a Continue is this same id again.
|
|
1338
|
+
this.cancelRequested.delete(runId);
|
|
1060
1339
|
// The run's slot is gone from busySlots() as of the deletes above — hand it
|
|
1061
1340
|
// to the workspace's oldest queued run, in ANY project. Every terminal path
|
|
1062
1341
|
// funnels through here, so this one call covers them all.
|
|
@@ -1075,8 +1354,10 @@ export class RunManager {
|
|
|
1075
1354
|
// failed/cancelled) — the one moment the finished-worktree count can grow.
|
|
1076
1355
|
// Enforce count-based retention (#483) here so a single hook covers every
|
|
1077
1356
|
// terminal path. Fire-and-forget: retention must never delay or throw into
|
|
1078
|
-
// the lifecycle
|
|
1079
|
-
|
|
1357
|
+
// the lifecycle — but it is REGISTERED (#200), so dispose() can still wait
|
|
1358
|
+
// for the git it spawns instead of leaving it writing into a data root the
|
|
1359
|
+
// caller has already started deleting.
|
|
1360
|
+
this.enforceRetention();
|
|
1080
1361
|
// The run's temp directory (#785) goes on the same terminal transition, and
|
|
1081
1362
|
// unconditionally — it is scratch, not an artifact, so unlike a worktree
|
|
1082
1363
|
// there is no keep-count to respect and nothing left to recover from it. A
|
|
@@ -1139,6 +1420,21 @@ export class RunManager {
|
|
|
1139
1420
|
*/
|
|
1140
1421
|
fireAutoResume(runId) {
|
|
1141
1422
|
this.autoResumeTimers.delete(runId);
|
|
1423
|
+
// A self-starting entry point that `pump()`'s gate cannot cover, and the difference is the
|
|
1424
|
+
// reason both gates exist. dispose() clears the timers so this cannot fire after it, but
|
|
1425
|
+
// `quiesce()` disposes LAST — so any timer armed BEFORE the drain (by an earlier pump, by
|
|
1426
|
+
// `recover()`, or by an ordinary usage-limit schedule whose deadline comes due) fires inside
|
|
1427
|
+
// it, with no pump involved at any point.
|
|
1428
|
+
//
|
|
1429
|
+
// What that would cost is RECORD WRITES, not a spawn: the call below defers for capacity, so
|
|
1430
|
+
// it enqueues rather than starting a turn — but `continueRun` first writes a `continue-N`
|
|
1431
|
+
// step and flips the record from `failed` to `queued`, and this method then stamps
|
|
1432
|
+
// `autoResumeAttempts`. Three writes into a data root the caller is tearing down, and a
|
|
1433
|
+
// user's `failed` run converted to `queued` for the trailing cancel pass to mark `cancelled`
|
|
1434
|
+
// (#200). A resume dropped here is not lost: the deadline lives on the record, and the next
|
|
1435
|
+
// manager's `reconcileAutoResumes` re-arms from it.
|
|
1436
|
+
if (this.disposed || this.quiescing)
|
|
1437
|
+
return;
|
|
1142
1438
|
const run = this.store.getRun(runId);
|
|
1143
1439
|
if (!run || run.status !== 'failed' || !run.autoResumeAt)
|
|
1144
1440
|
return;
|
|
@@ -1299,21 +1595,26 @@ export class RunManager {
|
|
|
1299
1595
|
*
|
|
1300
1596
|
* Public so a test can drive the wedge directly instead of waiting out the interval.
|
|
1301
1597
|
*
|
|
1302
|
-
* The sweep itself is `sweepStalledQueue`; this wrapper only publishes it to `
|
|
1598
|
+
* The sweep itself is `sweepStalledQueue`; this wrapper only publishes it to `writesInFlight`
|
|
1303
1599
|
* so dispose() can settle it (#125). Every entry point goes through here — the interval and the
|
|
1304
1600
|
* tests alike — so nothing can start a sweep that teardown cannot see. The schedule, the
|
|
1305
1601
|
* conditions and the effects are untouched.
|
|
1306
1602
|
*/
|
|
1307
1603
|
async rescueStalledQueue(now = Date.now()) {
|
|
1308
|
-
|
|
1604
|
+
// `quiescing` for the same reason `pump()` carries it, one phase earlier than `disposed`:
|
|
1605
|
+
// this is a self-starting entry point (an unref'd 60 s interval), and a drain that outlasts
|
|
1606
|
+
// one tick would otherwise get a sweep re-adopting queued records into the data root the
|
|
1607
|
+
// caller is tearing down. A skipped sweep costs nothing — re-adoption is idempotent and the
|
|
1608
|
+
// next process does it at boot, which is exactly what this method's own doc promises.
|
|
1609
|
+
if (this.disposed || this.quiescing)
|
|
1309
1610
|
return;
|
|
1310
1611
|
const sweep = this.sweepStalledQueue(now);
|
|
1311
|
-
this.
|
|
1612
|
+
this.writesInFlight.add(sweep);
|
|
1312
1613
|
try {
|
|
1313
1614
|
await sweep;
|
|
1314
1615
|
}
|
|
1315
1616
|
finally {
|
|
1316
|
-
this.
|
|
1617
|
+
this.writesInFlight.delete(sweep);
|
|
1317
1618
|
}
|
|
1318
1619
|
}
|
|
1319
1620
|
async sweepStalledQueue(now) {
|
|
@@ -1386,7 +1687,10 @@ export class RunManager {
|
|
|
1386
1687
|
if (Number.isFinite(at) && at > now)
|
|
1387
1688
|
deadline.add(key());
|
|
1388
1689
|
}
|
|
1389
|
-
else if (resumeInFlight(run)) {
|
|
1690
|
+
else if (resumeInFlight(run) && !resumeProven(run, now, this.resumeProofMs)) {
|
|
1691
|
+
// A resume whose turn has stayed live past the proof window has tested the account and
|
|
1692
|
+
// found it open; holding the queue for the rest of that turn is the #285 freeze. The
|
|
1693
|
+
// counter stays on the record — the cap still reads it — only the hold ends here.
|
|
1390
1694
|
inFlight.add(key());
|
|
1391
1695
|
}
|
|
1392
1696
|
}
|
|
@@ -1405,18 +1709,40 @@ export class RunManager {
|
|
|
1405
1709
|
if (!run)
|
|
1406
1710
|
return false;
|
|
1407
1711
|
const pending = run.autoResumeAt !== undefined || this.autoResumeTimers.has(runId);
|
|
1712
|
+
// A resume already running has neither a deadline nor a timer — `continueRun` retired both —
|
|
1713
|
+
// yet its counter is what holds the account, so clearing it is a release too (#285).
|
|
1714
|
+
const holding = run.autoResumeAttempts !== undefined;
|
|
1408
1715
|
this.clearAutoResume(runId);
|
|
1409
1716
|
if (pending) {
|
|
1410
1717
|
this.store.appendEvent(runId, {
|
|
1411
1718
|
type: 'note',
|
|
1412
1719
|
message: 'automatic resume cancelled for this task',
|
|
1413
1720
|
});
|
|
1414
|
-
// This run may have been the last thing holding its account's queue — nothing else will
|
|
1415
|
-
// notice, since the hold is derived and its release is not an event.
|
|
1416
|
-
void this.pump();
|
|
1417
1721
|
}
|
|
1722
|
+
// This run may have been the last thing holding its account's queue — nothing else will
|
|
1723
|
+
// notice, since the hold is derived and its release is not an event.
|
|
1724
|
+
if (pending || holding)
|
|
1725
|
+
void this.pump();
|
|
1418
1726
|
return true;
|
|
1419
1727
|
}
|
|
1728
|
+
/**
|
|
1729
|
+
* Pump the whole workspace once a live resume's proof window has closed (#285). The hold is
|
|
1730
|
+
* workspace-wide — one account can be queued in several projects — so this is `release()`,
|
|
1731
|
+
* not this manager's own pump. It carries no state: whether the hold has lifted is decided by
|
|
1732
|
+
* `accountHolds()` from the record when the pump reads it, so a turn that failed or finished
|
|
1733
|
+
* first just makes this a no-op pump. The slack keeps a timer that fires a millisecond early
|
|
1734
|
+
* from reading a window that has not quite closed yet.
|
|
1735
|
+
*/
|
|
1736
|
+
armResumeProofPump() {
|
|
1737
|
+
const timer = setTimeout(() => {
|
|
1738
|
+
this.resumeProofTimers.delete(timer);
|
|
1739
|
+
if (this.disposed)
|
|
1740
|
+
return;
|
|
1741
|
+
void this.semaphore.release();
|
|
1742
|
+
}, this.resumeProofMs + 250);
|
|
1743
|
+
timer.unref?.();
|
|
1744
|
+
this.resumeProofTimers.add(timer);
|
|
1745
|
+
}
|
|
1420
1746
|
/** Retire a pending resume — timer, deadline and counter. The counter goes too because every
|
|
1421
1747
|
* caller is a fresh epoch: a human Continue, or a resume that re-stamps its own count. */
|
|
1422
1748
|
clearAutoResume(runId) {
|
|
@@ -1431,13 +1757,57 @@ export class RunManager {
|
|
|
1431
1757
|
this.store.updateRun(runId, { autoResumeAt: undefined, autoResumeAttempts: undefined });
|
|
1432
1758
|
}
|
|
1433
1759
|
}
|
|
1434
|
-
/**
|
|
1435
|
-
*
|
|
1436
|
-
*
|
|
1437
|
-
|
|
1760
|
+
/**
|
|
1761
|
+
* Reclaim finished worktrees beyond the keep-limit (#483) — directory only, `xez/<id8>` branch
|
|
1762
|
+
* kept. Best-effort; a failure never affects run lifecycle. `review`/live runs are excluded by
|
|
1763
|
+
* the selector.
|
|
1764
|
+
*
|
|
1765
|
+
* The sweep itself is `reclaimRetiredWorktrees`; this wrapper only publishes it to
|
|
1766
|
+
* `writesInFlight` so dispose() can settle it, the same shape `rescueStalledQueue` already
|
|
1767
|
+
* uses. Callers still float it — retention must never delay or throw into a terminal
|
|
1768
|
+
* transition — but a floated promise nobody holds is one teardown cannot wait for, and this
|
|
1769
|
+
* one spawns `git worktree remove` and `git worktree prune` inside `repoRoot` (#200).
|
|
1770
|
+
*/
|
|
1771
|
+
enforceRetention() {
|
|
1772
|
+
if (this.disposed)
|
|
1773
|
+
return;
|
|
1774
|
+
const sweep = this.reclaimRetiredWorktrees();
|
|
1775
|
+
this.writesInFlight.add(sweep);
|
|
1776
|
+
// `.catch()` BEFORE `.finally()`, the same order `trackRun` uses: `p.finally(fn)` returns a
|
|
1777
|
+
// DERIVED promise that adopts p's rejection, so floating it leaves that copy unhandled. The
|
|
1778
|
+
// sweep cannot reject today (it swallows its own errors), which makes this latent rather than
|
|
1779
|
+
// live — and exactly the kind of latency a later edit turns into a process-level crash.
|
|
1780
|
+
void sweep
|
|
1781
|
+
.catch(() => undefined)
|
|
1782
|
+
.finally(() => {
|
|
1783
|
+
this.writesInFlight.delete(sweep);
|
|
1784
|
+
});
|
|
1785
|
+
}
|
|
1786
|
+
/**
|
|
1787
|
+
* The retention sweep proper. `disposed` is re-read after `resolveWorktreeRetention` — the one
|
|
1788
|
+
* await before any git spawn — for the reason the flag exists: a sweep already past its first
|
|
1789
|
+
* check when dispose() landed would otherwise start removing directories under a root its
|
|
1790
|
+
* caller is deleting. `shouldStop` carries the same question INTO the loop, because a sweep
|
|
1791
|
+
* removes one directory per iteration and a single check before the first one stops nothing on
|
|
1792
|
+
* a five-worktree pass that straddles a dispose.
|
|
1793
|
+
*
|
|
1794
|
+
* Bailing there reclaims nothing, and nothing is lost by it: retention is count-based and
|
|
1795
|
+
* re-enforced from scratch by the boot sweep (`index.ts`) and by the Settings → Worktrees
|
|
1796
|
+
* reclaim route, so a skipped pass costs one over-limit directory until the next boot. The
|
|
1797
|
+
* alternative — finishing the pass — spends two git spawns inside a directory whose owner has
|
|
1798
|
+
* just said it is done with it, plus a `worktreeReclaimedAt` stamp through `store.updateRun`
|
|
1799
|
+
* that schedules a debounced `runs.json` write from a store nobody owns any more. That is the
|
|
1800
|
+
* whole of #200, and the same hazard `armRepoHandle` is guarded against in
|
|
1801
|
+
* `server/project-context.ts`.
|
|
1802
|
+
*/
|
|
1803
|
+
async reclaimRetiredWorktrees() {
|
|
1438
1804
|
try {
|
|
1439
1805
|
const keep = await resolveWorktreeRetention(this.repoRoot);
|
|
1440
|
-
|
|
1806
|
+
if (this.disposed)
|
|
1807
|
+
return;
|
|
1808
|
+
await reclaimWorktrees(this.repoRoot, this.store, keep, {
|
|
1809
|
+
shouldStop: () => this.disposed,
|
|
1810
|
+
});
|
|
1441
1811
|
}
|
|
1442
1812
|
catch {
|
|
1443
1813
|
// retention is best-effort; swallow so terminal transitions never break.
|
|
@@ -1514,16 +1884,89 @@ export class RunManager {
|
|
|
1514
1884
|
return true;
|
|
1515
1885
|
}
|
|
1516
1886
|
const state = this.active.get(runId);
|
|
1517
|
-
if (!state)
|
|
1887
|
+
if (!state) {
|
|
1888
|
+
// Dequeued, but its body has not built an `ActiveRun` yet — there is no `cancelled` flag to
|
|
1889
|
+
// set and no `interrupt` to call. Leave the request where the body will find it
|
|
1890
|
+
// (`adoptActive`) instead of answering false and delivering nothing: that window is short
|
|
1891
|
+
// on the `execute()` path and real on the continuation one, which awaits
|
|
1892
|
+
// `rematerializeReclaimedWorktree` first, and missing it costs a whole uncancellable agent
|
|
1893
|
+
// turn (#200).
|
|
1894
|
+
if (this.starting.has(runId)) {
|
|
1895
|
+
this.cancelRequested.add(runId);
|
|
1896
|
+
return true;
|
|
1897
|
+
}
|
|
1518
1898
|
return false;
|
|
1899
|
+
}
|
|
1519
1900
|
state.cancelled = true;
|
|
1520
1901
|
this.clearIdleTimer(state);
|
|
1521
1902
|
state.interrupt();
|
|
1522
1903
|
return true;
|
|
1523
1904
|
}
|
|
1905
|
+
/**
|
|
1906
|
+
* Publish a run's freshly built `ActiveRun` and adopt any cancellation that arrived while it
|
|
1907
|
+
* was being built. Returns true when the caller must abandon the run.
|
|
1908
|
+
*
|
|
1909
|
+
* One helper for both bodies, because the two `active.set` sites are the two halves of the same
|
|
1910
|
+
* moment and had already drifted once (`state.autonomous`, #141). Ordering inside is
|
|
1911
|
+
* load-bearing: the flag is consumed BEFORE `active.set`, so a `cancel()` landing after this
|
|
1912
|
+
* returns finds a state to write to and takes the ordinary path.
|
|
1913
|
+
*/
|
|
1914
|
+
adoptActive(runId, state) {
|
|
1915
|
+
if (this.cancelRequested.delete(runId))
|
|
1916
|
+
state.cancelled = true;
|
|
1917
|
+
this.active.set(runId, state);
|
|
1918
|
+
this.starting.delete(runId);
|
|
1919
|
+
return state.cancelled;
|
|
1920
|
+
}
|
|
1921
|
+
/**
|
|
1922
|
+
* Publish a freshly started agent session on the run's `ActiveRun`, and deliver any
|
|
1923
|
+
* cancellation that arrived while the session was being BUILT.
|
|
1924
|
+
*
|
|
1925
|
+
* The twin of `adoptActive`, one phase later and for the same reason. `cancel()` stops a live
|
|
1926
|
+
* turn by calling `state.interrupt()`, and between the step loop's own `state.cancelled` check
|
|
1927
|
+
* and this moment that function is still the `() => undefined` placeholder: everything in
|
|
1928
|
+
* between — `configuredModelProvider`, `agentEnvForStep`, a team skill's `materializeSkillDir`
|
|
1929
|
+
* — is `await`ed, and a cancel landing in any of those gaps set the flag and delivered nothing.
|
|
1930
|
+
*
|
|
1931
|
+
* What that cost is a run that never ends, not a turn that runs one step too long (#199/#200).
|
|
1932
|
+
* The session spawns anyway; an interactive step passes `timeoutMs: 0`, so the runner has no
|
|
1933
|
+
* wall clock at all; and the turn-end handler computes `sessionOpen` as
|
|
1934
|
+
* `!state.cancelled && session.open`, so a cancelled run neither parks at `waiting` nor closes
|
|
1935
|
+
* the session on `XEZ:DONE`. The body then sits in `await session.result` for as long as the
|
|
1936
|
+
* agent CLI keeps stdin open — which for the bundled mock is forever. `quiesce()` issues its
|
|
1937
|
+
* cancel once per drain generation and only re-issues it after `Promise.allSettled` resolves,
|
|
1938
|
+
* so a single missed delivery is a permanent hang: measured as the 90 s teardown timeout on a
|
|
1939
|
+
* 2-core CI runner, where the widened gap makes the window easy to land in.
|
|
1940
|
+
*
|
|
1941
|
+
* ONE helper for both construction sites — `runAgentStep` and `runContinuation` — because they
|
|
1942
|
+
* are the same moment written twice and had already drifted (only one of them re-stamped
|
|
1943
|
+
* `currentStepId`). A third session site must come through here too, or it re-opens the hole.
|
|
1944
|
+
*/
|
|
1945
|
+
publishSession(runId, state, stepId, session) {
|
|
1946
|
+
state.session = session;
|
|
1947
|
+
state.sessionEverOpened = true;
|
|
1948
|
+
this.flushDeferred(runId);
|
|
1949
|
+
state.currentStepId = stepId;
|
|
1950
|
+
state.interrupt = () => session.interrupt();
|
|
1951
|
+
if (session.pid !== undefined)
|
|
1952
|
+
registerRunProcess(runId, session.pid);
|
|
1953
|
+
// The adopt. Ordering is load-bearing exactly as it is in `adoptActive`: `state.interrupt`
|
|
1954
|
+
// already points at this session, so a `cancel()` arriving one tick later takes the ordinary
|
|
1955
|
+
// path and this call is not a second, racing teardown.
|
|
1956
|
+
if (state.cancelled)
|
|
1957
|
+
session.interrupt();
|
|
1958
|
+
}
|
|
1524
1959
|
isActive(runId) {
|
|
1525
1960
|
return this.active.has(runId) || this.starting.has(runId) || this.queue.includes(runId);
|
|
1526
1961
|
}
|
|
1962
|
+
/** The id set behind `isActive` — `active ∪ starting ∪ queue`. `quiesce()` needs the whole
|
|
1963
|
+
* population rather than one membership test, and a second spelling of that union is exactly
|
|
1964
|
+
* how the two drift apart: `run-quiesce.test.ts` pins the observable half by putting one run
|
|
1965
|
+
* in `queue` and one in `active`, asserting `isActive` for both before `quiesce()` and for
|
|
1966
|
+
* neither after. A registry this set forgets would leave a live writer behind. */
|
|
1967
|
+
activeRunIds() {
|
|
1968
|
+
return new Set([...this.active.keys(), ...this.starting, ...this.queue]);
|
|
1969
|
+
}
|
|
1527
1970
|
/**
|
|
1528
1971
|
* Fold a queued run's persisted prompt — `run.task` plus everything stacked
|
|
1529
1972
|
* onto it (#472) — into the job input that is about to execute.
|
|
@@ -1904,7 +2347,14 @@ export class RunManager {
|
|
|
1904
2347
|
if (agentModelsLocked(this.repoRoot) && opts.model?.trim()) {
|
|
1905
2348
|
return { ok: false, error: AGENT_MODELS_LOCKED_ERROR };
|
|
1906
2349
|
}
|
|
1907
|
-
|
|
2350
|
+
// `isActive`, not `active.has`: this method now writes a `starting` entry of its own (below),
|
|
2351
|
+
// and `rematerializeReclaimedWorktree` can hold it there for a whole `git worktree add`.
|
|
2352
|
+
// Two Continues inside that window both passed an `active`-only guard, both enrolled the
|
|
2353
|
+
// same id, and the second `adoptActive` overwrote the first's `ActiveRun` — discarding a
|
|
2354
|
+
// cancellation already delivered to it and leaving one body driving a session the registry
|
|
2355
|
+
// no longer points at. Two spellings of "is this run live" is the drift `activeRunIds()`
|
|
2356
|
+
// was introduced to end (#200).
|
|
2357
|
+
if (this.isActive(runId))
|
|
1908
2358
|
return { ok: false, error: 'run is still active' };
|
|
1909
2359
|
const run = this.store.getRun(runId);
|
|
1910
2360
|
if (!run)
|
|
@@ -2009,15 +2459,25 @@ export class RunManager {
|
|
|
2009
2459
|
});
|
|
2010
2460
|
return { ok: true };
|
|
2011
2461
|
}
|
|
2012
|
-
|
|
2462
|
+
// In `starting` for the same reason `pump()` puts its dequeued runs there, and it is not
|
|
2463
|
+
// decoration: `runContinuation` awaits `rematerializeReclaimedWorktree` before it builds its
|
|
2464
|
+
// `ActiveRun`, so without this the run is in NEITHER registry for the width of a git spawn —
|
|
2465
|
+
// invisible to `isActive()`, unreachable by `cancel()`, and therefore a turn `quiesce()`
|
|
2466
|
+
// would start waiting for without ever having been able to stop it (#200).
|
|
2467
|
+
this.starting.add(runId);
|
|
2468
|
+
this.trackRun(this.runContinuation(runId, stepId, resume ? sessionStep.sessionId : undefined, targetRunner, prompt, images).catch((err) => {
|
|
2013
2469
|
const message = err instanceof Error ? err.message : String(err);
|
|
2014
2470
|
this.store.updateRun(runId, {
|
|
2015
2471
|
status: 'failed',
|
|
2016
2472
|
error: `continue crashed: ${message}`,
|
|
2017
2473
|
finishedAt: new Date().toISOString(),
|
|
2018
2474
|
});
|
|
2475
|
+
// Paired with the `starting.add` above, exactly as `pump()`'s handler pairs with its own:
|
|
2476
|
+
// a crash before `adoptActive` would otherwise leave the id in `starting` for good, and
|
|
2477
|
+
// `busySlots()` counts it — one leaked entry permanently narrows the parallel cap.
|
|
2478
|
+
this.starting.delete(runId);
|
|
2019
2479
|
this.dropActive(runId);
|
|
2020
|
-
});
|
|
2480
|
+
}));
|
|
2021
2481
|
return { ok: true };
|
|
2022
2482
|
}
|
|
2023
2483
|
async runContinuation(runId, stepId, sessionId, backend, prompt,
|
|
@@ -2091,8 +2551,18 @@ export class RunManager {
|
|
|
2091
2551
|
autonomous: record?.autonomous === true,
|
|
2092
2552
|
autoContinues: 0,
|
|
2093
2553
|
};
|
|
2094
|
-
|
|
2095
|
-
|
|
2554
|
+
// A cancel that arrived while the worktree was being re-materialized above is honoured here
|
|
2555
|
+
// rather than dropped (#200): stop before the spawn, exactly as a cancel one tick later
|
|
2556
|
+
// would have. `finishedAt` is stamped fresh because nothing else has stamped it — this run
|
|
2557
|
+
// ends without ever having opened a session.
|
|
2558
|
+
if (this.adoptActive(runId, state)) {
|
|
2559
|
+
const finishedAt = new Date().toISOString();
|
|
2560
|
+
this.store.updateStep(runId, stepId, { status: 'cancelled', finishedAt });
|
|
2561
|
+
this.store.updateRun(runId, { status: 'cancelled', finishedAt, currentStepId: undefined });
|
|
2562
|
+
this.store.appendEvent(runId, { type: 'lifecycle', message: 'run cancelled' });
|
|
2563
|
+
this.dropActive(runId);
|
|
2564
|
+
return;
|
|
2565
|
+
}
|
|
2096
2566
|
if (state.cwd === this.repoRoot) {
|
|
2097
2567
|
if (repositoryRootLockDisabled()) {
|
|
2098
2568
|
this.store.appendEvent(runId, {
|
|
@@ -2140,6 +2610,10 @@ export class RunManager {
|
|
|
2140
2610
|
backend,
|
|
2141
2611
|
});
|
|
2142
2612
|
this.store.appendEvent(runId, { type: 'step-start', stepId, name: 'Continue', kind: 'agent', iteration: 1 });
|
|
2613
|
+
// An automatic resume's turn is now live. If it is still running when the proof window
|
|
2614
|
+
// closes, its hold lifts (`resumeProven`) — and the queue behind it has to hear about that.
|
|
2615
|
+
if (this.store.getRun(runId)?.autoResumeAttempts !== undefined)
|
|
2616
|
+
this.armResumeProofPump();
|
|
2143
2617
|
let stepCost = 0;
|
|
2144
2618
|
let turnText = '';
|
|
2145
2619
|
let sessionError;
|
|
@@ -2181,7 +2655,10 @@ export class RunManager {
|
|
|
2181
2655
|
// coalescers; the v1 turn boundary flushes again (idempotent) so no
|
|
2182
2656
|
// buffered delta can outlive its turn.
|
|
2183
2657
|
sink.flushAll();
|
|
2184
|
-
|
|
2658
|
+
// Tracked, not floated free (#200): this escapes the run body it is fired from, and it
|
|
2659
|
+
// spawns `git diff --shortstat` inside a worktree under the data root. `quiesce()` waits
|
|
2660
|
+
// for it; dispose() deliberately does not, so no existing caller's timing changes.
|
|
2661
|
+
this.trackRun(this.recordTurnEnd(runId, turnText)); // titleSummary + diffStat (#389)
|
|
2185
2662
|
const sessionOpen = !state.cancelled && state.session?.open;
|
|
2186
2663
|
const done = sessionOpen && DONE_MARKER_RE.test(turnText.trimEnd());
|
|
2187
2664
|
// `XEZ:ASK` → the user is genuinely blocked; wins over `XEZ:MONITORING`
|
|
@@ -2232,11 +2709,12 @@ export class RunManager {
|
|
|
2232
2709
|
this.releaseSlot();
|
|
2233
2710
|
}
|
|
2234
2711
|
}
|
|
2235
|
-
// A turn that completed is the
|
|
2236
|
-
// it is what retires the consecutive-resume counter —
|
|
2237
|
-
//
|
|
2238
|
-
// 2026-08-03-auto-resume-after-usage-limit).
|
|
2239
|
-
//
|
|
2712
|
+
// A turn that completed is the strongest evidence the provider's window actually reopened,
|
|
2713
|
+
// so it is what retires the consecutive-resume counter — and with it any account hold
|
|
2714
|
+
// still standing for every other task queued behind it (spec
|
|
2715
|
+
// 2026-08-03-auto-resume-after-usage-limit). A long turn has usually lifted that hold
|
|
2716
|
+
// already, by staying live past `AUTO_RESUME_PROOF_MS` (#285). `settleSuccess` does the
|
|
2717
|
+
// same for a run that finishes outright; this covers the "parked for the user" ending.
|
|
2240
2718
|
if (this.store.getRun(runId)?.autoResumeAttempts !== undefined) {
|
|
2241
2719
|
this.store.updateRun(runId, { autoResumeAttempts: undefined });
|
|
2242
2720
|
}
|
|
@@ -2357,12 +2835,7 @@ export class RunManager {
|
|
|
2357
2835
|
resume: sessionId !== undefined,
|
|
2358
2836
|
timeoutMs: 0,
|
|
2359
2837
|
}, onEvent, { onUiEvent: (event) => this.handleRunnerUiEvent(runId, state, sink, event) });
|
|
2360
|
-
|
|
2361
|
-
state.sessionEverOpened = true;
|
|
2362
|
-
this.flushDeferred(runId);
|
|
2363
|
-
state.interrupt = () => session.interrupt();
|
|
2364
|
-
if (session.pid !== undefined)
|
|
2365
|
-
registerRunProcess(runId, session.pid);
|
|
2838
|
+
this.publishSession(runId, state, stepId, session);
|
|
2366
2839
|
const finishedAt = () => new Date().toISOString();
|
|
2367
2840
|
/**
|
|
2368
2841
|
* The turn's teardown — usage peaks, timers, and the turn-end autosave commit — runs
|
|
@@ -2464,8 +2937,17 @@ export class RunManager {
|
|
|
2464
2937
|
autonomous: input.autonomous === true,
|
|
2465
2938
|
autoContinues: 0,
|
|
2466
2939
|
};
|
|
2467
|
-
|
|
2468
|
-
|
|
2940
|
+
// Through the same helper as the continuation path, so the two `active.set` sites cannot
|
|
2941
|
+
// drift again (#141). The return is deliberately not branched on, and the reason is narrow:
|
|
2942
|
+
// `pump()` adds this run to `starting` and reaches the line above with no await in between,
|
|
2943
|
+
// so the materializing window is empty HERE and `cancelRequested` can hold nothing for it.
|
|
2944
|
+
//
|
|
2945
|
+
// Do not read that as "a cancellation adopted here is handled anyway". It is not: the first
|
|
2946
|
+
// git spawn on this path is `createWorktree`, well before the step loop that breaks on
|
|
2947
|
+
// `state.cancelled`. `requeueWhileHeld` below is passed `state` so its own `state?.cancelled`
|
|
2948
|
+
// guard is live rather than inert, but that guard only declines to re-queue. If an await
|
|
2949
|
+
// ever appears between `pump()`'s `starting.add` and this line, branch on the return.
|
|
2950
|
+
this.adoptActive(runId, state);
|
|
2469
2951
|
const emit = (event) => this.store.appendEvent(runId, event);
|
|
2470
2952
|
// Resolve the agent backend for this run: the task choice (GUI) wins over
|
|
2471
2953
|
// the config default. Per-step `runner` can still override it below.
|
|
@@ -2475,7 +2957,10 @@ export class RunManager {
|
|
|
2475
2957
|
// gate cannot be the only one, because dequeue is not the moment of no return. Nothing has
|
|
2476
2958
|
// happened yet here, so the run goes back to the queue untouched (spec
|
|
2477
2959
|
// 2026-08-03-auto-resume-after-usage-limit).
|
|
2478
|
-
|
|
2960
|
+
// `state` passed, exactly as the post-lease call site below passes it: without it the
|
|
2961
|
+
// helper's `state?.cancelled` guard is inert, and a run cancelled through `cancelRequested`
|
|
2962
|
+
// would be handed back to the queue as `queued` instead of stopping.
|
|
2963
|
+
if (this.requeueWhileHeld(runId, workflow, input, taskBackend, state))
|
|
2479
2964
|
return;
|
|
2480
2965
|
// Extra system prompt (R2 2.3): POST override > config default; echoed on
|
|
2481
2966
|
// the record so the UI/API can show what the run actually used.
|
|
@@ -2819,6 +3304,9 @@ export class RunManager {
|
|
|
2819
3304
|
const startTokens = stepRecord?.tokensUsed ?? 0;
|
|
2820
3305
|
let stepCost = stepRecord?.costUsd ?? 0;
|
|
2821
3306
|
let turnText = '';
|
|
3307
|
+
// The text of the step's last FINISHED turn — what `unfinishedStepReason` judges a non-final
|
|
3308
|
+
// step by (#317). Null until a turn ends, so a session that closed mid-turn is not done either.
|
|
3309
|
+
let lastTurnText = null;
|
|
2822
3310
|
let sessionError;
|
|
2823
3311
|
const sink = this.makeUiSink(runId, step.id);
|
|
2824
3312
|
const onEvent = (event) => {
|
|
@@ -2858,7 +3346,10 @@ export class RunManager {
|
|
|
2858
3346
|
// v2 `turn.completed` already flushed the coalescers; the v1 turn
|
|
2859
3347
|
// boundary flushes again (idempotent) as a backstop.
|
|
2860
3348
|
sink.flushAll();
|
|
2861
|
-
|
|
3349
|
+
// Tracked, not floated free (#200): this escapes the run body it is fired from, and it
|
|
3350
|
+
// spawns `git diff --shortstat` inside a worktree under the data root. `quiesce()` waits
|
|
3351
|
+
// for it; dispose() deliberately does not, so no existing caller's timing changes.
|
|
3352
|
+
this.trackRun(this.recordTurnEnd(runId, turnText)); // titleSummary + diffStat (#389)
|
|
2862
3353
|
const sessionOpen = !state.cancelled && state.session?.open;
|
|
2863
3354
|
const done = interactive && sessionOpen && DONE_MARKER_RE.test(turnText.trimEnd());
|
|
2864
3355
|
// `XEZ:ASK` → the user is blocked; wins over `XEZ:MONITORING`, loses to
|
|
@@ -2869,6 +3360,7 @@ export class RunManager {
|
|
|
2869
3360
|
!done &&
|
|
2870
3361
|
!ask &&
|
|
2871
3362
|
MONITORING_MARKER_RE.test(turnText.trimEnd());
|
|
3363
|
+
lastTurnText = turnText;
|
|
2872
3364
|
turnText = '';
|
|
2873
3365
|
for (const note of askNotes)
|
|
2874
3366
|
emit({ type: 'note', stepId: step.id, ...note });
|
|
@@ -2997,13 +3489,7 @@ export class RunManager {
|
|
|
2997
3489
|
state.currentStepId = undefined;
|
|
2998
3490
|
return err instanceof Error ? err.message : String(err);
|
|
2999
3491
|
}
|
|
3000
|
-
state.
|
|
3001
|
-
state.sessionEverOpened = true;
|
|
3002
|
-
this.flushDeferred(runId);
|
|
3003
|
-
state.currentStepId = step.id;
|
|
3004
|
-
state.interrupt = () => session.interrupt();
|
|
3005
|
-
if (session.pid !== undefined)
|
|
3006
|
-
registerRunProcess(runId, session.pid);
|
|
3492
|
+
this.publishSession(runId, state, step.id, session);
|
|
3007
3493
|
try {
|
|
3008
3494
|
const result = await session.result;
|
|
3009
3495
|
if (sessionError) {
|
|
@@ -3014,6 +3500,10 @@ export class RunManager {
|
|
|
3014
3500
|
// events to the RunManager — only it knows how the session settled).
|
|
3015
3501
|
sink.sessionEnded(state.cancelled ? 'cancelled' : 'end_turn');
|
|
3016
3502
|
this.store.updateStep(runId, step.id, { tokensUsed: startTokens + result.tokensUsed });
|
|
3503
|
+
// A session that ended cleanly is not a finished step (#317). The last step keeps its own
|
|
3504
|
+
// rules — it is interactive, parks at `waiting` and closes on `XEZ:DONE` or idle.
|
|
3505
|
+
if (!interactive && !state.cancelled)
|
|
3506
|
+
return unfinishedStepReason(lastTurnText);
|
|
3017
3507
|
return null;
|
|
3018
3508
|
}
|
|
3019
3509
|
catch (err) {
|
|
@@ -3167,7 +3657,12 @@ export class RunManager {
|
|
|
3167
3657
|
skillDescription = skills.find((s) => s.name === skillName)?.description;
|
|
3168
3658
|
}
|
|
3169
3659
|
const result = await generateRunName(this.repoRoot, { task, skillName, skillDescription, ...live });
|
|
3170
|
-
|
|
3660
|
+
// A torn-down manager stops writing (#200). The namer is deliberately NOT enrolled in
|
|
3661
|
+
// `writesInFlight`: it is a model call with its own `NAMER_TIMEOUT_MS`, and making
|
|
3662
|
+
// `dispose()` wait up to that long on it would be a far worse trade than losing a title.
|
|
3663
|
+
// This is the cheap half of the same guarantee — the call finishes on its own, and its
|
|
3664
|
+
// RESULT is simply not written into a store whose data root may already be gone.
|
|
3665
|
+
if (!result || this.disposed)
|
|
3171
3666
|
return;
|
|
3172
3667
|
const run = this.store.getRun(runId);
|
|
3173
3668
|
// Marker-owned state outranks the namer (spec 2026-07-18-task-ref-markers):
|