@codetreker/daynight 0.32.1 → 0.38.0

package/dist/commands/worker/cleanup.js

@@ -1,5 +1,8 @@
 import path from "node:path";
-import { actualTaskPane, executionContext, liveAgent, settleWorkerStops, workerOperation } from "./execution.js";
+import { CLEANUP_OPERATION_MS, cleanupOperation } from "../cleanup-operation.js";
+import { ensureCleanupSchedule } from "./cleanup-schedule.js";
+import { WorkerCleanupError } from "./cleanup-error.js";
+import { actualTaskPane, executionContext, liveAgent, settleWorkerStops } from "./execution.js";
 import { workerCleanupPrompt } from "./prompts.js";
 const key = (owner) => `${owner.projectId}/${owner.taskId}/${owner.claimId}`;
 export const createCleanupDriver = (deps) => {
@@ -7,47 +10,60 @@ export const createCleanupDriver = (deps) => {
     return {
         advance: async (input, resources, deleteWorktree, signal) => {
             const identity = key(input.owner);
-            let attempt = attempts.get(identity);
+            let attempt = deleteWorktree ? ensureCleanupSchedule(deps.store, input.owner, deps.now()) : attempts.get(identity);
             if (attempt === undefined) {
                 attempt = { deadline: deps.now() + 600_000, notified: false, submitted: false };
                 attempts.set(identity, attempt);
                 deps.emit({ kind: "cleanup-started", ...(input.task === undefined ? {} : { task: input.task }), message: `Cleaning claim ${input.owner.claimId}; this attempt has a ten-minute deadline.` });
             }
-            const stopContext = { ...input.owner, cwd: path.join(input.project.localPath, ".worktrees", "daynight", input.owner.taskId) };
-            const stopped = await settleWorkerStops(deps, input.owner, signal);
-            const pane = stopped ? undefined : actualTaskPane(input.owner, resources);
-            if (pane !== undefined) {
-                const agent = input.task?.agent ?? pane.kind ?? pane.nativeSession?.agent;
-                if (agent === undefined) {
-                    await workerOperation((operation) => deps.herdr.tasks.stopAndClose(stopContext, pane.binding, operation), signal);
-                }
-                else {
-                    const context = executionContext({ ...input, task: { agent } });
-                    if (liveAgent(pane) && !attempt.notified && deps.now() < attempt.deadline) {
-                        attempt.notified = true;
-                        try {
-                            await workerOperation((operation) => deps.herdr.tasks.submitPrompt(context, pane.binding, workerCleanupPrompt(input.owner, attempt.deadline), operation), signal, Math.max(1, Math.min(30_000, attempt.deadline - deps.now())));
-                            attempt.submitted = true;
-                        }
-                        catch (error) {
-                            deps.emit({ kind: "cleanup-notification-error", ...(input.task === undefined ? {} : { task: input.task }), message: "Cleanup notification failed; the current bounded cleanup attempt continues.", error });
+            const saveAttempt = () => {
+                if (deleteWorktree)
+                    deps.store.saveCleanupSchedule(input.owner, { ...deps.store.getCleanupSchedule(input.owner), deadline: attempt.deadline, notified: attempt.notified, submitted: attempt.submitted });
+            };
+            let stage = "agent-stop";
+            try {
+                const stopContext = { ...input.owner, cwd: path.join(input.project.localPath, ".worktrees", "daynight", input.owner.taskId) };
+                const stopped = await settleWorkerStops(deps, input.owner, signal, cleanupOperation);
+                const pane = stopped ? undefined : actualTaskPane(input.owner, resources);
+                if (pane !== undefined) {
+                    const agent = input.task?.agent ?? pane.kind ?? pane.nativeSession?.agent;
+                    if (agent === undefined) {
+                        await cleanupOperation((operation) => deps.herdr.tasks.stopAndClose(stopContext, pane.binding, operation), signal);
+                    }
+                    else {
+                        const context = executionContext({ ...input, task: { agent } });
+                        if (liveAgent(pane) && !attempt.notified && deps.now() < attempt.deadline) {
+                            attempt.notified = true;
+                            saveAttempt();
+                            try {
+                                await cleanupOperation((operation) => deps.herdr.tasks.submitPrompt(context, pane.binding, workerCleanupPrompt(input.owner, attempt.deadline), operation), signal, Math.max(1, Math.min(CLEANUP_OPERATION_MS, attempt.deadline - deps.now())));
+                                attempt.submitted = true;
+                                saveAttempt();
+                            }
+                            catch (error) {
+                                deps.emit({ kind: "cleanup-notification-error", ...(input.task === undefined ? {} : { task: input.task }), message: "Cleanup notification failed; the current bounded cleanup attempt continues.", error });
+                            }
+                            if (deps.now() < attempt.deadline)
+                                return false;
                         }
-                        if (deps.now() < attempt.deadline)
+                        const completed = attempt.submitted && pane.status === "idle";
+                        if (liveAgent(pane) && !completed && deps.now() < attempt.deadline)
                             return false;
+                        await cleanupOperation((operation) => deps.herdr.tasks.stopAndClose(context, pane.binding, operation), signal);
                     }
-                    const completed = attempt.submitted && pane.status === "idle";
-                    if (liveAgent(pane) && !completed && deps.now() < attempt.deadline)
-                        return false;
-                    await workerOperation((operation) => deps.herdr.tasks.stopAndClose(context, pane.binding, operation), signal);
                 }
+                stage = "worktree-removal";
+                if (deleteWorktree)
+                    await cleanupOperation((operation) => deps.git.removeWorktree(input.project, input.owner, operation), signal);
+                attempts.delete(identity);
+                deps.emit({ kind: deleteWorktree ? "cleaned" : "agent-stopped", ...(input.task === undefined ? {} : { task: input.task }), message: deleteWorktree ? "Stopped the task resources and deleted its worktree; the branch remains." : "Stopped the task resources and preserved its worktree." });
+                return true;
+            }
+            catch (error) {
+                throw new WorkerCleanupError(stage, error);
             }
-            if (deleteWorktree)
-                await workerOperation((operation) => deps.git.removeWorktree(input.project, input.owner, operation), signal, 180_000);
-            attempts.delete(identity);
-            deps.emit({ kind: deleteWorktree ? "cleaned" : "agent-stopped", ...(input.task === undefined ? {} : { task: input.task }), message: deleteWorktree ? "Stopped the task resources and deleted its worktree; the branch remains." : "Stopped the task resources and preserved its worktree." });
-            return true;
         },
-        remaining: (owner) => Math.max(0, (attempts.get(key(owner))?.deadline ?? deps.now()) - deps.now()),
+        remaining: (owner) => Math.max(0, (deps.store.getCleanupSchedule(owner)?.deadline ?? attempts.get(key(owner))?.deadline ?? deps.now()) - deps.now()),
     };
 };
 //# sourceMappingURL=cleanup.js.map

package/dist/commands/worker/cleanup.js.map

@@ -1 +1 @@
-{"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../src/commands/worker/cleanup.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,SAAS,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEjH,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAKnD,MAAM,GAAG,GAAG,CAAC,KAAoB,EAAU,EAAE,CAAC,GAAG,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;AAEpG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,IAA6C,EAAE,EAAE;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,OAAO;QACL,OAAO,EAAE,KAAK,EAAE,KAAmB,EAAE,SAAyC,EAAE,cAAuB,EAAE,MAAoB,EAAoB,EAAE;YACjJ,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAClC,IAAI,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,OAAO,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;gBAChF,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,kBAAkB,KAAK,CAAC,KAAK,CAAC,OAAO,2CAA2C,EAAE,CAAC,CAAC;YAC/L,CAAC;YACD,MAAM,WAAW,GAAG,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;YAC9H,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YACnE,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;YAC1E,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;gBACvB,MAAM,KAAK,GAA4B,KAAK,CAAC,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC;gBACnG,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACxB,MAAM,eAAe,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;gBACpH,CAAC;qBAAM,CAAC;oBACN,MAAM,OAAO,GAAG,gBAAgB,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;oBAChE,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;wBAC1E,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;wBACxB,IAAI,CAAC;4BACH,MAAM,eAAe,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,OAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;4BACjO,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;wBAC3B,CAAC;wBAAC,OAAO,KAAK,EAAE,CAAC;4BACf,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,6EAA6E,EAAE,KAAK,EAAE,CAAC,CAAC;wBAC9M,CAAC;wBACD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ;4BAAE,OAAO,KAAK,CAAC;oBAClD,CAAC;oBACD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC;oBAC9D,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ;wBAAE,OAAO,KAAK,CAAC;oBACjF,MAAM,eAAe,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;gBAChH,CAAC;YACH,CAAC;YACD,IAAI,cAAc;gBAAE,MAAM,eAAe,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;YAC1I,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,0EAA0E,CAAC,CAAC,CAAC,wDAAwD,EAAE,CAAC,CAAC;YAC9R,OAAO,IAAI,CAAC;QACd,CAAC;QACD,SAAS,EAAE,CAAC,KAAoB,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;KAC1H,CAAC;AACJ,CAAC,CAAC"}
+{"version":3,"file":"cleanup.js","sourceRoot":"","sources":["../../../src/commands/worker/cleanup.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AACjF,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,gBAAgB,EAAE,SAAS,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAEhG,OAAO,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAC;AAKnD,MAAM,GAAG,GAAG,CAAC,KAAoB,EAAU,EAAE,CAAC,GAAG,KAAK,CAAC,SAAS,IAAI,KAAK,CAAC,MAAM,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;AAEpG,MAAM,CAAC,MAAM,mBAAmB,GAAG,CAAC,IAA6C,EAAE,EAAE;IACnF,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA0B,CAAC;IACnD,OAAO;QACL,OAAO,EAAE,KAAK,EAAE,KAAmB,EAAE,SAAyC,EAAE,cAAuB,EAAE,MAAoB,EAAoB,EAAE;YACjJ,MAAM,QAAQ,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAClC,IAAI,OAAO,GAAG,cAAc,CAAC,CAAC,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACnH,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;gBAC1B,OAAO,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;gBAChF,QAAQ,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;gBAChC,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,kBAAkB,KAAK,CAAC,KAAK,CAAC,OAAO,2CAA2C,EAAE,CAAC,CAAC;YAC/L,CAAC;YACD,MAAM,WAAW,GAAG,GAAS,EAAE;gBAC7B,IAAI,cAAc;oBAAE,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,KAAK,CAAC,KAAK,CAAE,EAAE,QAAQ,EAAE,OAAQ,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAQ,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;YAC/M,CAAC,CAAC;YACF,IAAI,KAAK,GAAsC,YAAY,CAAC;YAC5D,IAAI,CAAC;gBACH,MAAM,WAAW,GAAG,EAAE,GAAG,KAAK,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC9H,MAAM,OAAO,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;gBACrF,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;gBAC1E,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACvB,MAAM,KAAK,GAA4B,KAAK,CAAC,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,aAAa,EAAE,KAAK,CAAC;oBACnG,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;wBACxB,MAAM,gBAAgB,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,WAAW,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;oBACrH,CAAC;yBAAM,CAAC;wBACN,MAAM,OAAO,GAAG,gBAAgB,CAAC,EAAE,GAAG,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;wBAChE,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;4BAC1E,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC;4BACxB,WAAW,EAAE,CAAC;4BACd,IAAI,CAAC;gCACH,MAAM,gBAAgB,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,mBAAmB,CAAC,KAAK,CAAC,KAAK,EAAE,OAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;gCAChP,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC;gCACzB,WAAW,EAAE,CAAC;4BAChB,CAAC;4BAAC,OAAO,KAAK,EAAE,CAAC;gCACf,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,4BAA4B,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,6EAA6E,EAAE,KAAK,EAAE,CAAC,CAAC;4BAC9M,CAAC;4BACD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ;gCAAE,OAAO,KAAK,CAAC;wBAClD,CAAC;wBACD,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,MAAM,CAAC;wBAC9D,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,QAAQ;4BAAE,OAAO,KAAK,CAAC;wBACjF,MAAM,gBAAgB,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;oBACjH,CAAC;gBACH,CAAC;gBACD,KAAK,GAAG,kBAAkB,CAAC;gBAC3B,IAAI,cAAc;oBAAE,MAAM,gBAAgB,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,cAAc,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE,MAAM,CAAC,CAAC;gBAClI,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBAC1B,IAAI,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,eAAe,EAAE,GAAG,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,CAAC,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,0EAA0E,CAAC,CAAC,CAAC,wDAAwD,EAAE,CAAC,CAAC;gBAC9R,OAAO,IAAI,CAAC;YACd,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAAC,MAAM,IAAI,kBAAkB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YAAC,CAAC;QACjE,CAAC;QACD,SAAS,EAAE,CAAC,KAAoB,EAAU,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,KAAK,CAAC,EAAE,QAAQ,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;KAC5K,CAAC;AACJ,CAAC,CAAC"}

package/dist/commands/worker/coordinator.js

@@ -1,9 +1,12 @@
+import { CLEANUP_OPERATION_MS, cleanupOperation } from "../cleanup-operation.js";
 import { readWorkerClaim } from "../../api/task-properties.js";
 import { requireImplementedWorkerBackend } from "./backend.js";
 import { randomUUID } from "node:crypto";
 import path from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
 import { OperationError } from "../../api/operation.js";
+import { ensureCleanupSchedule } from "./cleanup-schedule.js";
+import { WorkerCleanupError } from "./cleanup-error.js";
 import { createCleanupDriver } from "./cleanup.js";
 import { actualTaskPane, createExecutionDriver, executionContext, isWorkerCancellation, liveAgent, sameClaim, settleWorkerStops, workerOperation } from "./execution.js";
 import { HerdrAgentActionError, HerdrRpcError } from "./herdr.js";
@@ -24,7 +27,9 @@ export const createWorkerCoordinator = (options) => {
     const tails = new Map();
     const jobs = new Map();
     const controllers = new Map();
-    const ending = new Map();
+    const ending = new Map(options.store.listCleanupSchedules().map(record => [key(record), record]));
+    let capacityWarning = "";
+    const cleanupRetries = new Map();
     const failures = new Map();
     const snapshotWarnings = new Map();
     let nextRepository = 0;
@@ -72,10 +77,12 @@ export const createWorkerCoordinator = (options) => {
         const current = options.repositories.find((repository) => repository.project.projectId === task.projectId)?.configuration ?? options.globalConfiguration;
         return current.agent === task.agent ? { ...current, agentControlMode: task.agentControlMode ?? "herdr" } : { agent: task.agent, agentControlMode: task.agentControlMode ?? "herdr", model: null, args: [], recoveryLimit: current.recoveryLimit };
     };
+    // Cleanup-only backend access stays inside its claim job and retry schedule.
     const needsHerdr = () => options.globalConfiguration.agentControlMode !== "web"
         || options.repositories.some(repository => repository.configuration.agentControlMode !== "web")
-        || options.store.listUnfinishedTasks().some(task => task.agentControlMode !== "web");
-    const pendingStops = () => needsHerdr() ? options.herdr.tasks.pendingStops() : [];
+        || options.store.listUnfinishedTasks().some(task => task.agentControlMode !== "web" && !ending.has(key(task)));
+    const hasHerdrCleanup = () => [...ending.values()].some(owner => options.store.getTask(owner)?.agentControlMode !== "web" && options.webStore?.getTask(owner) == null);
+    const pendingStops = () => needsHerdr() || hasHerdrCleanup() ? options.herdr.tasks.pendingStops() : [];
     const ensureHerdr = async (signal) => { if (needsHerdr())
         await workerOperation(operation => options.herdr.ensureInstance(operation), signal); };
     const listHerdr = async (signal) => needsHerdr() ? workerOperation(operation => options.herdr.tasks.listResources(operation), signal) : [];
@@ -164,10 +171,42 @@ export const createWorkerCoordinator = (options) => {
         const job = serialize(identity, () => action(signal)).catch((error) => {
             if (isWorkerCancellation(error, signal))
                 return;
-            if (!terminal)
+            let cleanupMessage;
+            let reportedError = error;
+            let task = null;
+            if (terminal) {
+                const owner = ending.get(identityKey);
+                let count = cleanupRetries.get(identityKey)?.failures ?? 0;
+                let previous = null;
+                try {
+                    previous = options.store.getCleanupSchedule(owner);
+                    count = Math.max(count, previous?.failures ?? 0);
+                }
+                catch (storageError) {
+                    reportedError = new AggregateError([error, storageError], "Cleanup failed and its retry state could not be read.");
+                }
+                count = Math.min(count + 1, Number.MAX_SAFE_INTEGER);
+                const retryMs = Math.min(60_000, 1000 * 2 ** Math.min(count - 1, 6));
+                const stage = error instanceof WorkerCleanupError ? error.stage : "observation";
+                const nextAttemptAt = now() + retryMs;
+                // Storage failures must not reject the job or prevent shutdown from proving stop.
+                cleanupRetries.set(identityKey, { failures: count, nextAttemptAt });
+                try {
+                    previous ??= ensureCleanupSchedule(options.store, owner, now());
+                    options.store.saveCleanupSchedule(previous, { ...previous, failures: count, nextAttemptAt, stage, lastError: error instanceof Error ? error.message || error.name : String(error) || "Unknown cleanup error" });
+                    task = options.store.getTask(identity);
+                }
+                catch (storageError) {
+                    reportedError = new AggregateError([reportedError, storageError], "Cleanup failed; retry persistence is unconfirmed and in-process backoff remains active.");
+                }
+                const stopStatus = stage === "archive" || stage === "worktree-removal" || stage === "backend-close" ? "Agent stop confirmed; " : "";
+                cleanupMessage = `Cleanup ${key(owner)} failed at ${stage}; ${stopStatus}next retry ${new Date(nextAttemptAt).toISOString()} (${retryMs} ms). Resources retain capacity; correct the reported cause and normal reconciliation retries automatically.`;
+            }
+            else {
                 failures.set(identityKey, (failures.get(identityKey) ?? 0) + 1);
-            const task = options.store.getTask(identity);
-            options.emit({ kind: terminal ? "cleanup-error" : "execution-error", ...(task === null ? {} : { task }), message: terminal ? "The resource cleanup failed and will be retried from actual observations." : "The task operation failed; this process retains its recovery count.", error });
+                task = options.store.getTask(identity);
+            }
+            options.emit({ kind: terminal ? "cleanup-error" : "execution-error", ...(task === null ? {} : { task }), message: terminal ? cleanupMessage : "The task operation failed; this process retains its recovery count.", error: reportedError });
         }).finally(() => { if (jobs.get(identityKey) === job) {
             jobs.delete(identityKey);
             controllers.delete(identityKey);
@@ -187,8 +226,11 @@ export const createWorkerCoordinator = (options) => {
         return options.store.registerPending({ ...owner, agent, sessionId: pane !== undefined && liveAgent(pane) && pane.nativeSession !== null ? pane.nativeSession.sessionId : null });
     };
     const runCleanup = async (owner, deleteWorktree, signal) => {
+        const schedule = deleteWorktree ? ensureCleanupSchedule(options.store, owner, now()) : null;
         const local = options.store.getTask(owner);
         const webRecord = options.webStore?.getTask(owner);
+        if (webRecord != null && webRecord.worktreeId !== owner.worktreeId)
+            throw new Error("The Web cleanup claim has conflicting worktree identities.");
         if (local?.agentControlMode === "web" || webRecord != null) {
             if (!options.webBackend || !options.webStore)
                 throw new Error("The Web execution backend is unavailable.");
@@ -198,16 +240,30 @@ export const createWorkerCoordinator = (options) => {
                     throw new Error("The original native identity has no local Web stop evidence.");
                 if (!options.verifyWebUnregistered)
                     throw new Error("Web launch provenance cannot be verified.");
-                await options.verifyWebUnregistered(owner, signal ?? new AbortController().signal);
+                await options.verifyWebUnregistered(owner, signal ?? new AbortController().signal, CLEANUP_OPERATION_MS);
                 if (deleteWorktree) {
-                    await workerOperation(operation => options.git.removeWorktree(projectFor(owner), owner, operation), signal);
+                    await cleanupOperation(operation => options.git.removeWorktree(projectFor(owner), owner, operation), signal);
                     if (local)
                         options.store.completeTask(owner);
                 }
                 return;
             }
+            if (webRecord.status === "done") {
+                // A resource snapshot can finish after the cleanup job it observed.
+                // Completed stop evidence is reusable, but present Git resources still require attention.
+                if (!webRecord.cleanup?.agentStopped || !webRecord.cleanup.backendResourcesClosed || !webRecord.cleanup.worktreeDeleted)
+                    throw new Error("The completed Web execution has incomplete cleanup evidence.");
+                if (deleteWorktree) {
+                    const retained = await cleanupOperation(operation => options.git.findWorktree(projectFor(owner), owner, operation), signal);
+                    if (retained !== null)
+                        throw new Error("The completed Web execution still has a worktree; its resources were retained.");
+                    if (local !== null && local.status !== "done")
+                        options.store.completeTask(owner);
+                }
+                return;
+            }
             while (true) {
-                const result = await options.webBackend.cleanup(webRecord, deleteWorktree ? "terminal" : "shutdown", now() + 600_000, signal);
+                const result = await options.webBackend.cleanup(webRecord, deleteWorktree ? "terminal" : "shutdown", schedule?.deadline ?? now() + 600_000, signal);
                 if (result.cleanup?.agentStopped && result.cleanup.backendResourcesClosed && (!deleteWorktree || result.cleanup.worktreeDeleted)) {
                     if (deleteWorktree) {
                         options.webStore.completeTask(owner, now());
@@ -219,8 +275,9 @@ export const createWorkerCoordinator = (options) => {
                 await delay(1000, undefined, signal === undefined ? {} : { signal });
             }
         }
+        await cleanupOperation(operation => options.herdr.ensureInstance(operation), signal);
         while (true) {
-            const resources = await workerOperation((operation) => options.herdr.tasks.listResources(operation), signal);
+            const resources = await cleanupOperation((operation) => options.herdr.tasks.listResources(operation), signal);
             const task = recordMissingClaim(owner, actualTaskPane(owner, resources));
             const completed = await cleanup.advance({ owner, project: projectFor(owner), ...(task === null ? {} : { task }) }, resources, deleteWorktree, signal);
             if (completed) {
@@ -235,7 +292,22 @@ export const createWorkerCoordinator = (options) => {
         for (const [identityKey, owner] of ending) {
             if (jobs.has(identityKey))
                 continue;
-            startJob(owner, async (signal) => { await runCleanup(owner, true, signal); ending.delete(identityKey); }, true);
+            if (now() < (cleanupRetries.get(identityKey)?.nextAttemptAt ?? 0))
+                continue;
+            // Creation and ownership validation run inside the isolated job's error boundary.
+            let nextAttemptAt = 0;
+            try {
+                nextAttemptAt = options.store.getCleanupSchedule(owner)?.nextAttemptAt ?? 0;
+            }
+            catch { /* The job records the storage failure without stopping other claims. */ }
+            if (now() < nextAttemptAt)
+                continue;
+            startJob(owner, async (signal) => {
+                await runCleanup(owner, true, signal);
+                options.store.deleteCleanupSchedule(owner);
+                ending.delete(identityKey);
+                cleanupRetries.delete(identityKey);
+            }, true);
         }
     };
     const syncWorkspaceName = async (task, owner, title, signal) => {
@@ -271,37 +343,63 @@ export const createWorkerCoordinator = (options) => {
         if (!sameClaim(task, owner) || readWorkerClaim(remoteTask.properties).workerId !== options.workerId || readWorkerClaim(remoteTask.properties).agent !== task.agent || readWorkerClaim(remoteTask.properties).agentControlMode !== (task.agentControlMode ?? "herdr"))
             throw new Error("The local execution does not match the observed original worker claim.");
         if (task.agentControlMode === "web") {
-            if (!execute)
-                return;
             if (!options.webStore || !options.webBackend)
                 throw new Error("The Web execution backend is unavailable.");
             const configuration = configurationFor(task);
-            requireImplementedWorkerBackend(configuration);
-            await options.verifyWebServer?.(configuration.agent, signal);
-            if (options.webStore.getTask(task) === null) {
-                if (!options.verifyWebUnregistered)
-                    throw new Error("Web launch provenance cannot be verified.");
-                await options.verifyWebUnregistered(task, signal);
-                if (task.sessionId !== null || readWorkerClaim(remoteTask.properties).nativeSessionId !== null)
-                    throw new Error("The original native identity has no local Web execution evidence.");
-            }
-            let record = options.webStore.register({ ...owner, project: projectFor(task), serverOrigin: options.serverOrigin, workerId: options.workerId, userId: readWorkerClaim(remoteTask.properties).userId, configuration }, now());
-            if (record.cleanup !== null)
-                record = options.webStore.clearShutdown(record, now());
-            const result = await options.webBackend.advance(record, signal);
-            const nativeSessionId = result.resources.nativeSession?.sessionId ?? task.sessionId;
-            if (nativeSessionId !== null)
-                options.store.setSessionId(task, nativeSessionId);
-            const phase = "executing";
             const worker = readWorkerClaim(remoteTask.properties);
-            if (worker.phase !== phase || worker.nativeSessionId !== nativeSessionId || worker.herdr !== null) {
-                try {
-                    await options.remote.update(remoteTask, { phase, nativeSessionId, herdr: null }, signal);
+            let record = options.webStore.getTask(task);
+            if (record !== null && (record.worktreeId !== owner.worktreeId || record.workerId !== options.workerId || record.userId !== worker.userId || record.serverOrigin !== options.serverOrigin))
+                throw new Error("The durable Web execution does not match the observed original worker claim.");
+            let observed = false;
+            try {
+                if (!execute) {
+                    if (record !== null && record.recovery.pauseReason === null)
+                        record = options.webStore.pauseRecovery(record, "The Web execution failure limit was reached; explicit retry is required.", now());
                 }
-                catch (error) {
-                    if (isWorkerCancellation(error, signal))
-                        throw error;
-                    options.emit({ kind: "summary-error", task, message: "The Web execution summary was not confirmed; the next round will read it again.", error });
+                else if (record === null || record.recovery.pauseReason === null || record.recovery.retryAuthorized) {
+                    requireImplementedWorkerBackend(configuration);
+                    await options.verifyWebServer?.(configuration.agent, signal);
+                    if (record === null) {
+                        if (!options.verifyWebUnregistered)
+                            throw new Error("Web launch provenance cannot be verified.");
+                        await options.verifyWebUnregistered(task, signal);
+                        if (task.sessionId !== null || worker.nativeSessionId !== null)
+                            throw new Error("The original native identity has no local Web execution evidence.");
+                    }
+                    record = options.webStore.register({ ...owner, project: projectFor(task), serverOrigin: options.serverOrigin, workerId: options.workerId, userId: worker.userId, configuration }, now());
+                    if (record.cleanup !== null)
+                        record = options.webStore.clearShutdown(record, now());
+                    record = await options.webBackend.advance(record, signal);
+                    observed = true;
+                }
+            }
+            catch (error) {
+                if (isWorkerCancellation(error, signal))
+                    throw error;
+                record = options.webStore.getTask(task);
+                if (record !== null && record.cleanup === null && (record.recovery.retryAuthorized || (failures.get(key(task)) ?? 0) + 1 > configuration.recoveryLimit)) {
+                    record = options.webStore.pauseRecovery(record, error instanceof Error ? error.message : String(error), now());
+                }
+                throw error;
+            }
+            finally {
+                // Summary transport is independent of native recovery and must remain live while paused.
+                record = options.webStore.getTask(task);
+                if (!signal.aborted && (observed || !execute || record?.recovery.pauseReason != null)) {
+                    const nativeSessionId = record?.resources.nativeSession?.sessionId ?? task.sessionId ?? worker.nativeSessionId;
+                    if (nativeSessionId !== null)
+                        options.store.setSessionId(task, nativeSessionId);
+                    const phase = !execute || record?.recovery.pauseReason != null || record?.recovery.retryAuthorized
+                        || record?.webExecution?.activity === "initializing" ? "waiting" : "executing";
+                    if (worker.phase !== phase || worker.nativeSessionId !== nativeSessionId || worker.herdr !== null) {
+                        try {
+                            await options.remote.update(remoteTask, { phase, nativeSessionId, herdr: null }, signal);
+                        }
+                        catch (error) {
+                            if (!isWorkerCancellation(error, signal))
+                                options.emit({ kind: "summary-error", task, message: "The Web execution summary was not confirmed; the next round will read it again.", error });
+                        }
+                    }
                 }
             }
             return;
@@ -365,11 +463,11 @@ export const createWorkerCoordinator = (options) => {
             const owner = resources.owners.get(key(local));
             if (owner !== undefined)
                 ending.set(key(local), owner);
-            else if (!jobs.has(key(local))) {
+            else if (!jobs.has(key(local)) && !ending.has(key(local))) {
                 if (local.agentControlMode === "web") {
                     if (!options.verifyWebUnregistered || local.sessionId !== null)
                         throw new Error("The ended Web claim has no original resource evidence; its ownership record is retained.");
-                    await options.verifyWebUnregistered(local, normalSignal);
+                    await options.verifyWebUnregistered(local, normalSignal, CLEANUP_OPERATION_MS);
                 }
                 options.store.completeTask(local);
             }
@@ -399,6 +497,14 @@ export const createWorkerCoordinator = (options) => {
             if (!started)
                 break;
         }
+        const blocked = capacity(resources) >= options.maxConcurrent
+            ? [...ending.keys()].filter(identity => options.store.getCleanupSchedule(ending.get(identity))?.lastError != null).sort() : [];
+        const warning = blocked.join(",");
+        if (warning !== capacityWarning) {
+            capacityWarning = warning;
+            if (warning !== "")
+                options.emit({ kind: "cleanup-capacity-blocked", message: `Worker capacity is full; failed cleanup retains ${blocked.length} slot(s). ${blocked.map(identity => { const state = options.store.getCleanupSchedule(ending.get(identity)); return `${identity}: ${state.stage}, next retry ${new Date(state.nextAttemptAt).toISOString()}, ${state.lastError}`; }).join("; ")}. Correct the reported resource or archive failure; retries are automatic. No slot is released until cleanup is confirmed.` });
+        }
         if (capacity(resources) >= options.maxConcurrent || options.repositories.length === 0)
             return;
         const candidates = new Map();
@@ -471,21 +577,36 @@ export const createWorkerCoordinator = (options) => {
             const { task, remoteTask } = await readSource(identity);
             if (task.status !== "active")
                 throw new Error("Only an active original claim may be retried; pending work is started by worker run.");
+            const webRecord = task.agentControlMode === "web" ? options.webStore?.getTask(task) : null;
+            const worker = readWorkerClaim(remoteTask.properties);
+            if (task.agentControlMode === "web" && !webRecord)
+                throw new Error("Retry requires the original Web execution evidence; it cannot start a replacement assignment.");
+            if (webRecord && (webRecord.worktreeId !== worker.worktreeId || webRecord.workerId !== worker.workerId || webRecord.userId !== worker.userId || webRecord.serverOrigin !== options.serverOrigin))
+                throw new Error("The original Web execution ownership no longer matches the active claim.");
             failures.delete(key(identity));
-            if (task.agentControlMode === "web" && options.webStore?.getTask(task))
-                options.webStore.authorizeRecoveryRetry(task, now());
             await ensureHerdr(normalSignal);
-            await serialize(identity, () => ensure(task, remoteTask, normalSignal));
+            await serialize(identity, async () => {
+                if (task.agentControlMode === "web" && options.webStore) {
+                    let record = options.webStore.getTask(task);
+                    if (record !== null) {
+                        if (record.cleanup !== null)
+                            record = options.webStore.clearShutdown(record, now());
+                        options.webStore.authorizeRecoveryRetry(record, now());
+                    }
+                }
+                await ensure(task, remoteTask, normalSignal);
+            });
         },
         cleanupSource: async (identity, settings = {}) => {
             const { owner } = await readSource(identity);
             await ensureHerdr(normalSignal);
-            await serialize(identity, () => runCleanup(owner, settings.deleteWorktree === true, normalSignal));
+            await serialize(identity, async () => { await runCleanup(owner, settings.deleteWorktree === true, normalSignal); if (settings.deleteWorktree === true)
+                options.store.deleteCleanupSchedule(owner); });
             return owner;
         },
         completeSource: async (owner) => {
             await ensureHerdr(normalSignal);
-            await serialize(owner, () => runCleanup(owner, true, normalSignal));
+            await serialize(owner, async () => { await runCleanup(owner, true, normalSignal); options.store.deleteCleanupSchedule(owner); });
         },
         shutdown: async () => {
             cancel();
@@ -496,7 +617,7 @@ export const createWorkerCoordinator = (options) => {
                 if (task.agentControlMode === "web" && options.webStore?.getTask(task) == null) {
                     if (task.status === "pending" && task.sessionId === null && options.verifyWebUnregistered) {
                         try {
-                            await options.verifyWebUnregistered(task, new AbortController().signal);
+                            await options.verifyWebUnregistered(task, new AbortController().signal, CLEANUP_OPERATION_MS);
                             continue;
                         }
                         catch { /* Missing launch evidence must not be interpreted as a confirmed native stop. */ }
@@ -514,7 +635,11 @@ export const createWorkerCoordinator = (options) => {
                     owners.set(key(context), context);
             }
             try {
-                const workspaces = await listHerdr();
+                let workspaces = [];
+                if (needsHerdr() || hasHerdrCleanup()) {
+                    await cleanupOperation(operation => options.herdr.ensureInstance(operation));
+                    workspaces = await cleanupOperation(operation => options.herdr.tasks.listResources(operation));
+                }
                 for (const workspace of workspaces) {
                     if (options.store.getProject(workspace.owner) === null) {
                         complete = false;
@@ -528,23 +653,21 @@ export const createWorkerCoordinator = (options) => {
                 complete = false;
                 options.emit({ kind: "shutdown-error", message: "Could not inspect every owned Herdr workspace during shutdown.", error });
             }
-            await Promise.all(options.store.listProjects().map(async (project) => {
-                try {
-                    const resources = await snapshot(undefined, project);
-                    for (const [identity, owner] of resources.owners)
-                        if (resources.proven.has(identity))
-                            owners.set(identity, owner);
-                }
-                catch (error) {
-                    complete = false;
-                    options.emit({ kind: "shutdown-error", message: `Could not observe owned resources in project ${project.projectId} during shutdown.`, error });
-                }
-            }));
+            // Git inventory cannot prove whether an agent stopped. Terminal deletion stays pending.
+            for (const record of options.webStore?.listUnfinishedTasks() ?? [])
+                owners.set(key(record), record);
             await Promise.all([...owners.values()].map(async (owner) => {
                 try {
                     await serialize(owner, async () => {
                         const webRecord = options.webStore?.getTask(owner);
                         if (webRecord != null) {
+                            if (webRecord.worktreeId !== owner.worktreeId)
+                                throw new Error("The Web cleanup claim has conflicting worktree identities.");
+                            if (webRecord.status === "done") {
+                                if (!webRecord.cleanup?.agentStopped || !webRecord.cleanup.backendResourcesClosed || !webRecord.cleanup.worktreeDeleted)
+                                    throw new Error("The completed Web execution has incomplete cleanup evidence.");
+                                return;
+                            }
                             if (!options.webBackend)
                                 throw new Error("The Web execution backend is unavailable during shutdown.");
                             await options.webBackend.cleanup(webRecord, "shutdown", now(), undefined, true);
@@ -553,21 +676,21 @@ export const createWorkerCoordinator = (options) => {
                         if (options.store.getTask(owner)?.agentControlMode === "web")
                             return;
                         if (options.store.getTask(owner) === null) {
-                            const observed = await workerOperation((operation) => options.herdr.tasks.listResources(operation));
+                            const observed = await cleanupOperation((operation) => options.herdr.tasks.listResources(operation));
                             recordMissingClaim(owner, actualTaskPane(owner, observed));
                         }
-                        await settleWorkerStops(deps, owner);
-                        const resources = await workerOperation((operation) => options.herdr.tasks.listResources(operation));
+                        await settleWorkerStops(deps, owner, undefined, cleanupOperation);
+                        const resources = await cleanupOperation((operation) => options.herdr.tasks.listResources(operation));
                         const pane = actualTaskPane(owner, resources);
                         if (pane === undefined)
                             return;
                         const context = { ...owner, cwd: path.join(projectFor(owner).localPath, ".worktrees", "daynight", owner.taskId) };
-                        await workerOperation((operation) => options.herdr.tasks.stopAndClose(context, pane.binding, operation));
+                        await cleanupOperation((operation) => options.herdr.tasks.stopAndClose(context, pane.binding, operation));
                     });
                 }
                 catch (error) {
                     complete = false;
-                    options.emit({ kind: "shutdown-error", ...(options.store.getTask(owner) === null ? {} : { task: options.store.getTask(owner) }), message: "Could not stop the original task resources.", error });
+                    options.emit({ kind: "shutdown-error", ...(options.store.getTask(owner) === null ? {} : { task: options.store.getTask(owner) }), message: error instanceof WorkerCleanupError && error.stage === "backend-close" ? "The original Agent is stopped, but backend resource closure is unconfirmed." : "Could not confirm the original task resources stopped and closed.", error });
                 }
             }));
             return complete;