@pentoshi/clai 3.9.1 → 3.9.3

package/dist/tools/jobs.js

@@ -1,13 +1,14 @@
 import { spawn, execFileSync } from "node:child_process";
-import { createWriteStream, existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "node:fs";
+import { createWriteStream, existsSync, mkdirSync, readdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { mkdir, open, stat } from "node:fs/promises";
-import { basename, join } from "node:path";
+import { basename, join, sep } from "node:path";
 import { createHash, randomUUID } from "node:crypto";
 import { StringDecoder } from "node:string_decoder";
 import { redactSecrets } from "../llm/provider.js";
 import { safeCwd } from "../os/cwd.js";
 import { getJobsDir } from "../store/paths.js";
 import { resolveShell } from "./shell.js";
+import { terminateProcessTree } from "../os/process-tree.js";
 export function formatJobElapsed(job, now = Date.now()) {
     const startedAt = Date.parse(job.startedAt);
     const endedAt = job.endedAt ? Date.parse(job.endedAt) : now;
@@ -31,6 +32,15 @@ function commandDisplay(command) {
         return command;
     return command.display ?? [command.command, ...command.argv].map(displayArg).join(" ");
 }
+function launchFollowUp(id, responder) {
+    return responder
+        ? "Responder owns completion tracking for this self-completing job. " +
+            "Do not poll it with shell.tail/shell.jobs, sleep, or read its artifact. " +
+            "Continue other work; if not already delivered, its terminal result will be delivered automatically."
+        : "OS launch does not prove application readiness or continued liveness. " +
+            `This is a normal background job: use shell.tail {"id":"${id}"} with nextOffset and shell.jobs for status; ` +
+            "for a server or watcher, also run an application readiness probe. Do not launch a duplicate.";
+}
 const PER_FILE_BYTES = 1024 * 1024;
 const MAX_STREAM_BYTES = 16 * 1024 * 1024;
 const DEFAULT_TAIL_BYTES = 8_000;
@@ -42,10 +52,16 @@ const MAX_DURABLE_TERMINAL_JOBS = 80;
 const TERMINAL_JOB_MAX_AGE_MS = 48 * 60 * 60 * 1000;
 const ARCHIVED_UNSETTLED_MAX_AGE_MS = 24 * 60 * 60 * 1000;
 const MAX_ARCHIVED_UNSETTLED_NOTIFICATIONS = 40;
+const MAX_SUPERSEDED_REVISIONS = 5;
+const SETTLEMENT_MAX_BACKOFF_MS = 30_000;
+const LIVENESS_WATCH_INTERVAL_MS = 2_000;
+const SETTLEMENT_DEAD_LETTER_MS = 10 * 60_000;
 /** Max lines shell.jobs returns to the model (running first, then recent). */
 const LIST_JOBS_MAX_LINES = 40;
 /** Coalesce window for chatty stdout/stderr progress persistence + UI events. */
 const PROGRESS_FLUSH_MS = 250;
+/** Durable registry cadence for running-job progress (UI updates stay at 250ms). */
+const DURABLE_PROGRESS_FLUSH_MS = 5_000;
 /**
  * A live job with no in-process ChildProcess handle (resumed session, or a
  * handle that was released) is only declared "lost" after failing the liveness
@@ -55,6 +71,8 @@ const PROGRESS_FLUSH_MS = 250;
  * elapsed, and a ✗ status=lost that later flipped back to exit=0).
  */
 const LIVENESS_LOST_GRACE_MS = 8_000;
+/** Minimum spacing between liveness probes for the same job. */
+const LIVENESS_PROBE_INTERVAL_MS = 1_000;
 /**
  * Number of trailing bytes in `buf` that form an incomplete multi-byte UTF-8
  * sequence (a lead byte whose continuation bytes were cut off by the read
@@ -91,27 +109,75 @@ function processAlive(pid) {
         return error.code === "EPERM";
     }
 }
-function processIdentity(pid) {
-    if (!pid || process.platform === "win32")
+/** Identity is invariant for a pid's lifetime, so it caches safely. */
+const PROCESS_IDENTITY_TTL_MS = 15_000;
+const PROCESS_IDENTITY_CACHE_MAX = 512;
+const processIdentityCache = new Map();
+function readLinuxProcessStart(pid) {
+    try {
+        const raw = readFileSync(`/proc/${pid}/stat`, "utf8");
+        const commEnd = raw.lastIndexOf(")");
+        if (commEnd < 0)
+            return undefined;
+        const fields = raw.slice(commEnd + 2).trim().split(/\s+/);
+        const startTime = fields[19];
+        return startTime && startTime.length > 0 ? startTime : undefined;
+    }
+    catch {
         return undefined;
+    }
+}
+function readPsProcessStart(pid) {
     try {
-        // Start time only: it is invariant for a pid's lifetime and changes on
-        // reuse, so it is a sufficient pid-reuse guard. The command line is
-        // deliberately excluded — `sh -c "cmd"` execs into `cmd`, mutating
-        // `ps command=` mid-run, which made a live job fail its OWN identity check
-        // (stop refused with "process identity no longer matches", liveness falsely
-        // marked lost, and double-Esc cancel unable to kill the process).
-        const value = execFileSync("ps", ["-p", String(pid), "-o", "lstart="], {
+        // Start time only: the command line is deliberately excluded — `sh -c "cmd"`
+        // execs into `cmd`, mutating `ps command=` mid-run, which made a live job
+        // fail its OWN identity check.
+        return execFileSync("ps", ["-p", String(pid), "-o", "lstart="], {
             encoding: "utf8",
             timeout: 2_000,
             stdio: ["ignore", "pipe", "ignore"],
         }).trim();
-        return value ? createHash("sha256").update(value).digest("hex") : undefined;
     }
     catch {
         return undefined;
     }
 }
+function computeProcessIdentity(pid) {
+    const raw = process.platform === "linux"
+        ? readLinuxProcessStart(pid)
+        : readPsProcessStart(pid);
+    return raw ? createHash("sha256").update(raw).digest("hex") : undefined;
+}
+function pruneProcessIdentityCache(now) {
+    if (processIdentityCache.size <= PROCESS_IDENTITY_CACHE_MAX)
+        return;
+    for (const [pid, entry] of processIdentityCache) {
+        if (now - entry.at >= PROCESS_IDENTITY_TTL_MS)
+            processIdentityCache.delete(pid);
+    }
+    if (processIdentityCache.size > PROCESS_IDENTITY_CACHE_MAX) {
+        processIdentityCache.clear();
+    }
+}
+function forgetProcessIdentity(pid) {
+    if (pid)
+        processIdentityCache.delete(pid);
+}
+function processIdentity(pid, options = {}) {
+    if (!pid || process.platform === "win32")
+        return undefined;
+    const now = Date.now();
+    const cached = processIdentityCache.get(pid);
+    if (cached &&
+        !options.refresh &&
+        now - cached.at < PROCESS_IDENTITY_TTL_MS) {
+        return cached.value;
+    }
+    const value = computeProcessIdentity(pid);
+    processIdentityCache.set(pid, { value, at: now });
+    pruneProcessIdentityCache(now);
+    return value;
+}
 /**
  * Heuristic for legacy registry rows that were actually agent tool-stall
  * trackers (commandDisplay = "fs.list /path", "shell.jobs {}", …), not real
@@ -268,7 +334,9 @@ export class JobManager {
     authorizationTimers = new Map();
     responderLeases = new Map();
     settlementTimers = new Map();
-    settlementAttempts = new Map();
+    pendingSettlements = new Map();
+    consumedResponderResults = new Map();
+    livenessWatchTimer;
     finalizations = new Map();
     listeners = new Set();
     /**
@@ -278,6 +346,7 @@ export class JobManager {
      * observed alive again or finalized.
      */
     livenessMisses = new Map();
+    livenessCheckedAt = new Map();
     registryRetryTimer;
     registryPath;
     transientV2RegistryPath;
@@ -286,10 +355,84 @@ export class JobManager {
         this.registryPath = join(this.jobsDir, REGISTRY_FILE);
         this.transientV2RegistryPath = join(this.jobsDir, TRANSIENT_V2_REGISTRY_FILE);
         this.loadAndReconcile();
+        this.sweepOrphanArtifacts();
     }
     isDurable(job) {
         return job.kind !== "ephemeral";
     }
+    artifactPathsOf(job) {
+        return [
+            job.artifactPath,
+            job.stdoutArtifact,
+            job.stderrArtifact,
+            ...(job.artifacts?.stdout.chunks ?? []),
+            ...(job.artifacts?.stderr.chunks ?? []),
+        ].filter((path) => Boolean(path));
+    }
+    /** Delete a dropped job's artifact chunks, never one still referenced. */
+    removeJobArtifacts(job) {
+        const paths = this.artifactPathsOf(job).filter((path) => path.startsWith(`${this.jobsDir}${sep}`));
+        if (paths.length === 0)
+            return;
+        const referenced = new Set();
+        for (const other of this.jobs.values()) {
+            for (const path of this.artifactPathsOf(other))
+                referenced.add(path);
+        }
+        for (const path of new Set(paths)) {
+            if (referenced.has(path))
+                continue;
+            try {
+                rmSync(path, { force: true });
+            }
+            catch {
+                // Artifact cleanup is best-effort; a locked file is retried on boot.
+            }
+        }
+    }
+    /** Drop a job row plus its per-job caches and artifacts. */
+    forgetJob(id) {
+        const job = this.jobs.get(id);
+        this.jobs.delete(id);
+        for (const [notificationId, notification] of this.notifications) {
+            if (notification.jobId === id)
+                this.notifications.delete(notificationId);
+        }
+        this.consumedResponderResults.delete(id);
+        this.livenessMisses.delete(id);
+        this.livenessCheckedAt.delete(id);
+        if (!job)
+            return;
+        forgetProcessIdentity(job.pid);
+        this.removeJobArtifacts(job);
+    }
+    /** Remove artifact files whose job row no longer exists. */
+    sweepOrphanArtifacts() {
+        let names;
+        try {
+            names = readdirSync(this.jobsDir);
+        }
+        catch {
+            return;
+        }
+        const known = new Set(this.jobs.keys());
+        const now = Date.now();
+        for (const name of names) {
+            if (!/\.(stdout|stderr)\.log(\.\d+)?$/.test(name))
+                continue;
+            if ([...known].some((id) => name.includes(`-${id}.`)))
+                continue;
+            const path = join(this.jobsDir, name);
+            try {
+                if (now - statSync(path).mtimeMs <= TERMINAL_JOB_MAX_AGE_MS)
+                    continue;
+                rmSync(path, { force: true });
+            }
+            catch {
+                // Ignore unreadable/locked leftovers.
+            }
+        }
+    }
     isLive(job) {
         return (job.status === "running" ||
             job.status === "starting" ||
@@ -355,18 +498,17 @@ export class JobManager {
         if (!current || (leaseId && current !== leaseId))
             return;
         this.responderLeases.delete(sessionId);
-        const archivedAt = new Date().toISOString();
         let changed = false;
         for (const notification of this.notifications.values()) {
-            if (notification.ownerSessionId === sessionId &&
-                notification.responderLeaseId === current) {
-                this.claimedNotifications.delete(notification.id);
+            if (notification.ownerSessionId !== sessionId ||
+                notification.responderLeaseId !== current) {
+                continue;
             }
-            if (notification.ownerSessionId === sessionId &&
-                notification.responderLeaseId === current &&
-                !notification.acknowledgedAt &&
-                !notification.archivedAt) {
-                notification.archivedAt = archivedAt;
+            this.claimedNotifications.delete(notification.id);
+            // Releasing a runtime lease must not discard an unread durable result;
+            // detach it so the next lease for this session adopts it.
+            if (!notification.acknowledgedAt && !notification.archivedAt) {
+                delete notification.responderLeaseId;
                 changed = true;
             }
         }
@@ -399,10 +541,60 @@ export class JobManager {
                     JSON.stringify(stdoutArtifact) ||
                 JSON.stringify(existing.stderrArtifact) !==
                     JSON.stringify(stderrArtifact)));
-        const reopenSettlement = Boolean(existing?.acknowledgedAt && completionChanged);
-        const responderLeaseId = job.responderLeaseId ?? existing?.responderLeaseId;
-        const leaseIsActive = Boolean(responderLeaseId &&
-            this.responderLeases.get(job.ownerSessionId) === responderLeaseId);
+        const resultHash = createHash("sha256")
+            .update([
+            job.status,
+            endedAt,
+            String(job.exitCode ?? ""),
+            String(job.signal ?? ""),
+            JSON.stringify(stdoutArtifact),
+            JSON.stringify(stderrArtifact),
+        ].join("\0"))
+            .digest("hex")
+            .slice(0, 24);
+        const consumed = this.consumedResponderResults.get(job.id);
+        if (!existing && consumed?.resultHash === resultHash) {
+            return { created: false, updated: false };
+        }
+        const correctsConsumed = Boolean(!existing && consumed && consumed.resultHash !== resultHash);
+        if (correctsConsumed)
+            this.consumedResponderResults.delete(job.id);
+        // A materially different authoritative result is a new revision: every
+        // delivery/read/analysis marker belonged to the superseded result and must
+        // not suppress review of the corrected one.
+        const supersedes = Boolean(existing &&
+            completionChanged &&
+            (existing.resultHash === undefined || existing.resultHash !== resultHash));
+        const resultRevision = supersedes
+            ? (existing?.resultRevision ?? 1) + 1
+            : correctsConsumed
+                ? (consumed?.resultRevision ?? 1) + 1
+                : (existing?.resultRevision ?? 1);
+        const supersededRevisions = supersedes && existing
+            ? [
+                ...(existing.supersededRevisions ?? []),
+                {
+                    resultRevision: existing.resultRevision ?? 1,
+                    resultHash: existing.resultHash ?? "",
+                    status: existing.status,
+                    endedAt: existing.endedAt,
+                    ...(existing.exitCode !== undefined
+                        ? { exitCode: existing.exitCode }
+                        : {}),
+                    ...(existing.signal !== undefined ? { signal: existing.signal } : {}),
+                    ...(existing.deliveredAt ? { deliveredAt: existing.deliveredAt } : {}),
+                    ...(existing.readAt ? { readAt: existing.readAt } : {}),
+                    ...(existing.analyzedAt ? { analyzedAt: existing.analyzedAt } : {}),
+                    ...(existing.acknowledgedAt
+                        ? { acknowledgedAt: existing.acknowledgedAt }
+                        : {}),
+                    ...(existing.settledAt ? { settledAt: existing.settledAt } : {}),
+                },
+            ].slice(-MAX_SUPERSEDED_REVISIONS)
+            : existing?.supersededRevisions;
+        const carried = supersedes ? undefined : existing;
+        const activeLeaseId = this.responderLeases.get(job.ownerSessionId);
+        const responderLeaseId = activeLeaseId ?? job.responderLeaseId ?? existing?.responderLeaseId;
         const notification = {
             id: existing?.id ??
                 `completion:${createHash("sha256").update(`${job.ownerSessionId}\0${job.id}\0${job.startedAt}`).digest("hex").slice(0, 24)}`,
@@ -423,21 +615,34 @@ export class JobManager {
             ...(job.signal !== undefined ? { signal: job.signal } : {}),
             ...(job.monitor !== undefined ? { monitor: job.monitor } : {}),
             ...(responderLeaseId ? { responderLeaseId } : {}),
-            ...(existing?.deliveredAt ? { deliveredAt: existing.deliveredAt } : {}),
-            ...(existing?.readAt ? { readAt: existing.readAt } : {}),
-            ...((existing?.analyzedAt ?? existing?.acknowledgedAt)
-                ? { analyzedAt: existing?.analyzedAt ?? existing?.acknowledgedAt }
+            resultRevision,
+            resultHash,
+            ...(supersededRevisions?.length ? { supersededRevisions } : {}),
+            ...(carried?.deliveryStartedAt
+                ? { deliveryStartedAt: carried.deliveryStartedAt }
                 : {}),
-            ...(!reopenSettlement && existing?.acknowledgedAt
-                ? { acknowledgedAt: existing.acknowledgedAt }
+            ...(carried?.deliveredAt ? { deliveredAt: carried.deliveredAt } : {}),
+            ...(carried?.readAt ? { readAt: carried.readAt } : {}),
+            ...((carried?.analyzedAt ?? carried?.acknowledgedAt)
+                ? { analyzedAt: carried?.analyzedAt ?? carried?.acknowledgedAt }
                 : {}),
-            ...(existing?.settledAt ? { settledAt: existing.settledAt } : {}),
-            ...(!leaseIsActive
-                ? { archivedAt: existing?.archivedAt ?? endedAt }
-                : existing?.archivedAt
-                    ? { archivedAt: existing.archivedAt }
-                    : {}),
+            ...(carried?.acknowledgedAt
+                ? { acknowledgedAt: carried.acknowledgedAt }
+                : {}),
+            ...(carried?.settledAt ? { settledAt: carried.settledAt } : {}),
+            ...(carried?.discardedAt
+                ? {
+                    discardedAt: carried.discardedAt,
+                    ...(carried.discardReason
+                        ? { discardReason: carried.discardReason }
+                        : {}),
+                    wakeOnCompletion: false,
+                }
+                : {}),
+            ...(carried?.archivedAt ? { archivedAt: carried.archivedAt } : {}),
         };
+        if (supersedes)
+            this.claimedNotifications.delete(notification.id);
         const updated = Boolean(existing && JSON.stringify(existing) !== JSON.stringify(notification));
         this.notifications.set(notification.id, notification);
         return { notification, created: !existing, updated };
@@ -514,11 +719,22 @@ export class JobManager {
         }
     }
     /** Reconcile a restored process that no longer has a ChildProcess close event. */
-    refreshJobLiveness(job) {
+    refreshJobLiveness(job, options = {}) {
         if (!this.isLive(job) || this.processes.has(job.id)) {
             this.livenessMisses.delete(job.id);
+            this.livenessCheckedAt.delete(job.id);
+            return;
+        }
+        // Subscribers can ask for state many times per frame; one probe per job per
+        // interval is enough because the lost grace window is far longer.
+        const checkedAt = this.livenessCheckedAt.get(job.id);
+        const probeAt = Date.now();
+        if (!options.force &&
+            checkedAt !== undefined &&
+            probeAt - checkedAt < LIVENESS_PROBE_INTERVAL_MS) {
             return;
         }
+        this.livenessCheckedAt.set(job.id, probeAt);
         // Primary signal is the pid itself. If the process is alive we keep the job
         // running and only ever declare "lost" on a PROVEN pid reuse (both the
         // stored and the current identity are present AND differ). An identity read
@@ -532,6 +748,9 @@ export class JobManager {
                 return;
             }
         }
+        else {
+            forgetProcessIdentity(job.pid);
+        }
         // The process looks gone (or is a proven pid reuse). Require the miss to
         // persist across a grace window before finalizing — a single transient
         // hiccup must not kill a job whose real close event is still in flight.
@@ -570,47 +789,122 @@ export class JobManager {
             clearTimeout(existing);
         const timer = setTimeout(() => {
             this.settlementTimers.delete(job.id);
+            const revision = this.notificationForJob(job.id)?.resultRevision ?? 1;
             void import("../store/responder-settlement.js")
-                .then(({ settleResponderJob }) => settleResponderJob({ ...job }))
+                .then(({ settleResponderJob }) => settleResponderJob({ ...job }, { resultRevision: revision }))
                 .then((result) => {
                 if (result === "applied" || result === "noop") {
-                    this.settlementAttempts.delete(job.id);
-                    const notification = this.notificationForJob(job.id);
-                    if (notification && !notification.settledAt) {
-                        notification.settledAt = new Date().toISOString();
-                        if (!this.persistSync())
-                            this.scheduleRegistryRetry();
-                        this.emit({
-                            type: "notification",
-                            jobId: job.id,
-                            notificationId: notification.id,
-                        });
-                    }
+                    this.completeSettlement(job.id, revision);
                     return;
                 }
-                const attempt = (this.settlementAttempts.get(job.id) ?? 0) + 1;
-                this.settlementAttempts.set(job.id, attempt);
-                if (attempt < 8) {
-                    this.scheduleTaskSettlement(job, Math.min(5_000, 50 * 2 ** attempt));
-                }
-                else {
-                    this.settlementAttempts.delete(job.id);
-                }
+                this.retrySettlement(job, revision, result);
             })
-                .catch(() => {
-                const attempt = (this.settlementAttempts.get(job.id) ?? 0) + 1;
-                this.settlementAttempts.set(job.id, attempt);
-                if (attempt < 8) {
-                    this.scheduleTaskSettlement(job, Math.min(5_000, 50 * 2 ** attempt));
-                }
-                else {
-                    this.settlementAttempts.delete(job.id);
-                }
+                .catch((error) => {
+                this.retrySettlement(job, revision, error instanceof Error ? error.message : String(error));
             });
         }, delayMs);
         timer.unref?.();
         this.settlementTimers.set(job.id, timer);
     }
+    completeSettlement(jobId, resultRevision) {
+        this.pendingSettlements.delete(jobId);
+        const notification = this.notificationForJob(jobId);
+        if (!notification ||
+            (notification.resultRevision ?? 1) !== resultRevision ||
+            notification.settledAt) {
+            if (!this.persistSync())
+                this.scheduleRegistryRetry();
+            return;
+        }
+        notification.settledAt = new Date().toISOString();
+        if (!this.persistSync())
+            this.scheduleRegistryRetry();
+        this.emit({
+            type: "notification",
+            jobId,
+            notificationId: notification.id,
+        });
+    }
+    /**
+     * Retry until the projection lands. The marker is durable, so a crash or a
+     * long-lived plan-write conflict cannot silently strand a green child, and an
+     * unrecoverable case is dead-lettered with a visible reason.
+     */
+    retrySettlement(job, resultRevision, reason) {
+        const now = new Date().toISOString();
+        const previous = this.pendingSettlements.get(job.id);
+        const carryOver = previous && previous.resultRevision === resultRevision ? previous : undefined;
+        const attempts = (carryOver?.attempts ?? 0) + 1;
+        const firstAttemptAt = carryOver?.firstAttemptAt ?? now;
+        const elapsed = Date.now() - Date.parse(firstAttemptAt);
+        const deadLettered = Number.isFinite(elapsed) && elapsed > SETTLEMENT_DEAD_LETTER_MS;
+        this.pendingSettlements.set(job.id, {
+            jobId: job.id,
+            resultRevision,
+            attempts,
+            firstAttemptAt,
+            lastAttemptAt: now,
+            lastReason: reason,
+            ...(deadLettered ? { deadLetteredAt: now } : {}),
+        });
+        if (!this.persistSync())
+            this.scheduleRegistryRetry();
+        this.emit({ type: "job", jobId: job.id });
+        if (deadLettered)
+            return;
+        this.scheduleTaskSettlement(job, Math.min(SETTLEMENT_MAX_BACKOFF_MS, 50 * 2 ** Math.min(attempts, 10)));
+    }
+    /**
+     * One coalesced, unref'd watcher that reconciles restored live jobs without a
+     * UI read. Without it a headless session can leave a finished responder job
+     * "running" forever, because liveness was only refreshed by list/get calls.
+     */
+    scheduleLivenessWatch() {
+        if (this.livenessWatchTimer)
+            return;
+        const restored = [...this.jobs.values()].filter((job) => this.isDurable(job) && this.isLive(job) && !this.processes.has(job.id));
+        if (restored.length === 0)
+            return;
+        const timer = setInterval(() => {
+            const pending = [...this.jobs.values()].filter((job) => this.isDurable(job) && this.isLive(job) && !this.processes.has(job.id));
+            for (const job of pending)
+                this.refreshJobLiveness(job);
+            const remaining = [...this.jobs.values()].some((job) => this.isDurable(job) && this.isLive(job) && !this.processes.has(job.id));
+            if (remaining)
+                return;
+            clearInterval(timer);
+            this.livenessWatchTimer = undefined;
+        }, LIVENESS_WATCH_INTERVAL_MS);
+        timer.unref?.();
+        this.livenessWatchTimer = timer;
+    }
+    /** Ownership guard: a receipt may only be mutated by its owning session. */
+    ownsNotification(notification, sessionId) {
+        if (!notification)
+            return false;
+        return sessionId === undefined || notification.ownerSessionId === sessionId;
+    }
+    /** Receipt for a launch that was deduplicated by its delegation key. */
+    startJobReceipt(job, reason) {
+        return {
+            ok: true,
+            output: `Reusing background job id=${job.id} (${reason}) pid=${job.pid ?? "?"} status=${job.status}\n` +
+                `Command: ${job.commandDisplay}\nArtifact: ${job.stdoutArtifact}\n` +
+                `The duplicate launch was not started. ${launchFollowUp(job.id, job.responder === true)}`,
+            outputPath: job.stdoutArtifact,
+            backgroundJob: {
+                id: job.id,
+                status: job.status,
+                artifactPath: job.stdoutArtifact,
+                ...(job.responder ? { responder: true } : {}),
+                nextOffset: 0,
+            },
+        };
+    }
+    /** Terminal results whose plan child could not be settled yet. */
+    getPendingSettlements() {
+        return [...this.pendingSettlements.values()];
+    }
     /**
      * Register an in-flight tool for stall tracking only.
      * Never appears in shell.jobs and never touches the durable registry.
@@ -644,7 +938,7 @@ export class JobManager {
         if (!this.isLive(job))
             this.clearAuthorizationTimer(id);
         if (!this.isDurable(job) && !this.isLive(job)) {
-            this.jobs.delete(id);
+            this.forgetJob(id);
             this.emit({ type: "job", jobId: id });
             return;
         }
@@ -661,6 +955,14 @@ export class JobManager {
                 return { ok: false, output: `Engagement authorization for ${options.authorization.target} has expired.`, exitCode: 1 };
             }
         }
+        if (options?.delegationId) {
+            // A retried tool call must not spawn a second scanner: the delegation key
+            // identifies the work, so the existing job is returned unchanged.
+            const existing = [...this.jobs.values()].find((candidate) => candidate.delegationId === options.delegationId &&
+                candidate.ownerSessionId === (options.ownerSessionId ?? "unknown"));
+            if (existing)
+                return this.startJobReceipt(existing, "existing delegation");
+        }
         const id = randomUUID().slice(0, 8);
         const cwd = options?.cwd ?? safeCwd();
         try {
@@ -691,7 +993,7 @@ export class JobManager {
                 exitCode: 127,
             };
         }
-        await mkdir(this.jobsDir, { recursive: true });
+        await mkdir(this.jobsDir, { recursive: true, mode: 0o700 });
         const prefix = `${new Date().toISOString().replace(/[:.]/g, "-")}-${id}`;
         const stdoutArtifact = join(this.jobsDir, `${prefix}.stdout.log`);
         const stderrArtifact = join(this.jobsDir, `${prefix}.stderr.log`);
@@ -721,6 +1023,7 @@ export class JobManager {
             ...(options?.name ? { name: options.name } : {}),
             ...(options?.taskId ? { taskId: options.taskId } : {}),
             ...(options?.parentTaskId ? { parentTaskId: options.parentTaskId } : {}),
+            ...(options?.delegationId ? { delegationId: options.delegationId } : {}),
             ...(options?.wakeOnCompletion !== undefined ? { wakeOnCompletion: options.wakeOnCompletion } : {}),
             ...(options?.responder !== undefined ? { responder: options.responder } : {}),
             ...(monitor !== undefined ? { monitor } : {}),
@@ -735,7 +1038,7 @@ export class JobManager {
             await this.persist();
         }
         catch {
-            this.jobs.delete(id);
+            this.forgetJob(id);
             return {
                 ok: false,
                 output: "Background command launch error [PERSIST_FAILED]: could not persist the job registry; the command was not started.",
@@ -801,10 +1104,14 @@ export class JobManager {
                     launchError.syscall ? `syscall=${JSON.stringify(launchError.syscall)}` : undefined,
                     launchError.path ? `path=${JSON.stringify(launchError.path)}` : undefined,
                 ].filter((value) => Boolean(value));
+                const ownership = job.responder
+                    ? "Responder owns this terminal launch failure; do not poll or retry it. Its result will be delivered automatically."
+                    : "Inspect the failed job receipt before deciding whether changed evidence supports a retry.";
                 const detail = `${launchRetried ? "Automatic retry after a transient launch ENOENT also failed.\n" : ""}` +
                     `Background command launch error [${code}]: ${launchError.message}\n` +
                     `${fields.join("; ")}\n` +
-                    "The command did not start. Do not rewrite its syntax to work around this infrastructure error; verify the target and cwd.";
+                    "The command did not start. Do not rewrite its syntax to work around this infrastructure error; verify the target and cwd.\n" +
+                    ownership;
                 stderr.append(`${detail}\n`);
                 await this.finalizeJob(job, "failed", { exitCode: 127 });
                 return {
@@ -817,6 +1124,7 @@ export class JobManager {
                         status: "failed",
                         exitCode: 127,
                         artifactPath: stdoutArtifact,
+                        ...(job.responder ? { responder: true } : {}),
                         nextOffset: 0,
                     },
                 };
@@ -831,17 +1139,23 @@ export class JobManager {
             job.processGroupId = detached ? child.pid : undefined;
             job.status = "running";
             job.heartbeatAt = new Date().toISOString();
-            job.processIdentity = processIdentity(child.pid);
+            job.processIdentity = processIdentity(child.pid, { refresh: true });
             this.processes.set(id, child);
             this.writers.set(id, { stdout, stderr });
             this.scheduleAuthorizationExpiry(job);
             let lastProgressFlush = 0;
+            let lastDurableFlush = Date.now();
             let progressDirty = false;
             let progressTimer;
+            // UI freshness is cheap; rewriting the whole registry is not. Terminal
+            // transitions always persist authoritatively below.
             const flushProgress = () => {
                 progressDirty = false;
                 lastProgressFlush = Date.now();
-                this.persistSync();
+                if (lastProgressFlush - lastDurableFlush >= DURABLE_PROGRESS_FLUSH_MS) {
+                    lastDurableFlush = lastProgressFlush;
+                    this.persistSync();
+                }
                 this.emit({ type: "job", jobId: id });
             };
             // Coalesce high-frequency stdout/stderr chunks: a chatty job (scanner,
@@ -937,10 +1251,13 @@ export class JobManager {
             const earlyStatus = this.getJob(id)?.status;
             if (earlyStatus === "failed") {
                 await this.persist();
+                const failureFollowUp = job.responder
+                    ? "Responder owns this terminal failure; do not poll or retry it. Its result will be delivered automatically."
+                    : `Use shell.tail {"id":"${id}"} for captured stderr; do not retry unchanged.`;
                 return {
                     ok: false,
                     output: `Background job failed immediately: id=${id} exit=${job.exitCode ?? "?"}\n` +
-                        `Command: ${job.commandDisplay}\nUse shell.tail {"id":"${id}"} for captured stderr; do not retry unchanged.`,
+                        `Command: ${job.commandDisplay}\n${failureFollowUp}`,
                     exitCode: job.exitCode ?? 1,
                     outputPath: stdoutArtifact,
                     backgroundJob: {
@@ -948,6 +1265,7 @@ export class JobManager {
                         status: "failed",
                         exitCode: job.exitCode,
                         artifactPath: stdoutArtifact,
+                        ...(job.responder ? { responder: true } : {}),
                         nextOffset: 0,
                     },
                 };
@@ -957,14 +1275,14 @@ export class JobManager {
             return {
                 ok: true,
                 output: `OS process launch confirmed: id=${id} (canonical job ID) pid=${child.pid ?? "?"}\n` +
-                    `This does not prove application readiness or continued liveness; use shell.tail {"id":"${id}"} and a readiness probe.\n` +
-                    `Use shell.jobs for status; do not use the artifact filename as the id.\n` +
-                    `Command: ${job.commandDisplay}\nArtifact: ${stdoutArtifact}`,
+                    `Command: ${job.commandDisplay}\nArtifact: ${stdoutArtifact}\n` +
+                    launchFollowUp(id, job.responder === true),
                 outputPath: stdoutArtifact,
                 backgroundJob: {
                     id,
                     status: job.status,
                     artifactPath: stdoutArtifact,
+                    ...(job.responder ? { responder: true } : {}),
                     ...(options?.profile ? { profile: options.profile } : {}),
                     ...(options?.estimatedSeconds !== undefined ? { estimatedSeconds: options.estimatedSeconds } : {}),
                     nextOffset: 0,
@@ -983,10 +1301,12 @@ export class JobManager {
                     this.persistSync();
                     this.emit({ type: "job", jobId: id });
                 }
+                const followUp = launchFollowUp(id, job.responder === true);
                 return {
                     ok: alive,
                     output: `Background command launched as pid=${job.pid ?? "?"}, but job setup/persistence failed: ${detail}\n` +
-                        `${alive ? `The process is still running as job ${id}; do not launch a duplicate. Use shell.jobs/tail/stop.` : "The process is no longer running; inspect its artifacts before deciding whether to retry."}`,
+                        `${alive ? `The process is still running as job ${id}; do not launch a duplicate. ` : "The process has already stopped. "}` +
+                        followUp,
                     exitCode: alive ? undefined : job.exitCode,
                     outputPath: stdoutArtifact,
                     backgroundJob: {
@@ -994,6 +1314,7 @@ export class JobManager {
                         status: job.status,
                         exitCode: job.exitCode,
                         artifactPath: stdoutArtifact,
+                        ...(job.responder ? { responder: true } : {}),
                         nextOffset: 0,
                     },
                 };
@@ -1002,8 +1323,18 @@ export class JobManager {
             return {
                 ok: false,
                 output: `Background command launch error [UNKNOWN]: ${detail}\n` +
-                    `cwd=${JSON.stringify(cwd)}\nThe command did not start.`,
+                    `cwd=${JSON.stringify(cwd)}\nThe command did not start.\n` +
+                    launchFollowUp(id, job.responder === true),
                 exitCode: 127,
+                outputPath: stdoutArtifact,
+                backgroundJob: {
+                    id,
+                    status: "failed",
+                    exitCode: 127,
+                    artifactPath: stdoutArtifact,
+                    ...(job.responder ? { responder: true } : {}),
+                    nextOffset: 0,
+                },
             };
         }
     }
@@ -1039,7 +1370,27 @@ export class JobManager {
                     ? "alive"
                     : "unresponsive"
                 : "terminal";
-            return `[${job.id}] ${job.status} health=${health} exit=${job.exitCode ?? "?"} ${formatJobElapsed(job)}  ${job.commandDisplay.slice(0, 80)}`;
+            const notification = this.notificationForJob(job.id);
+            const receipt = notification
+                ? notification.acknowledgedAt
+                    ? "consumed"
+                    : notification.analyzedAt
+                        ? "analyzed"
+                        : notification.readAt
+                            ? "read"
+                            : notification.deliveredAt
+                                ? "delivered-unread"
+                                : notification.deliveryStartedAt
+                                    ? "delivering"
+                                    : "ready"
+                : this.consumedResponderResults.has(job.id)
+                    ? "consumed"
+                    : job.responder
+                        ? this.isLive(job)
+                            ? "running"
+                            : "pending"
+                        : undefined;
+            return `[${job.id}] ${job.status} health=${health} exit=${job.exitCode ?? "?"}${receipt ? ` responder=${receipt}` : ""} ${formatJobElapsed(job)}  ${job.commandDisplay.slice(0, 80)}`;
         };
         const header = sessionId
             ? `Session background jobs (${durable.length} total, session ${sessionId}):`
@@ -1082,6 +1433,7 @@ export class JobManager {
     getPendingNotifications(sessionId) {
         return [...this.notifications.values()]
             .filter((notification) => !notification.acknowledgedAt &&
+            !notification.discardedAt &&
             (!sessionId || notification.ownerSessionId === sessionId))
             .sort((a, b) => a.createdAt.localeCompare(b.createdAt));
     }
@@ -1091,22 +1443,24 @@ export class JobManager {
     claimNextResponderNotification(sessionId, leaseId) {
         if (this.responderLeases.get(sessionId) !== leaseId)
             return undefined;
-        const archivedAt = new Date().toISOString();
-        let archived = false;
+        let adopted = false;
         for (const notification of this.notifications.values()) {
+            // Receipts belong to the session, not to a runtime lease: a new lease
+            // adopts everything unread that an earlier lease left behind.
             if (notification.ownerSessionId === sessionId &&
                 !notification.acknowledgedAt &&
-                notification.responderLeaseId !== leaseId &&
-                !notification.archivedAt) {
-                notification.archivedAt = archivedAt;
-                archived = true;
+                !notification.archivedAt &&
+                !notification.discardedAt &&
+                notification.responderLeaseId !== leaseId) {
+                notification.responderLeaseId = leaseId;
+                adopted = true;
             }
         }
         const notification = [...this.notifications.values()]
             .filter((candidate) => candidate.ownerSessionId === sessionId &&
             candidate.responderLeaseId === leaseId &&
             !candidate.archivedAt &&
-            !candidate.deliveredAt &&
+            !candidate.discardedAt &&
             !candidate.readAt &&
             !candidate.analyzedAt &&
             !candidate.acknowledgedAt &&
@@ -1114,26 +1468,53 @@ export class JobManager {
             .sort((left, right) => left.createdAt.localeCompare(right.createdAt))[0];
         if (notification)
             this.claimedNotifications.add(notification.id);
-        if (archived && !this.persistSync())
+        if (adopted && !this.persistSync())
             this.scheduleRegistryRetry();
         return notification;
     }
     releaseResponderNotificationClaim(notificationId) {
         this.claimedNotifications.delete(notificationId);
     }
-    markDelivered(notificationId) {
+    /**
+     * Record that a delivery attempt started. This is deliberately weaker than
+     * `markDelivered`: an aborted or failed analysis turn must remain deliverable.
+     */
+    markDeliveryStarted(notificationId, sessionId) {
         const notification = this.notifications.get(notificationId);
-        if (!notification)
+        if (!this.ownsNotification(notification, sessionId))
+            return false;
+        if (notification.deliveryStartedAt)
+            return true;
+        notification.deliveryStartedAt = new Date().toISOString();
+        if (!this.persistSync()) {
+            notification.deliveryStartedAt = undefined;
+            return false;
+        }
+        this.emit({
+            type: "notification",
+            jobId: notification.jobId,
+            notificationId,
+        });
+        return true;
+    }
+    markDelivered(notificationId, sessionId) {
+        const notification = this.notifications.get(notificationId);
+        if (!this.ownsNotification(notification, sessionId))
             return false;
         if (!notification.deliveredAt) {
             notification.deliveredAt = new Date().toISOString();
+            notification.deliveryStartedAt ??= notification.deliveredAt;
             if (!this.persistSync()) {
                 notification.deliveredAt = undefined;
                 return false;
             }
             this.emit({ type: "notification", jobId: notification.jobId, notificationId });
         }
-        this.claimedNotifications.delete(notificationId);
+        if (notification.readAt ||
+            notification.analyzedAt ||
+            notification.acknowledgedAt) {
+            this.claimedNotifications.delete(notificationId);
+        }
         return true;
     }
     markRead(notificationId, sessionId) {
@@ -1141,7 +1522,8 @@ export class JobManager {
         if (!notification ||
             notification.ownerSessionId !== sessionId ||
             !notification.responder ||
-            notification.archivedAt) {
+            notification.archivedAt ||
+            notification.discardedAt) {
             return false;
         }
         const previousDeliveredAt = notification.deliveredAt;
@@ -1162,10 +1544,14 @@ export class JobManager {
         });
         return true;
     }
-    markAnalyzed(notificationId) {
+    markAnalyzed(notificationId, sessionId) {
         const notification = this.notifications.get(notificationId);
-        if (!notification?.deliveredAt)
+        if (!notification || !this.ownsNotification(notification, sessionId)) {
             return false;
+        }
+        if (!notification.deliveredAt && !notification.deliveryStartedAt) {
+            return false;
+        }
         this.claimedNotifications.delete(notificationId);
         if (!notification.analyzedAt) {
             notification.analyzedAt = new Date().toISOString();
@@ -1179,13 +1565,22 @@ export class JobManager {
         }
         return true;
     }
-    acknowledge(notificationId) {
+    acknowledge(notificationId, sessionId) {
         const notification = this.notifications.get(notificationId);
-        if (!notification?.analyzedAt)
+        if (!notification || !this.ownsNotification(notification, sessionId)) {
+            return false;
+        }
+        if (!notification.analyzedAt)
             return false;
         this.claimedNotifications.delete(notificationId);
         if (!notification.acknowledgedAt) {
             notification.acknowledgedAt = new Date().toISOString();
+            this.consumedResponderResults.set(notification.jobId, {
+                jobId: notification.jobId,
+                resultHash: notification.resultHash ?? "",
+                resultRevision: notification.resultRevision ?? 1,
+                acknowledgedAt: notification.acknowledgedAt,
+            });
             const persisted = this.persistSync();
             if (!persisted)
                 this.scheduleRegistryRetry();
@@ -1377,7 +1772,7 @@ export class JobManager {
         const identityMatches = () => {
             if (process.platform === "win32")
                 return processAlive(pid);
-            const current = processIdentity(pid);
+            const current = processIdentity(pid, { refresh: true });
             return Boolean(current && job.processIdentity && current === job.processIdentity);
         };
         if (targetAlive() && (!processAlive(pid) || !identityMatches())) {
@@ -1415,13 +1810,13 @@ export class JobManager {
             else if (!(processGroupId && processGroupVerified && allowVerifiedGroup)) {
                 return "identity-mismatch";
             }
-            try {
-                processGroupId ? process.kill(-processGroupId, signal) : process.kill(pid, signal);
-                return "sent";
-            }
-            catch {
+            const outcome = terminateProcessTree(pid, {
+                signal,
+                processGroupId,
+            });
+            if (outcome === "failed")
                 return targetAlive() ? "failed" : "gone";
-            }
+            return outcome;
         };
         const restoreRunning = () => {
             job.status = "running";
@@ -1536,15 +1931,16 @@ export class JobManager {
                 acknowledgedAt: notification.acknowledgedAt,
             });
             notification.wakeOnCompletion = false;
-            notification.deliveredAt ??= acknowledgedAt;
-            notification.acknowledgedAt = acknowledgedAt;
+            notification.discardedAt = acknowledgedAt;
+            notification.discardReason = "session-cancelled";
+            this.claimedNotifications.delete(notification.id);
         }
         const acknowledgementsPersisted = this.persistSync();
         if (!acknowledgementsPersisted) {
             for (const snapshot of notificationSnapshots) {
                 snapshot.notification.wakeOnCompletion = snapshot.wakeOnCompletion;
-                snapshot.notification.deliveredAt = snapshot.deliveredAt;
-                snapshot.notification.acknowledgedAt = snapshot.acknowledgedAt;
+                snapshot.notification.discardedAt = undefined;
+                snapshot.notification.discardReason = undefined;
             }
         }
         else {
@@ -1611,21 +2007,26 @@ export class JobManager {
             if (timer)
                 clearTimeout(timer);
             this.settlementTimers.delete(notification.jobId);
-            this.settlementAttempts.delete(notification.jobId);
+            this.pendingSettlements.delete(notification.jobId);
         }
         for (const [id, notification] of this.notifications) {
             const job = this.jobs.get(notification.jobId);
             if (!job ||
+                notification.discardedAt ||
                 (notification.acknowledgedAt && job.status !== "lost")) {
                 this.notifications.delete(id);
             }
         }
+        for (const [jobId] of this.consumedResponderResults) {
+            if (!this.jobs.has(jobId))
+                this.consumedResponderResults.delete(jobId);
+        }
         const durableTerminal = [];
         for (const [id, job] of this.jobs) {
             if (!this.isDurable(job)) {
                 // Safety: never keep ephemeral rows that are not live.
                 if (!this.isLive(job))
-                    this.jobs.delete(id);
+                    this.forgetJob(id);
                 continue;
             }
             if (this.isLive(job))
@@ -1638,7 +2039,7 @@ export class JobManager {
             }
             const ended = job.endedAt ? Date.parse(job.endedAt) : Date.parse(job.startedAt);
             if (Number.isFinite(ended) && now - ended > TERMINAL_JOB_MAX_AGE_MS) {
-                this.jobs.delete(id);
+                this.forgetJob(id);
                 continue;
             }
             durableTerminal.push(job);
@@ -1647,7 +2048,7 @@ export class JobManager {
             return;
         durableTerminal.sort((a, b) => b.startedAt.localeCompare(a.startedAt));
         for (const job of durableTerminal.slice(MAX_DURABLE_TERMINAL_JOBS)) {
-            this.jobs.delete(job.id);
+            this.forgetJob(job.id);
         }
     }
     loadAndReconcile() {
@@ -1667,6 +2068,23 @@ export class JobManager {
             catch { /* a corrupt current registry must not replay stale v1 state */ }
             if (!parsed)
                 return;
+            if (parsed.schemaVersion === 2 && Array.isArray(parsed.settlements)) {
+                for (const settlement of parsed.settlements) {
+                    if (!settlement || typeof settlement.jobId !== "string")
+                        continue;
+                    this.pendingSettlements.set(settlement.jobId, settlement);
+                }
+            }
+            if (parsed.schemaVersion === 2 && Array.isArray(parsed.consumedResults)) {
+                for (const consumed of parsed.consumedResults) {
+                    if (!consumed ||
+                        typeof consumed.jobId !== "string" ||
+                        typeof consumed.resultHash !== "string") {
+                        continue;
+                    }
+                    this.consumedResponderResults.set(consumed.jobId, consumed);
+                }
+            }
             if (parsed.schemaVersion === 2 && Array.isArray(parsed.notifications)) {
                 const notifiedJobs = new Set();
                 for (const notification of parsed.notifications) {
@@ -1676,7 +2094,9 @@ export class JobManager {
                         continue;
                     notifiedJobs.add(notification.jobId);
                     if (!notification.acknowledgedAt && !notification.archivedAt) {
-                        notification.archivedAt = new Date().toISOString();
+                        // A durable receipt must survive the restart it exists for. Drop the
+                        // dead runtime lease so the next activated lease can adopt it.
+                        delete notification.responderLeaseId;
                     }
                     this.notifications.set(notification.id, notification);
                 }
@@ -1713,6 +2133,7 @@ export class JobManager {
                 if (!job?.responder)
                     this.notifications.delete(id);
             }
+            this.scheduleLivenessWatch();
             this.pruneTerminalJobs();
             this.persistSync();
         }
@@ -1724,6 +2145,12 @@ export class JobManager {
             schemaVersion: 2,
             jobs: [...this.jobs.values()].filter((job) => this.isDurable(job)),
             notifications: [...this.notifications.values()],
+            ...(this.pendingSettlements.size
+                ? { settlements: [...this.pendingSettlements.values()] }
+                : {}),
+            ...(this.consumedResponderResults.size
+                ? { consumedResults: [...this.consumedResponderResults.values()] }
+                : {}),
         };
     }
     scheduleRegistryRetry() {
@@ -1739,7 +2166,7 @@ export class JobManager {
     }
     persistSync() {
         try {
-            mkdirSync(this.jobsDir, { recursive: true });
+            mkdirSync(this.jobsDir, { recursive: true, mode: 0o700 });
             const temp = `${this.registryPath}.${process.pid}.${randomUUID()}.tmp`;
             writeFileSync(temp, `${JSON.stringify(this.registry(), null, 2)}\n`, { mode: 0o600 });
             renameSync(temp, this.registryPath);