@pentoshi/clai 3.8.50 → 3.9.0

package/dist/tools/jobs.d.ts

@@ -16,6 +16,8 @@ export interface JobLinkMetadata {
     parentTaskId?: string | undefined;
     wakeOnCompletion?: boolean | undefined;
     monitor?: JobMonitorMetadata | undefined;
+    /** Runtime listening lease that authorized this responder delegation. */
+    responderLeaseId?: string | undefined;
     /**
      * Opt-in delegation to the Responder: fire-and-continue, plan subtask +
      * auto-wake on completion, and inclusion in the Responder inbox/UI. When
@@ -59,7 +61,7 @@ export interface BackgroundJob extends JobLinkMetadata {
         target: string;
         expiresAt?: string | undefined;
     } | undefined;
-    /** Optional execution deadline for finite durable jobs. */
+    /** Accepted when reading legacy registries but never armed or restored. */
     timeoutAt?: string | undefined;
 }
 export declare function formatJobElapsed(job: Pick<BackgroundJob, "startedAt" | "endedAt">, now?: number): string;
@@ -81,9 +83,12 @@ export interface ResponderNotification {
     wakeOnCompletion: boolean;
     responder: boolean;
     monitor?: JobMonitorMetadata | undefined;
+    responderLeaseId?: string | undefined;
     deliveredAt?: string | undefined;
     analyzedAt?: string | undefined;
     acknowledgedAt?: string | undefined;
+    archivedAt?: string | undefined;
+    settledAt?: string | undefined;
 }
 export type JobManagerChange = {
     type: "job";
@@ -113,7 +118,7 @@ export interface StartJobOptions extends JobLinkMetadata {
     ownerSessionId?: string | undefined;
     profile?: string | undefined;
     estimatedSeconds?: number | undefined;
-    /** Stop this finite job after the selected duration. */
+    /** Legacy compatibility input. Durable jobs no longer have generic deadlines. */
     timeoutMs?: number | undefined;
     authorization?: {
         target: string;
@@ -124,10 +129,14 @@ export declare class JobManager {
     private readonly jobsDir;
     private jobs;
     private notifications;
+    private readonly claimedNotifications;
     private processes;
     private writers;
     private abortControllers;
-    private deadlineTimers;
+    private authorizationTimers;
+    private responderLeases;
+    private settlementTimers;
+    private settlementAttempts;
     private finalizations;
     private listeners;
     /**
@@ -147,7 +156,11 @@ export declare class JobManager {
     private emit;
     subscribe(listener: JobManagerListener): () => void;
     private matchesSession;
-    private clearJobDeadline;
+    private clearAuthorizationTimer;
+    private scheduleAuthorizationExpiry;
+    activateResponderLease(sessionId: string): string;
+    getResponderLeaseId(sessionId: string | undefined): string | undefined;
+    releaseResponderLease(sessionId: string, leaseId?: string): void;
     private cloneReceipt;
     private notificationForJob;
     private ensureCompletionNotification;
@@ -155,8 +168,7 @@ export declare class JobManager {
     private finalizeJob;
     /** Reconcile a restored process that no longer has a ChildProcess close event. */
     private refreshJobLiveness;
-    /** Arm or restore a persisted finite-job deadline. */
-    private scheduleJobDeadline;
+    private scheduleTaskSettlement;
     /**
      * Register an in-flight tool for stall tracking only.
      * Never appears in shell.jobs and never touches the durable registry.
@@ -173,6 +185,8 @@ export declare class JobManager {
     getJob(id: string): BackgroundJob | undefined;
     getPendingNotifications(sessionId?: string): ResponderNotification[];
     pendingNotifications(sessionId?: string): ResponderNotification[];
+    claimNextResponderNotification(sessionId: string, leaseId: string): ResponderNotification | undefined;
+    releaseResponderNotificationClaim(notificationId: string): void;
     markDelivered(notificationId: string): boolean;
     markAnalyzed(notificationId: string): boolean;
     acknowledge(notificationId: string): boolean;

package/dist/tools/jobs.js

@@ -40,6 +40,8 @@ const TRANSIENT_V2_REGISTRY_FILE = "registry-v2.json";
 const MAX_DURABLE_TERMINAL_JOBS = 80;
 /** Drop terminal durable jobs older than this on load/list. */
 const TERMINAL_JOB_MAX_AGE_MS = 48 * 60 * 60 * 1000;
+const ARCHIVED_UNSETTLED_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+const MAX_ARCHIVED_UNSETTLED_NOTIFICATIONS = 40;
 /** Max lines shell.jobs returns to the model (running first, then recent). */
 const LIST_JOBS_MAX_LINES = 40;
 /** Coalesce window for chatty stdout/stderr progress persistence + UI events. */
@@ -238,10 +240,14 @@ export class JobManager {
     jobsDir;
     jobs = new Map();
     notifications = new Map();
+    claimedNotifications = new Set();
     processes = new Map();
     writers = new Map();
     abortControllers = new Map();
-    deadlineTimers = new Map();
+    authorizationTimers = new Map();
+    responderLeases = new Map();
+    settlementTimers = new Map();
+    settlementAttempts = new Map();
     finalizations = new Map();
     listeners = new Set();
     /**
@@ -288,11 +294,64 @@ export class JobManager {
             return true;
         return job.ownerSessionId === sessionId;
     }
-    clearJobDeadline(id) {
-        const timer = this.deadlineTimers.get(id);
+    clearAuthorizationTimer(id) {
+        const timer = this.authorizationTimers.get(id);
         if (timer)
             clearTimeout(timer);
-        this.deadlineTimers.delete(id);
+        this.authorizationTimers.delete(id);
+    }
+    scheduleAuthorizationExpiry(job) {
+        this.clearAuthorizationTimer(job.id);
+        const raw = job.authorization?.expiresAt;
+        if (!raw || !this.isLive(job))
+            return;
+        const expiresAt = Date.parse(raw);
+        if (!Number.isFinite(expiresAt))
+            return;
+        const timer = setTimeout(() => {
+            this.authorizationTimers.delete(job.id);
+            this.refreshJobLiveness(job);
+            if (this.isLive(job))
+                void this.stopJob(job.id, { graceMs: 1_000 });
+        }, Math.max(0, expiresAt - Date.now()));
+        timer.unref?.();
+        this.authorizationTimers.set(job.id, timer);
+    }
+    activateResponderLease(sessionId) {
+        const current = this.responderLeases.get(sessionId);
+        if (current)
+            return current;
+        const leaseId = randomUUID();
+        this.responderLeases.set(sessionId, leaseId);
+        this.emit({ type: "job", jobId: `responder:${sessionId}` });
+        return leaseId;
+    }
+    getResponderLeaseId(sessionId) {
+        return sessionId ? this.responderLeases.get(sessionId) : undefined;
+    }
+    releaseResponderLease(sessionId, leaseId) {
+        const current = this.responderLeases.get(sessionId);
+        if (!current || (leaseId && current !== leaseId))
+            return;
+        this.responderLeases.delete(sessionId);
+        const archivedAt = new Date().toISOString();
+        let changed = false;
+        for (const notification of this.notifications.values()) {
+            if (notification.ownerSessionId === sessionId &&
+                notification.responderLeaseId === current) {
+                this.claimedNotifications.delete(notification.id);
+            }
+            if (notification.ownerSessionId === sessionId &&
+                notification.responderLeaseId === current &&
+                !notification.acknowledgedAt &&
+                !notification.archivedAt) {
+                notification.archivedAt = archivedAt;
+                changed = true;
+            }
+        }
+        if (changed && !this.persistSync())
+            this.scheduleRegistryRetry();
+        this.emit({ type: "job", jobId: `responder:${sessionId}` });
     }
     cloneReceipt(receipt) {
         return { ...receipt, chunks: [...receipt.chunks] };
@@ -320,6 +379,9 @@ export class JobManager {
                 JSON.stringify(existing.stderrArtifact) !==
                     JSON.stringify(stderrArtifact)));
         const reopenSettlement = Boolean(existing?.acknowledgedAt && completionChanged);
+        const responderLeaseId = job.responderLeaseId ?? existing?.responderLeaseId;
+        const leaseIsActive = Boolean(responderLeaseId &&
+            this.responderLeases.get(job.ownerSessionId) === responderLeaseId);
         const notification = {
             id: existing?.id ??
                 `completion:${createHash("sha256").update(`${job.ownerSessionId}\0${job.id}\0${job.startedAt}`).digest("hex").slice(0, 24)}`,
@@ -339,6 +401,7 @@ export class JobManager {
             ...(job.exitCode !== undefined ? { exitCode: job.exitCode } : {}),
             ...(job.signal !== undefined ? { signal: job.signal } : {}),
             ...(job.monitor !== undefined ? { monitor: job.monitor } : {}),
+            ...(responderLeaseId ? { responderLeaseId } : {}),
             ...(existing?.deliveredAt ? { deliveredAt: existing.deliveredAt } : {}),
             ...((existing?.analyzedAt ?? existing?.acknowledgedAt)
                 ? { analyzedAt: existing?.analyzedAt ?? existing?.acknowledgedAt }
@@ -346,6 +409,12 @@ export class JobManager {
             ...(!reopenSettlement && existing?.acknowledgedAt
                 ? { acknowledgedAt: existing.acknowledgedAt }
                 : {}),
+            ...(existing?.settledAt ? { settledAt: existing.settledAt } : {}),
+            ...(!leaseIsActive
+                ? { archivedAt: existing?.archivedAt ?? endedAt }
+                : existing?.archivedAt
+                    ? { archivedAt: existing.archivedAt }
+                    : {}),
         };
         const updated = Boolean(existing && JSON.stringify(existing) !== JSON.stringify(notification));
         this.notifications.set(notification.id, notification);
@@ -366,6 +435,7 @@ export class JobManager {
         const correctsLost = job.status === "lost" && status !== "lost";
         if (this.isTerminalStatus(job.status) && !correctsLost) {
             const completion = this.ensureCompletionNotification(job);
+            this.scheduleTaskSettlement(job);
             if ((!completion.created && !completion.updated) ||
                 !completion.notification) {
                 return true;
@@ -382,7 +452,7 @@ export class JobManager {
             return persisted;
         }
         const finalization = (async () => {
-            this.clearJobDeadline(job.id);
+            this.clearAuthorizationTimer(job.id);
             this.livenessMisses.delete(job.id);
             const streamsFlushed = await this.closeWriters(job.id);
             job.status = streamsFlushed ? status : "failed";
@@ -397,6 +467,7 @@ export class JobManager {
             this.abortControllers.delete(job.id);
             this.processes.delete(job.id);
             const completion = this.ensureCompletionNotification(job);
+            this.scheduleTaskSettlement(job);
             this.pruneTerminalJobs();
             const persisted = this.persistSync();
             if (!persisted)
@@ -453,8 +524,9 @@ export class JobManager {
         this.livenessMisses.delete(job.id);
         job.status = "lost";
         job.endedAt = new Date().toISOString();
-        this.clearJobDeadline(job.id);
+        this.clearAuthorizationTimer(job.id);
         const completion = this.ensureCompletionNotification(job);
+        this.scheduleTaskSettlement(job);
         this.pruneTerminalJobs();
         if (!this.persistSync())
             this.scheduleRegistryRetry();
@@ -468,22 +540,54 @@ export class JobManager {
             });
         }
     }
-    /** Arm or restore a persisted finite-job deadline. */
-    scheduleJobDeadline(job) {
-        this.clearJobDeadline(job.id);
-        if (!job.timeoutAt || !this.isLive(job))
-            return;
-        const timeoutAt = Date.parse(job.timeoutAt);
-        if (!Number.isFinite(timeoutAt))
+    scheduleTaskSettlement(job, delayMs = 0) {
+        if (!job.responder || !this.isTerminalStatus(job.status))
             return;
+        const existing = this.settlementTimers.get(job.id);
+        if (existing)
+            clearTimeout(existing);
         const timer = setTimeout(() => {
-            this.deadlineTimers.delete(job.id);
-            this.refreshJobLiveness(job);
-            if (this.isLive(job))
-                void this.stopJob(job.id, { graceMs: 1_000 });
-        }, Math.max(0, timeoutAt - Date.now()));
+            this.settlementTimers.delete(job.id);
+            void import("../store/responder-settlement.js")
+                .then(({ settleResponderJob }) => settleResponderJob({ ...job }))
+                .then((result) => {
+                if (result === "applied" || result === "noop") {
+                    this.settlementAttempts.delete(job.id);
+                    const notification = this.notificationForJob(job.id);
+                    if (notification && !notification.settledAt) {
+                        notification.settledAt = new Date().toISOString();
+                        if (!this.persistSync())
+                            this.scheduleRegistryRetry();
+                        this.emit({
+                            type: "notification",
+                            jobId: job.id,
+                            notificationId: notification.id,
+                        });
+                    }
+                    return;
+                }
+                const attempt = (this.settlementAttempts.get(job.id) ?? 0) + 1;
+                this.settlementAttempts.set(job.id, attempt);
+                if (attempt < 8) {
+                    this.scheduleTaskSettlement(job, Math.min(5_000, 50 * 2 ** attempt));
+                }
+                else {
+                    this.settlementAttempts.delete(job.id);
+                }
+            })
+                .catch(() => {
+                const attempt = (this.settlementAttempts.get(job.id) ?? 0) + 1;
+                this.settlementAttempts.set(job.id, attempt);
+                if (attempt < 8) {
+                    this.scheduleTaskSettlement(job, Math.min(5_000, 50 * 2 ** attempt));
+                }
+                else {
+                    this.settlementAttempts.delete(job.id);
+                }
+            });
+        }, delayMs);
         timer.unref?.();
-        this.deadlineTimers.set(job.id, timer);
+        this.settlementTimers.set(job.id, timer);
     }
     /**
      * Register an in-flight tool for stall tracking only.
@@ -516,7 +620,7 @@ export class JobManager {
         this.abortControllers.delete(id);
         this.processes.delete(id);
         if (!this.isLive(job))
-            this.clearJobDeadline(id);
+            this.clearAuthorizationTimer(id);
         if (!this.isDurable(job) && !this.isLive(job)) {
             this.jobs.delete(id);
             this.emit({ type: "job", jobId: id });
@@ -598,10 +702,8 @@ export class JobManager {
             ...(options?.wakeOnCompletion !== undefined ? { wakeOnCompletion: options.wakeOnCompletion } : {}),
             ...(options?.responder !== undefined ? { responder: options.responder } : {}),
             ...(monitor !== undefined ? { monitor } : {}),
-            ...(options?.timeoutMs !== undefined
-                ? {
-                    timeoutAt: new Date(Date.now() + Math.max(1_000, Math.floor(options.timeoutMs))).toISOString(),
-                }
+            ...(options?.responderLeaseId
+                ? { responderLeaseId: options.responderLeaseId }
                 : {}),
             ...(options?.authorization ? { authorization: options.authorization } : {}),
         };
@@ -710,12 +812,7 @@ export class JobManager {
             job.processIdentity = processIdentity(child.pid);
             this.processes.set(id, child);
             this.writers.set(id, { stdout, stderr });
-            const expiresAt = options?.authorization?.expiresAt ? Date.parse(options.authorization.expiresAt) : Number.NaN;
-            const expiryTimer = Number.isFinite(expiresAt) && expiresAt > Date.now()
-                ? setTimeout(() => { void this.stopJob(id, { graceMs: 1_000 }); }, expiresAt - Date.now())
-                : undefined;
-            expiryTimer?.unref?.();
-            this.scheduleJobDeadline(job);
+            this.scheduleAuthorizationExpiry(job);
             let lastProgressFlush = 0;
             let progressDirty = false;
             let progressTimer;
@@ -765,8 +862,6 @@ export class JobManager {
             child.stdout?.on("data", (chunk) => recordOutput(stdout, chunk));
             child.stderr?.on("data", (chunk) => recordOutput(stderr, chunk));
             child.on("close", (code, signal) => {
-                if (expiryTimer)
-                    clearTimeout(expiryTimer);
                 stopProgressFlush();
                 const status = signal ? "killed" : code === 0 ? "exited" : "failed";
                 void this.finalizeJob(job, status, {
@@ -775,8 +870,6 @@ export class JobManager {
                 });
             });
             child.on("error", (error) => {
-                if (expiryTimer)
-                    clearTimeout(expiryTimer);
                 stopProgressFlush();
                 const code = error.code ?? "UNKNOWN";
                 try {
@@ -966,31 +1059,62 @@ export class JobManager {
     pendingNotifications(sessionId) {
         return this.getPendingNotifications(sessionId);
     }
+    claimNextResponderNotification(sessionId, leaseId) {
+        if (this.responderLeases.get(sessionId) !== leaseId)
+            return undefined;
+        const archivedAt = new Date().toISOString();
+        let archived = false;
+        for (const notification of this.notifications.values()) {
+            if (notification.ownerSessionId === sessionId &&
+                !notification.acknowledgedAt &&
+                notification.responderLeaseId !== leaseId &&
+                !notification.archivedAt) {
+                notification.archivedAt = archivedAt;
+                archived = true;
+            }
+        }
+        const notification = [...this.notifications.values()]
+            .filter((candidate) => candidate.ownerSessionId === sessionId &&
+            candidate.responderLeaseId === leaseId &&
+            !candidate.archivedAt &&
+            !candidate.deliveredAt &&
+            !candidate.analyzedAt &&
+            !candidate.acknowledgedAt &&
+            !this.claimedNotifications.has(candidate.id))
+            .sort((left, right) => left.createdAt.localeCompare(right.createdAt))[0];
+        if (notification)
+            this.claimedNotifications.add(notification.id);
+        if (archived && !this.persistSync())
+            this.scheduleRegistryRetry();
+        return notification;
+    }
+    releaseResponderNotificationClaim(notificationId) {
+        this.claimedNotifications.delete(notificationId);
+    }
     markDelivered(notificationId) {
         const notification = this.notifications.get(notificationId);
         if (!notification)
             return false;
         if (!notification.deliveredAt) {
-            const deliveredAt = new Date().toISOString();
-            notification.deliveredAt = deliveredAt;
+            notification.deliveredAt = new Date().toISOString();
             if (!this.persistSync()) {
                 notification.deliveredAt = undefined;
                 return false;
             }
             this.emit({ type: "notification", jobId: notification.jobId, notificationId });
         }
+        this.claimedNotifications.delete(notificationId);
         return true;
     }
     markAnalyzed(notificationId) {
         const notification = this.notifications.get(notificationId);
         if (!notification?.deliveredAt)
             return false;
+        this.claimedNotifications.delete(notificationId);
         if (!notification.analyzedAt) {
             notification.analyzedAt = new Date().toISOString();
-            if (!this.persistSync()) {
-                notification.analyzedAt = undefined;
-                return false;
-            }
+            if (!this.persistSync())
+                this.scheduleRegistryRetry();
             this.emit({
                 type: "notification",
                 jobId: notification.jobId,
@@ -1003,14 +1127,16 @@ export class JobManager {
         const notification = this.notifications.get(notificationId);
         if (!notification?.analyzedAt)
             return false;
+        this.claimedNotifications.delete(notificationId);
         if (!notification.acknowledgedAt) {
             notification.acknowledgedAt = new Date().toISOString();
-            if (!this.persistSync()) {
-                notification.acknowledgedAt = undefined;
-                return false;
+            const persisted = this.persistSync();
+            if (!persisted)
+                this.scheduleRegistryRetry();
+            if (persisted) {
+                this.pruneTerminalJobs();
+                this.persistSync();
             }
-            this.pruneTerminalJobs();
-            this.persistSync();
             this.emit({ type: "notification", jobId: notification.jobId, notificationId });
         }
         return true;
@@ -1048,6 +1174,7 @@ export class JobManager {
         if (metadata.responder !== undefined)
             job.responder = metadata.responder;
         const completion = this.ensureCompletionNotification(job);
+        this.scheduleTaskSettlement(job);
         if (completion.notification) {
             if (metadata.taskId !== undefined)
                 completion.notification.taskId = metadata.taskId;
@@ -1208,7 +1335,7 @@ export class JobManager {
         const childClose = child
             ? new Promise((resolve) => child.once("close", () => resolve()))
             : undefined;
-        this.clearJobDeadline(id);
+        this.clearAuthorizationTimer(id);
         const previousStatus = job.status;
         const previousWakeOnCompletion = job.wakeOnCompletion;
         if (options?.suppressWake)
@@ -1217,8 +1344,7 @@ export class JobManager {
         if (!this.persistSync()) {
             job.status = previousStatus;
             job.wakeOnCompletion = previousWakeOnCompletion;
-            if (!job.timeoutAt || Date.parse(job.timeoutAt) > Date.now())
-                this.scheduleJobDeadline(job);
+            this.scheduleAuthorizationExpiry(job);
             return { ok: false, output: `Failed to persist stop state for job "${id}"; no signal was sent.`, exitCode: 1 };
         }
         this.emit({ type: "job", jobId: id });
@@ -1243,8 +1369,7 @@ export class JobManager {
         };
         const restoreRunning = () => {
             job.status = "running";
-            if (!job.timeoutAt || Date.parse(job.timeoutAt) > Date.now())
-                this.scheduleJobDeadline(job);
+            this.scheduleAuthorizationExpiry(job);
             this.persistSync();
             this.emit({ type: "job", jobId: id });
         };
@@ -1414,6 +1539,24 @@ export class JobManager {
     /** Drop stale terminal durable jobs and all leftover ephemeral rows. */
     pruneTerminalJobs() {
         const now = Date.now();
+        const archivedUnsettled = [...this.notifications.values()]
+            .filter((notification) => Boolean(notification.archivedAt) &&
+            !notification.acknowledgedAt &&
+            !notification.settledAt)
+            .sort((left, right) => (right.archivedAt ?? right.createdAt).localeCompare(left.archivedAt ?? left.createdAt));
+        for (const [index, notification] of archivedUnsettled.entries()) {
+            const archivedAt = Date.parse(notification.archivedAt ?? notification.endedAt ?? notification.createdAt);
+            const expired = Number.isFinite(archivedAt) &&
+                now - archivedAt > ARCHIVED_UNSETTLED_MAX_AGE_MS;
+            if (index < MAX_ARCHIVED_UNSETTLED_NOTIFICATIONS && !expired)
+                continue;
+            this.notifications.delete(notification.id);
+            const timer = this.settlementTimers.get(notification.jobId);
+            if (timer)
+                clearTimeout(timer);
+            this.settlementTimers.delete(notification.jobId);
+            this.settlementAttempts.delete(notification.jobId);
+        }
         for (const [id, notification] of this.notifications) {
             const job = this.jobs.get(notification.jobId);
             if (!job ||
@@ -1432,8 +1575,11 @@ export class JobManager {
             if (this.isLive(job))
                 continue;
             const notification = this.notificationForJob(job.id);
-            if (notification && !notification.acknowledgedAt)
+            if (notification &&
+                !notification.acknowledgedAt &&
+                !(notification.archivedAt && notification.settledAt)) {
                 continue;
+            }
             const ended = job.endedAt ? Date.parse(job.endedAt) : Date.parse(job.startedAt);
             if (Number.isFinite(ended) && now - ended > TERMINAL_JOB_MAX_AGE_MS) {
                 this.jobs.delete(id);
@@ -1473,6 +1619,9 @@ export class JobManager {
                     if (notifiedJobs.has(notification.jobId))
                         continue;
                     notifiedJobs.add(notification.jobId);
+                    if (!notification.acknowledgedAt && !notification.archivedAt) {
+                        notification.archivedAt = new Date().toISOString();
+                    }
                     this.notifications.set(notification.id, notification);
                 }
             }
@@ -1497,9 +1646,11 @@ export class JobManager {
                         job.heartbeatAt = new Date().toISOString();
                     }
                 }
+                delete job.timeoutAt;
                 this.jobs.set(job.id, job);
                 this.ensureCompletionNotification(job);
-                this.scheduleJobDeadline(job);
+                this.scheduleAuthorizationExpiry(job);
+                this.scheduleTaskSettlement(job);
             }
             for (const [id, notification] of this.notifications) {
                 const job = this.jobs.get(notification.jobId);