@semiont/jobs 0.6.1 → 0.6.3

package/dist/index.d.ts

@@ -369,6 +369,23 @@ interface JobQueue {
  * starvation, 2026-09-15).
  */
 declare const JOB_QUEUE_EMITS: readonly ["job:queued"];
+/**
+ * How long a job's record survives after it reaches a terminal state
+ * (complete/failed/cancelled), and how often a driver enforces that.
+ *
+ * ONE pair for every driver. The window is a CONTRACT fact — a caller that
+ * reads a finished job's result gets the same day whichever backing store the
+ * stack runs — and the cadence is what turns the window from an aspiration
+ * into a promise. Restated per driver the two numbers drift, and the drift is
+ * invisible until someone compares two deployments.
+ *
+ * The METHOD that enforces retention is not on the interface and cannot be:
+ * it is an unlink in one driver and a stream purge in another. The NUMBER is
+ * here because it is the same number.
+ */
+declare const TERMINAL_JOB_RETENTION_MS: number;
+/** How often a driver sweeps for records past `TERMINAL_JOB_RETENTION_MS`. */
+declare const TERMINAL_JOB_SWEEP_INTERVAL_MS: number;
 
 /**
  * Job Queue Manager
@@ -500,9 +517,12 @@ declare class FsJobQueue implements JobQueue {
      */
     recoverStaleRunningJobs(): Promise<number>;
     /**
-     * Clean up old completed/failed jobs (older than retention period)
+     * Delete terminal jobs whose `completedAt` is older than `retentionMs`.
+     * The window is a parameter with no default: the ONE value the queue runs
+     * on is `TERMINAL_JOB_RETENTION_MS`, and a default here would be a second
+     * copy of it that nobody notices going stale.
      */
-    cleanupOldJobs(retentionHours?: number): Promise<number>;
+    cleanupOldJobs(retentionMs: number): Promise<number>;
     /**
      * Get job file path
      */
@@ -530,7 +550,10 @@ declare class FsJobQueue implements JobQueue {
  *  - **KV bucket `jobs`** is the AUTHORITATIVE operational state: one entry
  *    per job (`{ job, lastProgressAt }`), every transition a revision-CAS —
  *    which is what makes `claimJob` atomic: simultaneous claims race on one
- *    revision and exactly one update wins.
+ *    revision and exactly one update wins. It holds LIVE jobs plus a day of
+ *    concluded ones, not a history: `pruneTerminalJobs` is the retention the
+ *    fs driver's janitor holds, and without it every scan over the bucket
+ *    — `getStats` is one, on the metrics interval — grows forever.
  *  - **Stream `JOBS`** (subjects `jobs.<category>.<type>`, work-queue
  *    retention) is the DELIVERY vehicle and redelivery timer. A delivered
  *    message is the lease this process holds for a job; `working()`
@@ -598,6 +621,7 @@ declare class JetStreamJobQueue implements JobQueue {
     private reconciling;
     private tick;
     private ticking;
+    private cleanupTimer;
     private readonly lastProgressWrite;
     private readonly staleRunningMs;
     private readonly ackWaitMs;
@@ -642,6 +666,11 @@ declare class JetStreamJobQueue implements JobQueue {
     recordProgress(jobIdArg: JobId, progress: Record<string, unknown>): Promise<void>;
     cancelPendingJobs(category: 'annotation' | 'generation'): Promise<number>;
     cancelJob(jobIdArg: JobId): Promise<boolean>;
+    /**
+     * Counts by status. The three terminal counts are the RETENTION WINDOW, not
+     * lifetime totals — `pruneTerminalJobs` drops a record a day after it
+     * concluded — so they answer "recently finished", never "finished ever".
+     */
     getStats(): Promise<{
         pending: number;
         running: number;
@@ -656,6 +685,33 @@ declare class JetStreamJobQueue implements JobQueue {
      * through here; that is AckWait redelivery.
      */
     recoverStaleRunningJobs(): Promise<number>;
+    /**
+     * Terminal-job retention: the fs driver's hourly janitor, in this driver's
+     * terms. Without it the bucket keeps every job it has ever seen, and
+     * `getStats` — an OTel observable gauge, so a full bucket scan every metric
+     * interval — pays for all of them forever. Measured on a live stack
+     * 2026-09-24: 139 concluded records re-read every 30 seconds by a knowledge
+     * base that had done very little work.
+     *
+     * A SWEEP AND NOT A BUCKET TTL, deliberately. `ttl` on `views.kv()` is a
+     * BUCKET limit — the backing stream's `max_age` — so it expires every key
+     * by age, not the terminal ones by status: a pending job nobody claimed for
+     * a day, a running job whose worker went quiet, both evaporate alongside
+     * the finished ones. That is not a slower version of this bug, it is a
+     * worse one. This bucket is the AUTHORITATIVE record: with the entry gone
+     * `read` returns null, every CAS falls through to its `onMissing`, and the
+     * next delivery for that job is `term()`ed as unknown — the job disappears
+     * with no failure and no trace. The rule retention needs to express is
+     * about STATUS, and status is exactly what a bucket-wide clock cannot see.
+     * (A per-message TTL set at the terminal write would express it, but that
+     * is nats-server 2.11 plus a client that exposes it; nats.js 2.x does not.)
+     *
+     * Terminal is absorbing — no transition leaves `complete`/`failed`/
+     * `cancelled` — so a record this sweep reads as expired cannot come back to
+     * life under it, and the lease was settled at the transition that made it
+     * terminal.
+     */
+    pruneTerminalJobs(retentionMs: number): Promise<number>;
     /** The heartbeat body — extend live leases, settle concluded ones. */
     private reconcileHeld;
     /**
@@ -1077,5 +1133,5 @@ declare const WORKER_CONSUMED_BROADCASTS: readonly ["job:queued", "job:cancel-re
  */
 declare const WORKER_CHANNELS: readonly (keyof EventMap)[];
 
-export { AnnotationDetection, FsJobQueue, JOBS_STREAM_SUBJECTS, JOB_QUEUE_EMITS, JetStreamJobQueue, STALL_THRESHOLD_MS, WORKER_CHANNELS, WORKER_CONSUMED_BROADCASTS, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };
+export { AnnotationDetection, FsJobQueue, JOBS_STREAM_SUBJECTS, JOB_QUEUE_EMITS, JetStreamJobQueue, STALL_THRESHOLD_MS, TERMINAL_JOB_RETENTION_MS, TERMINAL_JOB_SWEEP_INTERVAL_MS, WORKER_CHANNELS, WORKER_CONSUMED_BROADCASTS, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };
 export type { AnyJob, AssessmentDetectionJob, AssessmentDetectionParams, AssessmentDetectionProgress, CancelledJob, CommentDetectionJob, CommentDetectionParams, CommentDetectionProgress, CompleteJob, DetectionJob, DetectionParams, DetectionProgress, FailedJob, GenerationJob, GenerationResult, HighlightDetectionJob, HighlightDetectionParams, HighlightDetectionProgress, JetStreamJobQueueOptions, JobMetadata, JobQueryFilters, JobQueue, JobStatus, JobType, OnProgress, PendingJob, ProcessorResult, RunningJob, TagDetectionJob, TagDetectionParams, TagDetectionProgress, YieldProgress };

package/dist/index.js

@@ -14,6 +14,8 @@ import '@semiont/http-transport';
 
 // src/job-queue-interface.ts
 var JOB_QUEUE_EMITS = ["job:queued"];
+var TERMINAL_JOB_RETENTION_MS = 24 * 60 * 60 * 1e3;
+var TERMINAL_JOB_SWEEP_INTERVAL_MS = 60 * 60 * 1e3;
 
 // src/will-retry.ts
 function willRetryAfter(metadata, failureClass) {
@@ -39,8 +41,6 @@ function mergeUnitCursors(existing, incoming, completed) {
 var REANNOUNCE_INTERVAL_MS = 3e4;
 var STALE_RUNNING_MS = 30 * 6e4;
 var PROGRESS_WRITE_MIN_INTERVAL_MS = 5e3;
-var RETENTION_HOURS = 24;
-var CLEANUP_INTERVAL_MS = 36e5;
 var FsJobQueue = class {
   constructor(state, logger, eventBus) {
     this.eventBus = eventBus;
@@ -82,12 +82,12 @@ var FsJobQueue = class {
     }
     if (!this.cleanupTimer) {
       this.cleanupTimer = setInterval(() => {
-        this.cleanupOldJobs(RETENTION_HOURS).catch((error) => {
+        this.cleanupOldJobs(TERMINAL_JOB_RETENTION_MS).catch((error) => {
           this.logger.warn("Job retention cleanup failed", {
             error: error instanceof Error ? error.message : String(error)
           });
         });
-      }, CLEANUP_INTERVAL_MS);
+      }, TERMINAL_JOB_SWEEP_INTERVAL_MS);
       this.cleanupTimer.unref?.();
     }
     this.logger.info("Job queue initialized");
@@ -450,10 +450,13 @@ var FsJobQueue = class {
     return recovered;
   }
   /**
-   * Clean up old completed/failed jobs (older than retention period)
+   * Delete terminal jobs whose `completedAt` is older than `retentionMs`.
+   * The window is a parameter with no default: the ONE value the queue runs
+   * on is `TERMINAL_JOB_RETENTION_MS`, and a default here would be a second
+   * copy of it that nobody notices going stale.
    */
-  async cleanupOldJobs(retentionHours = 24) {
-    const cutoffTime = Date.now() - retentionHours * 60 * 60 * 1e3;
+  async cleanupOldJobs(retentionMs) {
+    const cutoffTime = Date.now() - retentionMs;
     let deletedCount = 0;
     const cleanupStatuses = ["complete", "failed", "cancelled"];
     for (const status of cleanupStatuses) {
@@ -514,6 +517,8 @@ var FsJobQueue = class {
 var STREAM = "JOBS";
 var CONSUMER = "gateway-claims";
 var BUCKET = "jobs";
+var BUCKET_STREAM = `KV_${BUCKET}`;
+var bucketSubject = (key) => `$KV.${BUCKET}.${key}`;
 var PROGRESS_WRITE_MIN_INTERVAL_MS2 = 5e3;
 var MAX_CAS_ATTEMPTS = 20;
 var enc = new TextEncoder();
@@ -549,6 +554,7 @@ var JetStreamJobQueue = class {
   reconciling = false;
   tick = null;
   ticking = false;
+  cleanupTimer = null;
   lastProgressWrite = /* @__PURE__ */ new Map();
   staleRunningMs;
   ackWaitMs;
@@ -633,6 +639,14 @@ var JetStreamJobQueue = class {
       });
     }, this.tickMs);
     this.tick.unref?.();
+    this.cleanupTimer = setInterval(() => {
+      this.pruneTerminalJobs(TERMINAL_JOB_RETENTION_MS).catch((error) => {
+        this.logger.warn("Job retention cleanup failed", {
+          error: error instanceof Error ? error.message : String(error)
+        });
+      });
+    }, TERMINAL_JOB_SWEEP_INTERVAL_MS);
+    this.cleanupTimer.unref?.();
   }
   /** An outage is never silent: one line down, one line back, one line if the client gives up. */
   watchConnection() {
@@ -661,6 +675,8 @@ var JetStreamJobQueue = class {
     this.heartbeat = null;
     if (this.tick) clearInterval(this.tick);
     this.tick = null;
+    if (this.cleanupTimer) clearInterval(this.cleanupTimer);
+    this.cleanupTimer = null;
     this.held.clear();
     void this.nc?.close();
   }
@@ -940,6 +956,11 @@ var JetStreamJobQueue = class {
     if (done) this.settleLease(jobIdArg, "term");
     return done;
   }
+  /**
+   * Counts by status. The three terminal counts are the RETENTION WINDOW, not
+   * lifetime totals — `pruneTerminalJobs` drops a record a day after it
+   * concluded — so they answer "recently finished", never "finished ever".
+   */
   async getStats() {
     const stats = { pending: 0, running: 0, complete: 0, failed: 0, cancelled: 0 };
     for (const key of await this.allKeys()) {
@@ -972,6 +993,50 @@ var JetStreamJobQueue = class {
     }
     return recovered;
   }
+  /**
+   * Terminal-job retention: the fs driver's hourly janitor, in this driver's
+   * terms. Without it the bucket keeps every job it has ever seen, and
+   * `getStats` — an OTel observable gauge, so a full bucket scan every metric
+   * interval — pays for all of them forever. Measured on a live stack
+   * 2026-09-24: 139 concluded records re-read every 30 seconds by a knowledge
+   * base that had done very little work.
+   *
+   * A SWEEP AND NOT A BUCKET TTL, deliberately. `ttl` on `views.kv()` is a
+   * BUCKET limit — the backing stream's `max_age` — so it expires every key
+   * by age, not the terminal ones by status: a pending job nobody claimed for
+   * a day, a running job whose worker went quiet, both evaporate alongside
+   * the finished ones. That is not a slower version of this bug, it is a
+   * worse one. This bucket is the AUTHORITATIVE record: with the entry gone
+   * `read` returns null, every CAS falls through to its `onMissing`, and the
+   * next delivery for that job is `term()`ed as unknown — the job disappears
+   * with no failure and no trace. The rule retention needs to express is
+   * about STATUS, and status is exactly what a bucket-wide clock cannot see.
+   * (A per-message TTL set at the terminal write would express it, but that
+   * is nats-server 2.11 plus a client that exposes it; nats.js 2.x does not.)
+   *
+   * Terminal is absorbing — no transition leaves `complete`/`failed`/
+   * `cancelled` — so a record this sweep reads as expired cannot come back to
+   * life under it, and the lease was settled at the transition that made it
+   * terminal.
+   */
+  async pruneTerminalJobs(retentionMs) {
+    const cutoff = Date.now() - retentionMs;
+    let pruned = 0;
+    for (const key of await this.allKeys()) {
+      const envelope = await this.read(jobId(key));
+      if (!envelope) continue;
+      const { job } = envelope;
+      if (job.status !== "complete" && job.status !== "failed" && job.status !== "cancelled") continue;
+      if (Date.parse(job.completedAt) >= cutoff) continue;
+      await this.jsm.streams.purge(BUCKET_STREAM, { filter: bucketSubject(key) });
+      this.lastProgressWrite.delete(key);
+      pruned++;
+    }
+    if (pruned > 0) {
+      this.logger.info("Jobs cleaned up", { deletedCount: pruned });
+    }
+    return pruned;
+  }
   /** The heartbeat body — extend live leases, settle concluded ones. */
   async reconcileHeld() {
     for (const [id, held] of [...this.held]) {
@@ -2975,6 +3040,6 @@ var WORKER_CHANNELS = [
   ...WORKER_CONSUMED_BROADCASTS
 ];
 
-export { AnnotationDetection, FsJobQueue, JOBS_STREAM_SUBJECTS, JOB_QUEUE_EMITS, JetStreamJobQueue, STALL_THRESHOLD_MS, WORKER_CHANNELS, WORKER_CONSUMED_BROADCASTS, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };
+export { AnnotationDetection, FsJobQueue, JOBS_STREAM_SUBJECTS, JOB_QUEUE_EMITS, JetStreamJobQueue, STALL_THRESHOLD_MS, TERMINAL_JOB_RETENTION_MS, TERMINAL_JOB_SWEEP_INTERVAL_MS, WORKER_CHANNELS, WORKER_CONSUMED_BROADCASTS, generateResourceFromTopic, isCancelledJob, isCompleteJob, isFailedJob, isPendingJob, isRunningJob, processAssessmentJob, processCommentJob, processGenerationJob, processHighlightJob, processReferenceJob, processTagJob };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map