cairnq 0.13.0 → 0.14.0

package/src/retention.ts

@@ -1,5 +1,4 @@
-import type { TerminalStatus } from "./models.js";
-import { validatePurgeInput, type PurgeInput, type TaskStore } from "./store/base.js";
+import type { TaskStore } from "./store/base.js";
 
 /** Sweep every hour unless asked otherwise — often enough that a queue with a
  * day of retention never carries more than an hour of extra rows, rare enough
@@ -9,83 +8,17 @@ const DEFAULT_INTERVAL_MS = 3_600_000;
  * a backlog drains in few statements, small enough that each is a short write. */
 const DEFAULT_LIMIT = 1_000;
 
-/** Per-status cutoffs. A status left out is never swept — granular retention is
- * an explicit statement of what may go, not a default for what wasn't named. */
-export type RetentionCutoffs = Partial<Record<TerminalStatus, number>>;
-
-/**
- * One "these rows may go after this long" statement. Each field left out widens
- * what the rule covers; `olderThanMs` is the only required one.
- *
- * A rule is one `purge` call's filters, so the fields are exactly `PurgeInput`'s
- * — deliberately, since a rule the sweeper can express but the store cannot
- * enforce would be a lie about what is being deleted.
- */
-export interface RetentionRule {
-  /** Only this queue. Absent means every queue. */
-  queue?: string;
-  /** Only this terminal status. Absent means all three. */
-  status?: TerminalStatus;
-  /** Only this task name. Absent means every name. */
-  name?: string;
-  /** How long a row matching this rule is kept after it finished. */
-  olderThanMs: number;
-}
-
-export interface RetentionOptions {
-  /**
-   * How long a terminal task is kept after it finished. Required: there is no
-   * safe default for how long someone else's results stay readable.
-   *
-   * Three forms, widening as the deployment does:
-   *
-   * - A number keeps every terminal row the same time.
-   * - A per-status map tiers by outcome — a succeeded row is spent once its
-   *   result is consumed, a failed one is worth keeping for diagnosis:
-   *   `{ succeeded: 300_000, failed: 86_400_000 }`. A status left out is never
-   *   swept.
-   * - An array of rules tiers by anything `purge` can filter on, which is what a
-   *   store shared by two workloads needs — the recommended way for two
-   *   languages to coordinate is one installation, and an RPC queue read once
-   *   has nothing in common with a durable queue kept for a week:
-   *   `[{ queue: "rpc", olderThanMs: 300_000 },
-   *      { queue: "jobs", status: "failed", olderThanMs: 604_800_000 }]`.
-   *   Rules are independent, each its own sweep — nothing a rule does not match
-   *   is swept, and rules that overlap simply delete the same row once.
-   */
-  olderThanMs: number | RetentionCutoffs | RetentionRule[];
-  /** Time between sweeps. Default 3_600_000 (one hour). */
-  intervalMs?: number;
-  /** Rows deleted per statement while draining. Default 1_000. */
-  limit?: number;
-  /**
-   * Called for a sweep that threw. The next sweep runs on schedule regardless —
-   * a purge that failed because the database was busy is not a reason to stop
-   * retaining — so without this a store quietly stops being swept. Must not throw.
-   */
-  onError?: (err: unknown) => void;
-}
-
-/** The three `olderThanMs` forms as the one form the sweep runs on. The number
- * and the per-status map are the rule array's common cases spelled shorter, so
- * they are widened here rather than handled separately downstream. */
-function toRules(spec: number | RetentionCutoffs | RetentionRule[]): RetentionRule[] {
-  if (typeof spec === "number") return [{ olderThanMs: spec }];
-  if (Array.isArray(spec)) return spec;
-  return (Object.entries(spec) as [TerminalStatus, number][]).map(([status, olderThanMs]) => ({
-    status,
-    olderThanMs,
-  }));
-}
-
 /**
- * Deletes terminal tasks on a schedule, for as long as the handle is open.
+ * Deletes terminal tasks on a schedule, for as long as the handle is open —
+ * `purge(olderThanMs)` on a timer, which is the whole mechanism.
  *
  * `purge` exists because nothing else in CairnQ removes rows, and a queue whose
  * payloads carry real data — an image, a document, a batch of embeddings — turns
  * that into a disk leak measured in gigabytes per backfill. Every deployment
  * that runs longer than a demo needs the sweep; leaving it to an external
  * scheduler means the leak is the default and remembering is the opt-in.
+ * A deployment whose retention is tiered (per queue, per status) calls `purge`
+ * with those filters from its own scheduler instead.
  *
  * It sweeps in bounded batches with a yield between them, so draining a backlog
  * that accumulated while nothing was sweeping stays a sequence of short writes
@@ -112,41 +45,29 @@ export class RetentionSweeper {
   /** The loop itself, awaited by stop() so no purge outlives the store. */
   private loop: Promise<void> | null = null;
   private readonly intervalMs: number;
-  /** Rows per purge statement while draining — see DEFAULT_LIMIT. */
-  private readonly limit: number;
-  /** One purge per rule: a lone entry for a number, one per status for a map,
-   * one per element for an array. */
-  private readonly purgeInputs: PurgeInput[];
 
   constructor(
     private readonly store: TaskStore,
-    private readonly opts: RetentionOptions,
+    private readonly olderThanMs: number,
+    opts: { intervalMs?: number } = {},
   ) {
+    if (!Number.isFinite(olderThanMs) || olderThanMs < 0) {
+      throw new Error(`retentionMs must be >= 0, got ${olderThanMs}`);
+    }
     this.intervalMs = opts.intervalMs ?? DEFAULT_INTERVAL_MS;
     if (!Number.isFinite(this.intervalMs) || this.intervalMs < 1) {
-      throw new Error(`retention.intervalMs must be >= 1, got ${this.intervalMs}`);
-    }
-    this.limit = opts.limit ?? DEFAULT_LIMIT;
-    const rules = toRules(opts.olderThanMs);
-    // An empty map or array retains nothing and sweeps nothing — almost
-    // certainly a bug upstream of this call, so refuse it rather than silently
-    // never purging.
-    if (!rules.length) {
-      throw new Error("retention.olderThanMs must name at least one rule");
+      throw new Error(`retention intervalMs must be >= 1, got ${this.intervalMs}`);
     }
     this.arm();
-    this.purgeInputs = rules.map((rule) => ({ ...rule, limit: this.limit }));
-    // Fail fast on the store's own purge rules (terminal status, cutoff >= 0):
-    // the sweep runs an hour from now, and its errors only surface via onError.
-    for (const input of this.purgeInputs) validatePurgeInput(input);
   }
 
   start(): void {
     if (this.active) return;
     this.active = true;
-    this.stopping = false;
-    // A stopped sweeper can be started again, and the old signal is spent.
-    this.arm();
+    // No reset here: stop() clears `stopping` and re-arms the signal before it
+    // returns, and the constructor arms the first one — so a sweeper reaching
+    // this line always already has a fresh signal and a clear flag. (The Python
+    // twin has never had a reset here, for the same reason.)
     this.loop = this.run();
   }
 
@@ -162,6 +83,15 @@ export class RetentionSweeper {
     this.wake();
     await this.loop;
     this.loop = null;
+    // Cleared, and the signal re-armed, now that the loop is provably gone.
+    // `stopping` is how a sweep in flight cuts itself short, so leaving it set
+    // would silently truncate a later on-demand sweep() — a supported call — to
+    // its first batch. Leaving the signal spent would be the subtler half of the
+    // same bug: that sweep's between-batches yield would resolve on a microtask
+    // instead of handing the event loop back, so a long drain would starve the
+    // submits and claims sharing this process — exactly what the yield is for.
+    this.stopping = false;
+    this.arm();
   }
 
   private async run(): Promise<void> {
@@ -172,13 +102,9 @@ export class RetentionSweeper {
       if (this.stopping) return;
       try {
         await this.sweep();
-      } catch (err) {
-        try {
-          this.opts.onError?.(err);
-        } catch {
-          // A reporting hook must never take the sweep down with it — the same
-          // rule the worker's onError follows.
-        }
+      } catch {
+        // A purge that failed because the database was busy is not a reason to
+        // stop retaining — the next sweep runs on schedule regardless.
       }
     }
   }
@@ -190,27 +116,36 @@ export class RetentionSweeper {
    */
   async sweep(): Promise<number> {
     let deleted = 0;
-    for (const input of this.purgeInputs) {
-      for (;;) {
-        const ids = await this.store.purge(input);
-        deleted += ids.length;
-        if (this.stopping) return deleted;
-        if (ids.length < this.limit) break;
-        // Hand the loop back between batches: a large drain must not starve the
-        // submits and claims sharing this process.
-        await this.sleep(0);
-      }
+    for (;;) {
+      const ids = await this.store.purge({ olderThanMs: this.olderThanMs, limit: DEFAULT_LIMIT });
+      deleted += ids.length;
+      if (this.stopping) return deleted;
+      if (ids.length < DEFAULT_LIMIT) break;
+      // Hand the loop back between batches: a large drain must not starve the
+      // submits and claims sharing this process. Ref'd — see sleep().
+      await this.sleep(0, true);
     }
     return deleted;
   }
 
-  /** Sleep, interruptible by stop(). Unref'd: retention is housekeeping, and a
-   * pending sweep must never be the reason a process refuses to exit. */
-  private sleep(ms: number): Promise<void> {
+  /**
+   * Sleep, interruptible by stop().
+   *
+   * `holdProcess` is the same distinction the store draws between `claimWake`
+   * and `taskDoneWake`: a wait nobody is awaiting must not hold the process
+   * open, and a wait somebody IS awaiting must. The scheduled loop's interval is
+   * the first — retention is housekeeping, and a pending sweep must never be the
+   * reason a process refuses to exit. `sweep()`'s between-batches yield is the
+   * second: its caller is awaiting the drain, and an unref'd timer there let
+   * Node decide the loop was idle and exit mid-drain, leaving that promise
+   * unsettled forever (a maintenance command that swept two rows of seven and
+   * exited 13). Python's twin never had this, `asyncio.sleep` having no unref.
+   */
+  private sleep(ms: number, holdProcess = false): Promise<void> {
     let timer: NodeJS.Timeout;
     const nap = new Promise<void>((resolve) => {
       timer = setTimeout(resolve, ms);
-      timer.unref?.();
+      if (!holdProcess) timer.unref?.();
     });
     // Clear the timer whichever side wins, so a stop is never followed by a
     // leftover sweep timer.