cursedbelt 2.7.0 → 3.0.0

package/src/server/drivers/saasJobWorker.spec.ts

@@ -1,411 +0,0 @@
-/**
- * The SaaS worker runtime (08 §4a) contract suite — lease / execute / record /
- * cancel / drain, over a fake {@link SaasQueueBackend}.
- *
- * The properties tested here are the ones a "just run the handler" worker gets
- * wrong in a real at-least-once deployment: a redelivered job must EXECUTE once
- * (not merely record once), a cross-process cancel must actually abort the
- * running handler, a drain must let in-flight work finish, and the worker must
- * compose with {@link createSaasJobBroker} so "enqueue on the API node, run on
- * the worker, await anywhere" holds across the process boundary the store models.
- */
-import { describe, expect, it } from 'bun:test';
-import { createMemoryJobResultStore, type JobResultStore } from './jobResults';
-import { type JobHandler, type JobId } from './jobBroker';
-import { createSaasJobBroker } from './saasJobBroker';
-import type {
-  LeaseControl,
-  LeaseOptions,
-  LeaseProcessor,
-  SaasQueueBackend,
-  SaasQueuePayload,
-} from './saasJobBroker';
-import { JOB_HANDLER_UNREGISTERED, runSaasJobWorker } from './saasJobWorker';
-import { createSaasStorageProvider, resetSaasPool, type SaasPool } from 'cwip/storage';
-import {
-  createKnowledgeReindexSource,
-  createSearchJobHandlers,
-  type SearchJobHandlerOptions,
-} from '../services/searchJobs';
-import { KNOWLEDGE_CORPUS } from '../stream/searchBridge';
-import type { SearchDoc } from 'cwip/search-provider';
-
-const silentLogger = { info() {}, warn() {}, error() {}, debug() {} };
-
-const waitFor = async (predicate: () => boolean | Promise<boolean>, timeoutMs = 1000): Promise<void> => {
-  const start = Bun.nanoseconds();
-  while (!(await predicate())) {
-    if ((Bun.nanoseconds() - start) / 1e6 > timeoutMs) throw new Error('waitFor timed out');
-    await Bun.sleep(2);
-  }
-};
-
-/**
- * A controllable in-memory queue backend. Delivery is EXPLICIT (`deliver(id)`)
- * so a test can model at-least-once redelivery, concurrency and crash-before-ack
- * deterministically instead of racing a real timer.
- */
-class FakeQueue implements SaasQueueBackend {
-  readonly jobs = new Map<JobId, SaasQueuePayload>();
-  private seq = 0;
-  private processor: LeaseProcessor | null = null;
-  leaseConcurrency = 0;
-  readonly inFlight = new Set<Promise<void>>();
-  readonly subscribers = new Set<() => void>();
-  readonly removed: JobId[] = [];
-
-  async add(payload: SaasQueuePayload): Promise<JobId> {
-    this.seq += 1;
-    const id = `job-${this.seq}`;
-    this.jobs.set(id, payload);
-    return id;
-  }
-
-  async remove(id: JobId): Promise<void> {
-    this.removed.push(id);
-  }
-
-  subscribe(onCompletion: () => void): () => void {
-    this.subscribers.add(onCompletion);
-    return () => this.subscribers.delete(onCompletion);
-  }
-
-  lease(process: LeaseProcessor, options: LeaseOptions): LeaseControl {
-    this.processor = process;
-    this.leaseConcurrency = options.concurrency;
-    return {
-      close: async () => {
-        await Promise.allSettled([...this.inFlight]);
-        this.processor = null;
-      },
-    };
-  }
-
-  /** Deliver an enqueued job to the worker. Resolves when the lease is acked. */
-  deliver(id: JobId): Promise<void> {
-    if (!this.processor) throw new Error('deliver() before lease()');
-    const payload = this.jobs.get(id);
-    if (!payload) throw new Error(`deliver() unknown job ${id}`);
-    const p = this.processor({ id, ...payload }).finally(() => this.inFlight.delete(p));
-    this.inFlight.add(p);
-    return p;
-  }
-}
-
-const spec = (kind: string, payload: Record<string, unknown> = {}, tenantId = 't1'): SaasQueuePayload => ({
-  kind,
-  tenantId,
-  payload,
-});
-
-describe('runSaasJobWorker — execute + record', () => {
-  it('runs the registered handler and records a durable success before acking', async () => {
-    const queue = new FakeQueue();
-    const results = createMemoryJobResultStore();
-    const handler: JobHandler = (payload) => ({ echoed: (payload as { n: number }).n * 2 });
-    const worker = runSaasJobWorker({ queue, results, handlers: { double: handler }, logger: silentLogger });
-
-    const id = await queue.add(spec('double', { n: 21 }));
-    await queue.deliver(id); // resolves only after the outcome is recorded
-
-    await expect(results.read(id)).resolves.toMatchObject({ ok: true, output: { echoed: 42 }, tenantId: 't1' });
-    await worker.stop();
-  });
-
-  it('records an unregistered kind as a terminal failure instead of looping forever', async () => {
-    const queue = new FakeQueue();
-    const results = createMemoryJobResultStore();
-    const worker = runSaasJobWorker({ queue, results, handlers: {}, logger: silentLogger });
-
-    const id = await queue.add(spec('nobody-handles-this'));
-    await queue.deliver(id);
-
-    const recorded = await results.read(id);
-    expect(recorded?.ok).toBe(false);
-    expect(recorded?.error).toContain(JOB_HANDLER_UNREGISTERED);
-    await worker.stop();
-  });
-
-  it('records a thrown handler as a failure and still acks (no infinite retry on a deterministic bug)', async () => {
-    const queue = new FakeQueue();
-    const results = createMemoryJobResultStore();
-    const worker = runSaasJobWorker({
-      queue,
-      results,
-      handlers: {
-        boom: () => {
-          throw new Error('handler exploded');
-        },
-      },
-      logger: silentLogger,
-    });
-
-    const id = await queue.add(spec('boom'));
-    await queue.deliver(id); // resolves = acked, despite the throw
-
-    await expect(results.read(id)).resolves.toMatchObject({ ok: false, error: 'handler exploded' });
-    await worker.stop();
-  });
-});
-
-describe('runSaasJobWorker — exactly-once execution under at-least-once delivery', () => {
-  it('does not re-run a handler for a redelivered job that already completed', async () => {
-    const queue = new FakeQueue();
-    const results = createMemoryJobResultStore();
-    let runs = 0;
-    const worker = runSaasJobWorker({
-      queue,
-      results,
-      handlers: {
-        count: () => {
-          runs += 1;
-          return runs;
-        },
-      },
-      logger: silentLogger,
-    });
-
-    const id = await queue.add(spec('count'));
-    await queue.deliver(id); // first delivery — executes
-    await queue.deliver(id); // redelivery (e.g. an ack lost to a crash) — must NOT re-execute
-
-    expect(runs).toBe(1);
-    await expect(results.read(id)).resolves.toMatchObject({ ok: true, output: 1 });
-    await worker.stop();
-  });
-});
-
-describe('runSaasJobWorker — cooperative cancel across processes', () => {
-  it('aborts an in-flight handler when a cancel row appears in the shared store', async () => {
-    const queue = new FakeQueue();
-    const results = createMemoryJobResultStore();
-    let sawAbort = false;
-    const handler: JobHandler = (_payload, ctx) =>
-      new Promise((resolve) => {
-        ctx.signal.addEventListener(
-          'abort',
-          () => {
-            sawAbort = true;
-            resolve('stopped-cleanly');
-          },
-          { once: true },
-        );
-      });
-    const worker = runSaasJobWorker({
-      queue,
-      results,
-      handlers: { slow: handler },
-      logger: silentLogger,
-      cancelPollMs: 5,
-    });
-
-    const id = await queue.add(spec('slow'));
-    const acked = queue.deliver(id); // do not await — it parks on the signal
-    await waitFor(() => queue.inFlight.size === 1);
-
-    // Another process cancels: broker.cancel writes a `cancelled` row.
-    await results.record({ jobId: id, ok: false, error: 'cancelled', tenantId: 't1' });
-
-    await acked; // the worker's scan aborts the handler, which resolves
-    expect(sawAbort).toBe(true);
-    // First-writer-wins: the handler's clean resolve did NOT overwrite `cancelled`.
-    await expect(results.read(id)).resolves.toMatchObject({ ok: false, error: 'cancelled' });
-    await worker.stop();
-  });
-
-  it('never starts a handler for a job cancelled before it was leased', async () => {
-    const queue = new FakeQueue();
-    const results = createMemoryJobResultStore();
-    let started = false;
-    const worker = runSaasJobWorker({
-      queue,
-      results,
-      handlers: {
-        work: () => {
-          started = true;
-          return 'ran';
-        },
-      },
-      logger: silentLogger,
-    });
-
-    const id = await queue.add(spec('work'));
-    await results.record({ jobId: id, ok: false, error: 'cancelled', tenantId: 't1' }); // cancel wins the race
-    await queue.deliver(id);
-
-    expect(started).toBe(false);
-    await expect(results.read(id)).resolves.toMatchObject({ ok: false, error: 'cancelled' });
-    await worker.stop();
-  });
-});
-
-describe('runSaasJobWorker — drain on shutdown', () => {
-  it('lets in-flight jobs finish (does not abort them) and waits for them in stop()', async () => {
-    const queue = new FakeQueue();
-    const results = createMemoryJobResultStore();
-    let release!: () => void;
-    const gate = new Promise<void>((r) => {
-      release = r;
-    });
-    let abortedDuringDrain = false;
-    const handler: JobHandler = async (_payload, ctx) => {
-      await gate;
-      abortedDuringDrain = ctx.signal.aborted;
-      return 'finished';
-    };
-    const worker = runSaasJobWorker({ queue, results, handlers: { long: handler }, logger: silentLogger });
-
-    const id = await queue.add(spec('long'));
-    void queue.deliver(id);
-    await waitFor(() => queue.inFlight.size === 1);
-
-    const stopped = worker.stop(); // begins draining while the job is mid-flight
-    let stopResolved = false;
-    void stopped.then(() => {
-      stopResolved = true;
-    });
-    await Bun.sleep(10);
-    expect(stopResolved).toBe(false); // stop() must not resolve before the job settles
-
-    release();
-    await stopped;
-    expect(stopResolved).toBe(true);
-    expect(abortedDuringDrain).toBe(false); // drain != abort
-    await expect(results.read(id)).resolves.toMatchObject({ ok: true, output: 'finished' });
-  });
-
-  it('is idempotent — a second stop() resolves against the same drain', async () => {
-    const queue = new FakeQueue();
-    const results = createMemoryJobResultStore();
-    const worker = runSaasJobWorker({ queue, results, handlers: {}, logger: silentLogger });
-    await Promise.all([worker.stop(), worker.stop()]);
-  });
-});
-
-describe('runSaasJobWorker — misconfiguration', () => {
-  it('refuses a producer-only backend (no lease) at construction', () => {
-    const producerOnly: SaasQueueBackend = {
-      async add() {
-        return 'x';
-      },
-      async remove() {},
-    };
-    expect(() =>
-      runSaasJobWorker({ queue: producerOnly, results: createMemoryJobResultStore(), handlers: {} }),
-    ).toThrow(/SAAS_QUEUE_NOT_CONSUMABLE|producer-only/);
-  });
-});
-
-describe('runSaasJobWorker — a leased job pages through the pg-dialect repository (03 §2)', () => {
-  it('completes a reindex whose source reads through the Postgres executor', async () => {
-    // The gap this closes: before the dialect layer, every repository was
-    // SQLite-typed, so a leased `reindex` under saas died on its first page read
-    // — the worker runtime was proven, the work it leases was not.
-    const recorded: { sql: string; params: readonly unknown[] }[] = [];
-    const page = (from: number, count: number): Record<string, unknown>[] =>
-      Array.from({ length: count }, (_unused, index) => ({
-        id: `note_${from + index}`,
-        tenant_id: 't1',
-        user_id: 'u1',
-        item_type: 'note',
-        title: `Note ${from + index}`,
-        body_md: 'body',
-        // A pg driver hands 64-bit integers back as strings.
-        pinned: '0',
-        archived: '0',
-        created_at: '2026-07-24T00:00:00.000Z',
-        updated_at: `2026-07-24T00:00:0${9 - ((from + index) % 10)}.000Z`,
-      }));
-
-    let served = 0;
-    const pool: SaasPool = {
-      async withConnection(fn) {
-        return await fn({
-          async query(sql: string, params: readonly unknown[] = []) {
-            recorded.push({ sql, params });
-            if (!/FROM k_items/.test(sql)) return [];
-            // Two full pages then a short one: the handler must FOLLOW the
-            // cursor, which is where a half-ported repository stops.
-            served += 1;
-            return (served <= 2 ? page(served * 10, 3) : []) as never[];
-          },
-          async execute(sql: string, params: readonly unknown[] = []) {
-            recorded.push({ sql, params });
-            return { rowCount: 0 };
-          },
-        });
-      },
-    };
-
-    const storage = createSaasStorageProvider({ pool, config: { label: 'pg-worker', maxSockets: 4 } });
-    const indexed: string[] = [];
-    const handlers = createSearchJobHandlers({
-      search: {
-        async index(doc: SearchDoc) {
-          indexed.push(doc.id);
-        },
-        async query() {
-          return [];
-        },
-        async remove() {},
-        now: () => new Date().toISOString(),
-        async sweepStale() {
-          return 0;
-        },
-      } as unknown as SearchJobHandlerOptions['search'],
-      sources: [createKnowledgeReindexSource(storage)],
-      pageSize: 2,
-      epochSettleMs: 1,
-    });
-
-    const queue = new FakeQueue();
-    const results = createMemoryJobResultStore();
-    const worker = runSaasJobWorker({ queue, results, handlers, logger: silentLogger });
-    try {
-      const id = await queue.add(spec('reindex', { corpus: KNOWLEDGE_CORPUS }, 't1'));
-      await queue.deliver(id);
-
-      const outcome = await results.read(id);
-      expect(outcome?.ok).toBe(true);
-      expect((outcome?.output as { indexed: number } | undefined)?.indexed).toBe(4);
-      expect(indexed.length).toBe(4);
-
-      // Every page read ran as Postgres, inside the tenant's RLS scope.
-      const reads = recorded.filter((entry) => /FROM k_items/.test(entry.sql));
-      expect(reads.length).toBeGreaterThanOrEqual(3);
-      for (const read of reads) {
-        expect(read.sql.includes('?')).toBe(false);
-        expect(read.sql).toContain('$1');
-      }
-      expect(recorded.some((entry) => entry.sql.startsWith('SELECT set_config') && entry.params[1] === 't1')).toBe(true);
-    } finally {
-      await worker.stop();
-      await storage.close();
-      resetSaasPool();
-    }
-  });
-});
-
-describe('runSaasJobWorker + createSaasJobBroker compose across the process boundary', () => {
-  it('enqueue on the broker, execute on the worker, await the result anywhere', async () => {
-    const queue = new FakeQueue();
-    const results: JobResultStore = createMemoryJobResultStore();
-    // The "API node": enqueues and awaits, never executes.
-    const broker = createSaasJobBroker({ queue, results, logger: silentLogger, pollMs: 5 });
-    // The "worker process": executes, shares only the store + queue.
-    const worker = runSaasJobWorker({
-      queue,
-      results,
-      handlers: { sum: (p) => (p as { a: number; b: number }).a + (p as { a: number; b: number }).b },
-      logger: silentLogger,
-    });
-
-    const id = await broker.enqueue({ kind: 'sum', tenantId: 't1', payload: { a: 2, b: 40 } });
-    const completion = broker.awaitCompletion(id, { timeoutMs: 1000 });
-    await queue.deliver(id);
-
-    expect(await completion).toMatchObject({ jobId: id, ok: true, output: 42 });
-    broker.close();
-    await worker.stop();
-  });
-});

package/src/server/drivers/saasJobWorker.ts

@@ -1,241 +0,0 @@
-/**
- * The SaaS job WORKER runtime (08 §4a) — the far side of {@link
- * createSaasJobBroker}.
- *
- * The broker can `enqueue` and `awaitCompletion` across processes, but nothing
- * it ships EXECUTES a job: in the SaaS profile the enqueuer (an API node) and
- * the executor (a worker process) are deliberately different processes, which
- * is the whole point of a stateless multi-worker deployment. This module is the
- * executor. It leases payloads from the injected {@link SaasQueueBackend}
- * consumer side, runs the same `JobHandler` map the local broker runs, and
- * records the outcome in the shared {@link JobResultStore} — the one place every
- * API node reads through. The local {@link createLocalJobBroker} does the same
- * three things in one process; here they are split across the network, and the
- * result store is what keeps the completion contract identical.
- *
- * Four properties carry the correctness, and each is a failure mode a naive
- * "just run the handler" worker gets wrong in production:
- *
- *   * **Exactly-once EXECUTION, not just recording.** At-least-once delivery
- *     means the backend WILL redeliver a job whose worker died before acking.
- *     A handler with side effects (a charge, an email, an S3 write) run twice is
- *     a bug. So before running, the worker reads the store: a terminal row for
- *     this id means the job already settled (a crash after recording, or an
- *     external cancel) and the handler is SKIPPED — the lease is acked and the
- *     job disappears. Combined with first-writer-wins recording, a job executes
- *     to completion exactly once; only a crash in the narrow window between
- *     "handler returned" and "row written" re-runs it, and that window is the
- *     same one the local broker documents.
- *
- *   * **Record BEFORE ack.** The backend's own `completed` event (what wakes a
- *     waiting API node from its poll) fires when the lease is acked. If the
- *     worker acked first and recorded second, a waiter woken by that event would
- *     read the store a beat too early and see nothing. So the durable row is
- *     written first, always. See {@link recordWorkerResult}.
- *
- *   * **Cooperative cancel across processes.** `broker.cancel(id)` on an API
- *     node writes a `cancelled` row; the worker running that job is in another
- *     process and never saw the call. It learns of it by watching the shared
- *     store — a low-frequency scan of its in-flight ids — and aborts the
- *     handler's signal when a row appears under it. Same semantics as the local
- *     broker's per-job `AbortController`, achieved through the store instead of
- *     a shared heap.
- *
- *   * **Drain, don't abort, on shutdown.** A deploy sends SIGTERM; a worker that
- *     dropped its leases would strand every in-flight job until the lease
- *     timeout. {@link SaasJobWorker.stop} instead stops accepting new leases and
- *     lets the jobs already running finish (the lease timeout is the backstop
- *     for a handler that ignores the drain, never the mechanism).
- */
-import { AppError } from '../errors';
-import { defaultJobLogger } from '../jobs/logger';
-import type { JobLogger } from '../jobs/types';
-import type { JobHandler, JobId } from './jobBroker';
-import { type JobResultStore, recordWorkerResult } from './jobResults';
-import type { LeaseControl, LeasedJob, SaasQueueBackend } from './saasJobBroker';
-
-export interface SaasJobWorkerOptions {
-  /**
-   * The deployment's queue, which MUST expose the consumer-side `lease` — a
-   * producer-only backend cannot back a worker, and that is a boot error, not a
-   * silent no-op that leases nothing.
-   */
-  queue: SaasQueueBackend;
-  /** Shared across every API node and worker — the completion source of truth. */
-  results: JobResultStore;
-  /** The same handler map shape the local broker takes — `kind` selects one. */
-  handlers: Readonly<Record<string, JobHandler>>;
-  logger?: JobLogger;
-  /** Max jobs run through handlers concurrently. Default 4. */
-  concurrency?: number;
-  /**
-   * How often the worker scans the shared store for external cancellations of
-   * its in-flight jobs. Default 1000ms. Lower = faster cancel response at the
-   * cost of more reads; a push from {@link SaasQueueBackend.subscribe}, when the
-   * backend offers one, triggers an immediate scan on top of the interval.
-   */
-  cancelPollMs?: number;
-}
-
-export interface SaasJobWorker {
-  /**
-   * Drain and release. Stops accepting new leases, waits for the jobs already
-   * running to settle, then clears the cancel scan and any subscription.
-   * Idempotent — a second call (or a call racing SIGTERM) resolves against the
-   * same drain.
-   */
-  stop(): Promise<void>;
-}
-
-/** No handler registered for a leased kind — see {@link runSaasJobWorker}. */
-export const JOB_HANDLER_UNREGISTERED = 'JOB_HANDLER_UNREGISTERED';
-
-const errorMessage = (error: unknown): string =>
-  error instanceof Error ? error.message : typeof error === 'string' ? error : 'unknown error';
-
-export const runSaasJobWorker = (options: SaasJobWorkerOptions): SaasJobWorker => {
-  const logger = options.logger ?? defaultJobLogger;
-  const concurrency = options.concurrency ?? 4;
-  const cancelPollMs = options.cancelPollMs ?? 1000;
-
-  if (typeof options.queue.lease !== 'function')
-    throw new AppError('This SaasQueueBackend has no `lease` — it cannot back a worker (producer-only queue).', {
-      code: 'SAAS_QUEUE_NOT_CONSUMABLE',
-      status: 500,
-      category: 'config',
-      isOperational: false,
-    });
-
-  const { results } = options;
-
-  /** In-flight handlers by job id, so a cancel scan can abort the right one. */
-  const inFlight = new Map<JobId, AbortController>();
-
-  /**
-   * A row appearing under an in-flight id can only have been written by another
-   * process — this worker records a job's outcome AFTER the handler settles, so
-   * while a handler runs its id is unrecorded here. Any row it finds is an
-   * external `cancel`, and the handler's signal is aborted so a cancel-aware job
-   * actually stops rather than running to completion under a doomed lease.
-   */
-  /**
-   * `scanning` is not belt-and-braces. The store answers over a network now, so
-   * a scan can outlive its own interval tick; without the guard, a store slower
-   * than `cancelPollMs` would stack one concurrent scan per tick and issue a
-   * read storm precisely when it is already struggling.
-   */
-  let scanning = false;
-  const scanForCancellations = async (): Promise<void> => {
-    if (inFlight.size === 0 || scanning) return;
-    scanning = true;
-    try {
-      for (const [id, controller] of inFlight) {
-        if (controller.signal.aborted) continue;
-        // Re-check membership: the handler may have settled and been deleted
-        // while an earlier id's read was in flight.
-        if (!inFlight.has(id)) continue;
-        if (await results.read(id)) controller.abort(new DOMException(`Job ${id} cancelled`, 'AbortError'));
-      }
-    } finally {
-      scanning = false;
-    }
-  };
-
-  /** The interval and the queue subscription are both sync callers. */
-  const scanDetached = (): void => {
-    void scanForCancellations().catch((error) => {
-      logger.error('SaaS worker cancellation scan failed to read the result store', error);
-    });
-  };
-
-  const cancelTimer = setInterval(scanDetached, cancelPollMs);
-  // A bare interval keeps a Bun/Node process alive on its own; the worker's
-  // liveness is the lease loop, not this scan, so let the process exit when the
-  // real work is done.
-  (cancelTimer as { unref?: () => void }).unref?.();
-  const detachSubscribe = options.queue.subscribe?.(scanDetached) ?? null;
-
-  const process = async (job: LeasedJob): Promise<void> => {
-    // Exactly-once execution: a terminal row already under this id means the job
-    // settled elsewhere (a crash after recording, or a cancel before we leased
-    // it). Do NOT re-run a side-effecting handler — ack the lease and move on.
-    if (await results.read(job.id)) return;
-
-    const handler = options.handlers[job.kind];
-    if (!handler) {
-      // Fail closed and terminally. Redelivering forever on a missing handler is
-      // a poison-pill loop; a recorded failure lets a waiter see the truth and
-      // stops the churn. (A rolling deploy where only new workers know a new
-      // kind is the deployment's ordering problem, not a reason to spin.)
-      logger.error(`SaaS worker leased kind '${job.kind}' with no registered handler`, { jobId: job.id });
-      await recordWorkerResult(results, {
-        jobId: job.id,
-        ok: false,
-        error: `${JOB_HANDLER_UNREGISTERED}: '${job.kind}'`,
-        tenantId: job.tenantId,
-        instanceId: job.instanceId,
-      });
-      return;
-    }
-
-    const controller = new AbortController();
-    inFlight.set(job.id, controller);
-    // Catch a cancel that landed between the dedup read and now — the scan only
-    // sees ids already in `inFlight`, so close the gap by checking once here.
-    if (await results.read(job.id)) controller.abort(new DOMException(`Job ${job.id} cancelled`, 'AbortError'));
-
-    try {
-      const output = await handler(job.payload, {
-        jobId: job.id,
-        tenantId: job.tenantId,
-        instanceId: job.instanceId,
-        signal: controller.signal,
-        logger,
-      });
-      // First-writer-wins in the store means a late normal return cannot
-      // overwrite a `cancelled` row an API node already wrote — the store, not a
-      // local flag, is the arbiter of the settle race.
-      await recordWorkerResult(results, {
-        jobId: job.id,
-        ok: true,
-        output,
-        tenantId: job.tenantId,
-        instanceId: job.instanceId,
-      });
-    } catch (error) {
-      await recordWorkerResult(results, {
-        jobId: job.id,
-        ok: false,
-        error: errorMessage(error),
-        tenantId: job.tenantId,
-        instanceId: job.instanceId,
-      });
-    } finally {
-      inFlight.delete(job.id);
-    }
-    // Resolving acks the lease. The outcome is already durable, so a redelivery
-    // after this point (a crash mid-ack) is caught by the dedup read above.
-  };
-
-  // The non-null assertion is guarded by the SAAS_QUEUE_NOT_CONSUMABLE throw above. It
-  // carries no suppression comment on purpose: `lint/style/noNonNullAssertion` is off in
-  // this repo's config now, so the suppression that used to sit here was itself the error
-  // (`suppressions/unused`) and had been red on main.
-  const lease: LeaseControl = options.queue.lease!(process, { concurrency });
-
-  let stopping: Promise<void> | null = null;
-  return {
-    stop(): Promise<void> {
-      if (stopping) return stopping;
-      stopping = (async () => {
-        clearInterval(cancelTimer);
-        detachSubscribe?.();
-        // Drain: `close` stops new leases and resolves once in-flight jobs
-        // settle. In-flight handlers are NOT aborted here — a graceful deploy
-        // lets them finish; the lease timeout backstops any that ignore it.
-        await lease.close();
-      })();
-      return stopping;
-    },
-  };
-};