@ultimat3/jobs 30.9.0 → 31.0.0

package/src/jobs-retention.ts

@@ -32,22 +32,39 @@ import {
  * to that one — so each statement alone would keep the steps forever. `SQL_JOBS_RETENTION_STEPS`
  * settles a held run after this commits.
  *
+ * Each window is its OWN branch, seeking its own index in `updated_at` order and stopping at its
+ * cutoff: one walk over all four states with a window per state stepped, every batch, over each
+ * `dead` row inside its 90 days that sat between the `done` rows past their 14 — 32–40 ms a batch
+ * in production. The two answers are merged, oldest first, to one batch; a row a branch locked
+ * past the batch's `limit` is released at commit, untouched.
+ *
  * `held` is a LATERAL probe with a `limit`, not an `exists`: the planner flattens an `exists` into
  * a semi-join and, at a 1,000-row batch, hashes the whole of `x_jobs` for it; the lateral form is one
  * `x_jobs_run_idx` lookup per deleted run (EXPLAIN, 63,000 rows, 2026-10-17).
  */
 export const SQL_JOBS_RETENTION_BATCH = `
-with doomed as (
-  select j.id, j.run_id from x_jobs j
-   where ((j.state = 'done'
-           and j.updated_at < statement_timestamp() - ($1::bigint * interval '1 millisecond'))
-       or (j.state in ('failed', 'dead', 'cancelled')
-           and j.updated_at < statement_timestamp() - ($2::bigint * interval '1 millisecond')))
+with done_rows as (
+  select j.id, j.run_id, j.updated_at from x_jobs j
+   where j.state = 'done'
+     and j.updated_at < statement_timestamp() - ($1::bigint * interval '1 millisecond')
      and not exists (select 1 from x_outbox o where o.id = j.id and o.published_at is null)
      and not exists (select 1 from x_backfills b where b.run_id = j.run_id and b.status = 'running')
    order by j.updated_at
    limit $3
      for update of j skip locked
+), ended_rows as (
+  select j.id, j.run_id, j.updated_at from x_jobs j
+   where j.state in ('failed', 'dead', 'cancelled')
+     and j.updated_at < statement_timestamp() - ($2::bigint * interval '1 millisecond')
+     and not exists (select 1 from x_outbox o where o.id = j.id and o.published_at is null)
+     and not exists (select 1 from x_backfills b where b.run_id = j.run_id and b.status = 'running')
+   order by j.updated_at
+   limit $3
+     for update of j skip locked
+), doomed as (
+  select id, run_id from (select * from done_rows union all select * from ended_rows) d
+   order by updated_at
+   limit $3
 ), gone as (
   delete from x_jobs where id in (select id from doomed) returning run_id
 ), held as (

package/src/leases.ts

@@ -5,7 +5,7 @@
 // gives its slot back by expiry rather than by cleanup nobody runs.
 
 import type { Clock } from '@ultimat3/core';
-import { systemClock } from '@ultimat3/core';
+import { liveClock } from '@ultimat3/core';
 import { nowMs } from './clock';
 
 /** A granted slot. `slot` plus `holder` is what renew and release are addressed by. */
@@ -13,6 +13,13 @@ export interface HeldLease {
   readonly key: string;
   readonly slot: number;
   readonly holder: string;
+  /**
+   * The grant's FENCING number, assigned by the store on every acquire (a takeover too) and
+   * greater than every one it assigned before — `withLock`'s token. The store's counter and never
+   * a client clock: a host whose clock runs fast would otherwise win "refuse a smaller token".
+   * Optional so a store written before it still holds `job.concurrency`; `withLock` refuses one.
+   */
+  readonly fence?: number;
 }
 
 export interface LeaseStore {
@@ -63,9 +70,11 @@ export const LEASE_SWEEP_INTERVAL_MS = 60_000;
  * "a second worker is refused" test is a real test rather than a pg-only one.
  */
 export function memoryLeaseStore(options: MemoryLeaseStoreOptions = {}): MemoryLeaseStore {
-  const clock = options.clock ?? systemClock;
+  const clock = options.clock ?? liveClock;
   type Slots = Map<number, { holder: string; expiresAt: number }>;
   const slots = new Map<string, Slots>();
+  // One counter per store, as the pg store's one sequence: increasing across every key.
+  let fences = 0;
 
   /**
    * The live slots of `key`, expired ones dropped — and the KEY dropped with its last slot. A
@@ -101,7 +110,8 @@ export function memoryLeaseStore(options: MemoryLeaseStoreOptions = {}): MemoryL
         if (held.has(slot)) continue;
         held.set(slot, { holder, expiresAt: nowMs(clock) + ttlMs });
         slots.set(key, held);
-        return Promise.resolve({ key, slot, holder });
+        fences += 1;
+        return Promise.resolve({ key, slot, holder, fence: fences });
       }
       return Promise.resolve(undefined);
     },

package/src/ledger-book.ts

@@ -0,0 +1,168 @@
+// Where `ledger()` writes: the `LedgerBook` seam, the memory book a test reads, and the book over an
+// app's own entity. The TABLE is the app's (axiom 8: its accounts, its retention, its tenancy); the
+// mechanism is one idempotent statement keyed by `(postingKey, line)`, so a replay writes nothing.
+
+import type { Driver, EntityCore, RowWrite } from '@ultimat3/entity';
+import type { MoneyValue } from '@ultimat3/schema';
+import { LedgerEntryInvalidError, LedgerPostingConflictError } from './ledger-errors';
+
+/** One stored leg. `(tenant, postingKey, line)` is unique: the identity a replay lands on. */
+export interface LedgerLine {
+  /** The posting's `orgId`: a key names one posting PER TENANT. Absent for an untenanted book. */
+  readonly tenant?: string | undefined;
+  readonly postingKey: string;
+  readonly line: number;
+  readonly account: string;
+  readonly amount: MoneyValue;
+}
+
+export interface LedgerBook {
+  /**
+   * Store one posting's lines, all or none, and answer how many THIS call wrote: every line, or
+   * `0` when the key is already stored with exactly these lines. A key stored with any other lines
+   * — another count, another line — is `X_LEDGER_POSTING_CONFLICT`, and nothing is written: the
+   * new lines beside the stored ones would unbalance the book. Atomic: never half a posting.
+   */
+  write(lines: readonly LedgerLine[]): Promise<number>;
+}
+
+const sameAmount = (a: MoneyValue, b: MoneyValue): boolean =>
+  a.minor === b.minor && a.currency === b.currency && (a.scale ?? null) === (b.scale ?? null);
+
+/**
+ * Refuse unless `stored` (the key's lines in the book) are exactly `incoming`. `fresh` is how many
+ * of `incoming` this write itself inserted: all of them is a new posting, none a replay — any
+ * other number means part of the key was already there, which is a conflict whatever it now reads.
+ */
+export function assertSamePosting(
+  incoming: readonly LedgerLine[],
+  stored: readonly Pick<LedgerLine, 'line' | 'account' | 'amount'>[],
+  fresh: number,
+): void {
+  const key = incoming[0]?.postingKey ?? '';
+  const conflict = (line: number | null): never => {
+    throw new LedgerPostingConflictError({
+      key,
+      stored: stored.length - fresh,
+      incoming: incoming.length,
+      line,
+    });
+  };
+  if ((fresh !== 0 && fresh !== incoming.length) || stored.length !== incoming.length) {
+    conflict(null);
+  }
+  const byLine = new Map(stored.map((row) => [row.line, row]));
+  for (const line of incoming) {
+    const held = byLine.get(line.line);
+    if (
+      held === undefined ||
+      held.account !== line.account ||
+      !sameAmount(held.amount, line.amount)
+    ) {
+      conflict(line.line);
+    }
+  }
+}
+
+/** The memory book, plus what a test reads back. */
+export interface MemoryLedgerBook extends LedgerBook {
+  lines(): readonly LedgerLine[];
+  /** One account's balance in one currency, in minor units. */
+  balance(account: string, currency: string): number;
+}
+
+const lineKey = (line: Pick<LedgerLine, 'tenant' | 'postingKey' | 'line'>): string =>
+  `${line.tenant ?? ''}\u0000${line.postingKey}\u0000${line.line}`;
+
+/** Which posting a line belongs to: one per (tenant, key). A posting id, not a credential. */
+const postingOf = (line: Pick<LedgerLine, 'tenant' | 'postingKey'>): string =>
+  `${line.tenant ?? ''}\u0000${line.postingKey}`;
+
+export function memoryLedgerBook(): MemoryLedgerBook {
+  const stored = new Map<string, LedgerLine>();
+  return {
+    async write(lines) {
+      const first = lines[0];
+      const posting = first === undefined ? undefined : postingOf(first);
+      const held = [...stored.values()].filter((line) => postingOf(line) === posting);
+      // Judged before any write: a refused posting leaves the book as it was.
+      if (held.length > 0) {
+        assertSamePosting(lines, held, 0);
+        return 0;
+      }
+      for (const line of lines) stored.set(lineKey(line), line);
+      return lines.length;
+    },
+    lines: () => [...stored.values()],
+    balance: (account, currency) =>
+      [...stored.values()]
+        .filter((line) => line.account === account && line.amount.currency === currency)
+        .reduce((net, line) => net + line.amount.minor, 0),
+  };
+}
+
+export interface EntityLedgerBookOptions<Row extends LedgerLine> {
+  /** The driver the app's `database()` reads through, so tenancy and row observers hold. */
+  readonly driver: Driver;
+  /**
+   * The app's lines entity: `postingKey` text, `line` integer, `account` text, `amount` money, a
+   * UNIQUE index (or the primary key) on `(postingKey, line)` — on `(<tenant>, postingKey, line)`
+   * when the entity has a tenant column, which the posting's `orgId` fills — and whatever else it
+   * keeps defaulted. Without that unique Postgres refuses the statement (`42P10`).
+   */
+  readonly entity: EntityCore<Row>;
+}
+
+/**
+ * The book over an app entity: in ONE transaction, `insert … on conflict do nothing`, then the
+ * key's stored lines read back and judged (`assertSamePosting`) — a refusal rolls this write's
+ * lines back, and a concurrent write of the same key with other lines waits on the first's rows,
+ * then meets them committed and is refused.
+ */
+export function entityLedgerBook<Row extends LedgerLine>(
+  options: EntityLedgerBookOptions<Row>,
+): LedgerBook {
+  return {
+    write(lines) {
+      const tenant = options.entity.$tenantColumn;
+      const first = lines[0];
+      // A tenanted book holds one posting per (tenant, key): the org is the row's, never inferred.
+      if (tenant !== null && (first === undefined || first.tenant === undefined)) {
+        throw new LedgerEntryInvalidError({
+          key: first?.postingKey ?? '',
+          index: null,
+          field: 'orgId',
+          got: first?.tenant,
+        });
+      }
+      const scope: Record<string, unknown> = tenant === null ? {} : { [tenant]: first?.tenant };
+      // The lines ARE rows of `Row` minus its defaulted columns, which the statement fills.
+      const rows = lines.map(({ tenant: _, ...line }) => ({
+        ...scope,
+        ...line,
+      })) as unknown as readonly RowWrite<Row>[];
+      const target = (
+        tenant === null ? ['postingKey', 'line'] : [tenant, 'postingKey', 'line']
+      ) as (keyof Row & string)[];
+      const repo = options.driver.repo(options.entity);
+      return options.driver.transactor().run(async (tx) => {
+        const written = await repo.upsertAll(rows, { onConflict: target, onMatch: 'nothing', tx });
+        const held = await repo.findMany({
+          // The tenant named, not left to the run's scope: outside a job no scope exists, and an
+          // unscoped tenanted read is refused (`X_TENANCY_UNSCOPED`).
+          where: [
+            { column: 'postingKey', op: 'eq', value: first?.postingKey },
+            ...(tenant === null
+              ? []
+              : [{ column: tenant, op: 'eq' as const, value: first?.tenant }]),
+          ],
+          // One past what a match holds: a longer stored posting is seen as longer, never cut.
+          limit: lines.length + 1,
+          tx,
+        });
+        assertSamePosting(lines, held.rows, written.length);
+        return written.length;
+      });
+    },
+  };
+}

package/src/ledger-errors.ts

@@ -0,0 +1,48 @@
+// The two `X_LEDGER_*` refusals a posting can end on — both about the POSTING, raised before a line
+// is written, so a refused posting leaves the book exactly as it was. Amounts and accounts are the
+// app's own strings and integers; nothing `unknown` is interpolated (`describeValue` renders those).
+
+import { UltimateError } from '@ultimat3/core';
+import { describeValue } from '@ultimat3/schema';
+import './error-codes';
+
+/** One entry, or the posting's shape, is not something a balanced book can hold. */
+export class LedgerEntryInvalidError extends UltimateError {
+  constructor(input: { key: string; index: number | null; field: string; got: unknown }) {
+    const where = input.index === null ? 'the posting' : `entries[${input.index}]`;
+    super({
+      code: 'X_LEDGER_ENTRY_INVALID',
+      cause: `ledger posting ${JSON.stringify(input.key)}: ${where}.${input.field} is ${describeValue(input.got)}`,
+      fix: `{ account: 'cash', amount: { minor: 1299, currency: 'EUR' } }   # ${where} in the enqueue of the ledger job: a named account, integer minor units (fromDecimal('12.99', 'EUR') from @ultimat3/money), at least two entries`,
+      meta: { key: input.key, index: input.index, field: input.field },
+    });
+  }
+}
+
+/** The entries of one currency do not net to zero. `net` is that currency's sum, in minor units. */
+export class LedgerUnbalancedError extends UltimateError {
+  constructor(input: { key: string; currency: string; net: number }) {
+    super({
+      code: 'X_LEDGER_UNBALANCED',
+      cause: `ledger posting ${JSON.stringify(input.key)} nets ${input.net} minor units of ${input.currency}, not zero — every currency's entries must sum to zero`,
+      fix: `{ account: '<balancing account>', amount: { minor: ${-input.net}, currency: '${input.currency}' } }   # one more entry in the posting's entries, or correct the amounts`,
+      meta: { key: input.key, currency: input.currency, net: input.net },
+    });
+  }
+}
+
+/** A key already stored with other lines: writing beside them would unbalance the book. */
+export class LedgerPostingConflictError extends UltimateError {
+  constructor(input: { key: string; stored: number; incoming: number; line: number | null }) {
+    const differs =
+      input.line === null
+        ? `holds ${input.stored} line(s) and this posting has ${input.incoming}`
+        : `holds another line ${input.line}`;
+    super({
+      code: 'X_LEDGER_POSTING_CONFLICT',
+      cause: `ledger posting ${JSON.stringify(input.key)} is already in the book and ${differs} — a key names one posting, and its lines never change`,
+      fix: `{ key: ${JSON.stringify(`${input.key}:adjust-1`)}, entries: [ … ] }   # the posting in the enqueue of the ledger job: a correction under a new key, or the entries exactly as first posted`,
+      meta: { key: input.key, stored: input.stored, incoming: input.incoming, line: input.line },
+    });
+  }
+}

package/src/ledger.ts

@@ -0,0 +1,145 @@
+// `ledger()`: double-entry postings as a FACTORY over `job()`, never a ninth primitive. A posting
+// is balanced Money entries under one key; the job refuses an unbalanced or float posting before a
+// line is written, then writes every line in ONE durable step through the app's `LedgerBook` —
+// idempotent by `(key, line)`, so a retry, a replay or a second enqueue of one key posts it once.
+
+import type { MoneyValue, StandardSchemaV1 } from '@ultimat3/schema';
+import { isCurrencyCode, isMoneyScale, t } from '@ultimat3/schema';
+import type { DurationInput } from './clock';
+import type { JobHandle } from './job';
+import { job } from './job';
+import type { LedgerBook, LedgerLine } from './ledger-book';
+import { LedgerEntryInvalidError, LedgerUnbalancedError } from './ledger-errors';
+import type { RetryPolicy } from './retry';
+import { DEFAULT_RETRY } from './retry';
+import type { JobTenant } from './tenant';
+
+/** One leg: an account the app names and a signed amount. Debit or credit is the sign's, not ours. */
+export interface LedgerEntry {
+  readonly account: string;
+  readonly amount: MoneyValue;
+}
+
+/** What the job is enqueued with. `key` is the posting's identity — a payment id, a transfer id. */
+export interface LedgerPosting {
+  readonly key: string;
+  readonly entries: readonly LedgerEntry[];
+  /** The tenant the posting belongs to, read by `tenant: (p) => p.orgId ?? …` when the app has one. */
+  readonly orgId?: string | undefined;
+}
+
+export interface LedgerDefinition {
+  /** REQUIRED: a durable key — the queue row and the idempotency key carry it. */
+  readonly name: string;
+  readonly tenant: JobTenant<LedgerPosting>;
+  /** Where lines land — a thunk, read per run (`entityLedgerBook`, or the app's own). */
+  readonly book: () => LedgerBook;
+  readonly queue?: string;
+  readonly retry?: RetryPolicy;
+  readonly timeout?: DurationInput;
+}
+
+/** What one run answers: the lines the posting has, and how many THIS run wrote. */
+export interface LedgerReport {
+  readonly key: string;
+  readonly lines: number;
+  readonly written: number;
+}
+
+/** A posting's legs: at least two, since one entry can only balance at zero, which moves nothing. */
+export const MIN_LEDGER_ENTRIES = 2;
+
+/**
+ * Every rule `assertBalancedPosting` can say without adding anything up: a key, at least
+ * `MIN_LEDGER_ENTRIES` legs, a named account, a Money amount. Stated in the schema so a sample an
+ * exercise pass or a client draws from it is a posting the run could accept — only the per-currency
+ * sum, which no schema can express, is left to the run.
+ */
+const postingSchema: StandardSchemaV1<unknown, LedgerPosting> = t.object({
+  key: t.string.min(1),
+  entries: t.array(t.object({ account: t.string.min(1), amount: t.money }), {
+    min: MIN_LEDGER_ENTRIES,
+  }),
+  orgId: t.string.optional(),
+});
+
+function assertEntry(key: string, entry: LedgerEntry, index: number): void {
+  const refuse = (field: string, got: unknown): never => {
+    throw new LedgerEntryInvalidError({ key, index, field, got });
+  };
+  if (typeof entry.account !== 'string' || entry.account.trim() === '') {
+    refuse('account', entry.account);
+  }
+  const { minor, currency, scale } = entry.amount;
+  if (!Number.isSafeInteger(minor)) refuse('amount.minor', minor);
+  if (!isCurrencyCode(currency)) refuse('amount.currency', currency);
+  if (scale !== undefined && !isMoneyScale(scale)) refuse('amount.scale', scale);
+}
+
+/**
+ * Refuse a posting that is not balanced Money, before anything is written. Exported so the action
+ * that enqueues one refuses it at the request — the job refuses it again, since a queue row is input.
+ * Within one currency every entry carries the same `scale`: a sum across two scales is not a sum.
+ */
+export function assertBalancedPosting(posting: LedgerPosting): void {
+  const { key } = posting;
+  if (typeof key !== 'string' || key === '') {
+    throw new LedgerEntryInvalidError({ key: String(key), index: null, field: 'key', got: key });
+  }
+  if (posting.entries.length < MIN_LEDGER_ENTRIES) {
+    throw new LedgerEntryInvalidError({
+      key,
+      index: null,
+      field: 'entries.length',
+      got: posting.entries.length,
+    });
+  }
+  const sums = new Map<string, { net: number; scale: number | undefined }>();
+  posting.entries.forEach((entry, index) => {
+    assertEntry(key, entry, index);
+    const { minor, currency, scale } = entry.amount;
+    const seen = sums.get(currency);
+    if (seen !== undefined && seen.scale !== scale) {
+      throw new LedgerEntryInvalidError({ key, index, field: 'amount.scale', got: scale });
+    }
+    const net = (seen?.net ?? 0) + minor;
+    // Safe integers summed past 2^53 stop being exact: refused rather than rounded.
+    if (!Number.isSafeInteger(net)) {
+      throw new LedgerEntryInvalidError({ key, index, field: 'amount.minor', got: minor });
+    }
+    sums.set(currency, { net, scale });
+  });
+  for (const [currency, { net }] of [...sums].sort(([a], [b]) => (a < b ? -1 : 1))) {
+    if (net !== 0) throw new LedgerUnbalancedError({ key, currency, net });
+  }
+}
+
+/** The posting as the lines a book stores: one per entry, numbered from 0 in the posting's order. */
+export const ledgerLines = (posting: LedgerPosting): readonly LedgerLine[] =>
+  posting.entries.map((entry, line) => ({
+    ...(posting.orgId === undefined ? {} : { tenant: posting.orgId }),
+    postingKey: posting.key,
+    line,
+    account: entry.account,
+    amount: entry.amount,
+  }));
+
+export function ledger(definition: LedgerDefinition): JobHandle<LedgerPosting> {
+  return job<LedgerPosting>({
+    name: definition.name,
+    input: postingSchema,
+    tenant: definition.tenant,
+    // One live run per posting; the book's `(postingKey, line)` holds it after the run is gone.
+    idempotencyKey: (posting) => `${definition.name}:${posting.key}`,
+    retry: definition.retry ?? DEFAULT_RETRY,
+    ...(definition.queue === undefined ? {} : { queue: definition.queue }),
+    ...(definition.timeout === undefined ? {} : { timeout: definition.timeout }),
+    run: async ({ step, input }): Promise<LedgerReport> => {
+      assertBalancedPosting(input);
+      const lines = ledgerLines(input);
+      // ONE step: every line or none, and a replay answers the persisted count without a write.
+      const written = await step.run('post', () => definition.book().write(lines));
+      return { key: input.key, lines: lines.length, written };
+    },
+  });
+}

package/src/limits.ts

@@ -12,7 +12,7 @@
 // what `job.concurrency` is enforced with.
 
 import type { Clock } from '@ultimat3/core';
-import { assert, finiteCount, systemClock } from '@ultimat3/core';
+import { assert, finiteCount, liveClock } from '@ultimat3/core';
 import { nowMs } from './clock';
 
 export interface RateLimit {
@@ -114,6 +114,28 @@ const SWEEP_EVERY_MS = 60_000;
  */
 export const DEFAULT_MAX_LIMIT_TENANTS = 20_000;
 
+/**
+ * One tenant's rate window: start stamps in the order taken, the spent ones before `head`. A queue
+ * with a head index, so a stamp leaving is one increment and never a copy of the window.
+ */
+interface StartWindow {
+  stamps: number[];
+  head: number;
+}
+
+/** Stamps still in the window. */
+const live = (window: StartWindow): number => window.stamps.length - window.head;
+
+/**
+ * Drop the spent prefix once it is both large and most of the array: a copy amortised over at least
+ * as many acquires as it moves, so the window costs O(1) a claim and its memory stays O(live).
+ */
+const compact = (window: StartWindow): void => {
+  if (window.head < 1_024 || window.head * 2 < window.stamps.length) return;
+  window.stamps = window.stamps.slice(window.head);
+  window.head = 0;
+};
+
 /** Tenant key resolution. Structural on purpose: `jobs` must not import the auth types. */
 export function tenantKeyFrom(actor: { readonly orgId?: string } | undefined): string {
   return actor?.orgId ?? NO_TENANT;
@@ -121,7 +143,7 @@ export function tenantKeyFrom(actor: { readonly orgId?: string } | undefined): s
 
 export function concurrencyLimiter(
   config: LimitConfig,
-  clock: Clock = systemClock,
+  clock: Clock = liveClock,
   options: { readonly maxTenants?: number | undefined } = {},
 ): Limiter {
   const requested = options.maxTenants ?? DEFAULT_MAX_LIMIT_TENANTS;
@@ -169,7 +191,7 @@ export function concurrencyLimiter(
   // `inFlight` had nothing to answer it from and silently widened to "this tenant, everywhere".
   // Deleted at zero by `bump`, exactly like the two above, so it is bounded by live leases.
   const byQueueTenant = new Map<string, number>();
-  const starts = new Map<string, number[]>();
+  const starts = new Map<string, StartWindow>();
   const refusals = new Map<string, { reason: LimitReason; atMs: number }>();
   let global = 0;
   let lastSweepMs = Number.NEGATIVE_INFINITY;
@@ -202,14 +224,16 @@ export function concurrencyLimiter(
    */
   const sweep = (at: number): void => {
     lastSweepMs = at;
-    for (const [tenant, stamps] of starts) {
-      if (stamps.length === 0 || (stamps.at(-1) ?? 0) <= at - rateWindowMs) starts.delete(tenant);
+    for (const [tenant, window] of starts) {
+      if (live(window) === 0 || (window.stamps.at(-1) ?? 0) <= at - rateWindowMs) {
+        starts.delete(tenant);
+      }
     }
     for (const [key, refusal] of refusals) {
       if (refusal.atMs <= at - REFUSAL_TTL_MS) refusals.delete(key);
     }
     if (starts.size > maxTenants) {
-      const emptiest = [...starts.entries()].sort((a, b) => a[1].length - b[1].length);
+      const emptiest = [...starts.entries()].sort((a, b) => live(a[1]) - live(b[1]));
       for (const [tenant] of emptiest) {
         if (starts.size <= evictTo) break;
         starts.delete(tenant);
@@ -237,12 +261,24 @@ export function concurrencyLimiter(
   const rateBlocked = (tenant: string, at: number): boolean => {
     const rate = config.ratePerTenant;
     if (rate === undefined) return false;
-    const window = (starts.get(tenant) ?? []).filter((stamp) => stamp > at - rate.windowMs);
-    // An empty window is a tenant with no rate state at all — deleted rather than stored as `[]`,
-    // which is what kept one array per org that ever ran a job.
-    if (window.length === 0) starts.delete(tenant);
-    else starts.set(tenant, window);
-    return window.length >= rate.limit;
+    const window = starts.get(tenant);
+    if (window === undefined) return rate.limit <= 0;
+    // Spent stamps leave from the HEAD, never by copying the window: a filter per acquire made a
+    // window of n stamps O(n) a claim — 20k acquires were 3.9 s of event loop.
+    while (
+      window.head < window.stamps.length &&
+      (window.stamps[window.head] ?? 0) <= at - rate.windowMs
+    ) {
+      window.head += 1;
+    }
+    // An empty window is a tenant with no rate state at all — deleted rather than kept, which is
+    // what kept one array per org that ever ran a job.
+    if (live(window) === 0) {
+      starts.delete(tenant);
+      return rate.limit <= 0;
+    }
+    compact(window);
+    return live(window) >= rate.limit;
   };
 
   return {
@@ -270,7 +306,9 @@ export function concurrencyLimiter(
       bump(byTenant, tenant, 1);
       bump(byQueueTenant, refusalKey, 1);
       if (config.ratePerTenant !== undefined) {
-        starts.set(tenant, [...(starts.get(tenant) ?? []), at]);
+        const window = starts.get(tenant);
+        if (window === undefined) starts.set(tenant, { stamps: [at], head: 0 });
+        else window.stamps.push(at);
       }
       refusals.delete(refusalKey);
       maybeSweep(at);
@@ -296,12 +334,12 @@ export function concurrencyLimiter(
           if (!free() || config.ratePerTenant === undefined) return;
           // ONE stamp of this lease's own instant: two leases taken in the same millisecond wrote
           // equal stamps, and either is this one. Absent when a sweep already dropped the window.
-          const stamps = starts.get(tenant);
-          const index = stamps?.lastIndexOf(at) ?? -1;
-          if (stamps === undefined || index === -1) return;
-          const rest = stamps.filter((_stamp, position) => position !== index);
-          if (rest.length === 0) starts.delete(tenant);
-          else starts.set(tenant, rest);
+          const window = starts.get(tenant);
+          const index = window?.stamps.lastIndexOf(at) ?? -1;
+          if (window === undefined || index < window.head) return;
+          // Searched from the tail, where a lease handed back unrun almost always is.
+          window.stamps.splice(index, 1);
+          if (live(window) === 0) starts.delete(tenant);
         },
       };
     },

package/src/lock-errors.ts

@@ -0,0 +1,54 @@
+// The three `X_LOCK_*` refusals `withLock()` ends on: not taken within `wait`, lost mid-work, and
+// nothing to hold it in. Codes, titles and retry classes are `error-codes.ts`'s; the key is printed
+// JSON-quoted, since an app builds it from its own data (`invoice:${id}`).
+
+import { UltimateError } from '@ultimat3/core';
+import './error-codes';
+
+/** Another holder kept the key for the whole `wait` bound. Retryable: contention ends. */
+export class LockNotAcquiredError extends UltimateError {
+  constructor(input: { key: string; waitMs: number; holders: readonly string[] }) {
+    super({
+      code: 'X_LOCK_NOT_ACQUIRED',
+      cause: `withLock(${JSON.stringify(input.key)}) waited ${input.waitMs}ms and another holder (${input.holders.length === 0 ? 'released a moment ago' : input.holders.join(', ')}) kept the key throughout`,
+      fix: `withLock(${JSON.stringify(input.key)}, fn, { ttl, wait: '${Math.max(1, Math.ceil((input.waitMs * 2) / 1000))}s' })   # in the calling file: a longer wait:, or let the caller retry`,
+      meta: { key: input.key, waitMs: input.waitMs, holders: [...input.holders] },
+    });
+  }
+}
+
+/**
+ * A renewal was refused, or none landed within one `ttl`: the key may be someone else's now. The
+ * reason `fn`'s signal aborts with — a write fenced on the token is what stops the stale holder.
+ */
+export class LockLostError extends UltimateError {
+  constructor(input: { key: string; token: number; ttlMs: number }) {
+    super({
+      code: 'X_LOCK_LOST',
+      cause: `withLock(${JSON.stringify(input.key)}) lost the lock held as ${input.token}: no renewal landed within its ttl of ${input.ttlMs}ms, so another holder may have it`,
+      fix: `withLock(${JSON.stringify(input.key)}, fn, { ttl: '${Math.ceil((input.ttlMs * 4) / 1000)}s' })   # in the calling file: a longer ttl:, and fn hands lock.signal to every write it makes`,
+      meta: { key: input.key, token: input.token, ttlMs: input.ttlMs },
+    });
+  }
+}
+
+/**
+ * No job driver installed, one without a `leases` store, or a store that answers no `fence`: no
+ * fleet-wide place to hold a key behind a token the database assigned.
+ */
+export class LockStoreMissingError extends UltimateError {
+  constructor(input: { key: string; driver: string | undefined; fence?: boolean }) {
+    const why =
+      input.fence === true
+        ? 'the lease store granted the key with no fence, so there is no token a write can be refused by'
+        : input.driver === undefined
+          ? 'no job driver is installed'
+          : `the installed job driver "${input.driver}" has no leases`;
+    super({
+      code: 'X_LOCK_STORE_MISSING',
+      cause: `withLock(${JSON.stringify(input.key)}) has no lease store to lock in: ${why}`,
+      fix: 'jobs: postgresJobDriver()   # in apps/<app>/runtime.ts — or withLock(key, fn, { ttl, store }) in the calling file',
+      meta: { key: input.key, driver: input.driver ?? null },
+    });
+  }
+}