@ultimat3/action 1.2.0 → 3.0.0

package/src/idempotency-memory.ts

@@ -0,0 +1,148 @@
+/**
+ * The default idempotency store: process memory, bounded and swept. Correct for one web process
+ * and for tests, and refused at registration under a `shared` declaration — its `scope` says so.
+ * Shaped after `@ultimat3/http`'s `memoryRateLimitStore`, including the deliberate eviction order.
+ */
+import { uuid } from '@ultimat3/core';
+import type {
+  IdempotencyFailure,
+  IdempotencyRecord,
+  IdempotencyReservation,
+  IdempotencyScope,
+  IdempotencyStore,
+} from './idempotency';
+
+/**
+ * How long a key is remembered. A day is the window every payment API this shape exists to serve
+ * publishes, and it is a *bound*, not a promise of forever: a caller retrying two days later is
+ * making a new request, and the store says so by treating the record as missing.
+ */
+export const DEFAULT_IDEMPOTENCY_WINDOW_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * Hard bound on tracked keys. A key is `action:caller-supplied-string`, so its cardinality is the
+ * write rate, not the user count — at 500 idempotent writes a second an unbounded map is 43M
+ * immortal entries a day and an OOM. At ~250 bytes an entry this cap is a few megabytes, held.
+ */
+export const DEFAULT_MAX_IDEMPOTENCY_KEYS = 10_000;
+
+/** An idle store still sweeps this often, so a burst's records do not sit until the next one. */
+const SWEEP_EVERY_MS = 60_000;
+
+export interface MemoryIdempotencyStoreOptions {
+  readonly windowMs?: number | undefined;
+  readonly maxKeys?: number | undefined;
+  /** Injectable so a test can age a record without sleeping. */
+  readonly now?: (() => number) | undefined;
+}
+
+export class MemoryIdempotencyStore implements IdempotencyStore {
+  /** Declared, never inferred: this map is this process's and nothing else can reach it. */
+  readonly scope: IdempotencyScope = 'process';
+  readonly windowMs: number;
+  readonly #maxKeys: number;
+  readonly #evictTo: number;
+  readonly #now: () => number;
+  readonly #records = new Map<string, IdempotencyRecord>();
+  #lastSweepMs = Number.NEGATIVE_INFINITY;
+
+  constructor(options: MemoryIdempotencyStoreOptions = {}) {
+    this.windowMs = Math.max(1, Math.floor(options.windowMs ?? DEFAULT_IDEMPOTENCY_WINDOW_MS));
+    this.#maxKeys = Math.max(1, Math.floor(options.maxKeys ?? DEFAULT_MAX_IDEMPOTENCY_KEYS));
+    // Batched down to 90% so the eviction sort is paid once per 10% of the cap, not per write.
+    this.#evictTo = Math.max(1, Math.floor(this.#maxKeys * 0.9));
+    this.#now = options.now ?? ((): number => Date.now());
+  }
+
+  /** Records tracked right now — the bound, observable. */
+  get size(): number {
+    return this.#records.size;
+  }
+
+  reserve(key: string, requestHash: string): Promise<IdempotencyReservation> {
+    const nowMs = this.#now();
+    const existing = this.#records.get(key);
+    // Expired is missing. A record past the window answers exactly as a first-ever key does, so
+    // reclaiming it here is what makes the window mean something rather than being a comment.
+    if (existing !== undefined && !this.#expired(existing, nowMs)) {
+      return Promise.resolve({ record: existing, created: false });
+    }
+    const record: IdempotencyRecord = {
+      id: uuid(),
+      key,
+      requestHash,
+      status: 'in-flight',
+      value: undefined,
+      createdAt: nowMs,
+    };
+    this.#records.set(key, record);
+    this.#maintain(nowMs);
+    return Promise.resolve({ record, created: true });
+  }
+
+  /**
+   * Both settlements are FENCED on `in-flight`, as `SQL_IDEMPOTENCY_SETTLE` is and as
+   * `@ultimat3/jobs`' `SQL_ACK` is: a record past the window is reclaimed by the next caller, so a
+   * straggler from the reservation before it would otherwise overwrite a record it no longer owns
+   * and the next replay would answer one request with another's value. Both stores fence, or the
+   * guarantee is whichever store the deployment happens to install.
+   */
+  settle(key: string, value: unknown): Promise<void> {
+    const existing = this.#records.get(key);
+    if (existing?.status === 'in-flight') {
+      this.#records.set(key, { ...existing, status: 'settled', value });
+    }
+    return Promise.resolve();
+  }
+
+  fail(key: string, failure: IdempotencyFailure): Promise<void> {
+    const existing = this.#records.get(key);
+    if (existing?.status === 'in-flight') {
+      this.#records.set(key, { ...existing, status: 'failed', value: undefined, failure });
+    }
+    return Promise.resolve();
+  }
+
+  release(key: string): Promise<void> {
+    this.#records.delete(key);
+    return Promise.resolve();
+  }
+
+  get(key: string): Promise<IdempotencyRecord | undefined> {
+    const record = this.#records.get(key);
+    if (record === undefined || this.#expired(record, this.#now()))
+      return Promise.resolve(undefined);
+    return Promise.resolve(record);
+  }
+
+  #expired(record: IdempotencyRecord, nowMs: number): boolean {
+    return nowMs - record.createdAt >= this.windowMs;
+  }
+
+  /**
+   * Sweep, then evict — and the eviction order is part of the guarantee. Expired records go for
+   * free, because they already answer as missing. Only if that is not enough does the cap take
+   * live state, and then **`in-flight` records are the last to go**: one of those is the
+   * reservation that stops a concurrent duplicate from running the handler a second time, so
+   * dropping it is the double charge this store exists to prevent. That is the mirror of
+   * `memoryRateLimitStore` evicting the fullest bucket first — never swap either for an LRU.
+   */
+  #maintain(nowMs: number): void {
+    if (this.#records.size <= this.#maxKeys && nowMs - this.#lastSweepMs < SWEEP_EVERY_MS) return;
+    this.#lastSweepMs = nowMs;
+    for (const [key, record] of this.#records) {
+      if (this.#expired(record, nowMs)) this.#records.delete(key);
+    }
+    if (this.#records.size <= this.#maxKeys) return;
+    const settled = [...this.#records.entries()]
+      .filter(([, record]) => record.status !== 'in-flight')
+      .sort((a, b) => a[1].createdAt - b[1].createdAt);
+    for (const [key] of settled) {
+      if (this.#records.size <= this.#evictTo) break;
+      this.#records.delete(key);
+    }
+    // If nothing settled is left the map may still exceed the cap. That is deliberate and it is
+    // bounded by in-flight concurrency, not by the write rate — and every one of those records
+    // becomes sweepable the moment it ages past the window.
+  }
+}

package/src/idempotency-postgres.ts

@@ -0,0 +1,271 @@
+/**
+ * The shared idempotency store: one Postgres table, `insert … on conflict` for the atomicity.
+ * This is the store the memory default's own comment has promised since the primitive shipped —
+ * without it, `replicas: 3` means a retry that lands elsewhere re-runs a committed handler.
+ * Statements are spelled out so an agent can run the exact one it saw in a log.
+ */
+import { logger, uuid } from '@ultimat3/core';
+import type {
+  IdempotencyFailure,
+  IdempotencyRecord,
+  IdempotencyReservation,
+  IdempotencyScope,
+  IdempotencyStatus,
+  IdempotencyStore,
+} from './idempotency';
+import { DEFAULT_IDEMPOTENCY_WINDOW_MS } from './idempotency-memory';
+
+/**
+ * The one thing this store needs from the DB layer, declared structurally rather than imported.
+ * `@ultimat3/jobs` declares the same shape for the same reason: neither package owns the other's
+ * connection, and neither depends on a database package.
+ *
+ * **`Bun.sql` does not satisfy it** — verified against Bun 1.3.14: `Bun.sql.query` is `undefined`.
+ * `Bun.sql` is a tagged template whose positional form is `unsafe`, so `{ executor: Bun.sql }`
+ * would `TypeError` on the first reservation, which is the one call path that must not fail open.
+ * What satisfies it is a client that already speaks `(text, values)`, wrapped in one line —
+ * `@ultimat3/cli`'s `pgExecutorFor(client)` over `@ultimat3/db`'s `DbClient.query({ text, values })`
+ * is the framework's own — or a transaction handle, which is a client on its own connection.
+ */
+export interface PgExecutor {
+  query<R>(sql: string, params: readonly unknown[]): Promise<readonly R[]>;
+}
+
+/**
+ * The store's ONE install point, applied the way `SQL_JOBS_TABLE` is — by the boot, not by an app
+ * migration: `startQueue` runs both on every start, so `x dev`, the container's `web`/`worker` and
+ * the release-phase `ROLE=migrate` all apply it. `create table if not exists` is a no-op against a
+ * database that already has it, so a new column is added by `alter table … add column if not
+ * exists` and never by editing the `create`.
+ */
+export const SQL_IDEMPOTENCY_TABLE = `
+create table if not exists x_idempotency (
+  key          text        primary key,
+  id           uuid        not null,
+  request_hash text        not null,
+  status       text        not null default 'in-flight',
+  value        jsonb,
+  failure      jsonb,
+  created_at   timestamptz not null default now()
+);
+
+create index if not exists x_idempotency_created_at_idx on x_idempotency (created_at);
+`;
+
+/**
+ * The reservation, atomic in one statement. The `do update` fires ONLY for a row already outside
+ * the window — which answers as a missing one — so a returned row always means this caller owns
+ * the reservation and must run the handler. No row back means a live record exists and belongs to
+ * someone else.
+ */
+export const SQL_IDEMPOTENCY_RESERVE = `
+insert into x_idempotency (key, id, request_hash, status)
+values ($1, $2, $3, 'in-flight')
+on conflict (key) do update
+   set id           = excluded.id,
+       request_hash = excluded.request_hash,
+       status       = 'in-flight',
+       value        = null,
+       failure      = null,
+       created_at   = now()
+ where x_idempotency.created_at < now() - make_interval(secs => $4::double precision)
+returning key, id, request_hash, status, value, failure,
+          (extract(epoch from created_at) * 1000)::bigint as created_at
+`;
+
+export const SQL_IDEMPOTENCY_GET = `
+select key, id, request_hash, status, value, failure,
+       (extract(epoch from created_at) * 1000)::bigint as created_at
+  from x_idempotency
+ where key = $1
+   and created_at >= now() - make_interval(secs => $2::double precision)
+`;
+
+/**
+ * `and status = 'in-flight'` is a FENCE, not a filter — the one `@ultimat3/jobs`' `SQL_ACK` carries
+ * as `and state = 'running'`, for the same failure. A reservation whose window lapsed is reclaimed
+ * by the next caller (`do update` above), so a straggler from the first one arriving afterwards
+ * overwrote a record it no longer owned: the next replay under that key answered a retry with a
+ * value produced for a different request. `returning key` is what makes the refusal observable —
+ * an update matching no row is indistinguishable from one that matched, otherwise.
+ */
+export const SQL_IDEMPOTENCY_SETTLE = `
+update x_idempotency set status = 'settled', value = $2::jsonb, failure = null
+ where key = $1 and status = 'in-flight'
+returning key
+`;
+
+export const SQL_IDEMPOTENCY_FAIL = `
+update x_idempotency set status = 'failed', value = null, failure = $2::jsonb
+ where key = $1 and status = 'in-flight'
+returning key
+`;
+
+export const SQL_IDEMPOTENCY_RELEASE = `delete from x_idempotency where key = $1`;
+
+export const SQL_IDEMPOTENCY_PURGE = `
+delete from x_idempotency where created_at < now() - make_interval(secs => $1::double precision)
+`;
+
+interface IdempotencyRow {
+  readonly key: string;
+  readonly id: string;
+  readonly request_hash: string;
+  readonly status: string;
+  readonly value: unknown;
+  readonly failure: unknown;
+  /** `bigint`, which every Postgres client hands back as a string. */
+  readonly created_at: number | string;
+}
+
+export interface PostgresIdempotencyStoreOptions {
+  readonly executor: PgExecutor;
+  readonly windowMs?: number | undefined;
+}
+
+export interface PostgresIdempotencyStore extends IdempotencyStore {
+  readonly scope: IdempotencyScope;
+  readonly windowMs: number;
+  /**
+   * Delete every record past the window, and answer how many. The table is the one part of this
+   * store that does not bound itself — Postgres forgets nothing on its own — so an app runs this
+   * from a `task` on whatever cadence its write rate deserves.
+   */
+  purgeExpired(): Promise<number>;
+}
+
+/**
+ * **The boot installs this for you — an app declares the scope and nothing else.**
+ * `@ultimat3/cli`'s `startServices` builds a `PgExecutor` from the client it already resolved and
+ * calls `setIdempotencyStore(postgresIdempotencyStore({ executor }))` before `loadApp`, so the
+ * store is in place by the time `registerAction` evaluates a declaration against it. All an app
+ * owes is the one line `x new` scaffolds into `apps/web/server.ts`:
+ *
+ * ```ts
+ * configureIdempotency({ scope: 'shared' });
+ * ```
+ *
+ * Installing one by hand is for a host that boots the framework itself, and it needs a real
+ * `PgExecutor` — never `Bun.sql`, which has no `.query`. Wrap the client this process already
+ * opened, so a second pool is not opened against a URL the boot resolved once:
+ *
+ * ```ts
+ * const client = db();
+ * setIdempotencyStore(
+ *   postgresIdempotencyStore({
+ *     executor: { query: (text, values) => client.query({ text, values }) },
+ *   }),
+ * );
+ * ```
+ */
+export function postgresIdempotencyStore(
+  options: PostgresIdempotencyStoreOptions,
+): PostgresIdempotencyStore {
+  const windowMs = Math.max(1, Math.floor(options.windowMs ?? DEFAULT_IDEMPOTENCY_WINDOW_MS));
+  const windowSecs = windowMs / 1000;
+  const exec = options.executor;
+
+  const fetch = async (key: string): Promise<IdempotencyRecord | undefined> => {
+    const rows = await exec.query<IdempotencyRow>(SQL_IDEMPOTENCY_GET, [key, windowSecs]);
+    const row = rows[0];
+    return row === undefined ? undefined : toRecord(row);
+  };
+
+  return {
+    scope: 'shared',
+    windowMs,
+
+    async reserve(key, requestHash): Promise<IdempotencyReservation> {
+      // A bounded loop, not a `while (true)`: the only way the insert and the read can both come
+      // back empty is a concurrent `release`/`purgeExpired` deleting the row between them, and a
+      // caller losing that race twice is a store nobody should keep retrying against.
+      for (let attempt = 0; attempt < 3; attempt += 1) {
+        const claimed = await exec.query<IdempotencyRow>(SQL_IDEMPOTENCY_RESERVE, [
+          key,
+          uuid(),
+          requestHash,
+          windowSecs,
+        ]);
+        const row = claimed[0];
+        if (row !== undefined) return { record: toRecord(row), created: true };
+        const existing = await fetch(key);
+        if (existing !== undefined) return { record: existing, created: false };
+      }
+      // Reported as a fresh reservation rather than a throw would be wrong in the one direction
+      // that matters, so this is the honest answer: the caller sees the in-flight refusal.
+      return {
+        record: {
+          id: uuid(),
+          key,
+          requestHash,
+          status: 'in-flight',
+          value: undefined,
+          createdAt: Date.now(),
+        },
+        created: false,
+      };
+    },
+
+    async settle(key, value): Promise<void> {
+      const rows = await exec.query(SQL_IDEMPOTENCY_SETTLE, [key, JSON.stringify(value ?? null)]);
+      fenced(rows, key, 'settle');
+    },
+
+    async fail(key, failure: IdempotencyFailure): Promise<void> {
+      const rows = await exec.query(SQL_IDEMPOTENCY_FAIL, [key, JSON.stringify(failure)]);
+      fenced(rows, key, 'fail');
+    },
+
+    async release(key): Promise<void> {
+      await exec.query(SQL_IDEMPOTENCY_RELEASE, [key]);
+    },
+
+    get: fetch,
+
+    async purgeExpired(): Promise<number> {
+      const rows = await exec.query<{ readonly key: string }>(
+        `${SQL_IDEMPOTENCY_PURGE} returning key`,
+        [windowSecs],
+      );
+      return rows.length;
+    },
+  };
+}
+
+/**
+ * Logged, never thrown. A settlement lands after the handler has committed, so raising here would
+ * turn a durable write into the caller's error — the rule `withIdempotency` already follows for a
+ * store that refuses. An operator still has to see it: a fenced settle means this attempt's record
+ * belongs to another reservation, and the value this attempt produced is stored nowhere.
+ */
+function fenced(rows: readonly unknown[], key: string, statement: 'settle' | 'fail'): void {
+  if (rows.length > 0) return;
+  logger.warn('action.idempotency.settlement-fenced', { key, statement });
+}
+
+function toRecord(row: IdempotencyRow): IdempotencyRecord {
+  const failure = toFailure(row.failure);
+  return {
+    id: row.id,
+    key: row.key,
+    requestHash: row.request_hash,
+    status: row.status as IdempotencyStatus,
+    value: row.value,
+    ...(failure === undefined ? {} : { failure }),
+    createdAt: Number(row.created_at),
+  };
+}
+
+/** `jsonb` comes back as parsed JSON, so this is a shape check and never a second parse. */
+function toFailure(value: unknown): IdempotencyFailure | undefined {
+  if (typeof value !== 'object' || value === null) return undefined;
+  const record = value as Record<string, unknown>;
+  const code = record['code'];
+  const cause = record['cause'];
+  const fix = record['fix'];
+  if (typeof code !== 'string' || typeof cause !== 'string' || typeof fix !== 'string') {
+    return undefined;
+  }
+  const docs = record['docs'];
+  return { code, cause, fix, ...(typeof docs === 'string' ? { docs } : {}) };
+}

package/src/idempotency.ts

@@ -1,19 +1,50 @@
 /**
- * Idempotency for actions marked `idempotent`. A retried key replays the first
- * response; a concurrent duplicate is refused rather than run twice, because a
+ * Idempotency for actions marked `idempotent`: the store seam, where the deployment declares
+ * what it needs of one, and the replay-or-run gate. A retried key replays the first OUTCOME —
+ * a value or a failure — and a concurrent duplicate is refused rather than run twice, because a
  * double charge is worse than a 409.
  */
-import { uuid } from '@ultimat3/core';
-import { IdempotencyConflictError } from './errors';
+import { isUltimateError, logger } from '@ultimat3/core';
+import {
+  IdempotencyConflictError,
+  IdempotencyNotSharedError,
+  IdempotencyReplayedFailureError,
+} from './errors';
+import { MemoryIdempotencyStore } from './idempotency-memory';
 import { fingerprint } from './stable';
 
+/**
+ * Where a store's records live. Declared by the driver, never inferred — the same rule
+ * `@ultimat3/http`'s `RateLimitStore.scope` follows, and for a worse failure: N replicas each
+ * holding their own records means the retry that lands on replica B has never seen the key, so
+ * the handler runs a second time and the card is charged twice. Silently, with `x verify` green.
+ */
+export type IdempotencyScope = 'process' | 'shared';
+
+/**
+ * What a failed attempt left behind, flat enough to survive a database round trip. An `Error`
+ * object cannot: the record outlives the process that produced it, so the replay is rebuilt from
+ * the four fields `UltimateError` already promises rather than from a serialized stack.
+ */
+export interface IdempotencyFailure {
+  readonly code: string;
+  readonly cause: string;
+  readonly fix: string;
+  readonly docs?: string | undefined;
+}
+
+export type IdempotencyStatus = 'in-flight' | 'settled' | 'failed';
+
 export interface IdempotencyRecord {
   readonly id: string;
   readonly key: string;
   /** Fingerprint of the parsed input — a reused key with a new payload is a bug. */
   readonly requestHash: string;
-  readonly status: 'in-flight' | 'settled';
+  readonly status: IdempotencyStatus;
   readonly value: unknown;
+  /** Present exactly when `status === 'failed'` — what the replay re-throws. */
+  readonly failure?: IdempotencyFailure | undefined;
+  /** Epoch milliseconds. The store's own dedupe window is measured from here. */
   readonly createdAt: number;
 }
 
@@ -24,53 +55,58 @@ export interface IdempotencyReservation {
 }
 
 export interface IdempotencyStore {
+  /**
+   * Where these records live. Optional only so an existing external implementation still
+   * type-checks; an absent scope cannot be checked, so `assertIdempotencyScope` refuses it
+   * exactly as it refuses a wrong one — the same rule `RateLimiter.buckets` follows.
+   */
+  readonly scope?: IdempotencyScope | undefined;
+  /**
+   * How long a key is remembered. Outside it a record answers as a missing one, which is the
+   * only thing that keeps the store bounded: a key is caller-supplied, so an unbounded store
+   * is one immortal entry per write, forever.
+   */
+  readonly windowMs?: number | undefined;
   /** Atomically create-or-fetch the record for `key`. The atomicity is the point. */
   reserve(key: string, requestHash: string): Promise<IdempotencyReservation>;
   settle(key: string, value: unknown): Promise<void>;
-  /** Drop a reservation whose handler threw, so a retry can run. */
+  /**
+   * Settle a FAILURE, so the retry replays it instead of re-running a handler that may already
+   * have committed. Optional so an existing store still type-checks — and when it is absent the
+   * gate leaves the reservation standing rather than releasing it, because refusing the retry is
+   * the safe answer and re-running it is the double charge.
+   */
+  fail?(key: string, failure: IdempotencyFailure): Promise<void>;
+  /** Drop a reservation, so a retry can run. Only ever correct BEFORE the handler starts. */
   release(key: string): Promise<void>;
   get(key: string): Promise<IdempotencyRecord | undefined>;
 }
 
 /**
- * Default store: process memory. Correct for one web process and for tests;
- * production swaps in a Postgres-backed store behind the same interface (a
- * single `insert ... on conflict do nothing returning` gives the same atomicity).
+ * What the DEPLOYMENT requires of a store, against what the store provides. Two halves checked
+ * once at registration, exactly as `RateLimitConfig.scope` is checked against `RateLimitStore`.
  */
-export class MemoryIdempotencyStore implements IdempotencyStore {
-  readonly #records = new Map<string, IdempotencyRecord>();
-
-  async reserve(key: string, requestHash: string): Promise<IdempotencyReservation> {
-    const existing = this.#records.get(key);
-    if (existing !== undefined) return { record: existing, created: false };
-    const record: IdempotencyRecord = {
-      id: uuid(),
-      key,
-      requestHash,
-      status: 'in-flight',
-      value: undefined,
-      createdAt: Date.now(),
-    };
-    this.#records.set(key, record);
-    return { record, created: true };
-  }
+export interface IdempotencyConfig {
+  readonly scope: IdempotencyScope;
+}
 
-  async settle(key: string, value: unknown): Promise<void> {
-    const existing = this.#records.get(key);
-    if (existing === undefined) return;
-    this.#records.set(key, { ...existing, status: 'settled', value });
-  }
+/**
+ * One process is the only thing a framework can promise without being told; an app that runs
+ * more than one says so, and brings the store that makes it true.
+ */
+export const DEFAULT_IDEMPOTENCY_CONFIG: IdempotencyConfig = Object.freeze({ scope: 'process' });
 
-  async release(key: string): Promise<void> {
-    this.#records.delete(key);
-  }
+let config: IdempotencyConfig = DEFAULT_IDEMPOTENCY_CONFIG;
+let defaultStore: IdempotencyStore = new MemoryIdempotencyStore();
 
-  async get(key: string): Promise<IdempotencyRecord | undefined> {
-    return this.#records.get(key);
-  }
+/** Declared at boot, before `registerActions()`. Nothing infers a replica count. */
+export function configureIdempotency(next: IdempotencyConfig): void {
+  config = next;
 }
 
-let defaultStore: IdempotencyStore = new MemoryIdempotencyStore();
+export function idempotencyConfig(): IdempotencyConfig {
+  return config;
+}
 
 export function setIdempotencyStore(store: IdempotencyStore): void {
   defaultStore = store;
@@ -80,9 +116,26 @@ export function getIdempotencyStore(): IdempotencyStore {
   return defaultStore;
 }
 
-/** Keys are namespaced per action: the same key under two actions is two keys. */
-export function idempotencyKeyFor(actionName: string, key: string): string {
-  return `${actionName}:${key}`;
+/** Test-only. A process configures its idempotency once at boot and never reconfigures it. */
+export function resetIdempotency(): void {
+  config = DEFAULT_IDEMPOTENCY_CONFIG;
+  defaultStore = new MemoryIdempotencyStore();
+}
+
+/**
+ * Boot, never the first request — `registerAction` calls it, which every registration path funnels
+ * through and which necessarily runs before a route is mounted. A per-process store under a
+ * `'shared'` declaration is not a smaller guarantee, it is no guarantee: the retry that lands on
+ * another replica finds no record and re-runs the handler, so an `idempotent: true` action charges
+ * twice with nothing in any log to say it did. A store that declares no scope is refused too —
+ * what cannot be shown to be shared is not assumed to be.
+ */
+export function assertIdempotencyScope(
+  declared: IdempotencyConfig = idempotencyConfig(),
+  store: IdempotencyStore = getIdempotencyStore(),
+): void {
+  if (declared.scope !== 'shared') return;
+  if (store.scope !== 'shared') throw new IdempotencyNotSharedError(store.scope);
 }
 
 export interface IdempotentOutcome<T> {
@@ -91,9 +144,17 @@ export interface IdempotentOutcome<T> {
 }
 
 /**
- * Replay-or-run. Four outcomes: fresh run, replay of a settled record,
- * X_IDEMPOTENCY_CONFLICT for a payload mismatch, X_IDEMPOTENCY_CONFLICT for a
- * duplicate that is still in flight.
+ * Replay-or-run. Five outcomes: fresh run, replay of a settled record, replay of a FAILED record,
+ * X_IDEMPOTENCY_CONFLICT for a payload mismatch, X_IDEMPOTENCY_CONFLICT for a duplicate still in
+ * flight.
+ *
+ * **Everything `run()` throws is treated as possibly-committed.** `guard()` and `validateInput`
+ * both run before this gate is reached (`invoke.ts`), so by the time `run` is called the only
+ * things left are the handler and `validateOutput` — and the second of those throws *after* the
+ * first has committed. Releasing the reservation there is what turned a rounding change in an
+ * output schema into a second charge: `X_OUTPUT_INVALID` dropped the record, and the client's
+ * automatic retry re-ran a handler that had already taken the money. So the failure is SETTLED
+ * and replayed, and `release` is reserved for a pre-handler failure — of which this gate has none.
  */
 export async function withIdempotency<T>(
   store: IdempotencyStore,
@@ -108,15 +169,63 @@ export async function withIdempotency<T>(
   }
   if (!created) {
     if (record.status === 'in-flight') throw new IdempotencyConflictError(key, 'in-flight');
+    if (record.status === 'failed') throw new IdempotencyReplayedFailureError(key, record.failure);
     // The stored value is the previous return of this very handler.
     return { value: record.value as T, replayed: true };
   }
+  let value: T;
   try {
-    const value = await run();
-    await store.settle(key, value);
-    return { value, replayed: false };
+    value = await run();
   } catch (error) {
-    await store.release(key);
+    await settleFailure(store, key, error);
     throw error;
   }
+  // Outside the `try` on purpose: a `settle` that refuses is itself post-commit, and the record
+  // stays in flight rather than being released — a retry then gets a 409 it can act on instead of
+  // re-running a handler that has already committed.
+  await store.settle(key, value);
+  return { value, replayed: false };
+}
+
+/**
+ * Record the failure, and never let recording it replace the failure itself. A store that refuses
+ * here would otherwise surface as the caller's error, hiding the `X_OUTPUT_INVALID` or the
+ * handler's own throw that is the thing worth reading — the same rule `auditThrew` follows.
+ */
+async function settleFailure(store: IdempotencyStore, key: string, error: unknown): Promise<void> {
+  if (store.fail === undefined) {
+    // Deliberately NOT `release`. A store with no failure slot cannot say "this already ran", and
+    // the retry-safe reading of that is "refuse the retry", not "run it again".
+    logger.warn('action.idempotency.failure-unrecorded', { key });
+    return;
+  }
+  try {
+    await store.fail(key, failureOf(error));
+  } catch (sinkError) {
+    // Never the error's own text: rendering an `unknown` into a message is the second throw this
+    // branch exists to prevent. The logger takes it as a field and shapes it itself.
+    logger.error('action.idempotency.fail-refused', { key, error: sinkError });
+  }
+}
+
+/**
+ * A throw, flattened to the four fields a replay is rebuilt from. A non-`UltimateError` — a `row:`
+ * loader's `TypeError`, a driver's own error — carries no stable code, so the record takes
+ * `X_IDEMPOTENCY_REPLAYED_FAILURE` and says exactly that rather than inventing a code an app
+ * might match on.
+ */
+function failureOf(error: unknown): IdempotencyFailure {
+  if (!isUltimateError(error)) {
+    return {
+      code: 'X_IDEMPOTENCY_REPLAYED_FAILURE',
+      cause: 'the first attempt under this key threw a value that is not an UltimateError',
+      fix: 'read the first attempt in the logs, then send a fresh Idempotency-Key once the cause is fixed',
+    };
+  }
+  return {
+    code: error.code,
+    cause: error.cause,
+    fix: error.fix,
+    docs: error.docs,
+  };
 }