cursedbelt-server 1.0.0

package/src/server/storage/files/stack.ts

@@ -0,0 +1,284 @@
+import type { Database } from 'bun:sqlite';
+import type { Context, Hono } from 'hono';
+import { runMigrations } from '../../migration/runner';
+import { mountPrivateRoutes } from '../../private-media/mountPrivateRoutes';
+import {
+  createUnlockService,
+  type UnlockPasswordPolicy,
+  type UnlockService,
+} from '../../private-media/unlock';
+import { mountMediaRoutes } from '../mountMediaRoutes';
+import { createMediaUrls, type MediaUrls, SourceRegistry } from '../sources';
+import { createStreamTokenService, type StreamTokenService } from '../streamToken';
+import type { MediaCtx, MediaGallerySource, MediaSource, MediaStore, PhotoSource } from '../types';
+import {
+  createCatalogueMediaSource,
+  createCataloguePhotoSource,
+  createCatalogueVideoSource,
+} from './adapter';
+import { createSqliteFileCatalogue, type SqliteFileCatalogueOptions } from './catalogue';
+import {
+  type ChunkedUploadService,
+  createChunkedUploadService,
+  type MintChunkUpload,
+} from './chunkedUpload';
+import { createFileUrls, type FileUrls, mountFileRoutes } from './fileRoutes';
+import { FILE_CATALOGUE_MIGRATIONS } from './migrations';
+import { createStorageOpsLedger } from './storageOps';
+import type { FileCatalogue, PrivateFileCatalogue } from './types';
+
+// ── Files stack — the host recipe ───────────────────────────────────────────────
+// One call that assembles the whole files/media server stack a host would otherwise hand-wire:
+// migrations + SQLite catalogue over a MediaStore, the stream-token service, the media/file URL
+// builders, the catalogue-as-video-source registered in a SourceRegistry, the optional private
+// (encrypted) layer, and a `mount()` that wires every byte/upload route onto a Hono app. The auth
+// seams (session → MediaCtx / {userId, userSalt}) stay host-injected — the stack owns no auth.
+
+export interface FilesStackOptions {
+  /** The catalogue database. The stack runs FILE_CATALOGUE_MIGRATIONS on it itself. */
+  db: Database;
+  /** Byte backend (disk / binary-server / s3) — the swappable piece. */
+  store: MediaStore;
+  /** ffmpeg scratch dir for video processing. */
+  workDir: string;
+  /** Secret for the stream-token service (path-embedded byte-route auth). */
+  streamSecret: string;
+  /** Owner keyspace segment. Defaults to the userId (files + private are personal) — note the
+   * bare primitives default to tenantId; the stack picks the personal axis so every URL/route
+   * agrees with the private layer's per-user owner. */
+  owner?: (ctx: MediaCtx) => string;
+  /** Registered video-source id (also the media-token sourceId). Default 'files'. */
+  sourceId?: string;
+  /** Video source display title. Default 'Files'. */
+  sourceTitle?: string;
+  /** Registered photo-source id (the image-filtered view; also its media-token sourceId).
+   * Default 'photos'. */
+  photoSourceId?: string;
+  /** Photo source display title. Default 'Photos'. */
+  photoSourceTitle?: string;
+  /** Video byte-route mount + URL base (path-only). Default '/_media'. */
+  mediaBasePath?: string;
+  /** General file byte/upload route mount + URL base (path-only). Default '/_files'. */
+  filesBasePath?: string;
+  /** PUBLIC base for browser media URLs when it differs from the mount path — a split-origin
+   * deploy (SPA on the apex, API on a subdomain) passes the absolute origin+path here
+   * ('https://api.example.com/_media') while `mediaBasePath` stays the path-only mount. */
+  mediaPublicBase?: string;
+  /** PUBLIC base for browser file URLs (split-origin counterpart of `filesBasePath`). */
+  filesPublicBase?: string;
+  /** Private (encrypted) route mount. Default '/_private'. */
+  privateBasePath?: string;
+  /** HLS ladder heights for video ingest (createSqliteFileCatalogue passthrough). */
+  ladderHeights?: SqliteFileCatalogueOptions['ladderHeights'];
+  /** Cap on transcode kicks per file before it's marked failed (createSqliteFileCatalogue
+   * passthrough) — the boot-loop guard. Default 3. */
+  maxProcessingAttempts?: SqliteFileCatalogueOptions['maxProcessingAttempts'];
+  /** Transcode uploaded videos (createSqliteFileCatalogue passthrough). `false` ⇒ videos play
+   * raw, no ffmpeg — for memory-constrained hosts. Default true. */
+  transcodeVideos?: SqliteFileCatalogueOptions['transcodeVideos'];
+  /** Video ingest→derivatives implementation (createSqliteFileCatalogue passthrough) — inject
+   * a remote one (binary-server) when the byte store is remote. Default: local ffmpeg. */
+  videoProcessor?: SqliteFileCatalogueOptions['videoProcessor'];
+  /** Image thumbnail/display derivative implementation (createSqliteFileCatalogue passthrough).
+   * Default: local sharp with ffmpeg fallback. */
+  imageDerivatives?: SqliteFileCatalogueOptions['imageDerivatives'];
+  /** Enables the private (encrypted) stack: unlock service + /_private routes + private uploads.
+   * Omit ⇒ `unlock` is null and mount() wires only the public routes. `passwordPolicy` sets what
+   * a supplied password MEANS (derive = deniable sets, the default; gate = verify against the
+   * app's accounts + recoverable server-key sets — see UnlockPasswordPolicy). */
+  privateMedia?: { serverSecret: string; unlockSecret: string; passwordPolicy?: UnlockPasswordPolicy };
+  /** Re-kick transcodes a restart left stuck (catalogue.recoverProcessing) at build time.
+   * Default true. */
+  recoverProcessing?: boolean;
+  /** Backfill grid thumbnails at build time for finished PUBLIC rows missing a poster — videos
+   * (landed while transcodeVideos was off / failed before the poster step) AND images (e.g. HEIC
+   * whose sharp thumbnail failed before the ffmpeg fallback). Default false. */
+  backfillPosters?: boolean;
+  /** Wire LARGE-file chunked uploads (direct-to-backend). Give the backend-specific per-chunk URL
+   * minter (for binary-server: a signed `${base}/upload-chunk/${app}/${key}?token=`) and the stack
+   * builds the ChunkedUploadService over its own catalogue + a storage-ops ledger on `db`, then
+   * mount() auto-wires the /upload-init, /upload-complete, /upload-abort routes. Omit ⇒ chunked is
+   * off and clients fall back to the streamed single PUT (disk-only deploys). */
+  chunkedUpload?: {
+    mintChunkUpload: MintChunkUpload;
+    /** Byte-backend label recorded on each ledger op. Default 'binary'. */
+    backend?: string;
+    /** Per-chunk size. Default DEFAULT_UPLOAD_CHUNK_SIZE (8 MiB), always < MAX_SINGLE_UPLOAD_BYTES. */
+    chunkSize?: number;
+  };
+}
+
+/** The auth seams `mount()` needs from the host — the only pieces the stack cannot own. */
+export interface FilesStackMountAuth {
+  /** Session auth for uploads + listings (same guard mountMediaRoutes/mountFileRoutes take).
+   * Return a MediaCtx to allow, null to reject. */
+  authorizeUpload: (c: Context) => Promise<MediaCtx | null> | MediaCtx | null;
+  /** Session auth for the private layer → {userId, userSalt}. REQUIRED when `privateMedia` is
+   * configured (mount() throws otherwise); ignored when it isn't. */
+  authorizePrivate?: (c: Context) => Promise<{ userId: string; userSalt: string } | null>;
+  /** Wire large-file chunked uploads (mountFileRoutes' `chunked` passthrough). */
+  mintChunkUpload?: ChunkedUploadService;
+  /** Private-upload password header (default 'x-private-password'). */
+  passwordHeader?: string;
+  /** Private-upload unlock-token header (default 'x-unlock-token'). */
+  tokenHeader?: string;
+}
+
+/**
+ * The assembled stack. `catalogue` / `fileUrls` / `mediaUrls` / `videoSource` are intentionally
+ * named so a cursedshell host can pass the stack straight into `createFilesPlugin(stack)`
+ * structurally, and `{ registry }` into `createGalleryPlugin` — the stack does not (and must
+ * not, layering) import cursedshell.
+ */
+export interface FilesStack {
+  catalogue: FileCatalogue & PrivateFileCatalogue;
+  token: StreamTokenService;
+  mediaUrls: MediaUrls;
+  fileUrls: FileUrls;
+  videoSource: MediaSource;
+  /** The image-filtered sibling view (photo gallery data source). */
+  photoSource: PhotoSource;
+  /** The MIXED view — photos + videos in one feed (media gallery data source). */
+  mediaSource: MediaGallerySource;
+  registry: SourceRegistry;
+  /** Null unless `privateMedia` was configured. */
+  unlock: UnlockService | null;
+  /** The large-file chunked-upload service, or null unless `chunkedUpload` was configured. mount()
+   * auto-wires it into the file routes; exposed for observability (pending()) + tests. */
+  chunked: ChunkedUploadService | null;
+  /**
+   * Mount every byte/upload route the stack owns onto a Hono app: mountMediaRoutes
+   * (mediaBasePath) + mountFileRoutes (filesBasePath, incl. privateUpload when unlock exists) +
+   * mountPrivateRoutes (privateBasePath, when unlock exists). Auth stays host-injected.
+   */
+  mount(app: Hono, auth: FilesStackMountAuth): void;
+}
+
+/** Assemble the whole files/media server stack from one options bag. See {@link FilesStack}. */
+export function createFilesStack(opts: FilesStackOptions): FilesStack {
+  const owner = opts.owner ?? ((c: MediaCtx) => c.userId);
+  const sourceId = opts.sourceId ?? 'files';
+  const mediaBasePath = opts.mediaBasePath ?? '/_media';
+  const filesBasePath = opts.filesBasePath ?? '/_files';
+  const { db, store, workDir } = opts;
+
+  const token = createStreamTokenService({ secret: opts.streamSecret });
+  const mediaUrls = createMediaUrls({
+    baseUrl: opts.mediaPublicBase ?? mediaBasePath,
+    token,
+    owner,
+  });
+  const fileUrls = createFileUrls({ baseUrl: opts.filesPublicBase ?? filesBasePath, token, owner });
+
+  runMigrations(db, FILE_CATALOGUE_MIGRATIONS);
+  const catalogue = createSqliteFileCatalogue({
+    db,
+    store,
+    workDir,
+    owner,
+    ladderHeights: opts.ladderHeights,
+    maxProcessingAttempts: opts.maxProcessingAttempts,
+    transcodeVideos: opts.transcodeVideos,
+    videoProcessor: opts.videoProcessor,
+    imageDerivatives: opts.imageDerivatives,
+  });
+  if (opts.recoverProcessing !== false) catalogue.recoverProcessing();
+  if (opts.backfillPosters) {
+    catalogue.backfillVideoPosters();
+    catalogue.backfillImageThumbnails();
+  }
+
+  const videoSource = createCatalogueVideoSource({
+    id: sourceId,
+    title: opts.sourceTitle ?? 'Files',
+    catalogue,
+    urls: mediaUrls,
+  });
+  const photoSource = createCataloguePhotoSource({
+    id: opts.photoSourceId ?? 'photos',
+    title: opts.photoSourceTitle ?? 'Photos',
+    catalogue,
+    fileUrls,
+    mediaUrls,
+  });
+  const mediaSource = createCatalogueMediaSource({
+    id: opts.photoSourceId ?? 'photos',
+    title: 'Media',
+    catalogue,
+    urls: mediaUrls,
+    fileUrls,
+    // Videos in the mixed feed stream through the registered video source's routes.
+    videoSourceId: sourceId,
+  });
+  const registry = new SourceRegistry().register(videoSource);
+
+  const unlock = opts.privateMedia ? createUnlockService(opts.privateMedia) : null;
+
+  const chunked = opts.chunkedUpload
+    ? createChunkedUploadService({
+        catalogue,
+        ledger: createStorageOpsLedger(db),
+        backend: opts.chunkedUpload.backend ?? 'binary',
+        mintChunkUpload: opts.chunkedUpload.mintChunkUpload,
+        chunkSize: opts.chunkedUpload.chunkSize,
+        owner,
+      })
+    : null;
+
+  return {
+    catalogue,
+    token,
+    mediaUrls,
+    fileUrls,
+    videoSource,
+    photoSource,
+    mediaSource,
+    registry,
+    unlock,
+    chunked,
+    mount(app, auth) {
+      if (unlock && !auth.authorizePrivate) {
+        throw new Error(
+          'createFilesStack: privateMedia is configured but mount() got no authorizePrivate — ' +
+            'the private routes need a session→{userId, userSalt} resolver.',
+        );
+      }
+      mountMediaRoutes(app, {
+        store,
+        token,
+        basePath: mediaBasePath,
+        registry,
+        authorizeUpload: auth.authorizeUpload,
+      });
+      mountFileRoutes(app, {
+        store,
+        token,
+        basePath: filesBasePath,
+        catalogue,
+        authorizeUpload: auth.authorizeUpload,
+        // Prefer a mount-time override, else the service the stack built from `chunkedUpload` opts.
+        chunked: auth.mintChunkUpload ?? chunked ?? undefined,
+        privateUpload:
+          unlock && auth.authorizePrivate
+            ? {
+                unlock,
+                authorize: auth.authorizePrivate,
+                passwordHeader: auth.passwordHeader,
+                tokenHeader: auth.tokenHeader,
+                passwordPolicy: opts.privateMedia?.passwordPolicy,
+              }
+            : undefined,
+      });
+      if (unlock && auth.authorizePrivate) {
+        mountPrivateRoutes(app, {
+          store,
+          index: catalogue.asPrivateIndex(),
+          unlock,
+          basePath: opts.privateBasePath,
+          authorize: auth.authorizePrivate,
+          passwordPolicy: opts.privateMedia?.passwordPolicy,
+        });
+      }
+    },
+  };
+}

package/src/server/storage/files/storageOps.spec.ts

@@ -0,0 +1,126 @@
+import { Database } from 'bun:sqlite';
+import { beforeEach, describe, expect, test } from 'bun:test';
+import { runMigrations } from '../../migration/runner';
+import type { MediaStore } from '../types';
+import { FILE_CATALOGUE_MIGRATIONS } from './migrations';
+import {
+  createLedgeredDeletes,
+  createStorageOpsLedger,
+  createStorageReconciler,
+} from './storageOps';
+
+// A fake store whose remove() fails the first `failTimes` calls per key, then succeeds — models a
+// flaky remote backend (binary-server) so the reconciler's heal path is exercised deterministically.
+function flakyStore(failTimes: number): MediaStore & { removed: Set<string> } {
+  const removed = new Set<string>();
+  const attempts = new Map<string, number>();
+  const fail = (key: string) => {
+    const n = (attempts.get(key) ?? 0) + 1;
+    attempts.set(key, n);
+    if (n <= failTimes) throw new Error(`transient failure ${n} for ${key}`);
+  };
+  return {
+    removed,
+    async put() {},
+    async putStream() {},
+    async get() {
+      return null;
+    },
+    async remove(key) {
+      fail(key);
+      removed.add(key);
+    },
+    async removePrefix(prefix) {
+      fail(prefix);
+      removed.add(prefix);
+    },
+    async stat() {
+      return null;
+    },
+    async getRange() {
+      return null;
+    },
+  };
+}
+
+describe('storage-ops ledger', () => {
+  let db: Database;
+  let clock = 0;
+  beforeEach(() => {
+    db = new Database(':memory:');
+    runMigrations(db, FILE_CATALOGUE_MIGRATIONS);
+    clock = 0;
+  });
+
+  test('records pending, marks done/failed, and reports outstanding correctly', () => {
+    const ledger = createStorageOpsLedger(db, () => ++clock);
+    const a = ledger.record({ owner: 'u1', action: 'delete', key: 'fs/u1/a', backend: 'disk' });
+    const b = ledger.record({ owner: 'u1', action: 'delete', key: 'fs/u1/b', backend: 'disk' });
+
+    expect(ledger.get(a)?.status).toBe('pending');
+    ledger.markDone(a);
+    expect(ledger.get(a)?.status).toBe('done');
+    ledger.markFailed(b, 'boom');
+    expect(ledger.get(b)?.status).toBe('failed');
+    expect(ledger.get(b)?.attempts).toBe(1);
+
+    // `done` excluded; `failed` under the cap still outstanding.
+    expect(ledger.outstanding().map((o) => o.opId)).toEqual([b]);
+    // Over the attempt cap → dropped from outstanding (a poison op won't spin forever).
+    expect(ledger.outstanding({ maxAttempts: 1 })).toEqual([]);
+  });
+
+  test('`done` is a monotonic terminal state — a late failure can never flip it back', () => {
+    const ledger = createStorageOpsLedger(db, () => ++clock);
+    const op = ledger.record({ owner: 'u1', action: 'upload', key: 'fs/u1/x', backend: 'binary' });
+    ledger.markDone(op);
+    // A duplicate completion webhook racing the reconciler (or a stale failure) must NOT corrupt it.
+    ledger.markFailed(op, 'late webhook says failed');
+    expect(ledger.get(op)?.status).toBe('done');
+    expect(ledger.get(op)?.attempts).toBe(0);
+    ledger.markDone(op); // idempotent
+    expect(ledger.get(op)?.status).toBe('done');
+  });
+
+  test('ledgered deletes mark done on success and keep a failed row outstanding', async () => {
+    const ledger = createStorageOpsLedger(db, () => ++clock);
+    const store = flakyStore(0); // never fails
+    const del = createLedgeredDeletes(store, ledger, 'binary');
+    await del.removeFor('u1', 'f1', 'fs/u1/f1');
+    expect(store.removed.has('fs/u1/f1')).toBe(true);
+    expect(ledger.outstanding()).toEqual([]); // marked done
+
+    const flaky = createLedgeredDeletes(flakyStore(99), ledger, 'binary');
+    await expect(flaky.removeFor('u1', 'f2', 'fs/u1/f2')).rejects.toThrow();
+    // The failed delete survives for the reconciler.
+    expect(ledger.outstanding().map((o) => o.key)).toEqual(['fs/u1/f2']);
+  });
+
+  test('reconciler heals a transiently-failed delete on a later pass', async () => {
+    const ledger = createStorageOpsLedger(db, () => ++clock);
+    const store = flakyStore(1); // fails once per key, then succeeds
+    const del = createLedgeredDeletes(store, ledger, 'binary');
+
+    await expect(del.removeFor('u1', 'f3', 'fs/u1/f3')).rejects.toThrow(); // first attempt fails
+    expect(ledger.get(ledger.outstanding()[0].opId)?.status).toBe('failed');
+
+    // Reconcile → the retry succeeds (flaky store now lets it through) → healed, nothing left.
+    const reconciler = createStorageReconciler(store, ledger);
+    expect(await reconciler.reconcile()).toEqual({ healed: 1, failed: 0 });
+    expect(ledger.outstanding()).toEqual([]);
+    expect(store.removed.has('fs/u1/f3')).toBe(true);
+  });
+
+  test('reconciler bumps attempts and gives up past the cap (no infinite retry)', async () => {
+    const ledger = createStorageOpsLedger(db, () => ++clock);
+    const store = flakyStore(99); // always fails
+    ledger.record({ owner: 'u1', action: 'delete', key: 'fs/u1/dead', backend: 'binary' });
+
+    const reconciler = createStorageReconciler(store, ledger);
+    // Each pass bumps attempts; after 3 passes attempts reaches the cap and it drops out (no
+    // infinite retry of a poison op).
+    for (let i = 0; i < 3; i++) await reconciler.reconcile({ maxAttempts: 3 });
+    expect(ledger.outstanding({ maxAttempts: 3 })).toEqual([]);
+    expect(store.removed.has('fs/u1/dead')).toBe(false); // never actually removed
+  });
+});

package/src/server/storage/files/storageOps.ts

@@ -0,0 +1,213 @@
+import type { Database } from 'bun:sqlite';
+import type { MediaStore } from '../types';
+
+// ── Storage-operations ledger + reconciler ───────────────────────────────────────
+// A durable write-ahead log for CROSS-BOUNDARY byte operations (delete / delete-prefix / an
+// assembled chunked upload) against a remote backend like binary-server, where the call can
+// succeed on our side but the ack never arrives — or fail transiently — leaving bytes orphaned
+// or a delete un-applied. Before each such op we record a `pending` row; on success we mark it
+// `done`; a crash / failed call leaves it `pending`, and the reconciler re-drives it to
+// completion on the next boot (idempotent — remove/removePrefix must not throw on a missing key).
+//
+// Uploads that fail simply error the request (no ledger needed — nothing to heal). The ledger
+// targets the ops whose FAILURE silently leaks: a delete binary-server never applied, or an
+// assemble webhook we recorded but must confirm. The `storage_operations` table is created by the
+// FileCatalogue migration; binary-server echoes our `op_id` in its own event ledger so the two
+// can be reconciled against each other (see binary-server db.ts).
+
+export type StorageAction = 'upload' | 'delete' | 'delete_prefix';
+export type StorageOpStatus = 'pending' | 'done' | 'failed';
+
+export type StorageOpRow = {
+  opId: string;
+  owner: string;
+  fileId: string | null;
+  action: StorageAction;
+  key: string;
+  backend: string;
+  status: StorageOpStatus;
+  attempts: number;
+  error: string | null;
+  createdAt: number;
+  updatedAt: number;
+};
+
+type DbRow = {
+  op_id: string;
+  owner: string;
+  file_id: string | null;
+  action: string;
+  key: string;
+  backend: string;
+  status: string;
+  attempts: number;
+  error: string | null;
+  created_at: number;
+  updated_at: number;
+};
+
+const toRow = (r: DbRow): StorageOpRow => ({
+  opId: r.op_id,
+  owner: r.owner,
+  fileId: r.file_id,
+  action: r.action as StorageAction,
+  key: r.key,
+  backend: r.backend,
+  status: r.status as StorageOpStatus,
+  attempts: r.attempts,
+  error: r.error,
+  createdAt: r.created_at,
+  updatedAt: r.updated_at,
+});
+
+export interface StorageOpsLedger {
+  /** Record a pending op and return its id (caller may pass one, e.g. to share with binary-server
+   * as `x-op-id`; otherwise a uuid is minted). */
+  record(op: {
+    owner: string;
+    fileId?: string | null;
+    action: StorageAction;
+    key: string;
+    backend: string;
+    opId?: string;
+  }): string;
+  markDone(opId: string): void;
+  markFailed(opId: string, error: string): void;
+  get(opId: string): StorageOpRow | null;
+  /** Ops still needing work: `pending`, or `failed` under the attempt cap, created before `before`
+   * (default: everything). Oldest first. */
+  outstanding(opts?: { maxAttempts?: number; before?: number }): StorageOpRow[];
+}
+
+export function createStorageOpsLedger(
+  db: Database,
+  now: () => number = () => Date.now(),
+): StorageOpsLedger {
+  const insert = db.query(
+    `INSERT INTO storage_operations (op_id, owner, file_id, action, key, backend, status, attempts, created_at, updated_at)
+     VALUES ($op, $owner, $file, $action, $key, $backend, 'pending', 0, $now, $now)`,
+  );
+  // MONOTONIC terminal state: once an op is `done` it can never flip back (a duplicate completion
+  // webhook racing the reconciler, or a late failure, must not corrupt a settled success). `failed`
+  // is NOT terminal — the reconciler retries a failed delete and can heal it to `done`, or bump its
+  // attempts — so both writes guard only against overwriting `done`. (Ported from ca's resolveOp
+  // `CASE WHEN status='pending'` invariant, adapted to our two-writer model.)
+  const setDone = db.query(
+    `UPDATE storage_operations SET status = 'done', updated_at = $now WHERE op_id = $op AND status != 'done'`,
+  );
+  const setFailed = db.query(
+    `UPDATE storage_operations SET status = 'failed', attempts = attempts + 1, error = $error, updated_at = $now WHERE op_id = $op AND status != 'done'`,
+  );
+  const getOne = db.query<DbRow, { $op: string }>(
+    `SELECT * FROM storage_operations WHERE op_id = $op`,
+  );
+
+  return {
+    record(op) {
+      const opId = op.opId ?? crypto.randomUUID();
+      insert.run({
+        $op: opId,
+        $owner: op.owner,
+        $file: op.fileId ?? null,
+        $action: op.action,
+        $key: op.key,
+        $backend: op.backend,
+        $now: now(),
+      });
+      return opId;
+    },
+    markDone(opId) {
+      setDone.run({ $op: opId, $now: now() });
+    },
+    markFailed(opId, error) {
+      setFailed.run({ $op: opId, $error: error.slice(0, 500), $now: now() });
+    },
+    get(opId) {
+      const r = getOne.get({ $op: opId });
+      return r ? toRow(r) : null;
+    },
+    outstanding(opts = {}) {
+      const maxAttempts = opts.maxAttempts ?? 5;
+      const before = opts.before ?? now() + 1;
+      return db
+        .query<DbRow, { $max: number; $before: number }>(
+          `SELECT * FROM storage_operations
+           WHERE status != 'done' AND attempts < $max AND created_at < $before
+           ORDER BY created_at ASC`,
+        )
+        .all({ $max: maxAttempts, $before: before })
+        .map(toRow);
+    },
+  };
+}
+
+// Wrap a MediaStore so DELETE / DELETE-PREFIX are ledgered: a `pending` row is written first, the
+// op attempted, then marked `done`; a failure marks it `failed` and rethrows (the row survives for
+// the reconciler). Reads/writes pass straight through — only the leak-prone deletes are logged.
+export function createLedgeredDeletes(
+  store: MediaStore,
+  ledger: StorageOpsLedger,
+  backend: string,
+): Pick<MediaStore, 'remove' | 'removePrefix'> & {
+  removeFor(owner: string, fileId: string | null, key: string): Promise<void>;
+  removePrefixFor(owner: string, fileId: string | null, prefix: string): Promise<void>;
+} {
+  const run = async (
+    action: StorageAction,
+    owner: string,
+    fileId: string | null,
+    key: string,
+    exec: () => Promise<void>,
+  ) => {
+    const opId = ledger.record({ owner, fileId, action, key, backend });
+    try {
+      await exec();
+      ledger.markDone(opId);
+    } catch (e) {
+      ledger.markFailed(opId, e instanceof Error ? e.message : String(e));
+      throw e;
+    }
+  };
+  return {
+    remove: (key) => run('delete', '', null, key, () => store.remove(key)),
+    removePrefix: (prefix) =>
+      run('delete_prefix', '', null, prefix, () => store.removePrefix(prefix)),
+    removeFor: (owner, fileId, key) => run('delete', owner, fileId, key, () => store.remove(key)),
+    removePrefixFor: (owner, fileId, prefix) =>
+      run('delete_prefix', owner, fileId, prefix, () => store.removePrefix(prefix)),
+  };
+}
+
+export interface StorageReconciler {
+  /** Re-drive every outstanding op to completion (idempotent). Returns how many healed vs. still
+   * failing. Safe to call on boot and on an interval. */
+  reconcile(opts?: { maxAttempts?: number }): Promise<{ healed: number; failed: number }>;
+}
+
+export function createStorageReconciler(
+  store: MediaStore,
+  ledger: StorageOpsLedger,
+): StorageReconciler {
+  return {
+    async reconcile(opts = {}) {
+      let healed = 0;
+      let failed = 0;
+      for (const op of ledger.outstanding({ maxAttempts: opts.maxAttempts })) {
+        // Only DELETES are re-driven by re-issuing the store call (idempotent). An 'upload' op is
+        // healed by its completion webhook (or an app-level check against binary-server's own event
+        // ledger) — re-issuing store.remove here would wrongly delete the just-uploaded bytes.
+        if (op.action === 'upload') continue;
+        try {
+          if (op.action === 'delete') await store.remove(op.key);
+          else await store.removePrefix(op.key);
+          ledger.markDone(op.opId);
+          healed++;
+        } catch (e) {
+          ledger.markFailed(op.opId, e instanceof Error ? e.message : String(e));
+          failed++;
+        }
+      }
+      return { healed, failed };
+    },
+  };
+}