cursedbelt-server 4.40.4 → 4.41.0

package/dist/server/storage/binaryStore.d.ts

@@ -38,6 +38,8 @@ export interface BinaryStoreConfig {
     absenceTtlMs?: number;
     /** Injected so the memo's expiry is testable without a real clock. */
     now?: () => number;
+    /** Injected so a `paced` read's waits are testable without real time. Default: a timer. */
+    sleep?: (ms: number) => Promise<unknown>;
     /**
      * 🔴 How THIS PROCESS sends every request to binary-server. Default: the global `fetch`.
      *
@@ -154,7 +156,25 @@ export interface ReadOptions {
      * hour" is answered by reloading the picture.
      */
     fresh?: boolean;
+    /**
+     * 🔴 Wait out binary-server's read ceiling instead of throwing it (task 2505).
+     *
+     * The store allows a tenant a burst of reads, then a steady rate, and answers 429 with the
+     * `retry-after` it wants honoured (`apps/binary-server/src/rateLimit.ts`). A batch read —
+     * roms filing a disc's 256 parts, a hash backfill, an import — met it at read ~204 and threw
+     * the whole pass away. `paced` makes `get`/`meta`/`stat` sleep the named `retry-after` and ask
+     * again, for at most {@link BINARY_PACED_MAX_WAIT_MS} in total (or `{ maxWaitMs }`).
+     *
+     * Opt-in, because a page read a person is waiting on should fail fast rather than hang:
+     * without it a 429 throws {@link BinaryStoreRateLimitedError}, which carries the
+     * `retryAfterSec` so a caller pacing itself need not guess.
+     */
+    paced?: boolean | {
+        maxWaitMs?: number;
+    };
 }
+/** How long a `paced` read waits out 429s in total before it throws — see {@link ReadOptions.paced}. */
+export declare const BINARY_PACED_MAX_WAIT_MS = 120000;
 /** What the memos currently hold — for an owner surface, a smoke, and the tests. */
 export interface BinaryCacheStats {
     /** Keys known to be present. Never expires; see {@link BINARY_ABSENCE_TTL_MS}. */
@@ -436,3 +456,19 @@ export declare class BinaryStoreUnreachableError extends Error {
  * required vars are all present — callers fall back to their local backend.
  */
 export declare function readBinaryStoreEnv(appId: string, env?: Record<string, string | undefined>): BinaryStoreConfig | null;
+/**
+ * binary-server refused a READ for its rate ceiling (429) — a pace, not a verdict, and not an
+ * absence (nothing is memoized). Thrown by `get`/`meta`/`stat` without `paced`, or with `paced`
+ * once its wait budget is spent. The message keeps the `[binary-store] <op> <key>: 429` shape
+ * earlier callers matched on; `retryAfterSec` is what the store asked for.
+ */
+export declare class BinaryStoreRateLimitedError extends Error {
+    readonly retryAfterSec: number;
+    /** How long a `paced` read had already waited before giving up — 0 when it was not paced. */
+    readonly waitedMs: number;
+    readonly status = 429;
+    readonly code = "BYTE_STORE_RATE_LIMITED";
+    constructor(op: string, key: string, retryAfterSec: number, 
+    /** How long a `paced` read had already waited before giving up — 0 when it was not paced. */
+    waitedMs?: number);
+}

package/dist/server/storage/binaryStore.js

@@ -129,6 +129,8 @@ export function wantsFresh(req) {
  * that only just crosses the line. Being early is free; being late is a 413.
  */
 export const AUTO_CHUNK_THRESHOLD_BYTES = 64 * 1024 * 1024;
+/** How long a `paced` read waits out 429s in total before it throws — see {@link ReadOptions.paced}. */
+export const BINARY_PACED_MAX_WAIT_MS = 120_000;
 /**
  * 🔴 How long "the store does not hold this key" is believed — the NEGATIVE memo.
  *
@@ -263,6 +265,7 @@ export function createBinaryStore(cfg) {
     const send = cfg.fetch ?? ((input, init) => fetch(input, init));
     const autoChunkBytes = cfg.autoChunkBytes ?? AUTO_CHUNK_THRESHOLD_BYTES;
     const now = cfg.now ?? Date.now;
+    const sleep = cfg.sleep ?? ((ms) => new Promise((done) => setTimeout(done, ms)));
     const absenceTtlMs = cfg.absenceTtlMs ?? BINARY_ABSENCE_TTL_MS;
     /*
      * ── 🔴 The shared picture cache (2026-09-09) ───────────────────────────────
@@ -409,6 +412,28 @@ export function createBinaryStore(cfg) {
         }
         return { 'x-internal-secret': cfg.internalSecret };
     };
+    /**
+     * One read, with a 429 handled the one way every read handles it: waited out when the
+     * caller asked for `paced` and the budget allows, otherwise thrown as a typed error that
+     * keeps the `[binary-store] <op> <key>: 429` message batch callers already match.
+     */
+    const sendRead = async (op, key, url, init, opts) => {
+        const budget = opts?.paced
+            ? (typeof opts.paced === 'object' ? opts.paced.maxWaitMs : undefined) ?? BINARY_PACED_MAX_WAIT_MS
+            : 0;
+        let waited = 0;
+        for (let attempt = 0;; attempt++) {
+            const res = await send(url, init);
+            if (res.status !== 429)
+                return res;
+            const retryAfterSec = retryAfterSeconds(res.headers.get('retry-after'), attempt);
+            const ms = retryAfterSec * 1000;
+            if (waited + ms > budget)
+                throw new BinaryStoreRateLimitedError(op, key, retryAfterSec, waited);
+            await sleep(ms);
+            waited += ms;
+        }
+    };
     const store = {
         appId: cfg.appId,
         signToken,
@@ -523,7 +548,7 @@ export function createBinaryStore(cfg) {
             if (!opts?.fresh && knownAbsent(key))
                 return null;
             countWireRead(key);
-            const res = await send(await internalMediaUrl(key));
+            const res = await sendRead('get', key, await internalMediaUrl(key), undefined, opts);
             if (res.status === 404) {
                 rememberAbsent(key);
                 return null;
@@ -599,7 +624,7 @@ export function createBinaryStore(cfg) {
                     return null;
             }
             countWireRead(key);
-            const res = await send(await metaUrl(key));
+            const res = await sendRead('meta', key, await metaUrl(key), undefined, opts);
             if (res.status === 404) {
                 rememberAbsent(key);
                 return null;
@@ -639,11 +664,14 @@ export function createBinaryStore(cfg) {
                 if (m)
                     return { size: m.size };
             }
-            catch {
-                // A transport/5xx failure — the probe below is the second opinion.
+            catch (why) {
+                // A transport/5xx failure — the probe below is the second opinion. A 429 is not: the
+                // store answered "slow down", and a second read straight away only spends more of it.
+                if (why instanceof BinaryStoreRateLimitedError)
+                    throw why;
             }
             countWireRead(key);
-            const res = await send(await internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } });
+            const res = await sendRead('stat', key, await internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } }, opts);
             if (res.status === 404) {
                 rememberAbsent(key);
                 return null;
@@ -817,3 +845,34 @@ export function readBinaryStoreEnv(appId, env = process.env) {
         internalBaseUrl: env.BINARY_SERVER_INTERNAL_URL?.trim() || undefined,
     };
 }
+/**
+ * The seconds a 429 asks for. binary-server always names `retry-after` (1-30 s); a
+ * missing or junk one — an edge in front of it, an older server — backs off 1, 2, 4… s
+ * rather than hammering, capped at the same 30 s the server caps itself at.
+ */
+function retryAfterSeconds(header, attempt) {
+    const named = Number(header);
+    if (header !== null && Number.isFinite(named) && named >= 0)
+        return Math.min(30, Math.max(1, Math.ceil(named)));
+    return Math.min(30, 2 ** Math.min(attempt, 5));
+}
+/**
+ * binary-server refused a READ for its rate ceiling (429) — a pace, not a verdict, and not an
+ * absence (nothing is memoized). Thrown by `get`/`meta`/`stat` without `paced`, or with `paced`
+ * once its wait budget is spent. The message keeps the `[binary-store] <op> <key>: 429` shape
+ * earlier callers matched on; `retryAfterSec` is what the store asked for.
+ */
+export class BinaryStoreRateLimitedError extends Error {
+    retryAfterSec;
+    waitedMs;
+    status = 429;
+    code = 'BYTE_STORE_RATE_LIMITED';
+    constructor(op, key, retryAfterSec, 
+    /** How long a `paced` read had already waited before giving up — 0 when it was not paced. */
+    waitedMs = 0) {
+        super(`[binary-store] ${op} ${key}: 429`);
+        this.retryAfterSec = retryAfterSec;
+        this.waitedMs = waitedMs;
+        this.name = 'BinaryStoreRateLimitedError';
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedbelt-server",
-	"version": "4.40.4",
+	"version": "4.41.0",
 	"license": "ISC",
 	"type": "module",
 	"description": "The app-facing Bun/Hono server tier of the cursedbelt split — storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",

package/src/server/storage/binaryStore.spec.ts

@@ -45,8 +45,13 @@ import {
   storeUnreachableStatus,
   wantsFresh,
 } from './binaryStore.js';
+import * as binaryStoreModule from './binaryStore.js';
 import { binaryStoreFakeDefaults } from './binaryStoreFake.js';
 
+// Read off the namespace, not named imports, so the task-2505 specs run (and go red on the
+// BEHAVIOUR) against a build that predates these exports — which is what prove-red does.
+const { BINARY_PACED_MAX_WAIT_MS, BinaryStoreRateLimitedError } = binaryStoreModule;
+
 // One 2048-bit keypair for the whole file — generation is the slow part, and every
 // assertion below only needs *a* valid PKCS8 PEM and its public half.
 const { privateKey, publicKey } = generateKeyPairSync('rsa', { modulusLength: 2048 });
@@ -1019,3 +1024,80 @@ describe('🔴 a write the store could not ANSWER is a 503 that names it (2026-0
     expect(String((error as Error).message)).toBe('[binary-store] put k: 403');
   });
 });
+
+describe('🔴 a read the store PACES is waited out when asked, typed when not (task 2505)', () => {
+  // roms filed a 1 GiB disc by reading 256 parts back to back; binary-server's tenant ceiling
+  // answered 429 at part ~204 and the plain throw dropped the whole pass. Each app was about to
+  // hand-roll its own `/: 429$/` regex and guess the wait the store had already named.
+  const limited = (retryAfter: string | null = '2') =>
+    new Response('{"error":"rate_limited"}', {
+      status: 429,
+      headers: retryAfter === null ? {} : { 'retry-after': retryAfter },
+    });
+
+  /** Answers 429 `refusals` times, then the bytes; records every sleep. */
+  const pacedStore = (refusals: number, retryAfter: string | null = '2') => {
+    const slept: number[] = [];
+    let n = 0;
+    reply = () => (n++ < refusals ? limited(retryAfter) : ok(new Uint8Array([7, 8])));
+    const store = makeStore({
+      sleep: (ms) => {
+        slept.push(ms);
+        return Promise.resolve();
+      },
+    });
+    return { store, slept };
+  };
+
+  it('paced get sleeps the retry-after the store named and returns the bytes', async () => {
+    const { store, slept } = pacedStore(3);
+    expect(await store.get('parts/0', { paced: true })).toEqual(new Uint8Array([7, 8]));
+    expect(slept).toEqual([2000, 2000, 2000]);
+    expect(calls).toHaveLength(4);
+  });
+
+  it('without paced, a 429 is the typed error carrying retryAfterSec — and the old message', async () => {
+    const { store, slept } = pacedStore(1, '7');
+    const error = await store.get('parts/0').catch((e: unknown) => e);
+    expect(error).toBeInstanceOf(BinaryStoreRateLimitedError);
+    expect((error as BinaryStoreRateLimitedError).retryAfterSec).toBe(7);
+    expect((error as Error).message).toBe('[binary-store] get parts/0: 429');
+    expect(slept).toEqual([]);
+    expect(store.known('parts/0'), 'a pace is not an absence').toBe('unknown');
+  });
+
+  it('a paced read gives up once its wait budget is spent, saying how long it waited', async () => {
+    const { store, slept } = pacedStore(Number.POSITIVE_INFINITY, '30');
+    const error = await store.get('parts/0', { paced: { maxWaitMs: 65_000 } }).catch((e: unknown) => e);
+    expect(error).toBeInstanceOf(BinaryStoreRateLimitedError);
+    expect(slept).toEqual([30_000, 30_000]);
+    expect((error as BinaryStoreRateLimitedError).waitedMs).toBe(60_000);
+  });
+
+  it('the default budget is bounded', async () => {
+    const { store, slept } = pacedStore(Number.POSITIVE_INFINITY, '30');
+    await expect(store.get('parts/0', { paced: true })).rejects.toBeInstanceOf(BinaryStoreRateLimitedError);
+    expect(slept.reduce((a, b) => a + b, 0)).toBeLessThanOrEqual(BINARY_PACED_MAX_WAIT_MS);
+  });
+
+  it('no retry-after backs off 1, 2, 4 s rather than hammering', async () => {
+    const { store, slept } = pacedStore(3, null);
+    await store.get('parts/0', { paced: true });
+    expect(slept).toEqual([1000, 2000, 4000]);
+  });
+
+  it('meta waits it out too, and stat does not spend a second read probing after a 429', async () => {
+    const paced = pacedStore(1);
+    reply = (() => {
+      let n = 0;
+      return () => (n++ < 1 ? limited() : json({ size: 5, mime: 'x/y' }));
+    })();
+    expect((await paced.store.meta('m', { paced: true }))?.size).toBe(5);
+    expect(paced.slept).toEqual([2000]);
+
+    calls = [];
+    reply = () => limited();
+    await expect(makeStore().stat('s')).rejects.toBeInstanceOf(BinaryStoreRateLimitedError);
+    expect(calls, 'the Range probe must not follow a 429').toHaveLength(1);
+  });
+});

package/src/server/storage/binaryStore.ts

@@ -83,6 +83,8 @@ export interface BinaryStoreConfig {
   absenceTtlMs?: number;
   /** Injected so the memo's expiry is testable without a real clock. */
   now?: () => number;
+  /** Injected so a `paced` read's waits are testable without real time. Default: a timer. */
+  sleep?: (ms: number) => Promise<unknown>;
   /**
    * 🔴 How THIS PROCESS sends every request to binary-server. Default: the global `fetch`.
    *
@@ -254,8 +256,25 @@ export interface ReadOptions {
    * hour" is answered by reloading the picture.
    */
   fresh?: boolean;
+  /**
+   * 🔴 Wait out binary-server's read ceiling instead of throwing it (task 2505).
+   *
+   * The store allows a tenant a burst of reads, then a steady rate, and answers 429 with the
+   * `retry-after` it wants honoured (`apps/binary-server/src/rateLimit.ts`). A batch read —
+   * roms filing a disc's 256 parts, a hash backfill, an import — met it at read ~204 and threw
+   * the whole pass away. `paced` makes `get`/`meta`/`stat` sleep the named `retry-after` and ask
+   * again, for at most {@link BINARY_PACED_MAX_WAIT_MS} in total (or `{ maxWaitMs }`).
+   *
+   * Opt-in, because a page read a person is waiting on should fail fast rather than hang:
+   * without it a 429 throws {@link BinaryStoreRateLimitedError}, which carries the
+   * `retryAfterSec` so a caller pacing itself need not guess.
+   */
+  paced?: boolean | { maxWaitMs?: number };
 }
 
+/** How long a `paced` read waits out 429s in total before it throws — see {@link ReadOptions.paced}. */
+export const BINARY_PACED_MAX_WAIT_MS = 120_000;
+
 /** What the memos currently hold — for an owner surface, a smoke, and the tests. */
 export interface BinaryCacheStats {
   /** Keys known to be present. Never expires; see {@link BINARY_ABSENCE_TTL_MS}. */
@@ -526,6 +545,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
   const send = cfg.fetch ?? ((input: string | Request, init?: RequestInit) => fetch(input, init));
   const autoChunkBytes = cfg.autoChunkBytes ?? AUTO_CHUNK_THRESHOLD_BYTES;
   const now = cfg.now ?? Date.now;
+  const sleep = cfg.sleep ?? ((ms: number) => new Promise((done) => setTimeout(done, ms)));
   const absenceTtlMs = cfg.absenceTtlMs ?? BINARY_ABSENCE_TTL_MS;
 
   /*
@@ -700,6 +720,33 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
     return { 'x-internal-secret': cfg.internalSecret };
   };
 
+  /**
+   * One read, with a 429 handled the one way every read handles it: waited out when the
+   * caller asked for `paced` and the budget allows, otherwise thrown as a typed error that
+   * keeps the `[binary-store] <op> <key>: 429` message batch callers already match.
+   */
+  const sendRead = async (
+    op: string,
+    key: string,
+    url: string,
+    init: RequestInit | undefined,
+    opts: ReadOptions | undefined,
+  ): Promise<Response> => {
+    const budget = opts?.paced
+      ? (typeof opts.paced === 'object' ? opts.paced.maxWaitMs : undefined) ?? BINARY_PACED_MAX_WAIT_MS
+      : 0;
+    let waited = 0;
+    for (let attempt = 0; ; attempt++) {
+      const res = await send(url, init);
+      if (res.status !== 429) return res;
+      const retryAfterSec = retryAfterSeconds(res.headers.get('retry-after'), attempt);
+      const ms = retryAfterSec * 1000;
+      if (waited + ms > budget) throw new BinaryStoreRateLimitedError(op, key, retryAfterSec, waited);
+      await sleep(ms);
+      waited += ms;
+    }
+  };
+
   const store: BinaryStore = {
     appId: cfg.appId,
     signToken,
@@ -828,7 +875,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
       // short-circuit here — we want the bytes, and only the store has them.
       if (!opts?.fresh && knownAbsent(key)) return null;
       countWireRead(key);
-      const res = await send(await internalMediaUrl(key));
+      const res = await sendRead('get', key, await internalMediaUrl(key), undefined, opts);
       if (res.status === 404) {
         rememberAbsent(key);
         return null;
@@ -905,7 +952,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
         if (knownAbsent(key)) return null;
       }
       countWireRead(key);
-      const res = await send(await metaUrl(key));
+      const res = await sendRead('meta', key, await metaUrl(key), undefined, opts);
       if (res.status === 404) {
         rememberAbsent(key);
         return null;
@@ -942,11 +989,13 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
       try {
         const m = await store.meta(key, opts);
         if (m) return { size: m.size };
-      } catch {
-        // A transport/5xx failure — the probe below is the second opinion.
+      } catch (why) {
+        // A transport/5xx failure — the probe below is the second opinion. A 429 is not: the
+        // store answered "slow down", and a second read straight away only spends more of it.
+        if (why instanceof BinaryStoreRateLimitedError) throw why;
       }
       countWireRead(key);
-      const res = await send(await internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } });
+      const res = await sendRead('stat', key, await internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } }, opts);
       if (res.status === 404) {
         rememberAbsent(key);
         return null;
@@ -1145,3 +1194,35 @@ export function readBinaryStoreEnv(
     internalBaseUrl: env.BINARY_SERVER_INTERNAL_URL?.trim() || undefined,
   };
 }
+
+/**
+ * The seconds a 429 asks for. binary-server always names `retry-after` (1-30 s); a
+ * missing or junk one — an edge in front of it, an older server — backs off 1, 2, 4… s
+ * rather than hammering, capped at the same 30 s the server caps itself at.
+ */
+function retryAfterSeconds(header: string | null, attempt: number): number {
+  const named = Number(header);
+  if (header !== null && Number.isFinite(named) && named >= 0) return Math.min(30, Math.max(1, Math.ceil(named)));
+  return Math.min(30, 2 ** Math.min(attempt, 5));
+}
+
+/**
+ * binary-server refused a READ for its rate ceiling (429) — a pace, not a verdict, and not an
+ * absence (nothing is memoized). Thrown by `get`/`meta`/`stat` without `paced`, or with `paced`
+ * once its wait budget is spent. The message keeps the `[binary-store] <op> <key>: 429` shape
+ * earlier callers matched on; `retryAfterSec` is what the store asked for.
+ */
+export class BinaryStoreRateLimitedError extends Error {
+  readonly status = 429;
+  readonly code = 'BYTE_STORE_RATE_LIMITED';
+  constructor(
+    op: string,
+    key: string,
+    readonly retryAfterSec: number,
+    /** How long a `paced` read had already waited before giving up — 0 when it was not paced. */
+    readonly waitedMs = 0,
+  ) {
+    super(`[binary-store] ${op} ${key}: 429`);
+    this.name = 'BinaryStoreRateLimitedError';
+  }
+}