cursedbelt-server 4.0.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/bench/assert.d.ts +16 -3
- package/dist/server/bench/assert.js +54 -0
- package/dist/server/bench/cpuClock.js +21 -1
- package/dist/server/d1/fakeD1.d.ts +5 -0
- package/dist/server/d1/fakeD1.js +51 -18
- package/dist/server/d1/local.js +48 -3
- package/dist/server/d1/values.d.ts +19 -2
- package/dist/server/d1/values.js +21 -2
- package/dist/server/middleware/bodyLimit.d.ts +152 -0
- package/dist/server/middleware/bodyLimit.js +161 -0
- package/dist/server/storage/binaryStore.d.ts +385 -0
- package/dist/server/storage/binaryStore.js +739 -0
- package/dist/server/storage/binaryStoreFake.d.ts +56 -0
- package/dist/server/storage/binaryStoreFake.js +63 -0
- package/package.json +19 -1
- package/src/leafSubpathsImportNothing.spec.ts +37 -0
- package/src/server/bench/assert.ts +78 -3
- package/src/server/bench/cpuBudget.spec.ts +101 -1
- package/src/server/bench/cpuClock.ts +22 -1
- package/src/server/d1/fakeD1.ts +56 -18
- package/src/server/d1/local.ts +56 -3
- package/src/server/d1/sameShape.spec.ts +73 -0
- package/src/server/d1/values.ts +23 -2
- package/src/server/middleware/bodyLimit.spec.ts +238 -0
- package/src/server/middleware/bodyLimit.ts +210 -0
- package/src/server/storage/binaryStore.spec.ts +908 -0
- package/src/server/storage/binaryStore.ts +1049 -0
- package/src/server/storage/binaryStoreFake.ts +111 -0
|
@@ -0,0 +1,1049 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* binary-server client — the ONE way an app talks to the machine's binary store
|
|
3
|
+
* (`apps/binary-server`, :3099 behind binary-server.cursedalchemy.com).
|
|
4
|
+
*
|
|
5
|
+
* The owner's standing decision: binary-server is THE binary/media storage spot for
|
|
6
|
+
* every app — local disks must not accumulate image/blob payloads. Any app that loads
|
|
7
|
+
* or generates images stores them through this client, behind its own app-local seam
|
|
8
|
+
* (an `AttachmentStore`, a box-art cache, art artifacts).
|
|
9
|
+
*
|
|
10
|
+
* Protocol (see binary-server/src/server.ts + src/tokens.ts):
|
|
11
|
+
* - `PUT /upload/<app>/<key>?token=` uploads whole bytes (201).
|
|
12
|
+
* - `GET /media/<app>/<key>?token=` downloads (Range-capable; 404 when absent).
|
|
13
|
+
* - `DELETE /media/<app>/<key>?token=` is gated by BOTH `x-internal-secret` AND a
|
|
14
|
+
* per-op delete token (`op:"delete"`, tenant-scoped like read/write). Owner ruling #10.
|
|
15
|
+
* - Tokens are RS256 JWTs signed with the app's PRIVATE key (bs holds only the
|
|
16
|
+
* public half); `iss` must exactly match the registered issuer and `k` must be
|
|
17
|
+
* the app-prefixed path (`<app>/<key>`). Paths are decoded whole on the server,
|
|
18
|
+
* so keys are encoded PER SEGMENT with `/` kept literal.
|
|
19
|
+
*
|
|
20
|
+
* ── 🔴 Why this sits beside `createBinaryServerStore` rather than replacing it ──────
|
|
21
|
+
* `./binaryServerStore.ts` is 102 lines: a {@link MediaStore} adapter whose RS256 signer
|
|
22
|
+
* is INJECTED, deliberately, so that module carries no key material. This one is the
|
|
23
|
+
* other half of the same wire — it MINTS the token from a PKCS8 private PEM and owns the
|
|
24
|
+
* whole client surface: auto-chunking, the absence memo, Range reads, the repeat-key
|
|
25
|
+
* counter and tenant resolution. They are not the same module and neither is a worse
|
|
26
|
+
* version of the other; they share one contract, {@link FileTokenClaims}, so a token
|
|
27
|
+
* minted here can never drift out of shape from one minted there against the same
|
|
28
|
+
* binary-server.
|
|
29
|
+
*
|
|
30
|
+
* 🔴 It is published as the LEAF subpath `cursedbelt-server/binary-store`, never through
|
|
31
|
+
* the `./storage` barrel — that barrel drags the file-catalogue and transcode graph, and
|
|
32
|
+
* this module's whole promise is that an app pays nothing but `node:crypto` for it.
|
|
33
|
+
* `src/leafSubpathsImportNothing.spec.ts` is what holds that promise.
|
|
34
|
+
*
|
|
35
|
+
* ── Where it came from (2026-09-17) ────────────────────────────────────────────────
|
|
36
|
+
* Five apps carried a byte-for-byte copy of this file and three of them had drifted.
|
|
37
|
+
* Nothing could see it: a repo's gate proves that repo, so five green gates is what
|
|
38
|
+
* three forks of the fleet's binary-storage client looked like from the outside. The
|
|
39
|
+
* divergence was measured before the move and was entirely in prose and in one stale
|
|
40
|
+
* import specifier (`cursedbelt/server/storage`, the pre-split package name) — no
|
|
41
|
+
* behavioural difference in any of the three, so nothing was dropped to land the union.
|
|
42
|
+
*/
|
|
43
|
+
import { createPrivateKey, sign as cryptoSign, type KeyObject } from 'node:crypto';
|
|
44
|
+
import type { FileTokenClaims } from './types';
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* The claims binary-server verifies, re-exported from `./types` (the fleet's single
|
|
48
|
+
* source of truth — `cursedbelt-server/storage/types`). See there for the per-field
|
|
49
|
+
* docs. Kept exported here so the call sites that took it from the app-side copy of
|
|
50
|
+
* this module read unchanged.
|
|
51
|
+
*/
|
|
52
|
+
export type { FileTokenClaims };
|
|
53
|
+
|
|
54
|
+
export interface BinaryStoreConfig {
|
|
55
|
+
/** e.g. `http://127.0.0.1:3099` or `https://binary-server.cursedalchemy.com`. */
|
|
56
|
+
baseUrl: string;
|
|
57
|
+
/** Tenant id — the first path segment and the token scope, e.g. `notes`. */
|
|
58
|
+
appId: string;
|
|
59
|
+
/** Must byte-match the issuer registered in binary-server's `apps` table. */
|
|
60
|
+
issuer: string;
|
|
61
|
+
/** PKCS8 private PEM (raw, `\n`-escaped, or base64-wrapped — all accepted). */
|
|
62
|
+
privateKey: string;
|
|
63
|
+
/** Enables `remove`/`removePrefix` (sent as `x-internal-secret`). */
|
|
64
|
+
internalSecret?: string;
|
|
65
|
+
/**
|
|
66
|
+
* Where THIS PROCESS should reach binary-server, when that is somewhere cheaper than
|
|
67
|
+
* the public name — `http://127.0.0.1:3099` for a deployment sharing the Mac with it.
|
|
68
|
+
* Defaults to {@link BinaryStoreConfig.baseUrl}. Used by `get`, `meta` and `stat`;
|
|
69
|
+
* never by `mediaUrl`, whose output is handed to a browser.
|
|
70
|
+
*/
|
|
71
|
+
internalBaseUrl?: string;
|
|
72
|
+
/**
|
|
73
|
+
* Above this many bytes, {@link BinaryStore.put} takes the CHUNKED path by
|
|
74
|
+
* itself. Defaults to {@link AUTO_CHUNK_THRESHOLD_BYTES}; pass `Infinity` to
|
|
75
|
+
* turn the fallback off and get the pre-2026-08-08 behavior.
|
|
76
|
+
*/
|
|
77
|
+
autoChunkBytes?: number;
|
|
78
|
+
/**
|
|
79
|
+
* How long a "this key is not here" is believed. Defaults to
|
|
80
|
+
* {@link BINARY_ABSENCE_TTL_MS}; the constant carries the owner's reasoning and
|
|
81
|
+
* an app should have a measured reason to differ from it.
|
|
82
|
+
*/
|
|
83
|
+
absenceTtlMs?: number;
|
|
84
|
+
/** Injected so the memo's expiry is testable without a real clock. */
|
|
85
|
+
now?: () => number;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/*
|
|
89
|
+
* ── The process-wide repeat register (§ the repeat-key counter) ───────────────
|
|
90
|
+
*
|
|
91
|
+
* An app makes one store per tenant and a few make two, so the counter has to
|
|
92
|
+
* aggregate somewhere above an individual client for the answer to mean "what is
|
|
93
|
+
* THIS DEPLOYMENT doing". It is a module-level map for the same reason an install
|
|
94
|
+
* metrics recorder is module-level: the thing being measured is the process, and
|
|
95
|
+
* threading a collector through every `createBinaryStore` call site would make
|
|
96
|
+
* instrumenting an app an act of remembering.
|
|
97
|
+
*
|
|
98
|
+
* An app reports it on `/healthz`, which the fleet probe already fetches for every
|
|
99
|
+
* live deployment — so this reaches the incident sweep, and the owner's text channel,
|
|
100
|
+
* without anybody making a new request for it.
|
|
101
|
+
*/
|
|
102
|
+
const repeats = new Map<string, RepeatedBinaryKey & { at: number }>();
|
|
103
|
+
/** Bounded: a genuinely runaway app must not turn this into a leak. */
|
|
104
|
+
const MAX_TRACKED_REPEATS = 50;
|
|
105
|
+
|
|
106
|
+
function recordRepeat(tenant: string, key: string, count: number, at: number): void {
|
|
107
|
+
const id = `${tenant}/${key}`;
|
|
108
|
+
if (!repeats.has(id) && repeats.size >= MAX_TRACKED_REPEATS) return;
|
|
109
|
+
repeats.set(id, {
|
|
110
|
+
tenant,
|
|
111
|
+
key,
|
|
112
|
+
reads: count,
|
|
113
|
+
windowMs: BINARY_REPEAT_WINDOW_MS,
|
|
114
|
+
at,
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Which keys this process has read from the wire far too often, worst first —
|
|
120
|
+
* empty on every healthy deployment, which is what makes it a usable alarm.
|
|
121
|
+
*
|
|
122
|
+
* Entries older than one window are dropped as they are read, so a burst that
|
|
123
|
+
* stopped stops being reported rather than becoming a permanent accusation.
|
|
124
|
+
*/
|
|
125
|
+
export function repeatedBinaryKeys(now: () => number = Date.now): RepeatedBinaryKey[] {
|
|
126
|
+
const at = now();
|
|
127
|
+
const live: RepeatedBinaryKey[] = [];
|
|
128
|
+
for (const [id, row] of repeats) {
|
|
129
|
+
if (at - row.at >= BINARY_REPEAT_WINDOW_MS) {
|
|
130
|
+
repeats.delete(id);
|
|
131
|
+
continue;
|
|
132
|
+
}
|
|
133
|
+
live.push({ tenant: row.tenant, key: row.key, reads: row.reads, windowMs: row.windowMs });
|
|
134
|
+
}
|
|
135
|
+
return live.sort((a, b) => b.reads - a.reads);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** Test seam — the register is process-wide, so a spec must be able to clear it. */
|
|
139
|
+
export function resetRepeatedBinaryKeys(): void {
|
|
140
|
+
repeats.clear();
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Did this request ask for a FRESH answer — i.e. is it a hard refresh?
|
|
145
|
+
*
|
|
146
|
+
* 🔴 The owner's first escape hatch, and the reason it is a shared function rather
|
|
147
|
+
* than an inline header read in each app: *"if the no is remembered an hour then I
|
|
148
|
+
* could fix it but not be able to tell for an hour."* A browser hard-refresh sends
|
|
149
|
+
* `Cache-Control: no-cache` (older ones send `Pragma`), so reloading the picture is
|
|
150
|
+
* the repair — as long as every app spells the check the same way.
|
|
151
|
+
*/
|
|
152
|
+
export function wantsFresh(req: { headers: Headers } | Headers): boolean {
|
|
153
|
+
const headers = req instanceof Headers ? req : req.headers;
|
|
154
|
+
const cacheControl = headers.get('cache-control')?.toLowerCase() ?? '';
|
|
155
|
+
if (cacheControl.includes('no-cache')) return true;
|
|
156
|
+
return (headers.get('pragma')?.toLowerCase() ?? '').includes('no-cache');
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* 🔴 The size at which a single-request `put` stops being safe, FLEET-WIDE.
|
|
161
|
+
*
|
|
162
|
+
* binary-server is published through a Cloudflare Tunnel, and the edge refuses a
|
|
163
|
+
* request body over ~100 MB with a **413 before it ever reaches the origin**. So
|
|
164
|
+
* the ceiling is not bs's and not the caller's — it belongs to a hop neither end
|
|
165
|
+
* controls, and every app that stores a large blob inherits it.
|
|
166
|
+
*
|
|
167
|
+
* `apps/roms` hit it on 2026-08-08: eight genuine DS cartridges (128–512 MB)
|
|
168
|
+
* came back `[binary-store] put rom/<digest>: 413` and it read as eight
|
|
169
|
+
* mysterious ingest failures rather than as one limit. roms fixed it locally,
|
|
170
|
+
* which left the trap in place for every other app's attachments and for any
|
|
171
|
+
* future video/media path — so the threshold lives HERE now and `put` applies it
|
|
172
|
+
* itself.
|
|
173
|
+
*
|
|
174
|
+
* 64 MiB, not 100: the edge's number is approximate and includes headers and any
|
|
175
|
+
* transfer encoding, and the chunked path costs one extra request for a blob
|
|
176
|
+
* that only just crosses the line. Being early is free; being late is a 413.
|
|
177
|
+
*/
|
|
178
|
+
export const AUTO_CHUNK_THRESHOLD_BYTES = 64 * 1024 * 1024;
|
|
179
|
+
|
|
180
|
+
export interface PutOptions {
|
|
181
|
+
/** The user-facing display name ("My Dog Video") — binary-server stores it in its
|
|
182
|
+
* db ONLY (never on disk), for its inspector/UIs. Optional everywhere. */
|
|
183
|
+
title?: string;
|
|
184
|
+
/**
|
|
185
|
+
* This key IS the digest of these bytes, so they can never change under it.
|
|
186
|
+
* binary-server then serves the object `public, max-age=31536000, immutable`
|
|
187
|
+
* instead of the one-day default — no daily revalidation against the home
|
|
188
|
+
* uplink for bytes that are the same bytes by construction, and cache-busting
|
|
189
|
+
* needs no version segment because different bytes are a different key.
|
|
190
|
+
*
|
|
191
|
+
* Only pass it for a genuinely content-addressed key. A wrong `true` pins stale
|
|
192
|
+
* bytes at the edge and in every browser for a year.
|
|
193
|
+
*/
|
|
194
|
+
immutable?: boolean;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
export interface MediaUrlOptions {
|
|
198
|
+
filename?: string;
|
|
199
|
+
disposition?: 'inline' | 'attachment';
|
|
200
|
+
ttlSeconds?: number;
|
|
201
|
+
/**
|
|
202
|
+
* Mint a URL that is BYTE-IDENTICAL for every call inside the same window of
|
|
203
|
+
* this many seconds (the token's `iat`/`exp` are snapped to the window rather
|
|
204
|
+
* than to the clock), and valid for two windows so one minted at the very end
|
|
205
|
+
* of a window is still good.
|
|
206
|
+
*
|
|
207
|
+
* The problem it solves: a fresh token per call means a fresh URL per call, and
|
|
208
|
+
* a browser's HTTP cache keys on the URL — so an `<img src>` re-rendered or a
|
|
209
|
+
* ROM re-launched re-downloads bytes it already has, however long the
|
|
210
|
+
* Cache-Control says. (The Cloudflare edge is unaffected either way; its cache
|
|
211
|
+
* key strips `token`.) The cost is a longer-lived bearer URL, so keep the
|
|
212
|
+
* window as short as the caching win allows and leave it unset for anything
|
|
213
|
+
* whose leak matters more than its bytes.
|
|
214
|
+
*/
|
|
215
|
+
stableWindowSeconds?: number;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
export interface BlobMeta {
|
|
219
|
+
size: number;
|
|
220
|
+
mime: string | null;
|
|
221
|
+
/** The display name the uploader set (`PutOptions.title`), if any. */
|
|
222
|
+
title: string | null;
|
|
223
|
+
/** sha256 binary-server computed post-write. Audit-only — keys are app-chosen. */
|
|
224
|
+
checksum: string | null;
|
|
225
|
+
createdAt?: string;
|
|
226
|
+
updatedAt?: string;
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
export interface PutLargeOptions extends PutOptions {
|
|
230
|
+
/** Bytes per chunk. Default 16 MiB — big enough that a 500 MB upload is ~32 requests,
|
|
231
|
+
* small enough that neither side ever buffers a whole video. */
|
|
232
|
+
chunkBytes?: number;
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/** How a read should treat the memos — see {@link BINARY_ABSENCE_TTL_MS}. */
|
|
236
|
+
export interface ReadOptions {
|
|
237
|
+
/**
|
|
238
|
+
* Ignore both memos: ask the store, and refresh what we believe from its answer.
|
|
239
|
+
*
|
|
240
|
+
* This is the per-request escape hatch, and `wantsFresh(request)` is what turns a
|
|
241
|
+
* browser's hard refresh into it — so "I fixed the picture and cannot tell for an
|
|
242
|
+
* hour" is answered by reloading the picture.
|
|
243
|
+
*/
|
|
244
|
+
fresh?: boolean;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/** What the memos currently hold — for an owner surface, a smoke, and the tests. */
|
|
248
|
+
export interface BinaryCacheStats {
|
|
249
|
+
/** Keys known to be present. Never expires; see {@link BINARY_ABSENCE_TTL_MS}. */
|
|
250
|
+
present: number;
|
|
251
|
+
/** Keys known to be absent, and not yet lapsed. */
|
|
252
|
+
absent: number;
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/** One key this process has read from the wire far too often — see
|
|
256
|
+
* {@link BINARY_REPEAT_THRESHOLD}. */
|
|
257
|
+
export interface RepeatedBinaryKey {
|
|
258
|
+
/** binary-server tenant, i.e. which app is doing it. */
|
|
259
|
+
tenant: string;
|
|
260
|
+
key: string;
|
|
261
|
+
/** Wire reads inside the current window. Memo hits are NOT counted: the memo
|
|
262
|
+
* working is the fix, so a memoized key must go quiet. */
|
|
263
|
+
reads: number;
|
|
264
|
+
windowMs: number;
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export interface BinaryStore {
|
|
268
|
+
readonly appId: string;
|
|
269
|
+
/**
|
|
270
|
+
* Store an object. Above {@link AUTO_CHUNK_THRESHOLD_BYTES} this delegates to
|
|
271
|
+
* {@link BinaryStore.putLarge} by itself, because the limit that makes a
|
|
272
|
+
* single request fail belongs to the Cloudflare Tunnel in front of
|
|
273
|
+
* binary-server rather than to any caller — see that constant.
|
|
274
|
+
*/
|
|
275
|
+
put(key: string, bytes: Uint8Array, mime?: string, opts?: PutOptions): Promise<void>;
|
|
276
|
+
/**
|
|
277
|
+
* Upload in chunks (`PUT /upload-chunk`), for objects too large to buffer whole.
|
|
278
|
+
*
|
|
279
|
+
* `put` sends one request whose body binary-server reads entirely into memory before
|
|
280
|
+
* writing — fine for an attachment, wrong for a 500 MB video on a machine that is also
|
|
281
|
+
* running everything else. This sends bounded pieces and lets bs assemble them once the
|
|
282
|
+
* last one lands (it counts arrivals, so out-of-order delivery and retries are both safe).
|
|
283
|
+
*/
|
|
284
|
+
putLarge(
|
|
285
|
+
key: string,
|
|
286
|
+
source: Uint8Array | Blob,
|
|
287
|
+
mime?: string,
|
|
288
|
+
opts?: PutLargeOptions,
|
|
289
|
+
): Promise<void>;
|
|
290
|
+
/** `null` when the key does not exist. */
|
|
291
|
+
get(key: string, opts?: ReadOptions): Promise<Uint8Array | null>;
|
|
292
|
+
/**
|
|
293
|
+
* Everything binary-server knows about a blob (`GET /meta`) — `null` when unknown or
|
|
294
|
+
* deleted. The honest answer to "what IS this?": size, the stored mime, and the TITLE,
|
|
295
|
+
* none of which `stat` could report (it inferred a size from a Range probe and nothing
|
|
296
|
+
* else). Added to bs 2026-07-30 for exactly this.
|
|
297
|
+
*
|
|
298
|
+
* MEMOIZED since 2026-09-09 — a `null` for an hour, a hit for ever. See
|
|
299
|
+
* {@link BINARY_ABSENCE_TTL_MS} for both halves and for the two bypasses.
|
|
300
|
+
*/
|
|
301
|
+
meta(key: string, opts?: ReadOptions): Promise<BlobMeta | null>;
|
|
302
|
+
/** `null` when the key does not exist. Delegates to {@link BinaryStore.meta}. */
|
|
303
|
+
stat(key: string, opts?: ReadOptions): Promise<{ size: number } | null>;
|
|
304
|
+
/**
|
|
305
|
+
* Does the store hold this key? The cheap question, memoized — and the one every
|
|
306
|
+
* app was asking the expensive way.
|
|
307
|
+
*
|
|
308
|
+
* Prefer it over `stat`/`meta` when the answer is only ever used as a boolean:
|
|
309
|
+
* it says what it means, and it cannot tempt a caller into trusting a memoized
|
|
310
|
+
* `size` for a key whose bytes are replaceable.
|
|
311
|
+
*/
|
|
312
|
+
has(key: string, opts?: ReadOptions): Promise<boolean>;
|
|
313
|
+
/**
|
|
314
|
+
* What the memo ALREADY knows about a key, with no I/O and no promise.
|
|
315
|
+
*
|
|
316
|
+
* 🔴 The one question `has()` cannot answer, and the reason an app would otherwise keep
|
|
317
|
+
* a second copy of this cache. A listing route answers for hundreds of rows inside one
|
|
318
|
+
* request — roms' `/api/roms` draws 300 games — so it cannot await a round trip per row
|
|
319
|
+
* even when every one of them would hit the memo: `await` in a loop over 300 keys is 300
|
|
320
|
+
* microtask turns and a `Promise` each, on the box that also serves everything else.
|
|
321
|
+
* With this the render answers from what is already known and lets the per-card path
|
|
322
|
+
* fill in the rest.
|
|
323
|
+
*
|
|
324
|
+
* `"unknown"` is a real third answer and must not be collapsed into `"absent"`: it means
|
|
325
|
+
* *nobody has asked yet*, and a caller that treats it as "no" turns a cold process into
|
|
326
|
+
* one that permanently draws nothing.
|
|
327
|
+
*/
|
|
328
|
+
known(key: string): 'present' | 'absent' | 'unknown';
|
|
329
|
+
/**
|
|
330
|
+
* Forget every "the store does not have this" — all of them, or those whose key
|
|
331
|
+
* starts with `prefix`. Returns how many were dropped.
|
|
332
|
+
*
|
|
333
|
+
* 🔴 The owner's second escape hatch, and it is deliberately a REPAIR rather than
|
|
334
|
+
* a cache flush: the positive memos are untouched, so proving a fix costs nothing
|
|
335
|
+
* but the keys that were actually missing. An app exposes it behind its owner gate
|
|
336
|
+
* (`POST /api/media/forget-misses`) so "I put the picture back" can be verified in
|
|
337
|
+
* seconds instead of in an hour.
|
|
338
|
+
*/
|
|
339
|
+
forgetMisses(prefix?: string): number;
|
|
340
|
+
/** What the memos hold right now. */
|
|
341
|
+
cacheStats(): BinaryCacheStats;
|
|
342
|
+
remove(key: string): Promise<void>;
|
|
343
|
+
/** Wipes a whole container tree (`?container=1`). */
|
|
344
|
+
removePrefix(prefix: string): Promise<void>;
|
|
345
|
+
/** A signed, directly-fetchable download URL (for redirects / <img src>). */
|
|
346
|
+
mediaUrl(key: string, opts?: MediaUrlOptions): Promise<string>;
|
|
347
|
+
/**
|
|
348
|
+
* A signed URL for `master`, authorized by `prefix` — the only shape a multi-object tree
|
|
349
|
+
* (an HLS ladder) can be played from. See the implementation for why an exact-key token
|
|
350
|
+
* cannot work and why this is a separate method rather than a flag.
|
|
351
|
+
*/
|
|
352
|
+
mediaPrefixUrl(prefix: string, master: string, opts?: MediaUrlOptions): Promise<string>;
|
|
353
|
+
/**
|
|
354
|
+
* The raw `Response` for an object, from the origin THIS PROCESS should use — so an app
|
|
355
|
+
* can SERVE bytes itself, streaming, when the public edge will not.
|
|
356
|
+
*
|
|
357
|
+
* 🔴 The gap {@link BinaryStoreConfig.internalBaseUrl} left open. That note ends
|
|
358
|
+
* "redirected media — videos, large downloads — therefore still goes through the edge",
|
|
359
|
+
* and on 2026-09-08 that was the whole outage: Cloudflare spent its Workers day, every
|
|
360
|
+
* `/media/*` request answered 429, and apps/music could not play a note even though the
|
|
361
|
+
* bytes were on the same Mac as the server the browser was talking to.
|
|
362
|
+
*
|
|
363
|
+
* Two things make this a fallback rather than a second architecture:
|
|
364
|
+
* · It returns the RESPONSE, not the bytes. `get` buffers a whole object into memory,
|
|
365
|
+
* which is wrong for a 30 MB track and unthinkable for a video; the caller pipes
|
|
366
|
+
* `res.body` straight through and holds one chunk at a time.
|
|
367
|
+
* · `range` is forwarded and the origin's `206`/`content-range` come back untouched,
|
|
368
|
+
* because Range IS seeking. A proxy that drops it turns a seekable track into a
|
|
369
|
+
* download.
|
|
370
|
+
*/
|
|
371
|
+
fetchMedia(
|
|
372
|
+
key: string,
|
|
373
|
+
init?: { range?: string | null; method?: 'GET' | 'HEAD'; signal?: AbortSignal },
|
|
374
|
+
): Promise<Response>;
|
|
375
|
+
/** Escape hatch for protocols this module doesn't wrap (chunked uploads). */
|
|
376
|
+
signToken(claims: FileTokenClaims, ttlSeconds: number): Promise<string>;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
/**
|
|
380
|
+
* 🔴 How long "the store does not hold this key" is believed — the NEGATIVE memo.
|
|
381
|
+
*
|
|
382
|
+
* The positive one never expires and needs no constant: a key in this fleet names
|
|
383
|
+
* either a content digest or a timestamped upload, so the bytes under it cannot
|
|
384
|
+
* change and "it is there" cannot stop being true. An ABSENCE is different — it is
|
|
385
|
+
* a fact about right now, and art, web copies and derivatives all genuinely arrive
|
|
386
|
+
* later — so it has to lapse.
|
|
387
|
+
*
|
|
388
|
+
* An hour is the default because the owner named the cost of getting it wrong:
|
|
389
|
+
*
|
|
390
|
+
* > *"potentially we need a shorter time for remembering a 'no' in some
|
|
391
|
+
* > circumstances or at least a way to bypass it occasionally. For instance
|
|
392
|
+
* > collections showed a lot of missing thumbnails and images and if the no is
|
|
393
|
+
* > remembered an hour then I could fix it but not be able to tell for an hour."*
|
|
394
|
+
*
|
|
395
|
+
* So the hour is never the only answer. TWO bypasses ship with it, both cheap and
|
|
396
|
+
* both proven by {@link BinaryStore.forgetMisses}' tests:
|
|
397
|
+
*
|
|
398
|
+
* · a read with `{ fresh: true }` ignores both memos and re-asks. `wantsFresh()`
|
|
399
|
+
* turns a browser's hard refresh (`Cache-Control: no-cache`) into exactly that,
|
|
400
|
+
* so the owner's own reload IS the fix for one picture;
|
|
401
|
+
* · `forgetMisses(prefix?)` drops every negative memo, or a subtree of them, so an
|
|
402
|
+
* app can expose a one-click repair and prove it in seconds rather than in an
|
|
403
|
+
* hour.
|
|
404
|
+
*/
|
|
405
|
+
export const BINARY_ABSENCE_TTL_MS = 60 * 60 * 1000;
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* The window and the count that make a key's re-reads an INCIDENT rather than traffic.
|
|
409
|
+
*
|
|
410
|
+
* 🔴 Every quota incident this fleet has had was this exact shape and nothing was
|
|
411
|
+
* watching for it: roms boxart at ~200 reads per key per day (20,626 failed `/media`
|
|
412
|
+
* fetches in one day), family portraits at 2,449 per key per day, music cover art
|
|
413
|
+
* doing a server-side `get` that left the Mac to read a file on the Mac. In all three
|
|
414
|
+
* the giveaway was one key, over and over, from a `Bun/1.3.x` user agent — visible in
|
|
415
|
+
* Cloudflare's top-talker list and in nobody's code.
|
|
416
|
+
*
|
|
417
|
+
* 50 in five minutes is far above any legitimate server-side pattern (a page render
|
|
418
|
+
* asks for a key once; a warm pass asks once) and far below the rate that spends a
|
|
419
|
+
* day's allowance, so it fires early and it does not fire on a normal day.
|
|
420
|
+
*/
|
|
421
|
+
export const BINARY_REPEAT_WINDOW_MS = 5 * 60 * 1000;
|
|
422
|
+
export const BINARY_REPEAT_THRESHOLD = 50;
|
|
423
|
+
|
|
424
|
+
export const DOWNLOAD_TOKEN_TTL_SECONDS = 60;
|
|
425
|
+
export const UPLOAD_TOKEN_TTL_SECONDS = 600;
|
|
426
|
+
/** Deletes are server-to-server and near-instant; a short TTL suffices. The token proves the
|
|
427
|
+
* caller holds THIS tenant's private key and is authorized for THIS path — binary-server
|
|
428
|
+
* requires it alongside the internal secret (defense in depth, owner ruling #10). */
|
|
429
|
+
export const DELETE_TOKEN_TTL_SECONDS = 120;
|
|
430
|
+
|
|
431
|
+
/** register-app prints the key as base64(PKCS8 PEM); env files sometimes carry
|
|
432
|
+
* raw PEM or PEM with literal `\n` sequences. Accept all three. */
|
|
433
|
+
export function normalizePrivateKeyPem(raw: string): string {
|
|
434
|
+
const trimmed = raw.trim();
|
|
435
|
+
if (trimmed.includes('-----BEGIN')) return trimmed.replaceAll('\\n', '\n');
|
|
436
|
+
let decoded: string;
|
|
437
|
+
try {
|
|
438
|
+
decoded = Buffer.from(trimmed, 'base64').toString('utf8');
|
|
439
|
+
} catch {
|
|
440
|
+
throw new Error('[binary-store] private key is neither PEM nor base64');
|
|
441
|
+
}
|
|
442
|
+
if (!decoded.includes('-----BEGIN')) {
|
|
443
|
+
throw new Error('[binary-store] private key decodes to something that is not a PEM');
|
|
444
|
+
}
|
|
445
|
+
return decoded;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
const base64url = (bytes: Uint8Array | string): string =>
|
|
449
|
+
Buffer.from(bytes as Uint8Array).toString('base64url');
|
|
450
|
+
|
|
451
|
+
/** Keys may contain `/` as a real separator; each SEGMENT is percent-encoded
|
|
452
|
+
* because the server decodes the whole path before matching it to the token. */
|
|
453
|
+
const encodeKeyPath = (key: string): string => key.split('/').map(encodeURIComponent).join('/');
|
|
454
|
+
|
|
455
|
+
export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
456
|
+
const baseUrl = cfg.baseUrl.replace(/\/+$/, '');
|
|
457
|
+
/*
|
|
458
|
+
* 🔴 Where THIS PROCESS fetches bytes from, which is not where a browser fetches
|
|
459
|
+
* them from (2026-09-08).
|
|
460
|
+
*
|
|
461
|
+
* `baseUrl` is binary-server's public name, and it has to be: a phone has no other
|
|
462
|
+
* way to reach it. But an app's own SERVER calling `get`/`meta` went out to that
|
|
463
|
+
* public name too — through Cloudflare's Worker and back — to read bytes from a
|
|
464
|
+
* daemon on the same Mac. Two consequences, and the fleet met both on one day:
|
|
465
|
+
*
|
|
466
|
+
* 1. Every server-side read spent a request out of the Workers FREE plan's
|
|
467
|
+
* 100,000/day. family alone made 15,689 of them that day.
|
|
468
|
+
* 2. When that ceiling was reached Cloudflare answered 429 to everything, and
|
|
469
|
+
* family's people page drew no faces at all — because the app could not reach a
|
|
470
|
+
* store sitting 20cm away. The owner: "we can't let them block our images
|
|
471
|
+
* without a fallback."
|
|
472
|
+
*
|
|
473
|
+
* No extra copy is needed for that fallback. The bytes are already local, and an
|
|
474
|
+
* R2-backed object is reachable by binary-server directly — R2 is not gated by the
|
|
475
|
+
* Workers limit. So a deployment sharing a machine with binary-server names its
|
|
476
|
+
* loopback origin here and stops leaving the box to read its own files.
|
|
477
|
+
*
|
|
478
|
+
* `mediaUrl` is deliberately NOT affected: it mints a signed URL to hand to a
|
|
479
|
+
* BROWSER, and a browser cannot resolve 127.0.0.1 to this Mac. Redirected media —
|
|
480
|
+
* videos, large downloads — therefore still goes through the edge, which is correct,
|
|
481
|
+
* and is named as the remaining gap.
|
|
482
|
+
*/
|
|
483
|
+
const internalBaseUrl = (cfg.internalBaseUrl ?? cfg.baseUrl).replace(/\/+$/, '');
|
|
484
|
+
const autoChunkBytes = cfg.autoChunkBytes ?? AUTO_CHUNK_THRESHOLD_BYTES;
|
|
485
|
+
const now = cfg.now ?? Date.now;
|
|
486
|
+
const absenceTtlMs = cfg.absenceTtlMs ?? BINARY_ABSENCE_TTL_MS;
|
|
487
|
+
|
|
488
|
+
/*
|
|
489
|
+
* ── 🔴 The shared picture cache (2026-09-09) ───────────────────────────────
|
|
490
|
+
*
|
|
491
|
+
* Three apps had written three copies of this, each after its own outage, and a
|
|
492
|
+
* fourth (family) had none at all and was measured at 2,449 reads per key per day.
|
|
493
|
+
* It belongs here because this is the one seam every app's media passes through —
|
|
494
|
+
* the rule from music (remember a YES for ever) and the rule from roms (remember a
|
|
495
|
+
* NO for an hour) in the place that makes them true for everybody.
|
|
496
|
+
*
|
|
497
|
+
* The positive memo is a `BlobMeta`, not a bare flag, because the read it exists to
|
|
498
|
+
* remove is `meta()` — one index read at binary-server, but a whole Cloudflare Worker
|
|
499
|
+
* invocation from the outside, which is the currency the fleet actually ran out of.
|
|
500
|
+
*
|
|
501
|
+
* 🔴 What makes "for ever" safe is the KEY SHAPE, not optimism. A key here is a
|
|
502
|
+
* content digest (music, collections) or a timestamped upload (family portraits:
|
|
503
|
+
* `<space>/portrait/<id>-<base36 time>`), so a re-upload is a NEW key and nothing can
|
|
504
|
+
* change under an old one. The one shape that does NOT have that property is roms'
|
|
505
|
+
* box art, whose key hashes the game NAME — which is why `put` refreshes the memo
|
|
506
|
+
* rather than leaving it, and why a `get` that comes back empty DEMOTES a positive
|
|
507
|
+
* memo instead of trusting it. Between them, a key whose bytes were replaced or
|
|
508
|
+
* released out from under us self-heals on the next read that actually wanted bytes.
|
|
509
|
+
*/
|
|
510
|
+
const present = new Map<string, BlobMeta>();
|
|
511
|
+
const absentUntil = new Map<string, number>();
|
|
512
|
+
|
|
513
|
+
/** A live negative memo, or `false` (lapsed entries are dropped as they are met). */
|
|
514
|
+
const knownAbsent = (key: string): boolean => {
|
|
515
|
+
const until = absentUntil.get(key);
|
|
516
|
+
if (until === undefined) return false;
|
|
517
|
+
if (until > now()) return true;
|
|
518
|
+
absentUntil.delete(key);
|
|
519
|
+
return false;
|
|
520
|
+
};
|
|
521
|
+
const rememberPresent = (key: string, meta: BlobMeta): void => {
|
|
522
|
+
absentUntil.delete(key);
|
|
523
|
+
present.set(key, meta);
|
|
524
|
+
};
|
|
525
|
+
const rememberAbsent = (key: string): void => {
|
|
526
|
+
present.delete(key);
|
|
527
|
+
absentUntil.set(key, now() + absenceTtlMs);
|
|
528
|
+
};
|
|
529
|
+
/** Neither memo is trustworthy for this key any more — a write, or a delete. */
|
|
530
|
+
const forget = (key: string): void => {
|
|
531
|
+
present.delete(key);
|
|
532
|
+
absentUntil.delete(key);
|
|
533
|
+
};
|
|
534
|
+
|
|
535
|
+
/*
|
|
536
|
+
* The repeat-key counter (see BINARY_REPEAT_THRESHOLD). Counts WIRE reads only:
|
|
537
|
+
* a memo hit is the cure, so a key the memo answers has to fall silent, or the
|
|
538
|
+
* alarm would fire loudest on the apps that fixed themselves.
|
|
539
|
+
*/
|
|
540
|
+
const reads = new Map<string, { count: number; since: number }>();
|
|
541
|
+
const countWireRead = (key: string): void => {
|
|
542
|
+
const at = now();
|
|
543
|
+
const seen = reads.get(key);
|
|
544
|
+
if (!seen || at - seen.since >= BINARY_REPEAT_WINDOW_MS) {
|
|
545
|
+
reads.set(key, { count: 1, since: at });
|
|
546
|
+
return;
|
|
547
|
+
}
|
|
548
|
+
seen.count++;
|
|
549
|
+
// Below the line this is ordinary traffic, and the register must stay empty on a
|
|
550
|
+
// healthy deployment — an alarm that lists every key is one nobody reads.
|
|
551
|
+
if (seen.count < BINARY_REPEAT_THRESHOLD) return;
|
|
552
|
+
if (seen.count === BINARY_REPEAT_THRESHOLD) {
|
|
553
|
+
// Once per key per window, at the moment it crosses — a line per read would
|
|
554
|
+
// be the same storm in the log file.
|
|
555
|
+
console.warn(
|
|
556
|
+
`[binary-store] ${cfg.appId}/${key} has been read ${seen.count} times in ` +
|
|
557
|
+
`${Math.round((at - seen.since) / 1000)}s. Every one of those is a request out ` +
|
|
558
|
+
"of the fleet's daily Cloudflare allowance for bytes that cannot have changed — " +
|
|
559
|
+
'this caller should be holding the answer. See BINARY_REPEAT_THRESHOLD.',
|
|
560
|
+
);
|
|
561
|
+
}
|
|
562
|
+
recordRepeat(cfg.appId, key, seen.count, at);
|
|
563
|
+
};
|
|
564
|
+
let keyObject: KeyObject | null = null;
|
|
565
|
+
const privateKey = (): KeyObject => {
|
|
566
|
+
keyObject ??= createPrivateKey(normalizePrivateKeyPem(cfg.privateKey));
|
|
567
|
+
return keyObject;
|
|
568
|
+
};
|
|
569
|
+
|
|
570
|
+
const signToken = async (
|
|
571
|
+
claims: FileTokenClaims,
|
|
572
|
+
ttlSeconds: number,
|
|
573
|
+
/** Snap `iat` down to a multiple of this so the whole token is byte-stable
|
|
574
|
+
* inside one window — see {@link MediaUrlOptions.stableWindowSeconds}. */
|
|
575
|
+
windowSeconds?: number,
|
|
576
|
+
): Promise<string> => {
|
|
577
|
+
const clock = Math.floor(Date.now() / 1000);
|
|
578
|
+
const issuedAt =
|
|
579
|
+
windowSeconds && windowSeconds > 0
|
|
580
|
+
? Math.floor(clock / windowSeconds) * windowSeconds
|
|
581
|
+
: clock;
|
|
582
|
+
const header = base64url(JSON.stringify({ alg: 'RS256', typ: 'JWT' }));
|
|
583
|
+
const payload = base64url(
|
|
584
|
+
JSON.stringify({ iss: cfg.issuer, iat: issuedAt, exp: issuedAt + ttlSeconds, ...claims }),
|
|
585
|
+
);
|
|
586
|
+
const signature = cryptoSign('sha256', Buffer.from(`${header}.${payload}`), privateKey());
|
|
587
|
+
return `${header}.${payload}.${base64url(signature)}`;
|
|
588
|
+
};
|
|
589
|
+
|
|
590
|
+
const appKey = (key: string): string => `${cfg.appId}/${key}`;
|
|
591
|
+
const mediaUrl = async (key: string, opts?: MediaUrlOptions): Promise<string> => {
|
|
592
|
+
const window = opts?.stableWindowSeconds;
|
|
593
|
+
// Two windows, so a URL minted in the last second of one is still valid for a
|
|
594
|
+
// download that starts in the next.
|
|
595
|
+
const ttl =
|
|
596
|
+
window && window > 0 ? window * 2 : (opts?.ttlSeconds ?? DOWNLOAD_TOKEN_TTL_SECONDS);
|
|
597
|
+
const token = await signToken(
|
|
598
|
+
{ k: appKey(key), fn: opts?.filename, dl: opts?.disposition },
|
|
599
|
+
ttl,
|
|
600
|
+
window,
|
|
601
|
+
);
|
|
602
|
+
return `${baseUrl}/media/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}`;
|
|
603
|
+
};
|
|
604
|
+
|
|
605
|
+
/**
|
|
606
|
+
* A signed URL for the MASTER of a multi-object tree, authorized by PREFIX.
|
|
607
|
+
*
|
|
608
|
+
* ── 🔴 Why an exact-key token cannot work here ──────────────────────────────────────
|
|
609
|
+
* An HLS ladder is a tree — a master naming rung playlists naming segments — and a
|
|
610
|
+
* player resolves those names RELATIVE to the master's URL. A relative URI resolves
|
|
611
|
+
* against the base URL's PATH and discards its query string (RFC 3986, and every player
|
|
612
|
+
* behaves this way), so `720/index.m3u8` arrives at binary-server with no `?token=` and
|
|
613
|
+
* is correctly refused.
|
|
614
|
+
*
|
|
615
|
+
* binary-server solves the second half: it rewrites a playlist as it serves it, appending
|
|
616
|
+
* the SAME token to every URI the playlist names (`src/media/hlsPlaylist.ts`). That
|
|
617
|
+
* rewrite only fires for a PREFIX token, and deliberately so — copying an exact-key token
|
|
618
|
+
* onto a child URI produces a URL that 403s, which is the same broken video with an extra
|
|
619
|
+
* query parameter. So the first half has to be minted here, and this is it.
|
|
620
|
+
*
|
|
621
|
+
* 🔴 It grants nothing the bearer did not already have: the prefix is the tree, and every
|
|
622
|
+
* URI inside the tree resolves under it. But it IS broader than an exact key by
|
|
623
|
+
* construction, so it is deliberately a separate method with its own name rather than a
|
|
624
|
+
* flag on `mediaUrl` — a caller reaching for prefix authority should have to say so.
|
|
625
|
+
*/
|
|
626
|
+
const mediaPrefixUrl = async (
|
|
627
|
+
prefix: string,
|
|
628
|
+
master: string,
|
|
629
|
+
opts?: MediaUrlOptions,
|
|
630
|
+
): Promise<string> => {
|
|
631
|
+
const window = opts?.stableWindowSeconds;
|
|
632
|
+
const ttl =
|
|
633
|
+
window && window > 0 ? window * 2 : (opts?.ttlSeconds ?? DOWNLOAD_TOKEN_TTL_SECONDS);
|
|
634
|
+
// The claim is the PREFIX; the URL points at the master inside it.
|
|
635
|
+
const token = await signToken({ p: appKey(prefix) }, ttl, window);
|
|
636
|
+
return `${baseUrl}/media/${encodeKeyPath(appKey(master))}?token=${encodeURIComponent(token)}`;
|
|
637
|
+
};
|
|
638
|
+
|
|
639
|
+
/** The same signed URL, aimed at the origin THIS PROCESS should use. Never handed to
|
|
640
|
+
* a browser — see the note on `internalBaseUrl`. */
|
|
641
|
+
const internalMediaUrl = async (key: string, opts?: MediaUrlOptions): Promise<string> =>
|
|
642
|
+
(await mediaUrl(key, opts)).replace(baseUrl, internalBaseUrl);
|
|
643
|
+
|
|
644
|
+
/** `/meta` authorizes with the same token shape as `/media` — an exact-key claim. */
|
|
645
|
+
const metaUrl = async (key: string): Promise<string> => {
|
|
646
|
+
const token = await signToken({ k: appKey(key) }, DOWNLOAD_TOKEN_TTL_SECONDS);
|
|
647
|
+
// `/meta` is only ever asked by a server, so it always uses the internal origin.
|
|
648
|
+
return `${internalBaseUrl}/meta/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}`;
|
|
649
|
+
};
|
|
650
|
+
|
|
651
|
+
const internalHeaders = (): Record<string, string> => {
|
|
652
|
+
if (!cfg.internalSecret) {
|
|
653
|
+
throw new Error(
|
|
654
|
+
'[binary-store] remove requires internalSecret (BINARY_SERVER_INTERNAL_SECRET)',
|
|
655
|
+
);
|
|
656
|
+
}
|
|
657
|
+
return { 'x-internal-secret': cfg.internalSecret };
|
|
658
|
+
};
|
|
659
|
+
|
|
660
|
+
const store: BinaryStore = {
|
|
661
|
+
appId: cfg.appId,
|
|
662
|
+
signToken,
|
|
663
|
+
mediaUrl,
|
|
664
|
+
mediaPrefixUrl,
|
|
665
|
+
|
|
666
|
+
async put(key, bytes, mime, opts) {
|
|
667
|
+
// 🔴 The chunked fallback, applied HERE rather than at every call site —
|
|
668
|
+
// the limit is the Cloudflare Tunnel's, so it is a property of this
|
|
669
|
+
// client's destination and not of anyone's payload. See
|
|
670
|
+
// AUTO_CHUNK_THRESHOLD_BYTES.
|
|
671
|
+
if (bytes.byteLength > autoChunkBytes) {
|
|
672
|
+
return await store.putLarge(key, bytes, mime, opts);
|
|
673
|
+
}
|
|
674
|
+
const token = await signToken({ k: appKey(key) }, UPLOAD_TOKEN_TTL_SECONDS);
|
|
675
|
+
const title = opts?.title?.trim();
|
|
676
|
+
const titleQs = title ? `&title=${encodeURIComponent(title)}` : '';
|
|
677
|
+
const immutableQs = opts?.immutable ? '&immutable=1' : '';
|
|
678
|
+
const res = await fetch(
|
|
679
|
+
`${baseUrl}/upload/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}${titleQs}${immutableQs}`,
|
|
680
|
+
{
|
|
681
|
+
method: 'PUT',
|
|
682
|
+
headers: mime ? { 'content-type': mime } : undefined,
|
|
683
|
+
body: bytes as unknown as BodyInit,
|
|
684
|
+
},
|
|
685
|
+
);
|
|
686
|
+
if (!res.ok) {
|
|
687
|
+
// A 413 that reaches here despite the threshold above means the edge's
|
|
688
|
+
// real cap is lower than we think, or the caller disabled the
|
|
689
|
+
// fallback. Say what it is and what to do — the bare status is what
|
|
690
|
+
// made eight rejected DS cartridges read as eight unrelated mysteries
|
|
691
|
+
// on 2026-08-08.
|
|
692
|
+
if (res.status === 413) {
|
|
693
|
+
throw new Error(
|
|
694
|
+
`[binary-store] put ${key}: 413 — ${bytes.byteLength} bytes was refused before it ` +
|
|
695
|
+
'reached binary-server (the Cloudflare Tunnel in front of it caps a request body ' +
|
|
696
|
+
`at roughly 100 MB). Use putLarge(), or lower autoChunkBytes (currently ${autoChunkBytes}).`,
|
|
697
|
+
);
|
|
698
|
+
}
|
|
699
|
+
throw new Error(`[binary-store] put ${key}: ${res.status}`);
|
|
700
|
+
}
|
|
701
|
+
// 🔴 It exists now, and its BYTES may be new ones. Dropping both memos rather
|
|
702
|
+
// than asserting a presence is what keeps a name-addressed key (roms' box art
|
|
703
|
+
// hashes the game name, not the bytes) honest: the next reader re-asks once
|
|
704
|
+
// and re-learns, instead of trusting a size and mime from the previous file.
|
|
705
|
+
forget(key);
|
|
706
|
+
},
|
|
707
|
+
|
|
708
|
+
/*
|
|
709
|
+
* 🔴 Every chunk is READ INTO MEMORY before it is sent, and that is not a
|
|
710
|
+
* simplification — it is the fix for a hang that stopped the collections dropzone
|
|
711
|
+
* dead for a day (measured 2026-08-13, Bun 1.3.14).
|
|
712
|
+
*
|
|
713
|
+
* `Bun.file(path).slice(a, b)` is a lazy PARTIAL view of a file, and handing one to
|
|
714
|
+
* `fetch` as a body never sends: the request stalls until binary-server's 60s
|
|
715
|
+
* `idleTimeout` closes the socket, and the caller sees `EPIPE` / "socket connection
|
|
716
|
+
* was closed unexpectedly". Measured on one 20 MB file against the live server:
|
|
717
|
+
*
|
|
718
|
+
* Bun.file, 16 MiB chunks (2 parts) → FAIL after 56,164 ms
|
|
719
|
+
* Bun.file, 8 MiB chunks (3 parts) → FAIL after 60,000 ms
|
|
720
|
+
* Bun.file, one WHOLE-range slice → OK in 62 ms
|
|
721
|
+
* the same bytes already in memory → OK in 43 ms
|
|
722
|
+
*
|
|
723
|
+
* A whole-range slice works because it is not really a partial view, which is why
|
|
724
|
+
* every small file went through fine and only the ones that had to be split failed —
|
|
725
|
+
* i.e. exactly the videos this path exists for. 42 files / 35 GB failed 42/42 on
|
|
726
|
+
* every scheduled sweep for a day with nothing in binary-server's log, because from
|
|
727
|
+
* its side the request simply never arrived.
|
|
728
|
+
*
|
|
729
|
+
* The cost is one chunk resident at a time — 16 MiB — which is the bound `putLarge`
|
|
730
|
+
* always promised. It is NOT the bound the old code delivered: the lazy slice was
|
|
731
|
+
* never read at all.
|
|
732
|
+
*/
|
|
733
|
+
async putLarge(key, source, mime, opts) {
|
|
734
|
+
const chunkBytes = Math.max(1, opts?.chunkBytes ?? 16 * 1024 * 1024);
|
|
735
|
+
const blob = source instanceof Blob ? source : new Blob([source as BlobPart]);
|
|
736
|
+
const total = Math.max(1, Math.ceil(blob.size / chunkBytes));
|
|
737
|
+
// One session id for the whole upload: bs keys the chunk scratch by it and
|
|
738
|
+
// assembles as soon as it has counted `tc` arrivals.
|
|
739
|
+
const sid = `${Date.now().toString(36)}-${Math.trunc(Math.random() * 1e9).toString(36)}`;
|
|
740
|
+
const title = opts?.title?.trim();
|
|
741
|
+
const titleQs = title ? `&title=${encodeURIComponent(title)}` : '';
|
|
742
|
+
const immutableQs = opts?.immutable ? '&immutable=1' : '';
|
|
743
|
+
for (let ci = 0; ci < total; ci++) {
|
|
744
|
+
const token = await signToken(
|
|
745
|
+
{ k: appKey(key), sid, ci, tc: total },
|
|
746
|
+
UPLOAD_TOKEN_TTL_SECONDS,
|
|
747
|
+
);
|
|
748
|
+
const slice = new Uint8Array(
|
|
749
|
+
await blob
|
|
750
|
+
.slice(ci * chunkBytes, Math.min((ci + 1) * chunkBytes, blob.size))
|
|
751
|
+
.arrayBuffer(),
|
|
752
|
+
);
|
|
753
|
+
const res = await fetch(
|
|
754
|
+
`${baseUrl}/upload-chunk/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}${titleQs}${immutableQs}`,
|
|
755
|
+
{
|
|
756
|
+
method: 'PUT',
|
|
757
|
+
headers: mime ? { 'content-type': mime } : undefined,
|
|
758
|
+
body: slice,
|
|
759
|
+
},
|
|
760
|
+
);
|
|
761
|
+
if (!res.ok) {
|
|
762
|
+
throw new Error(`[binary-store] putLarge ${key} chunk ${ci + 1}/${total}: ${res.status}`);
|
|
763
|
+
}
|
|
764
|
+
}
|
|
765
|
+
forget(key); // see `put`
|
|
766
|
+
},
|
|
767
|
+
|
|
768
|
+
async get(key, opts) {
|
|
769
|
+
// 🔴 The negative memo short-circuits the BYTES too, and that is where most of
|
|
770
|
+
// the saving is: family drew a face by asking for bytes it had already been
|
|
771
|
+
// told were not there, hundreds of times a page. The positive memo cannot
|
|
772
|
+
// short-circuit here — we want the bytes, and only the store has them.
|
|
773
|
+
if (!opts?.fresh && knownAbsent(key)) return null;
|
|
774
|
+
countWireRead(key);
|
|
775
|
+
const res = await fetch(await internalMediaUrl(key));
|
|
776
|
+
if (res.status === 404) {
|
|
777
|
+
rememberAbsent(key);
|
|
778
|
+
return null;
|
|
779
|
+
}
|
|
780
|
+
if (!res.ok) {
|
|
781
|
+
// 🔴 NOT an absence. "The store could not be asked" and "the store does not
|
|
782
|
+
// have it" are different facts, and conflating them is the bug family met
|
|
783
|
+
// twice in one day — a 429 from a spent Cloudflare allowance rendering as a
|
|
784
|
+
// photograph that does not exist. Caching that would blank a page of faces
|
|
785
|
+
// for an hour, which is the one outcome this memo must never cause.
|
|
786
|
+
throw new Error(`[binary-store] get ${key}: ${res.status}`);
|
|
787
|
+
}
|
|
788
|
+
// A key we believed present that answers 404 is demoted above; one that
|
|
789
|
+
// answers bytes stays believed. See the memo header on why `put` is the
|
|
790
|
+
// other half of keeping a name-addressed key honest.
|
|
791
|
+
return new Uint8Array(await res.arrayBuffer());
|
|
792
|
+
},
|
|
793
|
+
|
|
794
|
+
async has(key, opts) {
|
|
795
|
+
return (await store.meta(key, opts)) !== null;
|
|
796
|
+
},
|
|
797
|
+
|
|
798
|
+
known(key) {
|
|
799
|
+
// Order matters: a lapsed absence is DROPPED by `knownAbsent`, so asking it
|
|
800
|
+
// first keeps this accessor from reporting an expiry the next real read would
|
|
801
|
+
// not honour. A positive memo has no expiry, so it needs no such care.
|
|
802
|
+
if (knownAbsent(key)) return 'absent';
|
|
803
|
+
return present.has(key) ? 'present' : 'unknown';
|
|
804
|
+
},
|
|
805
|
+
|
|
806
|
+
forgetMisses(prefix) {
|
|
807
|
+
if (prefix === undefined) {
|
|
808
|
+
const dropped = absentUntil.size;
|
|
809
|
+
absentUntil.clear();
|
|
810
|
+
return dropped;
|
|
811
|
+
}
|
|
812
|
+
let dropped = 0;
|
|
813
|
+
for (const key of [...absentUntil.keys()]) {
|
|
814
|
+
if (key.startsWith(prefix)) {
|
|
815
|
+
absentUntil.delete(key);
|
|
816
|
+
dropped++;
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
return dropped;
|
|
820
|
+
},
|
|
821
|
+
|
|
822
|
+
cacheStats() {
|
|
823
|
+
// Lapsed entries are dropped as they are met, so count only live ones —
|
|
824
|
+
// a stat that includes expired rows reads as a memo that never releases.
|
|
825
|
+
const at = now();
|
|
826
|
+
let absent = 0;
|
|
827
|
+
for (const until of absentUntil.values()) if (until > at) absent++;
|
|
828
|
+
return { present: present.size, absent };
|
|
829
|
+
},
|
|
830
|
+
|
|
831
|
+
async fetchMedia(key, init) {
|
|
832
|
+
// A one-hour stable window, matching what apps hand to browsers: the token is part
|
|
833
|
+
// of the URL, so a fresh one per request would defeat every cache between here and
|
|
834
|
+
// the object — see MediaUrlOptions.stableWindowSeconds.
|
|
835
|
+
const url = await internalMediaUrl(key, { stableWindowSeconds: 60 * 60 });
|
|
836
|
+
return await fetch(url, {
|
|
837
|
+
method: init?.method ?? 'GET',
|
|
838
|
+
// 🔴 Only when the caller HAS one. `Range: undefined` is fine, but a literal
|
|
839
|
+
// empty string is a malformed header that some origins answer 416 to.
|
|
840
|
+
...(init?.range ? { headers: { range: init.range } } : {}),
|
|
841
|
+
...(init?.signal ? { signal: init.signal } : {}),
|
|
842
|
+
});
|
|
843
|
+
},
|
|
844
|
+
|
|
845
|
+
async meta(key, opts) {
|
|
846
|
+
if (!opts?.fresh) {
|
|
847
|
+
const memo = present.get(key);
|
|
848
|
+
if (memo) return memo;
|
|
849
|
+
if (knownAbsent(key)) return null;
|
|
850
|
+
}
|
|
851
|
+
countWireRead(key);
|
|
852
|
+
const res = await fetch(await metaUrl(key));
|
|
853
|
+
if (res.status === 404) {
|
|
854
|
+
rememberAbsent(key);
|
|
855
|
+
return null;
|
|
856
|
+
}
|
|
857
|
+
// A throw leaves both memos untouched, deliberately — see `get`.
|
|
858
|
+
if (!res.ok) throw new Error(`[binary-store] meta ${key}: ${res.status}`);
|
|
859
|
+
const body = (await res.json()) as Partial<BlobMeta> & { size?: number };
|
|
860
|
+
const meta: BlobMeta = {
|
|
861
|
+
size: Number(body.size ?? 0),
|
|
862
|
+
mime: body.mime ?? null,
|
|
863
|
+
title: body.title ?? null,
|
|
864
|
+
checksum: body.checksum ?? null,
|
|
865
|
+
createdAt: body.createdAt,
|
|
866
|
+
updatedAt: body.updatedAt,
|
|
867
|
+
};
|
|
868
|
+
rememberPresent(key, meta);
|
|
869
|
+
return meta;
|
|
870
|
+
},
|
|
871
|
+
|
|
872
|
+
async stat(key, opts) {
|
|
873
|
+
// Prefer /meta: one index read on the server — no disk touch, no access-time
|
|
874
|
+
// bump, no body. It is the cheap answer and it is the one bs added for this.
|
|
875
|
+
//
|
|
876
|
+
// The fallback is NOT dead code, and it cannot be skipped on a null: a
|
|
877
|
+
// binary-server predating /meta answers 404 to /meta for an object it HAS, which
|
|
878
|
+
// by status alone is indistinguishable from "no such object". So a null is
|
|
879
|
+
// confirmed with the old Range probe, which every version supports. The extra
|
|
880
|
+
// round trip is paid only when the answer is "absent" or the server is old.
|
|
881
|
+
// 🔴 The MEMO is consulted for the whole of `stat`, not just for its `meta`
|
|
882
|
+
// half. Otherwise a key confirmed absent still paid the Range probe on every
|
|
883
|
+
// single call — which is most of what an absent key costs, and the reason
|
|
884
|
+
// this memo exists at all.
|
|
885
|
+
if (!opts?.fresh && knownAbsent(key)) return null;
|
|
886
|
+
try {
|
|
887
|
+
const m = await store.meta(key, opts);
|
|
888
|
+
if (m) return { size: m.size };
|
|
889
|
+
} catch {
|
|
890
|
+
// A transport/5xx failure — the probe below is the second opinion.
|
|
891
|
+
}
|
|
892
|
+
countWireRead(key);
|
|
893
|
+
const res = await fetch(await internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } });
|
|
894
|
+
if (res.status === 404) {
|
|
895
|
+
rememberAbsent(key);
|
|
896
|
+
return null;
|
|
897
|
+
}
|
|
898
|
+
if (!res.ok) throw new Error(`[binary-store] stat ${key}: ${res.status}`);
|
|
899
|
+
// 🔴 The object IS here and `/meta` said 404 — an old binary-server, which is
|
|
900
|
+
// the case this fallback exists for. `meta` recorded an absence on that 404 and
|
|
901
|
+
// it is WRONG, so it is dropped rather than left to blank the key for an hour.
|
|
902
|
+
// Not promoted to a positive memo either: a Range probe yields a size and
|
|
903
|
+
// nothing else, and a fabricated `mime: null` would be `meta()` inventing an
|
|
904
|
+
// answer it never received.
|
|
905
|
+
forget(key);
|
|
906
|
+
const contentRange = res.headers.get('content-range');
|
|
907
|
+
const totalBytes = contentRange
|
|
908
|
+
? Number(contentRange.split('/')[1])
|
|
909
|
+
: Number(res.headers.get('content-length'));
|
|
910
|
+
return Number.isFinite(totalBytes) ? { size: totalBytes } : null;
|
|
911
|
+
},
|
|
912
|
+
|
|
913
|
+
async remove(key) {
|
|
914
|
+
// Sign a per-op, path-scoped delete token: bs requires it AND the internal secret
|
|
915
|
+
// (owner ruling #10). `internalHeaders()` throws first if no secret is configured.
|
|
916
|
+
const headers = internalHeaders();
|
|
917
|
+
const token = await signToken({ k: appKey(key), op: 'delete' }, DELETE_TOKEN_TTL_SECONDS);
|
|
918
|
+
const res = await fetch(
|
|
919
|
+
`${baseUrl}/media/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}`,
|
|
920
|
+
{ method: 'DELETE', headers },
|
|
921
|
+
);
|
|
922
|
+
if (!res.ok && res.status !== 404) {
|
|
923
|
+
throw new Error(`[binary-store] remove ${key}: ${res.status}`);
|
|
924
|
+
}
|
|
925
|
+
// Gone, and we know it — a positive memo that survived a delete would hand out
|
|
926
|
+
// a URL to a 404 for the life of the process.
|
|
927
|
+
rememberAbsent(key);
|
|
928
|
+
},
|
|
929
|
+
|
|
930
|
+
async removePrefix(prefix) {
|
|
931
|
+
// The token's exact-key claim is the container's own path; bs derives the tree from it
|
|
932
|
+
// and the `?container=1` flag. Same two-barrier gate as remove().
|
|
933
|
+
const headers = internalHeaders();
|
|
934
|
+
const token = await signToken({ k: appKey(prefix), op: 'delete' }, DELETE_TOKEN_TTL_SECONDS);
|
|
935
|
+
const res = await fetch(
|
|
936
|
+
`${baseUrl}/media/${encodeKeyPath(appKey(prefix))}?container=1&token=${encodeURIComponent(token)}`,
|
|
937
|
+
{ method: 'DELETE', headers },
|
|
938
|
+
);
|
|
939
|
+
if (!res.ok && res.status !== 404) {
|
|
940
|
+
throw new Error(`[binary-store] removePrefix ${prefix}: ${res.status}`);
|
|
941
|
+
}
|
|
942
|
+
// A whole tree went; every memo under it is void. NOT recorded as absences —
|
|
943
|
+
// the keys are not enumerable from here, and a container is routinely refilled.
|
|
944
|
+
for (const key of [...present.keys()]) if (key.startsWith(prefix)) forget(key);
|
|
945
|
+
for (const key of [...absentUntil.keys()]) if (key.startsWith(prefix)) forget(key);
|
|
946
|
+
},
|
|
947
|
+
};
|
|
948
|
+
|
|
949
|
+
return store;
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
/**
|
|
953
|
+
* Which binary-server TENANT this process is — the app's own key, unless the environment
|
|
954
|
+
* names another.
|
|
955
|
+
*
|
|
956
|
+
* 🔴 It has to be a variable, and 2026-08-21 is the day that stopped being theoretical.
|
|
957
|
+
* A tenant is TWO facts that must agree: the key a request is SIGNED with, and the key the
|
|
958
|
+
* request PATH starts with (`authorize()` in binary-server refuses when
|
|
959
|
+
* `path.split("/")[0] !== token.app_id`). Apps supplied the second as a literal — `const
|
|
960
|
+
* TENANT = "collections"` — while `FILE_TOKEN_ISSUER`/`FILE_TOKEN_PRIVATE_KEY` came from
|
|
961
|
+
* the environment. That is fine on production, where the two happen to match, and it makes
|
|
962
|
+
* an ephemeral STAGE structurally unable to store a byte: `stage up` mints it a real
|
|
963
|
+
* `collections-stage` tenant and hands it that keypair, and every upload then signs as
|
|
964
|
+
* `collections-stage` while addressing `collections/…` and comes back
|
|
965
|
+
* `[binary-store] put file/<sha>: 403`. The stage looked like a working app with an empty
|
|
966
|
+
* gallery, which is exactly what an empty stage is supposed to look like.
|
|
967
|
+
*
|
|
968
|
+
* Reading it here means an app opts in by CALLING this instead of writing a literal, and
|
|
969
|
+
* `BINARY_STORE_TENANT` is already inside an app's stage-env shared-store denylist
|
|
970
|
+
* (`/^BINARY_STORE_(?!.*URL$)/`) — so a stage cannot INHERIT production's tenant, only be
|
|
971
|
+
* given its own. The fallback is the app's key, so an app that never sets it is unchanged.
|
|
972
|
+
*
|
|
973
|
+
* 🔴 **A BLANK tenant now throws, and that is the whole point of the check
|
|
974
|
+
* (2026-08-23).** `undefined` is a perfectly good JavaScript string once it is
|
|
975
|
+
* template-interpolated, so a caller that forgot the argument signed a token for
|
|
976
|
+
* `undefined/<key>` and addressed `PUT /upload/undefined/<key>` — at which point
|
|
977
|
+
* binary-server's `authorize()` refuses it correctly and answers a bare
|
|
978
|
+
* `{"error":"forbidden"}`. That reads as a broken CREDENTIAL, and it was filed as one
|
|
979
|
+
* (*"family.env's FILE_TOKEN_PRIVATE_KEY does not authorize against the family tenant"*).
|
|
980
|
+
* The keypair was correct the whole time and a Mac-side upload works; what was missing was
|
|
981
|
+
* the tenant id. bs cannot tell the two apart — it sees a token for a tenant it does not
|
|
982
|
+
* have — so the check has to live on this side of the wire, where the mistake is legible.
|
|
983
|
+
*/
|
|
984
|
+
export function binaryStoreTenant(
|
|
985
|
+
appId: string,
|
|
986
|
+
env: Record<string, string | undefined> = process.env,
|
|
987
|
+
): string {
|
|
988
|
+
const tenant = env.BINARY_STORE_TENANT?.trim() || appId?.trim();
|
|
989
|
+
if (!tenant || tenant === 'undefined' || tenant === 'null') {
|
|
990
|
+
throw new Error(
|
|
991
|
+
"[binary-store] no tenant id — pass the app's key as the first argument " +
|
|
992
|
+
'(`readBinaryStoreEnv("family")` / `createBinaryStore({ appId: "family" })`), ' +
|
|
993
|
+
'or set BINARY_STORE_TENANT. Without one every upload signs for ' +
|
|
994
|
+
`"${tenant ?? ''}/<key>" and binary-server answers 403 forbidden, which reads ` +
|
|
995
|
+
'as a bad credential rather than as a missing argument.',
|
|
996
|
+
);
|
|
997
|
+
}
|
|
998
|
+
return tenant;
|
|
999
|
+
}
|
|
1000
|
+
|
|
1001
|
+
/**
|
|
1002
|
+
* The tenant this process's binary store resolved, or `null` when it has no store.
|
|
1003
|
+
*
|
|
1004
|
+
* The value an app's `mediaTenant` option wants, and the reason it exists as a named
|
|
1005
|
+
* function rather than each app writing `readBinaryStoreEnv(x)?.appId`:
|
|
1006
|
+
*
|
|
1007
|
+
* 🔴 **`null` and a thrown error are different answers and both are correct.**
|
|
1008
|
+
* {@link binaryStoreTenant} throws when it cannot name a tenant, which is right at an upload
|
|
1009
|
+
* site — signing for `"undefined/<key>"` earns a bare 403 that reads as a bad credential. It is
|
|
1010
|
+
* wrong on `/healthz`, where the same throw would turn "this dev shell has no store" into a
|
|
1011
|
+
* 500 and take the app's health probe down with it. So this asks the same question through the
|
|
1012
|
+
* same resolution and answers `null` for the store-less case, which the health handler then
|
|
1013
|
+
* reports as an explicit "no store" rather than as silence.
|
|
1014
|
+
*
|
|
1015
|
+
* Pass the app's OWN tenant key — the constant its store is built from (`LIFE_BINARY_TENANT`,
|
|
1016
|
+
* `STUDIO_BINARY_TENANT`, …), never the app's display name. They are the same string for most
|
|
1017
|
+
* apps and that is precisely why the difference goes unnoticed when it is not.
|
|
1018
|
+
*/
|
|
1019
|
+
export function resolvedMediaTenant(
|
|
1020
|
+
appId: string,
|
|
1021
|
+
env: Record<string, string | undefined> = process.env,
|
|
1022
|
+
): string | null {
|
|
1023
|
+
return readBinaryStoreEnv(appId, env)?.appId ?? null;
|
|
1024
|
+
}
|
|
1025
|
+
|
|
1026
|
+
/**
|
|
1027
|
+
* The standard env surface every tenant's secrets file carries
|
|
1028
|
+
* (`$FORGE_STATE/secrets/<app>.env`, written at bs registration):
|
|
1029
|
+
* `BINARY_SERVER_URL`, `FILE_TOKEN_ISSUER`, `FILE_TOKEN_PRIVATE_KEY`, and
|
|
1030
|
+
* optionally `BINARY_SERVER_INTERNAL_SECRET`. Returns `null` unless the three
|
|
1031
|
+
* required vars are all present — callers fall back to their local backend.
|
|
1032
|
+
*/
|
|
1033
|
+
export function readBinaryStoreEnv(
|
|
1034
|
+
appId: string,
|
|
1035
|
+
env: Record<string, string | undefined> = process.env,
|
|
1036
|
+
): BinaryStoreConfig | null {
|
|
1037
|
+
const baseUrl = env.BINARY_SERVER_URL?.trim();
|
|
1038
|
+
const issuer = env.FILE_TOKEN_ISSUER?.trim();
|
|
1039
|
+
const privateKey = env.FILE_TOKEN_PRIVATE_KEY?.trim();
|
|
1040
|
+
if (!baseUrl || !issuer || !privateKey) return null;
|
|
1041
|
+
return {
|
|
1042
|
+
baseUrl,
|
|
1043
|
+
appId: binaryStoreTenant(appId, env),
|
|
1044
|
+
issuer,
|
|
1045
|
+
privateKey,
|
|
1046
|
+
internalSecret: env.BINARY_SERVER_INTERNAL_SECRET?.trim() || undefined,
|
|
1047
|
+
internalBaseUrl: env.BINARY_SERVER_INTERNAL_URL?.trim() || undefined,
|
|
1048
|
+
};
|
|
1049
|
+
}
|