cursedbelt-server 4.32.1 → 4.33.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/d1/fakeD1.js +2 -1
- package/dist/server/d1/pullD1.d.ts +5 -0
- package/dist/server/d1/pullD1.js +17 -5
- package/dist/server/storage/binaryStore.d.ts +12 -0
- package/dist/server/storage/binaryStore.js +9 -8
- package/package.json +2 -2
- package/src/server/d1/fakeD1.ts +1 -1
- package/src/server/d1/pullD1.spec.ts +57 -0
- package/src/server/d1/pullD1.ts +27 -5
- package/src/server/storage/binaryStore.spec.ts +36 -0
- package/src/server/storage/binaryStore.ts +21 -8
package/dist/server/d1/fakeD1.js
CHANGED
|
@@ -83,7 +83,8 @@ class FakeD1Statement {
|
|
|
83
83
|
this.params = params;
|
|
84
84
|
}
|
|
85
85
|
bind(...values) {
|
|
86
|
-
|
|
86
|
+
for (const [i, v] of values.entries())
|
|
87
|
+
assertD1Bindable(v, i);
|
|
87
88
|
// A real binding stores what it was given; the ArrayBuffer→Uint8Array conversion
|
|
88
89
|
// here is `bun:sqlite`'s requirement, not D1's, and happens below the emulated edge.
|
|
89
90
|
const stored = values.map((v) => (v instanceof ArrayBuffer ? new Uint8Array(v) : v));
|
|
@@ -129,6 +129,11 @@ export interface PullD1Options {
|
|
|
129
129
|
/** Run after the rows are loaded — where an app REBUILDS its search index (never copied). */
|
|
130
130
|
afterLoadSql?: string;
|
|
131
131
|
wrangler: Wrangler;
|
|
132
|
+
/**
|
|
133
|
+
* Where a page D1 refused is reported (default: stderr). Before 4.33.1 a halving was silent,
|
|
134
|
+
* so the only symptom of a slow spell was a backup that took hours.
|
|
135
|
+
*/
|
|
136
|
+
log?: (line: string) => void;
|
|
132
137
|
}
|
|
133
138
|
/** How a table is walked in pages: by a rowid keyset, or — a `WITHOUT ROWID` table — by offset. */
|
|
134
139
|
export interface TablePlan {
|
package/dist/server/d1/pullD1.js
CHANGED
|
@@ -202,8 +202,15 @@ export function pageSql(plan, limit, after) {
|
|
|
202
202
|
const order = plan.order.map(sqlIdent).join(', ');
|
|
203
203
|
return `SELECT ${values} FROM ${sqlIdent(plan.table)} ORDER BY ${order} LIMIT ${limit} OFFSET ${after.offset ?? 0}`;
|
|
204
204
|
}
|
|
205
|
-
/**
|
|
206
|
-
|
|
205
|
+
/**
|
|
206
|
+
* Every row of one table as `INSERT INTO …;` lines, read in pages that never block D1.
|
|
207
|
+
*
|
|
208
|
+
* A page D1 refuses is asked again at half the size, and 🔴 every page that SUCCEEDS doubles the
|
|
209
|
+
* size back toward {@link PAGE_ROWS}. Before 4.33.1 nothing grew it back: desk's first launchd
|
|
210
|
+
* backup (2026-09-25) hit a few slow calls early and read its 3,381-row table 15 rows per call —
|
|
211
|
+
* 225 two-minute calls, on course for hours. Only a refusal at ONE row is fatal.
|
|
212
|
+
*/
|
|
213
|
+
function dumpTable(wrangler, database, plan, log) {
|
|
207
214
|
const into = `INSERT INTO ${sqlIdent(plan.table)} (${plan.columns.map(sqlIdent).join(', ')}) VALUES (`;
|
|
208
215
|
const lines = [];
|
|
209
216
|
let limit = PAGE_ROWS;
|
|
@@ -214,11 +221,14 @@ function dumpTable(wrangler, database, plan) {
|
|
|
214
221
|
page = execute(wrangler, database, pageSql(plan, limit, after))[0] ?? [];
|
|
215
222
|
}
|
|
216
223
|
catch (error) {
|
|
224
|
+
const message = error.message;
|
|
217
225
|
if (limit > 1) {
|
|
218
|
-
|
|
226
|
+
const next = Math.max(1, Math.floor(limit / 2));
|
|
227
|
+
log(`pull D1: \`${plan.table}\` refused a page of ${limit} — asking for ${next}: ${message}`);
|
|
228
|
+
limit = next;
|
|
219
229
|
continue;
|
|
220
230
|
}
|
|
221
|
-
throw new Error(`could not read \`${plan.table}\` from D1: ${
|
|
231
|
+
throw new Error(`could not read \`${plan.table}\` from D1: ${message}`);
|
|
222
232
|
}
|
|
223
233
|
for (const row of page) {
|
|
224
234
|
const literals = plan.columns.map((_, i) => row[`c${i}`]);
|
|
@@ -230,6 +240,7 @@ function dumpTable(wrangler, database, plan) {
|
|
|
230
240
|
if (page.length < limit)
|
|
231
241
|
break;
|
|
232
242
|
after = plan.key ? { key: Number(page[page.length - 1]?.k) } : { offset: (after.offset ?? 0) + page.length };
|
|
243
|
+
limit = Math.min(PAGE_ROWS, limit * 2);
|
|
233
244
|
}
|
|
234
245
|
return { text: lines.join('\n'), rows: lines.length };
|
|
235
246
|
}
|
|
@@ -243,6 +254,7 @@ function dumpTable(wrangler, database, plan) {
|
|
|
243
254
|
*/
|
|
244
255
|
export async function pullD1ToSqlite(options) {
|
|
245
256
|
const { destination, database, wrangler } = options;
|
|
257
|
+
const log = options.log ?? ((line) => console.error(line));
|
|
246
258
|
let rows;
|
|
247
259
|
try {
|
|
248
260
|
rows = (execute(wrangler, database, "select name, sql from sqlite_master where type = 'table'")[0] ?? []);
|
|
@@ -265,7 +277,7 @@ export async function pullD1ToSqlite(options) {
|
|
|
265
277
|
const dumps = tables.map((table, i) => {
|
|
266
278
|
const cols = (columnSets[i] ?? []);
|
|
267
279
|
const plan = tablePlan(table, rows.find((r) => r.name === table)?.sql ?? null, cols);
|
|
268
|
-
return { table, ...dumpTable(wrangler, database, plan) };
|
|
280
|
+
return { table, ...dumpTable(wrangler, database, plan, log) };
|
|
269
281
|
});
|
|
270
282
|
if (dumps.every((d) => d.rows === 0)) {
|
|
271
283
|
throw new Error('the D1 pull contains no rows — refusing to snapshot an empty database as if it were the library');
|
|
@@ -38,6 +38,18 @@ export interface BinaryStoreConfig {
|
|
|
38
38
|
absenceTtlMs?: number;
|
|
39
39
|
/** Injected so the memo's expiry is testable without a real clock. */
|
|
40
40
|
now?: () => number;
|
|
41
|
+
/**
|
|
42
|
+
* 🔴 How THIS PROCESS sends every request to binary-server. Default: the global `fetch`.
|
|
43
|
+
*
|
|
44
|
+
* A Cloudflare Worker on the SAME ZONE as binary-server must pass its service binding to
|
|
45
|
+
* `binary-server-edge-gate` here (`(input, init) => env.BINARY_SERVER.fetch(input, init)`).
|
|
46
|
+
* A same-zone subrequest skips every other Worker's route and goes straight to the zone's
|
|
47
|
+
* ORIGIN — the byte tunnel into the owner's Mac — so a global `fetch` never reached the edge
|
|
48
|
+
* Worker, its R2 reads or its cache. Measured 2026-09-25: the owner powered the Mac off and
|
|
49
|
+
* every face on family's tree was gone, although family's bytes rest in R2. Auth met the same
|
|
50
|
+
* trap first (task 2150); this is the byte half of it.
|
|
51
|
+
*/
|
|
52
|
+
fetch?: (input: string | Request, init?: RequestInit) => Promise<Response>;
|
|
41
53
|
}
|
|
42
54
|
/**
|
|
43
55
|
* Which keys this process has read from the wire far too often, worst first —
|
|
@@ -260,6 +260,7 @@ export function createBinaryStore(cfg) {
|
|
|
260
260
|
* and is named as the remaining gap.
|
|
261
261
|
*/
|
|
262
262
|
const internalBaseUrl = (cfg.internalBaseUrl ?? cfg.baseUrl).replace(/\/+$/, '');
|
|
263
|
+
const send = cfg.fetch ?? ((input, init) => fetch(input, init));
|
|
263
264
|
const autoChunkBytes = cfg.autoChunkBytes ?? AUTO_CHUNK_THRESHOLD_BYTES;
|
|
264
265
|
const now = cfg.now ?? Date.now;
|
|
265
266
|
const absenceTtlMs = cfg.absenceTtlMs ?? BINARY_ABSENCE_TTL_MS;
|
|
@@ -425,7 +426,7 @@ export function createBinaryStore(cfg) {
|
|
|
425
426
|
const title = opts?.title?.trim();
|
|
426
427
|
const titleQs = title ? `&title=${encodeURIComponent(title)}` : '';
|
|
427
428
|
const immutableQs = opts?.immutable ? '&immutable=1' : '';
|
|
428
|
-
const res = await
|
|
429
|
+
const res = await send(`${baseUrl}/upload/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}${titleQs}${immutableQs}`, {
|
|
429
430
|
method: 'PUT',
|
|
430
431
|
headers: mime ? { 'content-type': mime } : undefined,
|
|
431
432
|
body: bytes,
|
|
@@ -494,7 +495,7 @@ export function createBinaryStore(cfg) {
|
|
|
494
495
|
const slice = new Uint8Array(await blob
|
|
495
496
|
.slice(ci * chunkBytes, Math.min((ci + 1) * chunkBytes, blob.size))
|
|
496
497
|
.arrayBuffer());
|
|
497
|
-
const res = await
|
|
498
|
+
const res = await send(`${baseUrl}/upload-chunk/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}${titleQs}${immutableQs}`, {
|
|
498
499
|
method: 'PUT',
|
|
499
500
|
headers: mime ? { 'content-type': mime } : undefined,
|
|
500
501
|
body: slice,
|
|
@@ -513,7 +514,7 @@ export function createBinaryStore(cfg) {
|
|
|
513
514
|
if (!opts?.fresh && knownAbsent(key))
|
|
514
515
|
return null;
|
|
515
516
|
countWireRead(key);
|
|
516
|
-
const res = await
|
|
517
|
+
const res = await send(await internalMediaUrl(key));
|
|
517
518
|
if (res.status === 404) {
|
|
518
519
|
rememberAbsent(key);
|
|
519
520
|
return null;
|
|
@@ -572,7 +573,7 @@ export function createBinaryStore(cfg) {
|
|
|
572
573
|
// of the URL, so a fresh one per request would defeat every cache between here and
|
|
573
574
|
// the object — see MediaUrlOptions.stableWindowSeconds.
|
|
574
575
|
const url = await internalMediaUrl(key, { stableWindowSeconds: 60 * 60 });
|
|
575
|
-
return await
|
|
576
|
+
return await send(url, {
|
|
576
577
|
method: init?.method ?? 'GET',
|
|
577
578
|
// 🔴 Only when the caller HAS one. `Range: undefined` is fine, but a literal
|
|
578
579
|
// empty string is a malformed header that some origins answer 416 to.
|
|
@@ -589,7 +590,7 @@ export function createBinaryStore(cfg) {
|
|
|
589
590
|
return null;
|
|
590
591
|
}
|
|
591
592
|
countWireRead(key);
|
|
592
|
-
const res = await
|
|
593
|
+
const res = await send(await metaUrl(key));
|
|
593
594
|
if (res.status === 404) {
|
|
594
595
|
rememberAbsent(key);
|
|
595
596
|
return null;
|
|
@@ -633,7 +634,7 @@ export function createBinaryStore(cfg) {
|
|
|
633
634
|
// A transport/5xx failure — the probe below is the second opinion.
|
|
634
635
|
}
|
|
635
636
|
countWireRead(key);
|
|
636
|
-
const res = await
|
|
637
|
+
const res = await send(await internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } });
|
|
637
638
|
if (res.status === 404) {
|
|
638
639
|
rememberAbsent(key);
|
|
639
640
|
return null;
|
|
@@ -658,7 +659,7 @@ export function createBinaryStore(cfg) {
|
|
|
658
659
|
// (owner ruling #10). `internalHeaders()` throws first if no secret is configured.
|
|
659
660
|
const headers = internalHeaders();
|
|
660
661
|
const token = await signToken({ k: appKey(key), op: 'delete' }, DELETE_TOKEN_TTL_SECONDS);
|
|
661
|
-
const res = await
|
|
662
|
+
const res = await send(`${baseUrl}/media/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}`, { method: 'DELETE', headers });
|
|
662
663
|
if (!res.ok && res.status !== 404) {
|
|
663
664
|
throw new Error(`[binary-store] remove ${key}: ${res.status}`);
|
|
664
665
|
}
|
|
@@ -671,7 +672,7 @@ export function createBinaryStore(cfg) {
|
|
|
671
672
|
// and the `?container=1` flag. Same two-barrier gate as remove().
|
|
672
673
|
const headers = internalHeaders();
|
|
673
674
|
const token = await signToken({ k: appKey(prefix), op: 'delete' }, DELETE_TOKEN_TTL_SECONDS);
|
|
674
|
-
const res = await
|
|
675
|
+
const res = await send(`${baseUrl}/media/${encodeKeyPath(appKey(prefix))}?container=1&token=${encodeURIComponent(token)}`, { method: 'DELETE', headers });
|
|
675
676
|
if (!res.ok && res.status !== 404) {
|
|
676
677
|
throw new Error(`[binary-store] removePrefix ${prefix}: ${res.status}`);
|
|
677
678
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cursedbelt-server",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.33.1",
|
|
4
4
|
"license": "ISC",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "The app-facing Bun/Hono server tier of the cursedbelt split — storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",
|
|
@@ -380,7 +380,7 @@
|
|
|
380
380
|
"@node-rs/argon2": "^2.0.2",
|
|
381
381
|
"@types/bun": "^1.3.14",
|
|
382
382
|
"@types/node": "^24",
|
|
383
|
-
"cursedbelt": "^5.
|
|
383
|
+
"cursedbelt": "^5.9.0",
|
|
384
384
|
"hono": "4.12.28",
|
|
385
385
|
"kysely": "^0.28.17",
|
|
386
386
|
"kysely-bun-sqlite": "^0.4.0",
|
package/src/server/d1/fakeD1.ts
CHANGED
|
@@ -88,7 +88,7 @@ class FakeD1Statement implements D1BindingStatement {
|
|
|
88
88
|
) {}
|
|
89
89
|
|
|
90
90
|
bind(...values: unknown[]): D1BindingStatement {
|
|
91
|
-
values.
|
|
91
|
+
for (const [i, v] of values.entries()) assertD1Bindable(v, i);
|
|
92
92
|
// A real binding stores what it was given; the ArrayBuffer→Uint8Array conversion
|
|
93
93
|
// here is `bun:sqlite`'s requirement, not D1's, and happens below the emulated edge.
|
|
94
94
|
const stored = values.map((v) => (v instanceof ArrayBuffer ? new Uint8Array(v) : v));
|
|
@@ -144,6 +144,7 @@ describe('pulling D1 down — paged SELECTs, table by table, never an export', (
|
|
|
144
144
|
requiredTable: 'items',
|
|
145
145
|
afterLoadSql: REBUILD,
|
|
146
146
|
wrangler: run,
|
|
147
|
+
log: () => {},
|
|
147
148
|
});
|
|
148
149
|
const opened = (dir: string) => new Database(join(dir, 'pulled.sqlite'), { readonly: true });
|
|
149
150
|
|
|
@@ -222,6 +223,62 @@ describe('pulling D1 down — paged SELECTs, table by table, never an export', (
|
|
|
222
223
|
);
|
|
223
224
|
});
|
|
224
225
|
|
|
226
|
+
test('🔴 one refused page does not shrink the rest of the table — the size doubles back to PAGE_ROWS (desk, 2026-09-25: 15 rows per call for hours)', async () => {
|
|
227
|
+
const dir = scratch();
|
|
228
|
+
const n = PAGE_ROWS * 3 + 7;
|
|
229
|
+
const source = live(
|
|
230
|
+
`WITH RECURSIVE s(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM s WHERE i < ${n}) INSERT INTO items_fts_map (item_id, text) SELECT 'i' || i, 't' FROM s; INSERT INTO items VALUES ('i1', 'x');`,
|
|
231
|
+
);
|
|
232
|
+
let failed = false;
|
|
233
|
+
const { run, seen } = d1(source, {
|
|
234
|
+
failWhen: (sql) => {
|
|
235
|
+
if (failed || !sql.includes('FROM "items_fts_map"')) return false;
|
|
236
|
+
failed = true;
|
|
237
|
+
return true;
|
|
238
|
+
},
|
|
239
|
+
});
|
|
240
|
+
const logged: string[] = [];
|
|
241
|
+
await pullD1ToSqlite({
|
|
242
|
+
destination: join(dir, 'pulled.sqlite'),
|
|
243
|
+
database: 'collections',
|
|
244
|
+
schemaSql: SCHEMA,
|
|
245
|
+
requiredTable: 'items',
|
|
246
|
+
afterLoadSql: REBUILD,
|
|
247
|
+
wrangler: run,
|
|
248
|
+
log: (line) => logged.push(line),
|
|
249
|
+
});
|
|
250
|
+
const db = opened(dir);
|
|
251
|
+
expect((db.query('select count(*) n from items_fts_map').get() as { n: number }).n).toBe(n);
|
|
252
|
+
db.close();
|
|
253
|
+
const limits = seen
|
|
254
|
+
.map((a) => a[a.indexOf('--command') + 1] as string)
|
|
255
|
+
.filter((sql) => sql.includes('FROM "items_fts_map"'))
|
|
256
|
+
.map((sql) => Number(/LIMIT (\d+)/.exec(sql)?.[1]));
|
|
257
|
+
// refused at 500, read at 250, then back to 500 for the rest.
|
|
258
|
+
expect(limits.slice(0, 3)).toEqual([PAGE_ROWS, PAGE_ROWS / 2, PAGE_ROWS]);
|
|
259
|
+
expect(limits.slice(2).every((limit) => limit === PAGE_ROWS)).toBe(true);
|
|
260
|
+
expect(logged).toHaveLength(1);
|
|
261
|
+
expect(logged[0]).toContain('D1 said no');
|
|
262
|
+
expect(logged[0]).toContain(`page of ${PAGE_ROWS}`);
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
test('a table D1 refuses on every call still THROWS with D1\'s own words, after logging each halving', async () => {
|
|
266
|
+
const source = live("INSERT INTO items VALUES ('i1', 'a');");
|
|
267
|
+
const logged: string[] = [];
|
|
268
|
+
await expect(
|
|
269
|
+
pullD1ToSqlite({
|
|
270
|
+
destination: join(scratch(), 'pulled.sqlite'),
|
|
271
|
+
database: 'collections',
|
|
272
|
+
schemaSql: SCHEMA,
|
|
273
|
+
requiredTable: 'items',
|
|
274
|
+
wrangler: d1(source, { failWhen: (sql) => sql.includes('FROM "items"') }).run,
|
|
275
|
+
log: (line) => logged.push(line),
|
|
276
|
+
}),
|
|
277
|
+
).rejects.toThrow(/could not read `items` from D1: ✘ D1 said no/);
|
|
278
|
+
// 500 → 250 → 125 → 62 → 31 → 15 → 7 → 3 → 1: eight halvings, then the refusal at one row.
|
|
279
|
+
expect(logged).toHaveLength(8);
|
|
280
|
+
});
|
|
281
|
+
|
|
225
282
|
test('🔴 a table D1 holds that the schema does not declare is created from D1\'s own DDL', async () => {
|
|
226
283
|
const dir = scratch();
|
|
227
284
|
const ddl = 'CREATE TABLE art_lookups (key TEXT PRIMARY KEY, art_key TEXT)';
|
package/src/server/d1/pullD1.ts
CHANGED
|
@@ -202,6 +202,11 @@ export interface PullD1Options {
|
|
|
202
202
|
/** Run after the rows are loaded — where an app REBUILDS its search index (never copied). */
|
|
203
203
|
afterLoadSql?: string;
|
|
204
204
|
wrangler: Wrangler;
|
|
205
|
+
/**
|
|
206
|
+
* Where a page D1 refused is reported (default: stderr). Before 4.33.1 a halving was silent,
|
|
207
|
+
* so the only symptom of a slow spell was a backup that took hours.
|
|
208
|
+
*/
|
|
209
|
+
log?: (line: string) => void;
|
|
205
210
|
}
|
|
206
211
|
|
|
207
212
|
const sqlIdent = (name: string): string => `"${name.replaceAll('"', '""')}"`;
|
|
@@ -250,8 +255,20 @@ export function pageSql(plan: TablePlan, limit: number, after: { key?: number; o
|
|
|
250
255
|
return `SELECT ${values} FROM ${sqlIdent(plan.table)} ORDER BY ${order} LIMIT ${limit} OFFSET ${after.offset ?? 0}`;
|
|
251
256
|
}
|
|
252
257
|
|
|
253
|
-
/**
|
|
254
|
-
|
|
258
|
+
/**
|
|
259
|
+
* Every row of one table as `INSERT INTO …;` lines, read in pages that never block D1.
|
|
260
|
+
*
|
|
261
|
+
* A page D1 refuses is asked again at half the size, and 🔴 every page that SUCCEEDS doubles the
|
|
262
|
+
* size back toward {@link PAGE_ROWS}. Before 4.33.1 nothing grew it back: desk's first launchd
|
|
263
|
+
* backup (2026-09-25) hit a few slow calls early and read its 3,381-row table 15 rows per call —
|
|
264
|
+
* 225 two-minute calls, on course for hours. Only a refusal at ONE row is fatal.
|
|
265
|
+
*/
|
|
266
|
+
function dumpTable(
|
|
267
|
+
wrangler: Wrangler,
|
|
268
|
+
database: string,
|
|
269
|
+
plan: TablePlan,
|
|
270
|
+
log: (line: string) => void,
|
|
271
|
+
): { text: string; rows: number } {
|
|
255
272
|
const into = `INSERT INTO ${sqlIdent(plan.table)} (${plan.columns.map(sqlIdent).join(', ')}) VALUES (`;
|
|
256
273
|
const lines: string[] = [];
|
|
257
274
|
let limit = PAGE_ROWS;
|
|
@@ -261,11 +278,14 @@ function dumpTable(wrangler: Wrangler, database: string, plan: TablePlan): { tex
|
|
|
261
278
|
try {
|
|
262
279
|
page = execute(wrangler, database, pageSql(plan, limit, after))[0] ?? [];
|
|
263
280
|
} catch (error) {
|
|
281
|
+
const message = (error as Error).message;
|
|
264
282
|
if (limit > 1) {
|
|
265
|
-
|
|
283
|
+
const next = Math.max(1, Math.floor(limit / 2));
|
|
284
|
+
log(`pull D1: \`${plan.table}\` refused a page of ${limit} — asking for ${next}: ${message}`);
|
|
285
|
+
limit = next;
|
|
266
286
|
continue;
|
|
267
287
|
}
|
|
268
|
-
throw new Error(`could not read \`${plan.table}\` from D1: ${
|
|
288
|
+
throw new Error(`could not read \`${plan.table}\` from D1: ${message}`);
|
|
269
289
|
}
|
|
270
290
|
for (const row of page) {
|
|
271
291
|
const literals = plan.columns.map((_, i) => row[`c${i}`]);
|
|
@@ -276,6 +296,7 @@ function dumpTable(wrangler: Wrangler, database: string, plan: TablePlan): { tex
|
|
|
276
296
|
}
|
|
277
297
|
if (page.length < limit) break;
|
|
278
298
|
after = plan.key ? { key: Number(page[page.length - 1]?.k) } : { offset: (after.offset ?? 0) + page.length };
|
|
299
|
+
limit = Math.min(PAGE_ROWS, limit * 2);
|
|
279
300
|
}
|
|
280
301
|
return { text: lines.join('\n'), rows: lines.length };
|
|
281
302
|
}
|
|
@@ -290,6 +311,7 @@ function dumpTable(wrangler: Wrangler, database: string, plan: TablePlan): { tex
|
|
|
290
311
|
*/
|
|
291
312
|
export async function pullD1ToSqlite(options: PullD1Options): Promise<void> {
|
|
292
313
|
const { destination, database, wrangler } = options;
|
|
314
|
+
const log = options.log ?? ((line: string) => console.error(line));
|
|
293
315
|
let rows: Array<{ name: string; sql: string | null }>;
|
|
294
316
|
try {
|
|
295
317
|
rows = (execute(wrangler, database, "select name, sql from sqlite_master where type = 'table'")[0] ?? []) as typeof rows;
|
|
@@ -313,7 +335,7 @@ export async function pullD1ToSqlite(options: PullD1Options): Promise<void> {
|
|
|
313
335
|
const dumps = tables.map((table, i) => {
|
|
314
336
|
const cols = (columnSets[i] ?? []) as Array<{ name: string; pk: number }>;
|
|
315
337
|
const plan = tablePlan(table, rows.find((r) => r.name === table)?.sql ?? null, cols);
|
|
316
|
-
return { table, ...dumpTable(wrangler, database, plan) };
|
|
338
|
+
return { table, ...dumpTable(wrangler, database, plan, log) };
|
|
317
339
|
});
|
|
318
340
|
if (dumps.every((d) => d.rows === 0)) {
|
|
319
341
|
throw new Error('the D1 pull contains no rows — refusing to snapshot an empty database as if it were the library');
|
|
@@ -673,6 +673,42 @@ describe('internalBaseUrl — reading bytes without leaving the box', () => {
|
|
|
673
673
|
});
|
|
674
674
|
});
|
|
675
675
|
|
|
676
|
+
describe('🔴 fetch — a same-zone Worker reads through its service binding, never the tunnel', () => {
|
|
677
|
+
it('sends EVERY method through the injected fetch and none through the global one', async () => {
|
|
678
|
+
// 2026-09-25: family's Worker read portraits with the global fetch, which on the
|
|
679
|
+
// cursedalchemy.com zone skips the edge Worker and lands on the Mac's byte tunnel —
|
|
680
|
+
// the owner turned the Mac off and every face on the tree went with it.
|
|
681
|
+
const bound: string[] = [];
|
|
682
|
+
const store = makeStore({
|
|
683
|
+
internalSecret: 's',
|
|
684
|
+
fetch: (input, init) => {
|
|
685
|
+
bound.push(`${init?.method ?? (input instanceof Request ? input.method : 'GET')} ${typeof input === 'string' ? input : input.url}`);
|
|
686
|
+
const url = typeof input === 'string' ? input : input.url;
|
|
687
|
+
if (url.includes('/meta/')) return Promise.resolve(json({ size: 1 }));
|
|
688
|
+
return Promise.resolve(ok(new Uint8Array([1]), { headers: { 'content-range': 'bytes 0-0/1' } }));
|
|
689
|
+
},
|
|
690
|
+
});
|
|
691
|
+
await store.put('a', new Uint8Array([1]));
|
|
692
|
+
await store.get('b');
|
|
693
|
+
await store.meta('c');
|
|
694
|
+
await store.stat('d');
|
|
695
|
+
await store.has('e');
|
|
696
|
+
await store.fetchMedia('f');
|
|
697
|
+
await store.remove('g');
|
|
698
|
+
await store.removePrefix('h/');
|
|
699
|
+
expect(calls).toEqual([]);
|
|
700
|
+
expect(bound.length).toBeGreaterThanOrEqual(8);
|
|
701
|
+
for (const line of bound) expect(line).toContain(BASE);
|
|
702
|
+
});
|
|
703
|
+
|
|
704
|
+
it('defaults to the global fetch when nothing is injected (a Bun host on the Mac)', async () => {
|
|
705
|
+
const store = makeStore();
|
|
706
|
+
reply = () => ok(new Uint8Array([1]));
|
|
707
|
+
await store.get('b');
|
|
708
|
+
expect(calls).toHaveLength(1);
|
|
709
|
+
});
|
|
710
|
+
});
|
|
711
|
+
|
|
676
712
|
describe('fetchMedia', () => {
|
|
677
713
|
it('forwards a Range header untouched — Range IS seeking', async () => {
|
|
678
714
|
const store = makeStore({ internalBaseUrl: INTERNAL });
|
|
@@ -83,6 +83,18 @@ export interface BinaryStoreConfig {
|
|
|
83
83
|
absenceTtlMs?: number;
|
|
84
84
|
/** Injected so the memo's expiry is testable without a real clock. */
|
|
85
85
|
now?: () => number;
|
|
86
|
+
/**
|
|
87
|
+
* 🔴 How THIS PROCESS sends every request to binary-server. Default: the global `fetch`.
|
|
88
|
+
*
|
|
89
|
+
* A Cloudflare Worker on the SAME ZONE as binary-server must pass its service binding to
|
|
90
|
+
* `binary-server-edge-gate` here (`(input, init) => env.BINARY_SERVER.fetch(input, init)`).
|
|
91
|
+
* A same-zone subrequest skips every other Worker's route and goes straight to the zone's
|
|
92
|
+
* ORIGIN — the byte tunnel into the owner's Mac — so a global `fetch` never reached the edge
|
|
93
|
+
* Worker, its R2 reads or its cache. Measured 2026-09-25: the owner powered the Mac off and
|
|
94
|
+
* every face on family's tree was gone, although family's bytes rest in R2. Auth met the same
|
|
95
|
+
* trap first (task 2150); this is the byte half of it.
|
|
96
|
+
*/
|
|
97
|
+
fetch?: (input: string | Request, init?: RequestInit) => Promise<Response>;
|
|
86
98
|
}
|
|
87
99
|
|
|
88
100
|
/*
|
|
@@ -511,6 +523,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
511
523
|
* and is named as the remaining gap.
|
|
512
524
|
*/
|
|
513
525
|
const internalBaseUrl = (cfg.internalBaseUrl ?? cfg.baseUrl).replace(/\/+$/, '');
|
|
526
|
+
const send = cfg.fetch ?? ((input: string | Request, init?: RequestInit) => fetch(input, init));
|
|
514
527
|
const autoChunkBytes = cfg.autoChunkBytes ?? AUTO_CHUNK_THRESHOLD_BYTES;
|
|
515
528
|
const now = cfg.now ?? Date.now;
|
|
516
529
|
const absenceTtlMs = cfg.absenceTtlMs ?? BINARY_ABSENCE_TTL_MS;
|
|
@@ -705,7 +718,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
705
718
|
const title = opts?.title?.trim();
|
|
706
719
|
const titleQs = title ? `&title=${encodeURIComponent(title)}` : '';
|
|
707
720
|
const immutableQs = opts?.immutable ? '&immutable=1' : '';
|
|
708
|
-
const res = await
|
|
721
|
+
const res = await send(
|
|
709
722
|
`${baseUrl}/upload/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}${titleQs}${immutableQs}`,
|
|
710
723
|
{
|
|
711
724
|
method: 'PUT',
|
|
@@ -785,7 +798,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
785
798
|
.slice(ci * chunkBytes, Math.min((ci + 1) * chunkBytes, blob.size))
|
|
786
799
|
.arrayBuffer(),
|
|
787
800
|
);
|
|
788
|
-
const res = await
|
|
801
|
+
const res = await send(
|
|
789
802
|
`${baseUrl}/upload-chunk/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}${titleQs}${immutableQs}`,
|
|
790
803
|
{
|
|
791
804
|
method: 'PUT',
|
|
@@ -807,7 +820,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
807
820
|
// short-circuit here — we want the bytes, and only the store has them.
|
|
808
821
|
if (!opts?.fresh && knownAbsent(key)) return null;
|
|
809
822
|
countWireRead(key);
|
|
810
|
-
const res = await
|
|
823
|
+
const res = await send(await internalMediaUrl(key));
|
|
811
824
|
if (res.status === 404) {
|
|
812
825
|
rememberAbsent(key);
|
|
813
826
|
return null;
|
|
@@ -868,7 +881,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
868
881
|
// of the URL, so a fresh one per request would defeat every cache between here and
|
|
869
882
|
// the object — see MediaUrlOptions.stableWindowSeconds.
|
|
870
883
|
const url = await internalMediaUrl(key, { stableWindowSeconds: 60 * 60 });
|
|
871
|
-
return await
|
|
884
|
+
return await send(url, {
|
|
872
885
|
method: init?.method ?? 'GET',
|
|
873
886
|
// 🔴 Only when the caller HAS one. `Range: undefined` is fine, but a literal
|
|
874
887
|
// empty string is a malformed header that some origins answer 416 to.
|
|
@@ -884,7 +897,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
884
897
|
if (knownAbsent(key)) return null;
|
|
885
898
|
}
|
|
886
899
|
countWireRead(key);
|
|
887
|
-
const res = await
|
|
900
|
+
const res = await send(await metaUrl(key));
|
|
888
901
|
if (res.status === 404) {
|
|
889
902
|
rememberAbsent(key);
|
|
890
903
|
return null;
|
|
@@ -925,7 +938,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
925
938
|
// A transport/5xx failure — the probe below is the second opinion.
|
|
926
939
|
}
|
|
927
940
|
countWireRead(key);
|
|
928
|
-
const res = await
|
|
941
|
+
const res = await send(await internalMediaUrl(key), { headers: { Range: 'bytes=0-0' } });
|
|
929
942
|
if (res.status === 404) {
|
|
930
943
|
rememberAbsent(key);
|
|
931
944
|
return null;
|
|
@@ -950,7 +963,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
950
963
|
// (owner ruling #10). `internalHeaders()` throws first if no secret is configured.
|
|
951
964
|
const headers = internalHeaders();
|
|
952
965
|
const token = await signToken({ k: appKey(key), op: 'delete' }, DELETE_TOKEN_TTL_SECONDS);
|
|
953
|
-
const res = await
|
|
966
|
+
const res = await send(
|
|
954
967
|
`${baseUrl}/media/${encodeKeyPath(appKey(key))}?token=${encodeURIComponent(token)}`,
|
|
955
968
|
{ method: 'DELETE', headers },
|
|
956
969
|
);
|
|
@@ -967,7 +980,7 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
967
980
|
// and the `?container=1` flag. Same two-barrier gate as remove().
|
|
968
981
|
const headers = internalHeaders();
|
|
969
982
|
const token = await signToken({ k: appKey(prefix), op: 'delete' }, DELETE_TOKEN_TTL_SECONDS);
|
|
970
|
-
const res = await
|
|
983
|
+
const res = await send(
|
|
971
984
|
`${baseUrl}/media/${encodeKeyPath(appKey(prefix))}?container=1&token=${encodeURIComponent(token)}`,
|
|
972
985
|
{ method: 'DELETE', headers },
|
|
973
986
|
);
|