cursedbelt-server 4.33.0 → 4.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/d1/fakeD1.js +2 -1
- package/dist/server/d1/pullD1.d.ts +5 -0
- package/dist/server/d1/pullD1.js +17 -5
- package/dist/server/storage/binaryStore.d.ts +23 -0
- package/dist/server/storage/binaryStore.js +43 -0
- package/package.json +1 -1
- package/src/server/d1/fakeD1.ts +1 -1
- package/src/server/d1/pullD1.spec.ts +57 -0
- package/src/server/d1/pullD1.ts +27 -5
- package/src/server/storage/binaryStore.spec.ts +57 -0
- package/src/server/storage/binaryStore.ts +52 -2
package/dist/server/d1/fakeD1.js
CHANGED
|
@@ -83,7 +83,8 @@ class FakeD1Statement {
|
|
|
83
83
|
this.params = params;
|
|
84
84
|
}
|
|
85
85
|
bind(...values) {
|
|
86
|
-
|
|
86
|
+
for (const [i, v] of values.entries())
|
|
87
|
+
assertD1Bindable(v, i);
|
|
87
88
|
// A real binding stores what it was given; the ArrayBuffer→Uint8Array conversion
|
|
88
89
|
// here is `bun:sqlite`'s requirement, not D1's, and happens below the emulated edge.
|
|
89
90
|
const stored = values.map((v) => (v instanceof ArrayBuffer ? new Uint8Array(v) : v));
|
|
@@ -129,6 +129,11 @@ export interface PullD1Options {
|
|
|
129
129
|
/** Run after the rows are loaded — where an app REBUILDS its search index (never copied). */
|
|
130
130
|
afterLoadSql?: string;
|
|
131
131
|
wrangler: Wrangler;
|
|
132
|
+
/**
|
|
133
|
+
* Where a page D1 refused is reported (default: stderr). Before 4.33.1 a halving was silent,
|
|
134
|
+
* so the only symptom of a slow spell was a backup that took hours.
|
|
135
|
+
*/
|
|
136
|
+
log?: (line: string) => void;
|
|
132
137
|
}
|
|
133
138
|
/** How a table is walked in pages: by a rowid keyset, or — a `WITHOUT ROWID` table — by offset. */
|
|
134
139
|
export interface TablePlan {
|
package/dist/server/d1/pullD1.js
CHANGED
|
@@ -202,8 +202,15 @@ export function pageSql(plan, limit, after) {
|
|
|
202
202
|
const order = plan.order.map(sqlIdent).join(', ');
|
|
203
203
|
return `SELECT ${values} FROM ${sqlIdent(plan.table)} ORDER BY ${order} LIMIT ${limit} OFFSET ${after.offset ?? 0}`;
|
|
204
204
|
}
|
|
205
|
-
/**
|
|
206
|
-
|
|
205
|
+
/**
|
|
206
|
+
* Every row of one table as `INSERT INTO …;` lines, read in pages that never block D1.
|
|
207
|
+
*
|
|
208
|
+
* A page D1 refuses is asked again at half the size, and 🔴 every page that SUCCEEDS doubles the
|
|
209
|
+
* size back toward {@link PAGE_ROWS}. Before 4.33.1 nothing grew it back: desk's first launchd
|
|
210
|
+
* backup (2026-09-25) hit a few slow calls early and read its 3,381-row table 15 rows per call —
|
|
211
|
+
* 225 two-minute calls, on course for hours. Only a refusal at ONE row is fatal.
|
|
212
|
+
*/
|
|
213
|
+
function dumpTable(wrangler, database, plan, log) {
|
|
207
214
|
const into = `INSERT INTO ${sqlIdent(plan.table)} (${plan.columns.map(sqlIdent).join(', ')}) VALUES (`;
|
|
208
215
|
const lines = [];
|
|
209
216
|
let limit = PAGE_ROWS;
|
|
@@ -214,11 +221,14 @@ function dumpTable(wrangler, database, plan) {
|
|
|
214
221
|
page = execute(wrangler, database, pageSql(plan, limit, after))[0] ?? [];
|
|
215
222
|
}
|
|
216
223
|
catch (error) {
|
|
224
|
+
const message = error.message;
|
|
217
225
|
if (limit > 1) {
|
|
218
|
-
|
|
226
|
+
const next = Math.max(1, Math.floor(limit / 2));
|
|
227
|
+
log(`pull D1: \`${plan.table}\` refused a page of ${limit} — asking for ${next}: ${message}`);
|
|
228
|
+
limit = next;
|
|
219
229
|
continue;
|
|
220
230
|
}
|
|
221
|
-
throw new Error(`could not read \`${plan.table}\` from D1: ${
|
|
231
|
+
throw new Error(`could not read \`${plan.table}\` from D1: ${message}`);
|
|
222
232
|
}
|
|
223
233
|
for (const row of page) {
|
|
224
234
|
const literals = plan.columns.map((_, i) => row[`c${i}`]);
|
|
@@ -230,6 +240,7 @@ function dumpTable(wrangler, database, plan) {
|
|
|
230
240
|
if (page.length < limit)
|
|
231
241
|
break;
|
|
232
242
|
after = plan.key ? { key: Number(page[page.length - 1]?.k) } : { offset: (after.offset ?? 0) + page.length };
|
|
243
|
+
limit = Math.min(PAGE_ROWS, limit * 2);
|
|
233
244
|
}
|
|
234
245
|
return { text: lines.join('\n'), rows: lines.length };
|
|
235
246
|
}
|
|
@@ -243,6 +254,7 @@ function dumpTable(wrangler, database, plan) {
|
|
|
243
254
|
*/
|
|
244
255
|
export async function pullD1ToSqlite(options) {
|
|
245
256
|
const { destination, database, wrangler } = options;
|
|
257
|
+
const log = options.log ?? ((line) => console.error(line));
|
|
246
258
|
let rows;
|
|
247
259
|
try {
|
|
248
260
|
rows = (execute(wrangler, database, "select name, sql from sqlite_master where type = 'table'")[0] ?? []);
|
|
@@ -265,7 +277,7 @@ export async function pullD1ToSqlite(options) {
|
|
|
265
277
|
const dumps = tables.map((table, i) => {
|
|
266
278
|
const cols = (columnSets[i] ?? []);
|
|
267
279
|
const plan = tablePlan(table, rows.find((r) => r.name === table)?.sql ?? null, cols);
|
|
268
|
-
return { table, ...dumpTable(wrangler, database, plan) };
|
|
280
|
+
return { table, ...dumpTable(wrangler, database, plan, log) };
|
|
269
281
|
});
|
|
270
282
|
if (dumps.every((d) => d.rows === 0)) {
|
|
271
283
|
throw new Error('the D1 pull contains no rows — refusing to snapshot an empty database as if it were the library');
|
|
@@ -405,6 +405,29 @@ export declare function binaryStoreTenant(appId: string, env?: Record<string, st
|
|
|
405
405
|
* apps and that is precisely why the difference goes unnoticed when it is not.
|
|
406
406
|
*/
|
|
407
407
|
export declare function resolvedMediaTenant(appId: string, env?: Record<string, string | undefined>): string | null;
|
|
408
|
+
/**
|
|
409
|
+
* binary-server's origin did not ANSWER a write — as opposed to answering "no".
|
|
410
|
+
*
|
|
411
|
+
* binary-server itself never answers 502, 503 or 504; those, and Cloudflare's 52x family (530 is a
|
|
412
|
+
* tunnel with no connector), are the edge speaking for an origin it could not reach — the owner's
|
|
413
|
+
* Mac off, asleep, or restarting. A thrown `fetch` is the same fact. Since 2026-09-26 the edge
|
|
414
|
+
* Worker takes an R2-backed tenant's whole upload itself in that case
|
|
415
|
+
* (`apps/binary-server/edge-worker/src/upload.ts`), so for desk and family this is what is left
|
|
416
|
+
* when BOTH halves failed; for a tenant whose bytes rest on the Mac it is simply "the Mac is off".
|
|
417
|
+
*/
|
|
418
|
+
export declare function storeUnreachableStatus(status: number): boolean;
|
|
419
|
+
/**
|
|
420
|
+
* The write could not reach the store. Carries `getResponse()`, so a Hono app that lets it escape
|
|
421
|
+
* a route answers a 503 that names the file store — Hono's default error handler returns
|
|
422
|
+
* `err.getResponse()` for any error that has one — instead of a bare `500 Internal Server Error`.
|
|
423
|
+
* The message keeps the `[binary-store] <op> <key>:` head every batch caller already prints.
|
|
424
|
+
*/
|
|
425
|
+
export declare class BinaryStoreUnreachableError extends Error {
|
|
426
|
+
readonly status = 503;
|
|
427
|
+
readonly code = "BYTE_STORE_UNREACHABLE";
|
|
428
|
+
constructor(op: string, key: string, detail: string);
|
|
429
|
+
getResponse(): Response;
|
|
430
|
+
}
|
|
408
431
|
/**
|
|
409
432
|
* The standard env surface every tenant's secrets file carries
|
|
410
433
|
* (`$FORGE_STATE/secrets/<app>.env`, written at bs registration):
|
|
@@ -430,7 +430,11 @@ export function createBinaryStore(cfg) {
|
|
|
430
430
|
method: 'PUT',
|
|
431
431
|
headers: mime ? { 'content-type': mime } : undefined,
|
|
432
432
|
body: bytes,
|
|
433
|
+
}).catch((cause) => {
|
|
434
|
+
throw new BinaryStoreUnreachableError('put', key, cause instanceof Error ? cause.message : String(cause));
|
|
433
435
|
});
|
|
436
|
+
if (storeUnreachableStatus(res.status))
|
|
437
|
+
throw new BinaryStoreUnreachableError('put', key, String(res.status));
|
|
434
438
|
if (!res.ok) {
|
|
435
439
|
// A 413 that reaches here despite the threshold above means the edge's
|
|
436
440
|
// real cap is lower than we think, or the caller disabled the
|
|
@@ -499,7 +503,12 @@ export function createBinaryStore(cfg) {
|
|
|
499
503
|
method: 'PUT',
|
|
500
504
|
headers: mime ? { 'content-type': mime } : undefined,
|
|
501
505
|
body: slice,
|
|
506
|
+
}).catch((cause) => {
|
|
507
|
+
throw new BinaryStoreUnreachableError('putLarge', key, cause instanceof Error ? cause.message : String(cause));
|
|
502
508
|
});
|
|
509
|
+
if (storeUnreachableStatus(res.status)) {
|
|
510
|
+
throw new BinaryStoreUnreachableError('putLarge', key, `chunk ${ci + 1}/${total}: ${res.status}`);
|
|
511
|
+
}
|
|
503
512
|
if (!res.ok) {
|
|
504
513
|
throw new Error(`[binary-store] putLarge ${key} chunk ${ci + 1}/${total}: ${res.status}`);
|
|
505
514
|
}
|
|
@@ -752,6 +761,40 @@ export function binaryStoreTenant(appId, env = process.env) {
|
|
|
752
761
|
export function resolvedMediaTenant(appId, env = process.env) {
|
|
753
762
|
return readBinaryStoreEnv(appId, env)?.appId ?? null;
|
|
754
763
|
}
|
|
764
|
+
/**
|
|
765
|
+
* binary-server's origin did not ANSWER a write — as opposed to answering "no".
|
|
766
|
+
*
|
|
767
|
+
* binary-server itself never answers 502, 503 or 504; those, and Cloudflare's 52x family (530 is a
|
|
768
|
+
* tunnel with no connector), are the edge speaking for an origin it could not reach — the owner's
|
|
769
|
+
* Mac off, asleep, or restarting. A thrown `fetch` is the same fact. Since 2026-09-26 the edge
|
|
770
|
+
* Worker takes an R2-backed tenant's whole upload itself in that case
|
|
771
|
+
* (`apps/binary-server/edge-worker/src/upload.ts`), so for desk and family this is what is left
|
|
772
|
+
* when BOTH halves failed; for a tenant whose bytes rest on the Mac it is simply "the Mac is off".
|
|
773
|
+
*/
|
|
774
|
+
export function storeUnreachableStatus(status) {
|
|
775
|
+
return status === 502 || status === 503 || status === 504 || (status >= 520 && status <= 530);
|
|
776
|
+
}
|
|
777
|
+
/**
|
|
778
|
+
* The write could not reach the store. Carries `getResponse()`, so a Hono app that lets it escape
|
|
779
|
+
* a route answers a 503 that names the file store — Hono's default error handler returns
|
|
780
|
+
* `err.getResponse()` for any error that has one — instead of a bare `500 Internal Server Error`.
|
|
781
|
+
* The message keeps the `[binary-store] <op> <key>:` head every batch caller already prints.
|
|
782
|
+
*/
|
|
783
|
+
export class BinaryStoreUnreachableError extends Error {
|
|
784
|
+
status = 503;
|
|
785
|
+
code = 'BYTE_STORE_UNREACHABLE';
|
|
786
|
+
constructor(op, key, detail) {
|
|
787
|
+
super(`[binary-store] ${op} ${key}: the file store did not answer (${detail}) — nothing was saved; ` +
|
|
788
|
+
'everything already stored still opens. Try again when the home server is on.');
|
|
789
|
+
this.name = 'BinaryStoreUnreachableError';
|
|
790
|
+
}
|
|
791
|
+
getResponse() {
|
|
792
|
+
return new Response(JSON.stringify({
|
|
793
|
+
error: this.code,
|
|
794
|
+
message: 'The file store did not answer, so nothing was saved. Everything already stored still opens — try again in a moment.',
|
|
795
|
+
}), { status: 503, headers: { 'content-type': 'application/json; charset=utf-8', 'retry-after': '30' } });
|
|
796
|
+
}
|
|
797
|
+
}
|
|
755
798
|
/**
|
|
756
799
|
* The standard env surface every tenant's secrets file carries
|
|
757
800
|
* (`$FORGE_STATE/secrets/<app>.env`, written at bs registration):
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cursedbelt-server",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.34.0",
|
|
4
4
|
"license": "ISC",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "The app-facing Bun/Hono server tier of the cursedbelt split — storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",
|
package/src/server/d1/fakeD1.ts
CHANGED
|
@@ -88,7 +88,7 @@ class FakeD1Statement implements D1BindingStatement {
|
|
|
88
88
|
) {}
|
|
89
89
|
|
|
90
90
|
bind(...values: unknown[]): D1BindingStatement {
|
|
91
|
-
values.
|
|
91
|
+
for (const [i, v] of values.entries()) assertD1Bindable(v, i);
|
|
92
92
|
// A real binding stores what it was given; the ArrayBuffer→Uint8Array conversion
|
|
93
93
|
// here is `bun:sqlite`'s requirement, not D1's, and happens below the emulated edge.
|
|
94
94
|
const stored = values.map((v) => (v instanceof ArrayBuffer ? new Uint8Array(v) : v));
|
|
@@ -144,6 +144,7 @@ describe('pulling D1 down — paged SELECTs, table by table, never an export', (
|
|
|
144
144
|
requiredTable: 'items',
|
|
145
145
|
afterLoadSql: REBUILD,
|
|
146
146
|
wrangler: run,
|
|
147
|
+
log: () => {},
|
|
147
148
|
});
|
|
148
149
|
const opened = (dir: string) => new Database(join(dir, 'pulled.sqlite'), { readonly: true });
|
|
149
150
|
|
|
@@ -222,6 +223,62 @@ describe('pulling D1 down — paged SELECTs, table by table, never an export', (
|
|
|
222
223
|
);
|
|
223
224
|
});
|
|
224
225
|
|
|
226
|
+
test('🔴 one refused page does not shrink the rest of the table — the size doubles back to PAGE_ROWS (desk, 2026-09-25: 15 rows per call for hours)', async () => {
|
|
227
|
+
const dir = scratch();
|
|
228
|
+
const n = PAGE_ROWS * 3 + 7;
|
|
229
|
+
const source = live(
|
|
230
|
+
`WITH RECURSIVE s(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM s WHERE i < ${n}) INSERT INTO items_fts_map (item_id, text) SELECT 'i' || i, 't' FROM s; INSERT INTO items VALUES ('i1', 'x');`,
|
|
231
|
+
);
|
|
232
|
+
let failed = false;
|
|
233
|
+
const { run, seen } = d1(source, {
|
|
234
|
+
failWhen: (sql) => {
|
|
235
|
+
if (failed || !sql.includes('FROM "items_fts_map"')) return false;
|
|
236
|
+
failed = true;
|
|
237
|
+
return true;
|
|
238
|
+
},
|
|
239
|
+
});
|
|
240
|
+
const logged: string[] = [];
|
|
241
|
+
await pullD1ToSqlite({
|
|
242
|
+
destination: join(dir, 'pulled.sqlite'),
|
|
243
|
+
database: 'collections',
|
|
244
|
+
schemaSql: SCHEMA,
|
|
245
|
+
requiredTable: 'items',
|
|
246
|
+
afterLoadSql: REBUILD,
|
|
247
|
+
wrangler: run,
|
|
248
|
+
log: (line) => logged.push(line),
|
|
249
|
+
});
|
|
250
|
+
const db = opened(dir);
|
|
251
|
+
expect((db.query('select count(*) n from items_fts_map').get() as { n: number }).n).toBe(n);
|
|
252
|
+
db.close();
|
|
253
|
+
const limits = seen
|
|
254
|
+
.map((a) => a[a.indexOf('--command') + 1] as string)
|
|
255
|
+
.filter((sql) => sql.includes('FROM "items_fts_map"'))
|
|
256
|
+
.map((sql) => Number(/LIMIT (\d+)/.exec(sql)?.[1]));
|
|
257
|
+
// refused at 500, read at 250, then back to 500 for the rest.
|
|
258
|
+
expect(limits.slice(0, 3)).toEqual([PAGE_ROWS, PAGE_ROWS / 2, PAGE_ROWS]);
|
|
259
|
+
expect(limits.slice(2).every((limit) => limit === PAGE_ROWS)).toBe(true);
|
|
260
|
+
expect(logged).toHaveLength(1);
|
|
261
|
+
expect(logged[0]).toContain('D1 said no');
|
|
262
|
+
expect(logged[0]).toContain(`page of ${PAGE_ROWS}`);
|
|
263
|
+
});
|
|
264
|
+
|
|
265
|
+
test('a table D1 refuses on every call still THROWS with D1\'s own words, after logging each halving', async () => {
|
|
266
|
+
const source = live("INSERT INTO items VALUES ('i1', 'a');");
|
|
267
|
+
const logged: string[] = [];
|
|
268
|
+
await expect(
|
|
269
|
+
pullD1ToSqlite({
|
|
270
|
+
destination: join(scratch(), 'pulled.sqlite'),
|
|
271
|
+
database: 'collections',
|
|
272
|
+
schemaSql: SCHEMA,
|
|
273
|
+
requiredTable: 'items',
|
|
274
|
+
wrangler: d1(source, { failWhen: (sql) => sql.includes('FROM "items"') }).run,
|
|
275
|
+
log: (line) => logged.push(line),
|
|
276
|
+
}),
|
|
277
|
+
).rejects.toThrow(/could not read `items` from D1: ✘ D1 said no/);
|
|
278
|
+
// 500 → 250 → 125 → 62 → 31 → 15 → 7 → 3 → 1: eight halvings, then the refusal at one row.
|
|
279
|
+
expect(logged).toHaveLength(8);
|
|
280
|
+
});
|
|
281
|
+
|
|
225
282
|
test('🔴 a table D1 holds that the schema does not declare is created from D1\'s own DDL', async () => {
|
|
226
283
|
const dir = scratch();
|
|
227
284
|
const ddl = 'CREATE TABLE art_lookups (key TEXT PRIMARY KEY, art_key TEXT)';
|
package/src/server/d1/pullD1.ts
CHANGED
|
@@ -202,6 +202,11 @@ export interface PullD1Options {
|
|
|
202
202
|
/** Run after the rows are loaded — where an app REBUILDS its search index (never copied). */
|
|
203
203
|
afterLoadSql?: string;
|
|
204
204
|
wrangler: Wrangler;
|
|
205
|
+
/**
|
|
206
|
+
* Where a page D1 refused is reported (default: stderr). Before 4.33.1 a halving was silent,
|
|
207
|
+
* so the only symptom of a slow spell was a backup that took hours.
|
|
208
|
+
*/
|
|
209
|
+
log?: (line: string) => void;
|
|
205
210
|
}
|
|
206
211
|
|
|
207
212
|
const sqlIdent = (name: string): string => `"${name.replaceAll('"', '""')}"`;
|
|
@@ -250,8 +255,20 @@ export function pageSql(plan: TablePlan, limit: number, after: { key?: number; o
|
|
|
250
255
|
return `SELECT ${values} FROM ${sqlIdent(plan.table)} ORDER BY ${order} LIMIT ${limit} OFFSET ${after.offset ?? 0}`;
|
|
251
256
|
}
|
|
252
257
|
|
|
253
|
-
/**
|
|
254
|
-
|
|
258
|
+
/**
|
|
259
|
+
* Every row of one table as `INSERT INTO …;` lines, read in pages that never block D1.
|
|
260
|
+
*
|
|
261
|
+
* A page D1 refuses is asked again at half the size, and 🔴 every page that SUCCEEDS doubles the
|
|
262
|
+
* size back toward {@link PAGE_ROWS}. Before 4.33.1 nothing grew it back: desk's first launchd
|
|
263
|
+
* backup (2026-09-25) hit a few slow calls early and read its 3,381-row table 15 rows per call —
|
|
264
|
+
* 225 two-minute calls, on course for hours. Only a refusal at ONE row is fatal.
|
|
265
|
+
*/
|
|
266
|
+
function dumpTable(
|
|
267
|
+
wrangler: Wrangler,
|
|
268
|
+
database: string,
|
|
269
|
+
plan: TablePlan,
|
|
270
|
+
log: (line: string) => void,
|
|
271
|
+
): { text: string; rows: number } {
|
|
255
272
|
const into = `INSERT INTO ${sqlIdent(plan.table)} (${plan.columns.map(sqlIdent).join(', ')}) VALUES (`;
|
|
256
273
|
const lines: string[] = [];
|
|
257
274
|
let limit = PAGE_ROWS;
|
|
@@ -261,11 +278,14 @@ function dumpTable(wrangler: Wrangler, database: string, plan: TablePlan): { tex
|
|
|
261
278
|
try {
|
|
262
279
|
page = execute(wrangler, database, pageSql(plan, limit, after))[0] ?? [];
|
|
263
280
|
} catch (error) {
|
|
281
|
+
const message = (error as Error).message;
|
|
264
282
|
if (limit > 1) {
|
|
265
|
-
|
|
283
|
+
const next = Math.max(1, Math.floor(limit / 2));
|
|
284
|
+
log(`pull D1: \`${plan.table}\` refused a page of ${limit} — asking for ${next}: ${message}`);
|
|
285
|
+
limit = next;
|
|
266
286
|
continue;
|
|
267
287
|
}
|
|
268
|
-
throw new Error(`could not read \`${plan.table}\` from D1: ${
|
|
288
|
+
throw new Error(`could not read \`${plan.table}\` from D1: ${message}`);
|
|
269
289
|
}
|
|
270
290
|
for (const row of page) {
|
|
271
291
|
const literals = plan.columns.map((_, i) => row[`c${i}`]);
|
|
@@ -276,6 +296,7 @@ function dumpTable(wrangler: Wrangler, database: string, plan: TablePlan): { tex
|
|
|
276
296
|
}
|
|
277
297
|
if (page.length < limit) break;
|
|
278
298
|
after = plan.key ? { key: Number(page[page.length - 1]?.k) } : { offset: (after.offset ?? 0) + page.length };
|
|
299
|
+
limit = Math.min(PAGE_ROWS, limit * 2);
|
|
279
300
|
}
|
|
280
301
|
return { text: lines.join('\n'), rows: lines.length };
|
|
281
302
|
}
|
|
@@ -290,6 +311,7 @@ function dumpTable(wrangler: Wrangler, database: string, plan: TablePlan): { tex
|
|
|
290
311
|
*/
|
|
291
312
|
export async function pullD1ToSqlite(options: PullD1Options): Promise<void> {
|
|
292
313
|
const { destination, database, wrangler } = options;
|
|
314
|
+
const log = options.log ?? ((line: string) => console.error(line));
|
|
293
315
|
let rows: Array<{ name: string; sql: string | null }>;
|
|
294
316
|
try {
|
|
295
317
|
rows = (execute(wrangler, database, "select name, sql from sqlite_master where type = 'table'")[0] ?? []) as typeof rows;
|
|
@@ -313,7 +335,7 @@ export async function pullD1ToSqlite(options: PullD1Options): Promise<void> {
|
|
|
313
335
|
const dumps = tables.map((table, i) => {
|
|
314
336
|
const cols = (columnSets[i] ?? []) as Array<{ name: string; pk: number }>;
|
|
315
337
|
const plan = tablePlan(table, rows.find((r) => r.name === table)?.sql ?? null, cols);
|
|
316
|
-
return { table, ...dumpTable(wrangler, database, plan) };
|
|
338
|
+
return { table, ...dumpTable(wrangler, database, plan, log) };
|
|
317
339
|
});
|
|
318
340
|
if (dumps.every((d) => d.rows === 0)) {
|
|
319
341
|
throw new Error('the D1 pull contains no rows — refusing to snapshot an empty database as if it were the library');
|
|
@@ -34,6 +34,7 @@ import {
|
|
|
34
34
|
BINARY_REPEAT_THRESHOLD,
|
|
35
35
|
BINARY_REPEAT_WINDOW_MS,
|
|
36
36
|
type BinaryStoreConfig,
|
|
37
|
+
BinaryStoreUnreachableError,
|
|
37
38
|
binaryStoreTenant,
|
|
38
39
|
createBinaryStore,
|
|
39
40
|
normalizePrivateKeyPem,
|
|
@@ -41,6 +42,7 @@ import {
|
|
|
41
42
|
repeatedBinaryKeys,
|
|
42
43
|
resetRepeatedBinaryKeys,
|
|
43
44
|
resolvedMediaTenant,
|
|
45
|
+
storeUnreachableStatus,
|
|
44
46
|
wantsFresh,
|
|
45
47
|
} from './binaryStore.js';
|
|
46
48
|
import { binaryStoreFakeDefaults } from './binaryStoreFake.js';
|
|
@@ -962,3 +964,58 @@ describe('binaryStoreFakeDefaults', () => {
|
|
|
962
964
|
expect(await fake.signToken({ k: 'c/a' }, 60)).toBe('fake-token');
|
|
963
965
|
});
|
|
964
966
|
});
|
|
967
|
+
|
|
968
|
+
describe('🔴 a write the store could not ANSWER is a 503 that names it (2026-09-26)', () => {
|
|
969
|
+
it('502-504 and 52x/530 are the origin unreachable; everything else is its own answer', () => {
|
|
970
|
+
for (const status of [502, 503, 504, 520, 522, 524, 530]) expect(storeUnreachableStatus(status)).toBe(true);
|
|
971
|
+
for (const status of [400, 403, 404, 413, 422, 500, 507]) expect(storeUnreachableStatus(status)).toBe(false);
|
|
972
|
+
});
|
|
973
|
+
|
|
974
|
+
it('put on a 530 throws the typed error, keeping the [binary-store] head batch callers print', async () => {
|
|
975
|
+
reply = () => new Response('tunnel gone', { status: 530 });
|
|
976
|
+
const error = await makeStore().put('art/a.png', new Uint8Array([1])).catch((e: unknown) => e);
|
|
977
|
+
expect(error).toBeInstanceOf(BinaryStoreUnreachableError);
|
|
978
|
+
expect(String((error as Error).message)).toStartWith('[binary-store] put art/a.png: the file store did not answer (530)');
|
|
979
|
+
expect((error as BinaryStoreUnreachableError).status).toBe(503);
|
|
980
|
+
});
|
|
981
|
+
|
|
982
|
+
it('a fetch that THROWS is the same fact', async () => {
|
|
983
|
+
const store = makeStore({ fetch: () => Promise.reject(new Error('connection refused')) });
|
|
984
|
+
const error = await store.put('k', new Uint8Array([1])).catch((e: unknown) => e);
|
|
985
|
+
expect(error).toBeInstanceOf(BinaryStoreUnreachableError);
|
|
986
|
+
expect(String((error as Error).message)).toContain('connection refused');
|
|
987
|
+
});
|
|
988
|
+
|
|
989
|
+
it('putLarge says which chunk found the store gone', async () => {
|
|
990
|
+
reply = () => new Response('', { status: 502 });
|
|
991
|
+
const error = await makeStore()
|
|
992
|
+
.putLarge('v.mp4', new Uint8Array(10), 'video/mp4', { chunkBytes: 4 })
|
|
993
|
+
.catch((e: unknown) => e);
|
|
994
|
+
expect(error).toBeInstanceOf(BinaryStoreUnreachableError);
|
|
995
|
+
expect(String((error as Error).message)).toContain('chunk 1/3: 502');
|
|
996
|
+
});
|
|
997
|
+
|
|
998
|
+
it('🔴 getResponse() is the 503 a Hono app answers with when the error escapes a route', async () => {
|
|
999
|
+
const res = new BinaryStoreUnreachableError('put', 'k', '530').getResponse();
|
|
1000
|
+
expect(res.status).toBe(503);
|
|
1001
|
+
expect(res.headers.get('retry-after')).toBe('30');
|
|
1002
|
+
expect(((await res.json()) as { error: string }).error).toBe('BYTE_STORE_UNREACHABLE');
|
|
1003
|
+
});
|
|
1004
|
+
|
|
1005
|
+
it('…and a real Hono app DOES answer it, with no onError of its own', async () => {
|
|
1006
|
+
const { Hono } = await import('hono');
|
|
1007
|
+
const app = new Hono().post('/upload', () => {
|
|
1008
|
+
throw new BinaryStoreUnreachableError('put', 'k', '530');
|
|
1009
|
+
});
|
|
1010
|
+
const res = await app.request('/upload', { method: 'POST' });
|
|
1011
|
+
expect(res.status).toBe(503);
|
|
1012
|
+
expect(((await res.json()) as { error: string }).error).toBe('BYTE_STORE_UNREACHABLE');
|
|
1013
|
+
});
|
|
1014
|
+
|
|
1015
|
+
it("the origin's own refusals are unchanged — a 403 is still the plain status error", async () => {
|
|
1016
|
+
reply = () => new Response('', { status: 403 });
|
|
1017
|
+
const error = await makeStore().put('k', new Uint8Array([1])).catch((e: unknown) => e);
|
|
1018
|
+
expect(error).not.toBeInstanceOf(BinaryStoreUnreachableError);
|
|
1019
|
+
expect(String((error as Error).message)).toBe('[binary-store] put k: 403');
|
|
1020
|
+
});
|
|
1021
|
+
});
|
|
@@ -725,7 +725,10 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
725
725
|
headers: mime ? { 'content-type': mime } : undefined,
|
|
726
726
|
body: bytes as unknown as BodyInit,
|
|
727
727
|
},
|
|
728
|
-
)
|
|
728
|
+
).catch((cause: unknown) => {
|
|
729
|
+
throw new BinaryStoreUnreachableError('put', key, cause instanceof Error ? cause.message : String(cause));
|
|
730
|
+
});
|
|
731
|
+
if (storeUnreachableStatus(res.status)) throw new BinaryStoreUnreachableError('put', key, String(res.status));
|
|
729
732
|
if (!res.ok) {
|
|
730
733
|
// A 413 that reaches here despite the threshold above means the edge's
|
|
731
734
|
// real cap is lower than we think, or the caller disabled the
|
|
@@ -805,7 +808,12 @@ export function createBinaryStore(cfg: BinaryStoreConfig): BinaryStore {
|
|
|
805
808
|
headers: mime ? { 'content-type': mime } : undefined,
|
|
806
809
|
body: slice,
|
|
807
810
|
},
|
|
808
|
-
)
|
|
811
|
+
).catch((cause: unknown) => {
|
|
812
|
+
throw new BinaryStoreUnreachableError('putLarge', key, cause instanceof Error ? cause.message : String(cause));
|
|
813
|
+
});
|
|
814
|
+
if (storeUnreachableStatus(res.status)) {
|
|
815
|
+
throw new BinaryStoreUnreachableError('putLarge', key, `chunk ${ci + 1}/${total}: ${res.status}`);
|
|
816
|
+
}
|
|
809
817
|
if (!res.ok) {
|
|
810
818
|
throw new Error(`[binary-store] putLarge ${key} chunk ${ci + 1}/${total}: ${res.status}`);
|
|
811
819
|
}
|
|
@@ -1071,6 +1079,48 @@ export function resolvedMediaTenant(
|
|
|
1071
1079
|
return readBinaryStoreEnv(appId, env)?.appId ?? null;
|
|
1072
1080
|
}
|
|
1073
1081
|
|
|
1082
|
+
/**
|
|
1083
|
+
* binary-server's origin did not ANSWER a write — as opposed to answering "no".
|
|
1084
|
+
*
|
|
1085
|
+
* binary-server itself never answers 502, 503 or 504; those, and Cloudflare's 52x family (530 is a
|
|
1086
|
+
* tunnel with no connector), are the edge speaking for an origin it could not reach — the owner's
|
|
1087
|
+
* Mac off, asleep, or restarting. A thrown `fetch` is the same fact. Since 2026-09-26 the edge
|
|
1088
|
+
* Worker takes an R2-backed tenant's whole upload itself in that case
|
|
1089
|
+
* (`apps/binary-server/edge-worker/src/upload.ts`), so for desk and family this is what is left
|
|
1090
|
+
* when BOTH halves failed; for a tenant whose bytes rest on the Mac it is simply "the Mac is off".
|
|
1091
|
+
*/
|
|
1092
|
+
export function storeUnreachableStatus(status: number): boolean {
|
|
1093
|
+
return status === 502 || status === 503 || status === 504 || (status >= 520 && status <= 530);
|
|
1094
|
+
}
|
|
1095
|
+
|
|
1096
|
+
/**
|
|
1097
|
+
* The write could not reach the store. Carries `getResponse()`, so a Hono app that lets it escape
|
|
1098
|
+
* a route answers a 503 that names the file store — Hono's default error handler returns
|
|
1099
|
+
* `err.getResponse()` for any error that has one — instead of a bare `500 Internal Server Error`.
|
|
1100
|
+
* The message keeps the `[binary-store] <op> <key>:` head every batch caller already prints.
|
|
1101
|
+
*/
|
|
1102
|
+
export class BinaryStoreUnreachableError extends Error {
|
|
1103
|
+
readonly status = 503;
|
|
1104
|
+
readonly code = 'BYTE_STORE_UNREACHABLE';
|
|
1105
|
+
constructor(op: string, key: string, detail: string) {
|
|
1106
|
+
super(
|
|
1107
|
+
`[binary-store] ${op} ${key}: the file store did not answer (${detail}) — nothing was saved; ` +
|
|
1108
|
+
'everything already stored still opens. Try again when the home server is on.',
|
|
1109
|
+
);
|
|
1110
|
+
this.name = 'BinaryStoreUnreachableError';
|
|
1111
|
+
}
|
|
1112
|
+
getResponse(): Response {
|
|
1113
|
+
return new Response(
|
|
1114
|
+
JSON.stringify({
|
|
1115
|
+
error: this.code,
|
|
1116
|
+
message:
|
|
1117
|
+
'The file store did not answer, so nothing was saved. Everything already stored still opens — try again in a moment.',
|
|
1118
|
+
}),
|
|
1119
|
+
{ status: 503, headers: { 'content-type': 'application/json; charset=utf-8', 'retry-after': '30' } },
|
|
1120
|
+
);
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
|
|
1074
1124
|
/**
|
|
1075
1125
|
* The standard env surface every tenant's secrets file carries
|
|
1076
1126
|
* (`$FORGE_STATE/secrets/<app>.env`, written at bs registration):
|