cursedbelt-server 4.43.3 → 4.44.0

package/dist/server/d1/pullD1.d.ts

@@ -155,7 +155,24 @@ export interface PullD1Options {
      * so the only symptom of a slow spell was a backup that took hours.
      */
     log?: (line: string) => void;
+    /** How a {@link TRANSIENT_WAITS_MS} wait is slept (default `Bun.sleepSync`). Injected so a spec waits for nothing. */
+    sleep?: (ms: number) => void;
 }
+/**
+ * The waits before asking again after a {@link isTransient} failure — ~1.5 minutes in all, then
+ * the failure is real. Measured 2026-10-09T15:35Z: this Mac's network dropped for under a minute
+ * just as collections' and flix's nightly backups ran (both at :35), wrangler answered the ONE
+ * columns call with `{"error":{"text":"fetch failed"}}`, and both nights were refused with no
+ * snapshot (tasks 2985–2987). patterns' log holds the same `fetch failed` from September.
+ */
+export declare const TRANSIENT_WAITS_MS: readonly number[];
+/**
+ * A failure of the WIRE between this Mac and Cloudflare, not an answer from D1 — which asking
+ * again a little later fixes. Only these are waited out — a refusal D1 actually made (a page too
+ * large, bad SQL, auth) is never retried here, so a page refused for its size still halves at
+ * once instead of sleeping ~1.5 minutes per halving.
+ */
+export declare const isTransient: (message: string) => boolean;
 /** How a table is walked in pages: by a rowid keyset, or — a `WITHOUT ROWID` table — by offset. */
 export interface TablePlan {
     table: string;

package/dist/server/d1/pullD1.js

@@ -186,10 +186,43 @@ export function backupTables(rows) {
         .filter((name) => name === 'sqlite_sequence' || !name.startsWith('sqlite_'))
         .sort();
 }
+/**
+ * The waits before asking again after a {@link isTransient} failure — ~1.5 minutes in all, then
+ * the failure is real. Measured 2026-10-09T15:35Z: this Mac's network dropped for under a minute
+ * just as collections' and flix's nightly backups ran (both at :35), wrangler answered the ONE
+ * columns call with `{"error":{"text":"fetch failed"}}`, and both nights were refused with no
+ * snapshot (tasks 2985–2987). patterns' log holds the same `fetch failed` from September.
+ */
+export const TRANSIENT_WAITS_MS = [5_000, 20_000, 60_000];
+/**
+ * A failure of the WIRE between this Mac and Cloudflare, not an answer from D1 — which asking
+ * again a little later fixes. Only these are waited out — a refusal D1 actually made (a page too
+ * large, bad SQL, auth) is never retried here, so a page refused for its size still halves at
+ * once instead of sleeping ~1.5 minutes per halving.
+ */
+export const isTransient = (message) => /fetch failed|ECONNRESET|ECONNREFUSED|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket hang up|network connection was lost/i.test(message);
 const sqlIdent = (name) => `"${name.replaceAll('"', '""')}"`;
 const sqlText = (text) => `'${text.replaceAll("'", "''")}'`;
-/** One `wrangler d1 execute --remote --json` — every statement's result rows, in order. */
-function execute(wrangler, database, sql) {
+/**
+ * One `wrangler d1 execute --remote --json` — every statement's result rows, in order — asked
+ * again after each of {@link TRANSIENT_WAITS_MS} while the failure {@link isTransient}.
+ */
+function execute(caller, database, sql) {
+    for (let attempt = 0;; attempt++) {
+        try {
+            return executeOnce(caller.wrangler, database, sql);
+        }
+        catch (error) {
+            const message = error.message;
+            const wait = TRANSIENT_WAITS_MS[attempt];
+            if (wait === undefined || !isTransient(message))
+                throw error;
+            caller.log(`pull D1: the wire failed (${message.replace(/\s+/g, ' ').slice(0, 120)}) — asking again in ${wait / 1000}s`);
+            caller.sleep(wait);
+        }
+    }
+}
+function executeOnce(wrangler, database, sql) {
     const r = wrangler(['d1', 'execute', database, '--remote', '--env=', '--json', '--command', sql]);
     if (r.code !== 0)
         throw new Error(r.stderr.trim() || r.stdout.trim() || `wrangler exited ${r.code}`);
@@ -236,7 +269,7 @@ export function pageSql(plan, limit, after) {
  * backup (2026-09-25) hit a few slow calls early and read its 3,381-row table 15 rows per call —
  * 225 two-minute calls, on course for hours. Only a refusal at ONE row is fatal.
  */
-function dumpTable(wrangler, database, plan, log) {
+function dumpTable(caller, database, plan) {
     const into = `INSERT INTO ${sqlIdent(plan.table)} (${plan.columns.map(sqlIdent).join(', ')}) VALUES (`;
     const lines = [];
     let limit = PAGE_ROWS;
@@ -244,13 +277,13 @@ function dumpTable(wrangler, database, plan, log) {
     for (;;) {
         let page;
         try {
-            page = execute(wrangler, database, pageSql(plan, limit, after))[0] ?? [];
+            page = execute(caller, database, pageSql(plan, limit, after))[0] ?? [];
         }
         catch (error) {
             const message = error.message;
             if (limit > 1) {
                 const next = Math.max(1, Math.floor(limit / 2));
-                log(`pull D1: \`${plan.table}\` refused a page of ${limit} — asking for ${next}: ${message}`);
+                caller.log(`pull D1: \`${plan.table}\` refused a page of ${limit} — asking for ${next}: ${message}`);
                 limit = next;
                 continue;
             }
@@ -279,11 +312,15 @@ function dumpTable(wrangler, database, plan, log) {
  * magic-byte check and `integrity_check` and land on the shelf with today's date on it.
  */
 export async function pullD1ToSqlite(options) {
-    const { destination, database, wrangler } = options;
-    const log = options.log ?? ((line) => console.error(line));
+    const { destination, database } = options;
+    const caller = {
+        wrangler: options.wrangler,
+        log: options.log ?? ((line) => console.error(line)),
+        sleep: options.sleep ?? ((ms) => Bun.sleepSync(ms)),
+    };
     let rows;
     try {
-        rows = (execute(wrangler, database, "select name, sql from sqlite_master where type = 'table'")[0] ?? []);
+        rows = (execute(caller, database, "select name, sql from sqlite_master where type = 'table'")[0] ?? []);
     }
     catch (error) {
         throw new Error(`could not list D1's tables: ${error.message}`);
@@ -295,7 +332,7 @@ export async function pullD1ToSqlite(options) {
     // Every table's columns in ONE call: one statement each, answered in order.
     let columnSets;
     try {
-        columnSets = execute(wrangler, database, tables.map((t) => `SELECT name, pk FROM pragma_table_info(${sqlText(t)});`).join(' '));
+        columnSets = execute(caller, database, tables.map((t) => `SELECT name, pk FROM pragma_table_info(${sqlText(t)});`).join(' '));
     }
     catch (error) {
         throw new Error(`could not read D1's columns: ${error.message}`);
@@ -303,7 +340,7 @@ export async function pullD1ToSqlite(options) {
     const dumps = tables.map((table, i) => {
         const cols = (columnSets[i] ?? []);
         const plan = tablePlan(table, rows.find((r) => r.name === table)?.sql ?? null, cols);
-        return { table, ...dumpTable(wrangler, database, plan, log) };
+        return { table, ...dumpTable(caller, database, plan) };
     });
     if (dumps.every((d) => d.rows === 0)) {
         throw new Error('the D1 pull contains no rows — refusing to snapshot an empty database as if it were the library');

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "cursedbelt-server",
-	"version": "4.43.3",
+	"version": "4.44.0",
 	"license": "ISC",
 	"type": "module",
 	"description": "The app-facing Bun/Hono server tier of the cursedbelt split — storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",

package/src/server/d1/pullD1.spec.ts

@@ -279,6 +279,80 @@ describe('pulling D1 down — paged SELECTs, table by table, never an export', (
 		expect(logged).toHaveLength(8);
 	});
 
+	describe('🔴 a blip on the wire is waited out, not a lost night (collections + flix, 2026-10-09T15:35Z)', () => {
+		const FETCH_FAILED = '{\n  "error": {\n    "text": "fetch failed"\n  }\n}';
+		/** `d1`, but the first `blips` calls whose SQL `when` matches fail the way wrangler did that night. */
+		const blipping = (source: Database, when: (sql: string) => boolean, blips: number, said = FETCH_FAILED) => {
+			const real = d1(source);
+			let left = blips;
+			const run: Wrangler = (args) => {
+				const sql = args[args.indexOf('--command') + 1] as string;
+				if (args[1] === 'execute' && left > 0 && when(sql)) {
+					left--;
+					return { code: 1, stdout: said, stderr: '' };
+				}
+				return real.run(args);
+			};
+			return { run, seen: real.seen };
+		};
+		const pullWaiting = (dir: string, run: Wrangler, slept: number[], logged: string[] = []) =>
+			pullD1ToSqlite({
+				destination: join(dir, 'pulled.sqlite'),
+				database: 'collections',
+				schemaSql: SCHEMA,
+				requiredTable: 'items',
+				wrangler: run,
+				log: (line) => logged.push(line),
+				sleep: (ms) => slept.push(ms),
+			});
+		const columns = (sql: string) => sql.includes('pragma_table_info');
+		/** `TRANSIENT_WAITS_MS`, written out: the spec pins the numbers rather than importing them. */
+		const WAITS = [5_000, 20_000, 60_000];
+
+		test('the columns call that failed that night is asked again after a wait, and the snapshot lands', async () => {
+			const dir = scratch();
+			const source = live("INSERT INTO items VALUES ('i1', 'a');");
+			const slept: number[] = [];
+			const logged: string[] = [];
+			await pullWaiting(dir, blipping(source, columns, 2).run, slept, logged);
+			expect(slept).toEqual(WAITS.slice(0, 2));
+			expect(logged[0]).toContain('fetch failed');
+			const db = opened(dir);
+			expect((db.query('select name from items').get() as { name: string }).name).toBe('a');
+			db.close();
+		});
+
+		test('a page that blips is asked again at the SAME size — a wire failure is not a page too large', async () => {
+			const source = live("INSERT INTO items VALUES ('i1', 'a');");
+			// wrangler's coloured form, as patterns' log has it.
+			const { run, seen } = blipping(source, (sql) => sql.includes('FROM "items"'), 1, '✘ \u001b[1mfetch failed\u001b[0m');
+			const slept: number[] = [];
+			await pullWaiting(scratch(), run, slept);
+			const limits = seen
+				.map((a) => a[a.indexOf('--command') + 1] as string)
+				.filter((sql) => sql.includes('FROM "items"'))
+				.map((sql) => Number(/LIMIT (\d+)/.exec(sql)?.[1]));
+			expect(limits).toEqual([PAGE_ROWS]);
+			expect(slept).toEqual([WAITS[0] as number]);
+		});
+
+		test('a wire that stays down still REFUSES, with its own words, after every wait', async () => {
+			const source = live("INSERT INTO items VALUES ('i1', 'a');");
+			const slept: number[] = [];
+			await expect(pullWaiting(scratch(), blipping(source, columns, 99).run, slept)).rejects.toThrow(
+				/could not read D1's columns: .*fetch failed/s,
+			);
+			expect(slept).toEqual(WAITS);
+		});
+
+		test('a refusal D1 itself made is never waited on', async () => {
+			const source = live("INSERT INTO items VALUES ('i1', 'a');");
+			const slept: number[] = [];
+			await expect(pullWaiting(scratch(), d1(source, { failWhen: columns }).run, slept)).rejects.toThrow(/D1 said no/);
+			expect(slept).toEqual([]);
+		});
+	});
+
 	test('🔴 a table D1 holds that the schema does not declare is created from D1\'s own DDL', async () => {
 		const dir = scratch();
 		const ddl = 'CREATE TABLE art_lookups (key TEXT PRIMARY KEY, art_key TEXT)';

package/src/server/d1/pullD1.ts

@@ -229,13 +229,59 @@ export interface PullD1Options {
 	 * so the only symptom of a slow spell was a backup that took hours.
 	 */
 	log?: (line: string) => void;
+	/** How a {@link TRANSIENT_WAITS_MS} wait is slept (default `Bun.sleepSync`). Injected so a spec waits for nothing. */
+	sleep?: (ms: number) => void;
+}
+
+/**
+ * The waits before asking again after a {@link isTransient} failure — ~1.5 minutes in all, then
+ * the failure is real. Measured 2026-10-09T15:35Z: this Mac's network dropped for under a minute
+ * just as collections' and flix's nightly backups ran (both at :35), wrangler answered the ONE
+ * columns call with `{"error":{"text":"fetch failed"}}`, and both nights were refused with no
+ * snapshot (tasks 2985–2987). patterns' log holds the same `fetch failed` from September.
+ */
+export const TRANSIENT_WAITS_MS: readonly number[] = [5_000, 20_000, 60_000];
+
+/**
+ * A failure of the WIRE between this Mac and Cloudflare, not an answer from D1 — which asking
+ * again a little later fixes. Only these are waited out — a refusal D1 actually made (a page too
+ * large, bad SQL, auth) is never retried here, so a page refused for its size still halves at
+ * once instead of sleeping ~1.5 minutes per halving.
+ */
+export const isTransient = (message: string): boolean =>
+	/fetch failed|ECONNRESET|ECONNREFUSED|ETIMEDOUT|ENOTFOUND|EAI_AGAIN|socket hang up|network connection was lost/i.test(
+		message,
+	);
+
+/** What every {@link execute} needs: the wrangler to run, where to log, and how to wait. */
+interface Caller {
+	wrangler: Wrangler;
+	log: (line: string) => void;
+	sleep: (ms: number) => void;
 }
 
 const sqlIdent = (name: string): string => `"${name.replaceAll('"', '""')}"`;
 const sqlText = (text: string): string => `'${text.replaceAll("'", "''")}'`;
 
-/** One `wrangler d1 execute --remote --json` — every statement's result rows, in order. */
-function execute(wrangler: Wrangler, database: string, sql: string): Array<Array<Record<string, unknown>>> {
+/**
+ * One `wrangler d1 execute --remote --json` — every statement's result rows, in order — asked
+ * again after each of {@link TRANSIENT_WAITS_MS} while the failure {@link isTransient}.
+ */
+function execute(caller: Caller, database: string, sql: string): Array<Array<Record<string, unknown>>> {
+	for (let attempt = 0; ; attempt++) {
+		try {
+			return executeOnce(caller.wrangler, database, sql);
+		} catch (error) {
+			const message = (error as Error).message;
+			const wait = TRANSIENT_WAITS_MS[attempt];
+			if (wait === undefined || !isTransient(message)) throw error;
+			caller.log(`pull D1: the wire failed (${message.replace(/\s+/g, ' ').slice(0, 120)}) — asking again in ${wait / 1000}s`);
+			caller.sleep(wait);
+		}
+	}
+}
+
+function executeOnce(wrangler: Wrangler, database: string, sql: string): Array<Array<Record<string, unknown>>> {
 	const r = wrangler(['d1', 'execute', database, '--remote', '--env=', '--json', '--command', sql]);
 	if (r.code !== 0) throw new Error(r.stderr.trim() || r.stdout.trim() || `wrangler exited ${r.code}`);
 	let parsed: Array<{ results?: Array<Record<string, unknown>> }>;
@@ -291,12 +337,7 @@ export function pageSql(plan: TablePlan, limit: number, after: { key?: number; o
  * backup (2026-09-25) hit a few slow calls early and read its 3,381-row table 15 rows per call —
  * 225 two-minute calls, on course for hours. Only a refusal at ONE row is fatal.
  */
-function dumpTable(
-	wrangler: Wrangler,
-	database: string,
-	plan: TablePlan,
-	log: (line: string) => void,
-): { text: string; rows: number } {
+function dumpTable(caller: Caller, database: string, plan: TablePlan): { text: string; rows: number } {
 	const into = `INSERT INTO ${sqlIdent(plan.table)} (${plan.columns.map(sqlIdent).join(', ')}) VALUES (`;
 	const lines: string[] = [];
 	let limit = PAGE_ROWS;
@@ -304,12 +345,12 @@ function dumpTable(
 	for (;;) {
 		let page: Array<Record<string, unknown>>;
 		try {
-			page = execute(wrangler, database, pageSql(plan, limit, after))[0] ?? [];
+			page = execute(caller, database, pageSql(plan, limit, after))[0] ?? [];
 		} catch (error) {
 			const message = (error as Error).message;
 			if (limit > 1) {
 				const next = Math.max(1, Math.floor(limit / 2));
-				log(`pull D1: \`${plan.table}\` refused a page of ${limit} — asking for ${next}: ${message}`);
+				caller.log(`pull D1: \`${plan.table}\` refused a page of ${limit} — asking for ${next}: ${message}`);
 				limit = next;
 				continue;
 			}
@@ -338,11 +379,15 @@ function dumpTable(
  * magic-byte check and `integrity_check` and land on the shelf with today's date on it.
  */
 export async function pullD1ToSqlite(options: PullD1Options): Promise<void> {
-	const { destination, database, wrangler } = options;
-	const log = options.log ?? ((line: string) => console.error(line));
+	const { destination, database } = options;
+	const caller: Caller = {
+		wrangler: options.wrangler,
+		log: options.log ?? ((line: string) => console.error(line)),
+		sleep: options.sleep ?? ((ms: number) => Bun.sleepSync(ms)),
+	};
 	let rows: Array<{ name: string; sql: string | null }>;
 	try {
-		rows = (execute(wrangler, database, "select name, sql from sqlite_master where type = 'table'")[0] ?? []) as typeof rows;
+		rows = (execute(caller, database, "select name, sql from sqlite_master where type = 'table'")[0] ?? []) as typeof rows;
 	} catch (error) {
 		throw new Error(`could not list D1's tables: ${(error as Error).message}`);
 	}
@@ -356,14 +401,14 @@ export async function pullD1ToSqlite(options: PullD1Options): Promise<void> {
 	// Every table's columns in ONE call: one statement each, answered in order.
 	let columnSets: Array<Array<Record<string, unknown>>>;
 	try {
-		columnSets = execute(wrangler, database, tables.map((t) => `SELECT name, pk FROM pragma_table_info(${sqlText(t)});`).join(' '));
+		columnSets = execute(caller, database, tables.map((t) => `SELECT name, pk FROM pragma_table_info(${sqlText(t)});`).join(' '));
 	} catch (error) {
 		throw new Error(`could not read D1's columns: ${(error as Error).message}`);
 	}
 	const dumps = tables.map((table, i) => {
 		const cols = (columnSets[i] ?? []) as Array<{ name: string; pk: number }>;
 		const plan = tablePlan(table, rows.find((r) => r.name === table)?.sql ?? null, cols);
-		return { table, ...dumpTable(wrangler, database, plan, log) };
+		return { table, ...dumpTable(caller, database, plan) };
 	});
 	if (dumps.every((d) => d.rows === 0)) {
 		throw new Error('the D1 pull contains no rows — refusing to snapshot an empty database as if it were the library');