cursedbelt-server 4.30.1 → 4.32.0

package/src/server/d1/pullD1.spec.ts

@@ -12,6 +12,7 @@ import {
 	type Wrangler,
 	wranglerArgv,
 } from './localBackup.js';
+import { PAGE_ROWS, pageSql, tablePlan } from './pullD1.js';
 
 const dirs: string[] = [];
 const scratch = (): string => {
@@ -82,7 +83,7 @@ describe('servedRuntime', () => {
 	});
 });
 
-describe('pulling D1 down — table by table, because a whole-database export refuses fts5', () => {
+describe('pulling D1 down — paged SELECTs, table by table, never an export', () => {
 	const MASTER = [
 		{ name: '_cf_KV', sql: 'CREATE TABLE _cf_KV (key TEXT PRIMARY KEY)' },
 		{ name: 'items', sql: 'CREATE TABLE items ( id TEXT PRIMARY KEY )' },
@@ -108,21 +109,32 @@ describe('pulling D1 down — table by table, because a whole-database export re
 		expect(backupTables(MASTER)).toEqual(['items', 'items_fts_map', 'meta', 'sqlite_sequence']);
 	});
 
-	const fake = (rowsFor: (table: string) => string | null): { run: Wrangler; exported: string[]; seen: string[][] } => {
-		const exported: string[] = [];
+	/**
+	 * A fake `wrangler` backed by a REAL sqlite: `d1 execute` runs the SQL against `source` and
+	 * answers the way wrangler's `--json` does (noise, then one result set per statement), and
+	 * `d1 export` — which must never be called any more — is recorded so a test can say so.
+	 */
+	const d1 = (source: Database, opts: { failWhen?: (sql: string) => boolean; master?: typeof MASTER } = {}) => {
 		const seen: string[][] = [];
 		const run: Wrangler = (args) => {
 			seen.push(args);
-			if (args[1] === 'execute') return { code: 0, stdout: `noise\n${JSON.stringify([{ results: MASTER }])}`, stderr: '' };
-			const table = args[args.indexOf('--table') + 1] as string;
-			const out = args[args.indexOf('--output') + 1] as string;
-			const rows = rowsFor(table);
-			if (rows === null) return { code: 1, stdout: '', stderr: `✘ export of ${table} failed` };
-			exported.push(table);
-			writeFileSync(out, rows);
-			return { code: 0, stdout: '', stderr: '' };
+			if (args[1] !== 'execute') return { code: 0, stdout: 'Downloaded successfully', stderr: '' };
+			const sql = args[args.indexOf('--command') + 1] as string;
+			if (opts.failWhen?.(sql)) return { code: 1, stdout: '', stderr: '✘ D1 said no' };
+			if (sql.startsWith('select name, sql from sqlite_master')) {
+				return { code: 0, stdout: `noise\n${JSON.stringify([{ results: opts.master ?? MASTER }])}`, stderr: '' };
+			}
+			const statements = sql.split(/;\s*/).filter(Boolean);
+			const results = statements.map((statement) => ({ results: source.query(statement).all() }));
+			return { code: 0, stdout: `noise\n${JSON.stringify(results)}`, stderr: '' };
 		};
-		return { run, exported, seen };
+		return { run, seen, exports: () => seen.filter((a) => a[1] === 'export').length };
+	};
+	const live = (setup = ''): Database => {
+		const db = new Database(':memory:');
+		db.exec(SCHEMA);
+		if (setup) db.exec(setup);
+		return db;
 	};
 	const pull = (dir: string, run: Wrangler) =>
 		pullD1ToSqlite({
@@ -133,77 +145,99 @@ describe('pulling D1 down — table by table, because a whole-database export re
 			afterLoadSql: REBUILD,
 			wrangler: run,
 		});
+	const opened = (dir: string) => new Database(join(dir, 'pulled.sqlite'), { readonly: true });
 
 	test('a pull is a real, searchable sqlite file built from the schema and the rows, of the named database', async () => {
 		const dir = scratch();
-		const { run, exported, seen } = fake((table) =>
-			table === 'items'
-				? "INSERT INTO items (id, name) VALUES ('itm_1', 'Travels 5388.jpg');"
-				: table === 'items_fts_map'
-					? "INSERT INTO items_fts_map (rowid, item_id, text) VALUES (1, 'itm_1', 'travels 5388');"
-					: '',
+		const { run, seen } = d1(
+			live("INSERT INTO items VALUES ('itm_1', 'Travels 5388.jpg'); INSERT INTO items_fts_map (item_id, text) VALUES ('itm_1', 'travels 5388');"),
 		);
 		await pull(dir, run);
-		expect(exported).toEqual(['items', 'items_fts_map', 'meta', 'sqlite_sequence']);
 		expect(seen.every((args) => args[2] === 'collections')).toBe(true);
-		const db = new Database(join(dir, 'pulled.sqlite'), { readonly: true });
+		const db = opened(dir);
 		const hits = db.query("select count(*) as n from items_fts where items_fts match '5388'").get() as { n: number };
+		expect((db.query('select seq from sqlite_sequence').get() as { seq: number }).seq).toBe(1);
 		db.close();
 		expect(hits.n).toBe(1);
 	});
 
-	test('🔴 a table D1 holds that the schema does not declare is created from D1\'s own DDL', async () => {
+	test('🔴 it NEVER runs `wrangler d1 export` — an export blocks production D1 (family, 2026-09-25: Error 1101)', async () => {
 		const dir = scratch();
-		const master = [...MASTER, { name: 'art_lookups', sql: 'CREATE TABLE art_lookups (key TEXT PRIMARY KEY, art_key TEXT)' }];
-		const run: Wrangler = (args) => {
-			if (args[1] === 'execute') return { code: 0, stdout: JSON.stringify([{ results: master }]), stderr: '' };
-			const table = args[args.indexOf('--table') + 1] as string;
-			const out = args[args.indexOf('--output') + 1] as string;
-			writeFileSync(
-				out,
-				table === 'items' ? "INSERT INTO items (id) VALUES ('i1');" : table === 'art_lookups' ? "INSERT INTO art_lookups VALUES ('k', 'art/1');" : '',
-			);
-			return { code: 0, stdout: '', stderr: '' };
-		};
+		const { run, seen, exports } = d1(live("INSERT INTO items VALUES ('i1', 'a');"));
 		await pull(dir, run);
-		const db = new Database(join(dir, 'pulled.sqlite'), { readonly: true });
-		const row = db.query('select art_key from art_lookups').get() as { art_key: string } | null;
-		db.close();
-		expect(row?.art_key).toBe('art/1');
+		expect(exports()).toBe(0);
+		expect(seen.every((args) => args[1] === 'execute' && args.includes('--remote'))).toBe(true);
+		// Every statement it sends is a read.
+		const sent = seen.map((args) => args[args.indexOf('--command') + 1] as string);
+		expect(sent.every((sql) => /^\s*select/i.test(sql))).toBe(true);
 	});
 
-	test('🔴 a database without the required table is refused as the wrong one', async () => {
+	test('🔴 types survive exactly — REAL 5.0 stays REAL, INTEGER 5 stays INTEGER, a BLOB stays its bytes, quotes and NULL too', async () => {
 		const dir = scratch();
-		const { run } = fake(() => '');
-		await expect(
-			pullD1ToSqlite({ destination: join(dir, 'x.sqlite'), database: 'x', schemaSql: SCHEMA, requiredTable: 'tracks', wrangler: run }),
-		).rejects.toThrow(/lists no `tracks` table/);
+		const source = live(
+			"CREATE TABLE typed (a, b, c, d, e); INSERT INTO typed VALUES (5.0, 5, x'00ff10', 'it''s\nlines', NULL); INSERT INTO items VALUES ('i1', 'x');",
+		);
+		const master = [...MASTER, { name: 'typed', sql: 'CREATE TABLE typed (a, b, c, d, e)' }];
+		await pull(dir, d1(source, { master }).run);
+		const db = opened(dir);
+		const row = db.query('select typeof(a) ta, typeof(b) tb, hex(c) hc, d, typeof(e) te, a + 0.1 s from typed').get();
+		db.close();
+		expect(row).toEqual({ ta: 'real', tb: 'integer', hc: '00FF10', d: "it's\nlines", te: 'null', s: 5.1 });
 	});
 
-	test('an export that says success and writes NO file is asked again — and refuses if it never does', async () => {
+	test('a table larger than one page is read in full on the rowid keyset — and a table with a column NAMED rowid pages on _rowid_', async () => {
 		const dir = scratch();
-		let flaky = 1;
-		const run: Wrangler = (args) => {
-			if (args[1] === 'execute') return { code: 0, stdout: JSON.stringify([{ results: MASTER }]), stderr: '' };
-			const table = args[args.indexOf('--table') + 1] as string;
-			const out = args[args.indexOf('--output') + 1] as string;
-			// `meta` "succeeds" once without writing — measured on collections, 2026-09-23.
-			if (table === 'meta' && flaky-- > 0) return { code: 0, stdout: 'Downloaded successfully', stderr: '' };
-			writeFileSync(out, table === 'items' ? "INSERT INTO items (id) VALUES ('i1');" : '');
-			return { code: 0, stdout: '', stderr: '' };
-		};
+		const n = PAGE_ROWS * 2 + 7;
+		const source = live(
+			`WITH RECURSIVE s(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM s WHERE i < ${n}) INSERT INTO items_fts_map (item_id, text) SELECT 'i' || i, 't' FROM s; INSERT INTO items VALUES ('i1', 'x');`,
+		);
+		const { run, seen } = d1(source);
 		await pull(dir, run);
-		const never: Wrangler = (args) =>
-			args[1] === 'execute'
-				? { code: 0, stdout: JSON.stringify([{ results: MASTER }]), stderr: '' }
-				: { code: 0, stdout: 'Downloaded successfully', stderr: '' };
-		await expect(pull(scratch(), never)).rejects.toThrow(/reported success 3 times and wrote no file/);
+		const db = opened(dir);
+		expect((db.query('select count(*) n from items_fts_map').get() as { n: number }).n).toBe(n);
+		db.close();
+		const mapPages = seen.map((a) => a[a.indexOf('--command') + 1] as string).filter((sql) => sql.includes('FROM "items_fts_map"'));
+		expect(mapPages).toHaveLength(3);
+		expect(mapPages[1]).toContain('WHERE _rowid_ >');
+	});
+
+	test('a WITHOUT ROWID table pages by its primary key and OFFSET', () => {
+		const plan = tablePlan('kv', 'CREATE TABLE kv (k TEXT PRIMARY KEY, v) WITHOUT ROWID', [
+			{ name: 'k', pk: 1 },
+			{ name: 'v', pk: 0 },
+		]);
+		expect(plan.key).toBeNull();
+		expect(pageSql(plan, 10, { offset: 20 })).toBe('SELECT quote("k") AS c0, quote("v") AS c1 FROM "kv" ORDER BY "k" LIMIT 10 OFFSET 20');
+	});
+
+	test('a page D1 refuses is halved and asked again; a table that cannot be read at one row THROWS', async () => {
+		const dir = scratch();
+		const source = live("INSERT INTO items VALUES ('i1', 'a'); INSERT INTO meta VALUES ('k', 'v');");
+		await pull(dir, d1(source, { failWhen: (sql) => sql.includes('FROM "meta"') && sql.includes(`LIMIT ${PAGE_ROWS}`) }).run);
+		const db = opened(dir);
+		expect((db.query('select value from meta').get() as { value: string }).value).toBe('v');
+		db.close();
+		await expect(pull(scratch(), d1(source, { failWhen: (sql) => sql.includes('FROM "meta"') }).run)).rejects.toThrow(
+			/could not read `meta` from D1/,
+		);
 	});
 
-	test('🔴 one table failing to export THROWS — never a snapshot missing that table', async () => {
+	test('🔴 a table D1 holds that the schema does not declare is created from D1\'s own DDL', async () => {
+		const dir = scratch();
+		const ddl = 'CREATE TABLE art_lookups (key TEXT PRIMARY KEY, art_key TEXT)';
+		const source = live(`${ddl}; INSERT INTO art_lookups VALUES ('k', 'art/1'); INSERT INTO items VALUES ('i1', 'x');`);
+		await pull(dir, d1(source, { master: [...MASTER, { name: 'art_lookups', sql: ddl }] }).run);
+		const db = opened(dir);
+		const row = db.query('select art_key from art_lookups').get() as { art_key: string } | null;
+		db.close();
+		expect(row?.art_key).toBe('art/1');
+	});
+
+	test('🔴 a database without the required table is refused as the wrong one', async () => {
 		const dir = scratch();
-		const { run } = fake((table) => (table === 'meta' ? null : "INSERT INTO items (id) VALUES ('x');"));
-		await expect(pull(dir, run)).rejects.toThrow(/--table meta failed/);
+		await expect(
+			pullD1ToSqlite({ destination: join(dir, 'x.sqlite'), database: 'x', schemaSql: SCHEMA, requiredTable: 'tracks', wrangler: d1(live()).run }),
+		).rejects.toThrow(/lists no `tracks` table/);
 	});
 
 	test("🔴 bun:sqlite's exec stops at a NUL with no error — the defect, pinned (task 2128)", () => {
@@ -214,38 +248,27 @@ describe('pulling D1 down — table by table, because a whole-database export re
 		db.close();
 	});
 
-	test('🔴 a NUL-padded export still loads EVERY table (vault, 2026-09-23: vault_keys came back empty)', async () => {
-		const dir = scratch();
-		const { run } = fake((table) =>
-			table === 'items'
-				? `INSERT INTO items (id, name) VALUES ('itm_1', 'a');\n${'\0'.repeat(988)}`
-				: table === 'meta'
-					? "INSERT INTO meta (key, value) VALUES ('k', 'v');\nINSERT INTO meta (key, value) VALUES ('k2', 'v2');"
-					: '',
-		);
-		await pull(dir, run);
-		const db = new Database(join(dir, 'pulled.sqlite'), { readonly: true });
-		expect((db.query('SELECT COUNT(*) AS n FROM meta').get() as { n: number }).n).toBe(2);
-		db.close();
-	});
-
-	test('🔴 a load SHORT of its dump — whatever truncated it — is refused, and leaves no snapshot', async () => {
+	test('🔴 a load SHORT of what was read — whatever truncated it — is refused, and leaves no snapshot', async () => {
 		const dir = scratch();
-		// A NUL INSIDE the SQL (not trailing, so not stripped) ends the exec mid-table.
-		const { run } = fake((table) =>
-			table === 'items' ? "INSERT INTO items (id) VALUES ('a');\n\0\nINSERT INTO items (id) VALUES ('b');" : '',
-		);
-		await expect(pull(dir, run)).rejects.toThrow(/items: 1 of 2/);
+		// Whatever truncates a load — task 2128's NUL padding, or the next thing — the table ends
+		// up holding fewer rows than were read. A row deleted after the load is exactly that shape.
+		const source = live("INSERT INTO items VALUES ('a', 'x'); INSERT INTO items VALUES ('b', 'y');");
+		const truncating = pullD1ToSqlite({
+			destination: join(dir, 'pulled.sqlite'),
+			database: 'collections',
+			schemaSql: SCHEMA,
+			requiredTable: 'items',
+			afterLoadSql: "DELETE FROM items WHERE id = 'b'",
+			wrangler: d1(source).run,
+		});
+		await expect(truncating).rejects.toThrow(/items: 1 of 2/);
 		expect(existsSync(join(dir, 'pulled.sqlite'))).toBe(false);
 	});
 
-	test('the per-table dumps are deleted — on success AND on a refusal', async () => {
+	test('no dump file is left beside the snapshot', async () => {
 		const dir = scratch();
-		const { run } = fake((table) => (table === 'items' ? "INSERT INTO items (id) VALUES ('a');" : ''));
-		await pull(dir, run);
+		await pull(dir, d1(live("INSERT INTO items VALUES ('a', 'x');")).run);
 		expect(readdirSync(dir).filter((f) => f.endsWith('.sql'))).toEqual([]);
-		const bad = fake((table) => (table === 'meta' ? null : "INSERT INTO items (id) VALUES ('x');"));
-		await expect(pull(scratch(), bad.run)).rejects.toThrow();
 	});
 
 	test('stripTrailingNuls strips only TRAILING NULs; wranglerArgv is this bun, never a bare bunx', () => {
@@ -258,9 +281,7 @@ describe('pulling D1 down — table by table, because a whole-database export re
 		expect(wranglerArgv('/x/bun')).toEqual(['/x/bun', 'x', 'wrangler']);
 	});
 
-	test('🔴 an export with no rows at all THROWS — never an empty library dated today', async () => {
-		const dir = scratch();
-		const { run } = fake(() => '');
-		await expect(pull(dir, run)).rejects.toThrow(/no rows/);
+	test('🔴 a pull with no rows at all THROWS — never an empty library dated today', async () => {
+		await expect(pull(scratch(), d1(live()).run)).rejects.toThrow(/no rows/);
 	});
 });

package/src/server/d1/pullD1.ts

@@ -18,13 +18,30 @@
  * the cutover (a GREEN backup of nothing current — the worst failure a backup has), guessing
  * D1 snapshots a stale copy while the Mac is live.
  *
- * ## 🔴 Why table by table, and through `wrangler`
+ * ## 🔴 Why paged `SELECT`s, and never `wrangler d1 export`
  *
- * `wrangler d1 export` REFUSES a whole database holding a virtual table — *"cannot export
- * databases with Virtual Tables (fts5)"*, measured on collections' first post-cutover night —
- * and a per-table export is allowed. `wrangler` rather than the REST API because the export
- * is typed SQL: the REST API's JSON cannot tell the REAL `5.0` from the INTEGER `5`
- * (`./http.ts` has the measurement), and a restore must put back what was there.
+ * **A D1 export blocks the database from serving anything else for as long as it runs**
+ * (Cloudflare's own docs). Measured on family 2026-09-25 00:32–00:33 UTC: a manual `bun run
+ * backup` — then a per-table `wrangler d1 export` — held production for ~28 s at a time, and
+ * 14 requests to the family Worker sat waiting and threw Error 1101 (wall p50 28 s, CPU p50
+ * 1.6 ms). Two earlier throws at 13:56/13:58 had the same shape. So every nightly backup of
+ * every app on this seam was a scheduled outage.
+ *
+ * Since 4.32.0 the rows are read with ordinary `wrangler d1 execute --remote` SELECTs, a page
+ * of {@link PAGE_ROWS} rows at a time on a rowid keyset, which D1 serves between other
+ * requests like any read. Each value comes back as SQLite's own `quote()` of it — an SQL
+ * literal, as TEXT — so the REAL `5.0` stays `5.0` and the INTEGER `5` stays `5` (the reason
+ * this used `export` and not the REST API's JSON: `./http.ts` has the measurement), a BLOB is
+ * `X'…'`, and `quote()` round-trips a double exactly. The dump is then the same
+ * `INSERT INTO …` text an export wrote, so everything after it is unchanged.
+ *
+ * Still table by table: a whole-database export refused fts5 (*"cannot export databases with
+ * Virtual Tables"*), and a virtual table's shadow tables are an index, rebuilt, never copied.
+ *
+ * 🔴 The trade: an export was one point in time; pages are not. A row written to one table
+ * while another is being paged may be in the snapshot without its partner. For a nightly
+ * backup that is the right trade against taking production down, and the row-count check
+ * below still refuses a load short of what was read.
  */
 
 import { Database } from 'bun:sqlite';
@@ -97,8 +114,13 @@ export async function servedRuntime(
 	return null;
 }
 
-/** How many times one table's export is asked for when wrangler says success and writes nothing. */
-const EXPORT_ATTEMPTS = 3;
+/**
+ * Rows per paged `SELECT`. The fleet's largest table held 290 rows when this was set (collections'
+ * `items`, 2026-09-25) and one `wrangler` call costs ~1 s, so a page this size is one call per
+ * table for every app today. A page D1 refuses — too large a response — is halved and asked
+ * again, down to one row, before it is a refusal.
+ */
+export const PAGE_ROWS = 500;
 
 /** One `wrangler` invocation, as {@link pullD1ToSqlite} needs it. Injected so a spec runs none. */
 export type Wrangler = (args: string[]) => { code: number; stdout: string; stderr: string };
@@ -125,6 +147,8 @@ export function spawnWrangler(options: { cwd: string; env?: Record<string, strin
 
 /**
  * 🔴 Trailing NUL bytes off a `wrangler d1 export` file, in place. Returns how many were removed.
+ * {@link pullD1ToSqlite} no longer exports (see the header), so nothing here calls this since
+ * 4.32.0; it stays exported for anyone still reading an export file by hand.
  *
  * Measured 2026-09-23 on vault's D1 (wrangler 4.136.3): `wrangler d1 export --table … --output`
  * INTERMITTENTLY ends the file in hundreds of NUL bytes (`entries` +988, `oplog` +1,369, `sync_ops`
@@ -180,22 +204,98 @@ export interface PullD1Options {
 	wrangler: Wrangler;
 }
 
+const sqlIdent = (name: string): string => `"${name.replaceAll('"', '""')}"`;
+const sqlText = (text: string): string => `'${text.replaceAll("'", "''")}'`;
+
+/** One `wrangler d1 execute --remote --json` — every statement's result rows, in order. */
+function execute(wrangler: Wrangler, database: string, sql: string): Array<Array<Record<string, unknown>>> {
+	const r = wrangler(['d1', 'execute', database, '--remote', '--env=', '--json', '--command', sql]);
+	if (r.code !== 0) throw new Error(r.stderr.trim() || r.stdout.trim() || `wrangler exited ${r.code}`);
+	const parsed = JSON.parse(r.stdout.slice(r.stdout.indexOf('['))) as Array<{ results?: Array<Record<string, unknown>> }>;
+	return parsed.map((statement) => statement.results ?? []);
+}
+
+/** How a table is walked in pages: by a rowid keyset, or — a `WITHOUT ROWID` table — by offset. */
+export interface TablePlan {
+	table: string;
+	columns: string[];
+	/** The rowid alias to page on, or `null` for a table with none (then `order` + OFFSET). */
+	key: string | null;
+	order: string[];
+}
+
+/**
+ * The paging plan for one table. The key is the first of `rowid`, `_rowid_`, `oid` that no
+ * declared column shadows — `items_fts_map` declares a column NAMED `rowid`, so it pages on
+ * `_rowid_`, which is the same value.
+ */
+export function tablePlan(table: string, ddl: string | null, cols: ReadonlyArray<{ name: string; pk: number }>): TablePlan {
+	const columns = cols.map((c) => c.name);
+	if (columns.length === 0) throw new Error(`D1 reports no columns for \`${table}\``);
+	const lower = new Set(columns.map((c) => c.toLowerCase()));
+	const withoutRowid = /\bWITHOUT\s+ROWID\b/i.test(ddl ?? '');
+	const key = withoutRowid ? null : (['rowid', '_rowid_', 'oid'].find((alias) => !lower.has(alias)) ?? null);
+	const pk = cols.filter((c) => c.pk > 0).sort((a, b) => a.pk - b.pk).map((c) => c.name);
+	return { table, columns, key, order: pk.length > 0 ? pk : columns };
+}
+
+/** The SELECT for one page: the key (if any) and every column as its `quote()`d SQL literal. */
+export function pageSql(plan: TablePlan, limit: number, after: { key?: number; offset?: number }): string {
+	const values = plan.columns.map((c, i) => `quote(${sqlIdent(c)}) AS c${i}`).join(', ');
+	if (plan.key) {
+		const where = after.key === undefined ? '' : ` WHERE ${plan.key} > ${after.key}`;
+		return `SELECT ${plan.key} AS k, ${values} FROM ${sqlIdent(plan.table)}${where} ORDER BY ${plan.key} LIMIT ${limit}`;
+	}
+	const order = plan.order.map(sqlIdent).join(', ');
+	return `SELECT ${values} FROM ${sqlIdent(plan.table)} ORDER BY ${order} LIMIT ${limit} OFFSET ${after.offset ?? 0}`;
+}
+
+/** Every row of one table as `INSERT INTO …;` lines, read in pages that never block D1. */
+function dumpTable(wrangler: Wrangler, database: string, plan: TablePlan): { text: string; rows: number } {
+	const into = `INSERT INTO ${sqlIdent(plan.table)} (${plan.columns.map(sqlIdent).join(', ')}) VALUES (`;
+	const lines: string[] = [];
+	let limit = PAGE_ROWS;
+	let after: { key?: number; offset?: number } = {};
+	for (;;) {
+		let page: Array<Record<string, unknown>>;
+		try {
+			page = execute(wrangler, database, pageSql(plan, limit, after))[0] ?? [];
+		} catch (error) {
+			if (limit > 1) {
+				limit = Math.max(1, Math.floor(limit / 2));
+				continue;
+			}
+			throw new Error(`could not read \`${plan.table}\` from D1: ${(error as Error).message}`);
+		}
+		for (const row of page) {
+			const literals = plan.columns.map((_, i) => row[`c${i}`]);
+			if (literals.some((v) => typeof v !== 'string')) {
+				throw new Error(`D1 returned a non-literal value paging \`${plan.table}\` — refusing a snapshot that may not restore`);
+			}
+			lines.push(`${into}${literals.join(', ')});`);
+		}
+		if (page.length < limit) break;
+		after = plan.key ? { key: Number(page[page.length - 1]?.k) } : { offset: (after.offset ?? 0) + page.length };
+	}
+	return { text: lines.join('\n'), rows: lines.length };
+}
+
 /**
- * Pull the live D1 database into a real SQLite file.
+ * Pull the live D1 database into a real SQLite file, reading it with paged SELECTs that never
+ * block production (see the header — this used to be `wrangler d1 export`, which did).
  *
  * 🔴 It THROWS rather than returning an empty database. Every layer beneath treats a readable
- * SQLite file as success, so an export that produced nothing would sail through the
+ * SQLite file as success, so a pull that produced nothing would sail through the
  * magic-byte check and `integrity_check` and land on the shelf with today's date on it.
  */
 export async function pullD1ToSqlite(options: PullD1Options): Promise<void> {
 	const { destination, database, wrangler } = options;
-	const listed = wrangler([
-		'd1', 'execute', database, '--remote', '--env=', '--json',
-		'--command', "select name, sql from sqlite_master where type = 'table'",
-	]);
-	if (listed.code !== 0) throw new Error(`could not list D1's tables: ${listed.stderr.trim() || listed.stdout.trim()}`);
-	const json = listed.stdout.slice(listed.stdout.indexOf('['));
-	const rows = (JSON.parse(json) as Array<{ results: Array<{ name: string; sql: string | null }> }>)[0]?.results ?? [];
+	let rows: Array<{ name: string; sql: string | null }>;
+	try {
+		rows = (execute(wrangler, database, "select name, sql from sqlite_master where type = 'table'")[0] ?? []) as typeof rows;
+	} catch (error) {
+		throw new Error(`could not list D1's tables: ${(error as Error).message}`);
+	}
 	const tables = backupTables(rows);
 	if (!tables.includes(options.requiredTable)) {
 		throw new Error(
@@ -203,78 +303,56 @@ export async function pullD1ToSqlite(options: PullD1Options): Promise<void> {
 		);
 	}
 
-	let sql = '';
-	const dumps: { table: string; text: string }[] = [];
+	// Every table's columns in ONE call: one statement each, answered in order.
+	let columnSets: Array<Array<Record<string, unknown>>>;
 	try {
-		for (const table of tables) {
-			const dump = `${destination}.${table}.sql`;
-			// 🔴 `wrangler d1 export` has been measured answering SUCCESS and writing no file —
-			// collections' nightly of 2026-09-23T15:35Z refused on `derivative_retries`, and the same
-			// command wrote the file (32 bytes, an empty table) an hour later. So a missing file is
-			// asked for again, twice, before it is a refusal; a failure that persists still refuses,
-			// because a snapshot missing a table is not a backup.
-			let exported = { code: 1, stdout: '', stderr: 'not attempted' };
-			for (let attempt = 1; attempt <= EXPORT_ATTEMPTS; attempt++) {
-				exported = wrangler(['d1', 'export', database, '--remote', '--env=', '--table', table, '--no-schema', '--output', dump, '-y']);
-				if (exported.code !== 0 || existsSync(dump)) break;
-			}
-			if (exported.code !== 0) {
-				throw new Error(`wrangler d1 export --table ${table} failed: ${exported.stderr.trim() || exported.stdout.trim()}`);
-			}
-			if (!existsSync(dump)) {
-				throw new Error(
-					`wrangler d1 export --table ${table} reported success ${EXPORT_ATTEMPTS} times and wrote no file at ${dump}`,
-				);
-			}
-			stripTrailingNuls(dump);
-			const text = readFileSync(dump, 'utf8');
-			dumps.push({ table, text });
-			sql += `${text}\n`;
+		columnSets = execute(wrangler, database, tables.map((t) => `SELECT name, pk FROM pragma_table_info(${sqlText(t)});`).join(' '));
+	} catch (error) {
+		throw new Error(`could not read D1's columns: ${(error as Error).message}`);
+	}
+	const dumps = tables.map((table, i) => {
+		const cols = (columnSets[i] ?? []) as Array<{ name: string; pk: number }>;
+		const plan = tablePlan(table, rows.find((r) => r.name === table)?.sql ?? null, cols);
+		return { table, ...dumpTable(wrangler, database, plan) };
+	});
+	if (dumps.every((d) => d.rows === 0)) {
+		throw new Error('the D1 pull contains no rows — refusing to snapshot an empty database as if it were the library');
+	}
+
+	const db = new Database(destination, { create: true });
+	try {
+		// `exec`, not `run`: each is many statements and `run` would execute only the first,
+		// leaving a file with a schema and no rows that looks perfectly valid.
+		db.exec(options.schemaSql);
+		// 🔴 A table D1 holds that the schema does not declare is created from D1's OWN DDL, never
+		// dropped: measured on music's first post-cutover pull (2026-09-23), `art_lookups` — a
+		// cache its art job creates where it first needs it — made the whole pull refuse with
+		// `no such table`, because its rows were exported and its table was not in the schema.
+		const have = new Set(
+			(db.query("SELECT name FROM sqlite_master WHERE type = 'table'").all() as { name: string }[]).map((r) => r.name),
+		);
+		for (const row of rows) {
+			if (!tables.includes(row.name) || have.has(row.name) || !row.sql) continue;
+			db.exec(row.sql.replace(/^\s*CREATE\s+TABLE\s+(?!IF\s+NOT\s+EXISTS)/i, 'CREATE TABLE IF NOT EXISTS '));
 		}
-		if (!/INSERT INTO/i.test(sql)) {
-			throw new Error('the D1 export contains no rows — refusing to snapshot an empty database as if it were the library');
+		// One table at a time, so a fault is located rather than smeared over the whole load.
+		for (const { text } of dumps) if (text) db.exec(text);
+		if (options.afterLoadSql) db.exec(options.afterLoadSql);
+		// 🔴 Every table holds as many rows as were read from D1, or this is not a backup: the check
+		// that makes a silently truncated load (task 2128's NUL padding, or whatever does it next)
+		// a REFUSAL. A row count only; nothing here reads what the rows hold.
+		const short: string[] = [];
+		for (const { table, rows: expected } of dumps) {
+			if (table === 'sqlite_sequence') continue;
+			const loaded = (db.query(`SELECT COUNT(*) AS n FROM ${sqlIdent(table)}`).get() as { n: number }).n;
+			if (loaded < expected) short.push(`${table}: ${loaded} of ${expected}`);
 		}
-
-		const db = new Database(destination, { create: true });
-		try {
-			// `exec`, not `run`: each is many statements and `run` would execute only the first,
-			// leaving a file with a schema and no rows that looks perfectly valid.
-			db.exec(options.schemaSql);
-			// 🔴 A table D1 holds that the schema does not declare is created from D1's OWN DDL, never
-			// dropped: measured on music's first post-cutover pull (2026-09-23), `art_lookups` — a
-			// cache its art job creates where it first needs it — made the whole pull refuse with
-			// `no such table`, because its rows were exported and its table was not in the schema.
-			const have = new Set(
-				(db.query("SELECT name FROM sqlite_master WHERE type = 'table'").all() as { name: string }[]).map((r) => r.name),
-			);
-			for (const row of rows) {
-				if (!tables.includes(row.name) || have.has(row.name) || !row.sql) continue;
-				db.exec(row.sql.replace(/^\s*CREATE\s+TABLE\s+(?!IF\s+NOT\s+EXISTS)/i, 'CREATE TABLE IF NOT EXISTS '));
-			}
-			// One table at a time, so a fault is located rather than smeared over the whole load.
-			for (const { text } of dumps) if (text.trim()) db.exec(text);
-			if (options.afterLoadSql) db.exec(options.afterLoadSql);
-			// 🔴 Every table holds as many rows as its dump has INSERTs, or this is not a backup: the
-			// check that makes a silently truncated load — the NUL padding, or whatever does it next —
-			// a REFUSAL. A row count only; nothing here reads what the rows hold.
-			const short: string[] = [];
-			for (const { table, text } of dumps) {
-				if (table === 'sqlite_sequence') continue;
-				const expected = insertLines(text);
-				const loaded = (db.query(`SELECT COUNT(*) AS n FROM "${table.replaceAll('"', '""')}"`).get() as { n: number }).n;
-				if (loaded < expected) short.push(`${table}: ${loaded} of ${expected}`);
-			}
-			if (short.length > 0) {
-				db.close();
-				rmSync(destination, { force: true });
-				throw new Error(`the D1 pull did not load every exported row — ${short.join('; ')}. Refusing the snapshot.`);
-			}
-		} finally {
+		if (short.length > 0) {
 			db.close();
+			rmSync(destination, { force: true });
+			throw new Error(`the D1 pull did not load every row it read — ${short.join('; ')}. Refusing the snapshot.`);
 		}
 	} finally {
-		// The per-table dumps are a second, unmanaged copy of the data that no retention rule
-		// would ever delete — eighteen were found beside vault's snapshot (task 2128).
-		for (const table of tables) rmSync(`${destination}.${table}.sql`, { force: true });
+		db.close();
 	}
 }

package/src/server/master-lock/accountsPage.ts

@@ -33,7 +33,7 @@
  * only be watching one of them.
  */
 import { MASTER_LOCK_PATHS, MASTER_LOCK_PREFIX } from "cursedbelt-core/master-lock";
-import { MASTER_LOCK_DERIVE_SOURCE, MASTER_LOCK_REVEAL_SOURCE, type LockPageOptions, withRevealToggles } from "./lockPage.js";
+import { MASTER_LOCK_DERIVE_SOURCE, MASTER_LOCK_MIN_PASSWORD_LENGTH, MASTER_LOCK_REVEAL_SOURCE, type LockPageOptions, withRevealToggles } from "./lockPage.js";
 
 /**
  * The page's own two assets.
@@ -54,9 +54,9 @@ const escapeHtml = (value: string): string =>
 		(ch) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" })[ch] ?? ch,
 	);
 
-/** The shortest password worth calling one. Matched by every `minlength` below and by the
- *  lock page's enrollment form, so a rotation cannot quietly weaken what enrollment demands. */
-export const MASTER_LOCK_MIN_PASSWORD_LENGTH = 10;
+/** The shortest password worth calling one — defined beside the lock page's enrollment form
+ *  (`lockPage.ts`) and re-exported, so a rotation cannot quietly differ from what enrollment demands. */
+export { MASTER_LOCK_MIN_PASSWORD_LENGTH } from "./lockPage.js";
 
 /**
  * The document.