@coywolf/emdash 0.0.0-stage → 0.35.13

package/src/backups/dump.test.mjs

@@ -0,0 +1,287 @@
+// Unit tests for the D1 dump: rows bigger than D1's statement limit restore intact.
+// Run: node --test src/backups/dump.test.mjs   (Node 22.6+ strips the types from dump.ts)
+import assert from "node:assert/strict";
+import { DatabaseSync } from "node:sqlite";
+import { test } from "node:test";
+
+const { dumpDatabase, splitText, MAX_STATEMENT_BYTES, nextPageSize, PAGE_TARGET_BYTES } = await import("./dump.ts");
+
+/** The slice of the D1 binding that dumpDatabase uses, over node:sqlite. */
+function d1(db) {
+	return {
+		prepare(sql) {
+			let params = [];
+			const stmt = {
+				bind(...args) {
+					params = args;
+					return stmt;
+				},
+				async all() {
+					return { results: db.prepare(sql).all(...params) };
+				},
+			};
+			return stmt;
+		},
+	};
+}
+
+function seed() {
+	const db = new DatabaseSync(":memory:");
+	db.exec(`CREATE TABLE options (name TEXT PRIMARY KEY, value TEXT, revision INTEGER);
+		CREATE TABLE ec_posts (id TEXT PRIMARY KEY, slug TEXT, content TEXT, excerpt TEXT);
+		CREATE TABLE log (line TEXT);`);
+	const insert = (sql, ...v) => db.prepare(sql).run(...v);
+	// 250 KB with quotes, CR/LF, emoji (surrogate pairs) and accents, so pieces split on awkward characters.
+	const big = Array.from({ length: 5000 }, (_, i) => `Line ${i}: it's "quoted" — café 🦊\r\n`).join("");
+	insert("INSERT INTO ec_posts VALUES (?, ?, ?, ?)", "p1", "long", big, big.slice(0, 120_000));
+	insert("INSERT INTO ec_posts VALUES (?, ?, ?, ?)", "p2", "short", "hello\nworld", null);
+	insert("INSERT INTO options VALUES (?, ?, ?)", "site:title", "Coywolf", 1);
+	insert("INSERT INTO options VALUES (?, ?, ?)", "plugin:coywolf-pack:cache:discovery:llms", big, 1);
+	insert("INSERT INTO log VALUES (?)", big);
+	return { db, big };
+}
+
+test("oversized rows are split into statements under the limit and restore intact", async () => {
+	const { db, big } = seed();
+	const { sql } = await dumpDatabase(d1(db));
+	const statements = sql.trim().split("\n");
+	for (const s of statements) assert.ok(new TextEncoder().encode(s).length <= MAX_STATEMENT_BYTES, `statement of ${s.length} chars`);
+	assert.ok(statements.some((s) => s.startsWith('UPDATE "ec_posts"')), "long post written in pieces");
+
+	const restored = new DatabaseSync(":memory:");
+	for (const s of statements) restored.exec(s);
+	const post = restored.prepare("SELECT * FROM ec_posts WHERE id = 'p1'").get();
+	assert.equal(post.content, big);
+	assert.equal(post.excerpt, big.slice(0, 120_000));
+	assert.equal(restored.prepare("SELECT content FROM ec_posts WHERE id = 'p2'").get().content, "hello\nworld");
+	assert.equal(restored.prepare("SELECT line FROM log").get().line, big, "table without a primary key uses rowid");
+});
+
+test("plugin caches are left out; other options are kept", async () => {
+	const { db } = seed();
+	const { sql, rows } = await dumpDatabase(d1(db));
+	assert.ok(!sql.includes("cache:discovery:llms"));
+	assert.ok(sql.includes("site:title"));
+	assert.equal(rows, 4);
+});
+
+test("splitText never cuts a surrogate pair and rejoins exactly", () => {
+	const text = "🦊".repeat(40_000);
+	const pieces = splitText(text);
+	assert.ok(pieces.length > 1);
+	assert.equal(pieces.join(""), text);
+	for (const p of pieces) assert.ok(!/^[\udc00-\udfff]/.test(p));
+});
+
+test("rows are paged by rowid across pages, in rowid order, and restore intact", async () => {
+	const db = new DatabaseSync(":memory:");
+	db.exec(`CREATE TABLE items (id TEXT PRIMARY KEY, n INTEGER);
+		CREATE TABLE plain (line TEXT);
+		CREATE TABLE odd (oid TEXT, v INTEGER);
+		CREATE TABLE kv (k TEXT PRIMARY KEY, v TEXT) WITHOUT ROWID;`);
+	for (let i = 0; i < 1234; i++) {
+		db.prepare("INSERT INTO items VALUES (?, ?)").run(`id-${String(1234 - i).padStart(4, "0")}`, i);
+		db.prepare("INSERT INTO plain VALUES (?)").run(`line ${i}`);
+		db.prepare("INSERT INTO odd VALUES (?, ?)").run(`o${i}`, i);
+		db.prepare("INSERT INTO kv VALUES (?, ?)").run(`k${i}`, `v${i}`);
+	}
+	// Gaps in the rowids, as deletes leave them.
+	db.exec("DELETE FROM items WHERE n % 7 = 0; DELETE FROM plain WHERE rowid BETWEEN 400 AND 650;");
+	const queries = [];
+	const binding = d1(db);
+	const spy = { prepare: (sql) => (queries.push(sql), binding.prepare(sql)) };
+	const { sql, rows } = await dumpDatabase(spy);
+	assert.ok(queries.some((q) => /FROM "items" WHERE rowid > \? ORDER BY rowid LIMIT \?/.test(q)), "paged by rowid");
+	assert.ok(queries.some((q) => /FROM "odd" ORDER BY rowid LIMIT \? OFFSET \?/.test(q)), "a column named oid falls back to OFFSET");
+	assert.ok(queries.some((q) => /FROM "kv" LIMIT \? OFFSET \?/.test(q)), "WITHOUT ROWID uses OFFSET");
+	assert.ok(!sql.includes("__coywolf_rowid"), "the paging cursor is never written");
+
+	const restored = new DatabaseSync(":memory:");
+	for (const s of sql.trim().split("\n")) restored.exec(s);
+	for (const table of ["items", "plain", "odd", "kv"]) {
+		assert.deepEqual(restored.prepare(`SELECT * FROM ${table} ORDER BY 1`).all(), db.prepare(`SELECT * FROM ${table} ORDER BY 1`).all(), table);
+	}
+	// Same order as before (rowid order): the n column of items rises.
+	const ns = [...sql.matchAll(/INSERT INTO "items" \("id","n"\) VALUES\('[^']+',(\d+)\);/g)].map((m) => Number(m[1]));
+	assert.deepEqual(ns, [...ns].sort((a, b) => a - b));
+	assert.equal(rows, db.prepare("SELECT (SELECT count(*) FROM items) + (SELECT count(*) FROM plain) + 1234 + 1234 AS n").get().n);
+});
+
+/** An R2 bucket that keeps what's put and records multipart uploads. */
+function fakeBucket() {
+	const store = new Map();
+	const uploads = [];
+	return {
+		store,
+		uploads,
+		async put(key, body) {
+			store.set(key, Buffer.from(body));
+		},
+		async createMultipartUpload(key) {
+			const parts = [];
+			const upload = { key, parts, completed: false, aborted: false };
+			uploads.push(upload);
+			return {
+				async uploadPart(n, body) {
+					parts.push({ n, size: body.byteLength, body: Buffer.from(body) });
+					return { partNumber: n, etag: `e${n}` };
+				},
+				async complete(list) {
+					upload.completed = true;
+					store.set(key, Buffer.concat(list.map((p) => parts.find((x) => x.n === p.partNumber).body)));
+				},
+				async abort() {
+					upload.aborted = true;
+				},
+			};
+		},
+	};
+}
+
+test("the dump streams into the bucket: multipart parts of one size, same bytes as the whole", async () => {
+	const { writeDumpStream } = await import("./store.ts");
+	const { dumpLines } = await import("./dump.ts");
+	const { gunzipSync } = await import("node:zlib");
+	const { createHash } = await import("node:crypto");
+	const { db } = seed();
+	const lines = [];
+	for await (const line of dumpLines(d1(db))) lines.push(line);
+	async function* replay() {
+		yield* lines;
+	}
+
+	const big = fakeBucket();
+	const result = await writeDumpStream(big, "2026-10-05T1200Z", "site.sql.gz", replay(), 4096);
+	const stored = big.store.get("d1/2026-10-05T1200Z/site.sql.gz");
+	assert.equal(big.uploads.length, 1);
+	const sizes = big.uploads[0].parts.map((p) => p.size);
+	assert.ok(sizes.length > 2, `${sizes.length} parts`);
+	assert.ok(sizes.slice(0, -1).every((s) => s === 4096), "every part but the last is exactly one part size");
+	assert.equal(result.bytes, stored.length);
+	assert.equal(result.sha256, createHash("sha256").update(stored).digest("hex"));
+	assert.equal(gunzipSync(stored).toString("utf8"), `${lines.join("\n")}\n`);
+
+	const small = fakeBucket();
+	await writeDumpStream(small, "2026-10-05T1200Z", "site.sql.gz", replay());
+	assert.equal(small.uploads.length, 0, "under one part: a single put");
+	assert.equal(gunzipSync(small.store.get("d1/2026-10-05T1200Z/site.sql.gz")).toString("utf8"), (await dumpDatabase(d1(db))).sql);
+});
+
+test("a failed dump aborts the upload and reports the dump's error", async () => {
+	const { writeDumpStream } = await import("./store.ts");
+	const bucket = fakeBucket();
+	async function* failing() {
+		for (let i = 0; i < 4000; i++) yield `${Math.random().toString(36)}${Math.random().toString(36)}`;
+		throw new Error("D1 went away");
+	}
+	await assert.rejects(() => writeDumpStream(bucket, "2026-10-05T1200Z", "x.sql.gz", failing(), 1024), /D1 went away/);
+	assert.equal(bucket.uploads.length, 1);
+	assert.equal(bucket.uploads[0].aborted, true);
+	assert.equal(bucket.store.size, 0);
+});
+
+test("a failed upload stops the dump instead of hanging", async () => {
+	const { writeDumpStream } = await import("./store.ts");
+	let aborted = false;
+	const bucket = {
+		async put() {},
+		async createMultipartUpload() {
+			return {
+				uploadPart: async () => {
+					throw new Error("R2 unavailable");
+				},
+				complete: async () => {},
+				abort: async () => {
+					aborted = true;
+				},
+			};
+		},
+	};
+	let produced = 0;
+	async function* endless() {
+		for (;;) {
+			produced++;
+			yield `${Math.random().toString(36)}${Math.random().toString(36)}${Math.random().toString(36)}`;
+		}
+	}
+	await assert.rejects(() => writeDumpStream(bucket, "2026-10-05T1200Z", "x.sql.gz", endless(), 1024), /R2 unavailable/);
+	assert.equal(aborted, true);
+	const stopped = produced;
+	await new Promise((resolve) => setTimeout(resolve, 20));
+	assert.equal(produced, stopped, "the dump stops reading rows");
+});
+
+test("reads are sized by bytes: big rows mean fewer rows per read, never more than 500", () => {
+	assert.equal(nextPageSize(0, 0), 500);
+	assert.equal(nextPageSize(500 * 200, 500), 500, "small rows: the full page");
+	assert.equal(nextPageSize(10 * 1024 * 1024, 10), Math.floor(PAGE_TARGET_BYTES / (1024 * 1024)), "1 MB rows: about 8 a read");
+	assert.equal(nextPageSize(50 * 1024 * 1024, 1), 1, "never below one row");
+});
+
+test("a read D1 refuses as too large is asked again for fewer rows; offset paging skips nothing", async () => {
+	// 600 rows of ~40 KB each in a table paged by OFFSET (a column named rowid shadows the real one),
+	// with a fake D1 that refuses any result over 5 MB, as D1 does over 50 MB.
+	const db = new DatabaseSync(":memory:");
+	db.exec("CREATE TABLE odd (rowid TEXT, body TEXT)");
+	const body = "x".repeat(40_000);
+	for (let i = 0; i < 600; i++) db.prepare("INSERT INTO odd VALUES (?, ?)").run(`r${i}`, `${i}:${body}`);
+	const base = d1(db);
+	let refused = 0;
+	const limited = {
+		prepare(sql) {
+			const stmt = base.prepare(sql);
+			let params = [];
+			return {
+				bind(...args) {
+					params = args;
+					stmt.bind(...args);
+					return this;
+				},
+				async all() {
+					const out = await stmt.all();
+					const size = JSON.stringify(out.results).length;
+					if (/FROM "odd"/.test(sql) && size > 5_000_000) {
+						refused++;
+						throw new Error("D1_ERROR: result too large");
+					}
+					return out;
+				},
+			};
+		},
+	};
+	const { sql, rows } = await dumpDatabase(limited);
+	assert.ok(refused >= 1, "the first 500-row read was refused");
+	assert.equal(rows, 600);
+	const restored = new DatabaseSync(":memory:");
+	for (const s of sql.trim().split("\n")) restored.exec(s);
+	assert.equal(restored.prepare("SELECT count(*) AS n FROM odd").get().n, 600, "every row once");
+	assert.equal(restored.prepare("SELECT count(DISTINCT rowid) AS n FROM odd").get().n, 600, "no row twice");
+});
+
+test("other read errors aren't retried with fewer rows: they fail the backup at once", async () => {
+	const db = new DatabaseSync(":memory:");
+	db.exec("CREATE TABLE t (id TEXT PRIMARY KEY, v TEXT)");
+	db.prepare("INSERT INTO t VALUES ('a', 'b')").run();
+	const base = d1(db);
+	let reads = 0;
+	const failing = {
+		prepare(sql) {
+			const stmt = base.prepare(sql);
+			return {
+				bind(...args) {
+					stmt.bind(...args);
+					return this;
+				},
+				async all() {
+					if (/FROM "t"/.test(sql)) {
+						reads++;
+						throw new Error("D1_ERROR: Network connection lost.");
+					}
+					return stmt.all();
+				},
+			};
+		},
+	};
+	await assert.rejects(dumpDatabase(failing), /Network connection lost/);
+	assert.equal(reads, 1, "asked once, not again with fewer rows");
+});

package/src/backups/dump.ts

@@ -0,0 +1,288 @@
+/**
+ * Restorable SQL dump of a D1 database, built from inside the Worker.
+ *
+ * Same output order as scripts/backup/d1-backup.sh, so either kind of dump
+ * restores with scripts/backup/d1-restore.sh:
+ *
+ *   1. CREATE TABLE for every regular table
+ *   2. FTS5 search tables and the triggers that maintain them
+ *   3. INSERT rows, one statement per line (the search triggers re-index
+ *      content rows as they load)
+ *   4. indexes and the remaining triggers
+ *
+ * FTS5 shadow tables are skipped: SQLite recreates them with the virtual table.
+ *
+ * D1 rejects any statement over 100 KB (SQLITE_TOOBIG), but the site writes
+ * rows with bound parameters, so a long post can be bigger than that. A row
+ * whose INSERT would be too long is written with its long text values cut
+ * short, followed by UPDATEs that append the rest in pieces under the limit.
+ */
+
+interface MasterRow {
+	type: string;
+	name: string;
+	tbl_name: string;
+	sql: string | null;
+}
+
+const PAGE_SIZE = 500;
+/**
+ * About how much a read of rows should return. D1 refuses query results over 50 MB (from
+ * November 4, 2026); 500 rows of a table with very large rows (long posts, revisions) could
+ * pass that, so each read is sized from the bytes the previous one returned.
+ */
+export const PAGE_TARGET_BYTES = 8 * 1024 * 1024;
+/** How D1 words a result over its size limit (its message has said "too large", "exceeds" or "limit" with a size). */
+const RESULT_TOO_LARGE = /too large|too big|exceed|size limit|limit.*(bytes|MB)|SQLITE_TOOBIG/i;
+
+/** Rows to ask for next, from the size of the rows just read. */
+export function nextPageSize(pageBytes: number, rows: number): number {
+	if (rows <= 0) return PAGE_SIZE;
+	const perRow = Math.max(1, pageBytes / rows);
+	return Math.max(1, Math.min(PAGE_SIZE, Math.floor(PAGE_TARGET_BYTES / perRow)));
+}
+const SEARCH_PREFIX = "_emdash_fts_";
+/** Longest statement written, in UTF-8 bytes; D1's limit is 100,000. */
+export const MAX_STATEMENT_BYTES = 90_000;
+/** Most UTF-8 bytes of escaped text per piece of a split value. */
+const PIECE_BYTES = 60_000;
+/** In a row too long for one INSERT, text values longer than this are appended separately. */
+const SHORT_BYTES = 2_000;
+/**
+ * Line breaks are written as '||char(10)||', two operators each, and SQLite
+ * refuses expressions deeper than 1,000: at most this many per statement.
+ */
+const MAX_BREAKS = 400;
+const breaks = (s: string) => s.match(/[\r\n]/g)?.length ?? 0;
+/** Option rows that the plugin rebuilds on demand; not worth backing up. */
+const SKIP_OPTION = /^plugin:coywolf-pack:cache:/;
+
+function isInternal(name: string): boolean {
+	return name.startsWith("sqlite_") || name.startsWith("_cf_");
+}
+
+function quoteIdent(name: string): string {
+	return `"${name.replace(/"/g, '""')}"`;
+}
+
+/** SQL literal for a D1 value, keeping every statement on one line (like sqlite3 .dump). */
+export function sqlLiteral(value: unknown): string {
+	if (value === null || value === undefined) return "NULL";
+	if (typeof value === "number") return Number.isFinite(value) ? String(value) : "NULL";
+	if (typeof value === "bigint") return value.toString();
+	if (typeof value === "boolean") return value ? "1" : "0";
+	if (value instanceof ArrayBuffer || ArrayBuffer.isView(value) || Array.isArray(value)) {
+		const bytes =
+			value instanceof ArrayBuffer
+				? new Uint8Array(value)
+				: Array.isArray(value)
+					? Uint8Array.from(value as number[])
+					: new Uint8Array((value as ArrayBufferView).buffer, (value as ArrayBufferView).byteOffset, (value as ArrayBufferView).byteLength);
+		let hex = "";
+		for (const b of bytes) hex += b.toString(16).padStart(2, "0");
+		return `X'${hex}'`;
+	}
+	// Newlines become ||char(10)|| so each INSERT stays on one line.
+	return `'${escapeText(String(value))}'`;
+}
+
+function escapeText(text: string): string {
+	return text.replace(/'/g, "''").replace(/\r/g, "'||char(13)||'").replace(/\n/g, "'||char(10)||'");
+}
+
+const utf8 = new TextEncoder();
+const bytes = (s: string) => utf8.encode(s).length;
+
+/** Split text into pieces whose escaped form stays under PIECE_BYTES (never inside a surrogate pair). */
+export function splitText(text: string): string[] {
+	const pieces: string[] = [];
+	let start = 0;
+	let size = 0;
+	let lines = 0;
+	for (let i = 0; i < text.length; i++) {
+		const code = text.charCodeAt(i);
+		const pair = code >= 0xd800 && code <= 0xdbff && i + 1 < text.length;
+		const char = pair ? text.slice(i, i + 2) : text[i];
+		const cost = bytes(escapeText(char));
+		const isBreak = char === "\n" || char === "\r";
+		if ((size + cost > PIECE_BYTES || (isBreak && lines >= MAX_BREAKS)) && i > start) {
+			pieces.push(text.slice(start, i));
+			start = i;
+			size = 0;
+			lines = 0;
+		}
+		size += cost;
+		if (isBreak) lines++;
+		if (pair) i++;
+	}
+	pieces.push(text.slice(start));
+	return pieces;
+}
+
+/**
+ * One row as statements: a single INSERT when it fits, otherwise an INSERT
+ * with each long text value's first piece, then UPDATEs appending the rest.
+ * `key` names the columns that identify the row (primary key, or rowid).
+ */
+export function rowStatements(table: string, row: Record<string, unknown>, key: Record<string, unknown>): string[] {
+	const columns = Object.keys(row).filter((c) => c !== ROWID);
+	const insert = (values: Record<string, unknown>) =>
+		`INSERT INTO ${table} (${columns.map(quoteIdent).join(",")}) VALUES(${columns.map((c) => sqlLiteral(values[c])).join(",")});`;
+	const whole = insert(row);
+	const lineCount = columns.reduce((n, c) => n + (typeof row[c] === "string" ? breaks(row[c] as string) : 0), 0);
+	if (bytes(whole) <= MAX_STATEMENT_BYTES && lineCount <= MAX_BREAKS) return [whole];
+
+	// Long text values start empty and are appended piece by piece, so the
+	// INSERT holds only short values however many long columns the row has.
+	const first: Record<string, unknown> = { ...row };
+	const rest: Array<[string, string]> = [];
+	for (const c of columns) {
+		const value = row[c];
+		if (typeof value !== "string" || (bytes(sqlLiteral(value)) <= SHORT_BYTES && breaks(value) <= 20)) continue;
+		first[c] = "";
+		for (const piece of splitText(value)) rest.push([c, piece]);
+	}
+	const where = Object.entries(key)
+		.map(([c, v]) => `${c === ROWID ? "rowid" : quoteIdent(c)}=${sqlLiteral(v)}`)
+		.join(" AND ");
+	return [
+		insert(first),
+		...rest.map(([c, piece]) => `UPDATE ${table} SET ${quoteIdent(c)}=${quoteIdent(c)}||${sqlLiteral(piece)} WHERE ${where};`),
+	];
+}
+
+/** Alias for rowid in dump queries of tables without a primary key. */
+const ROWID = "__coywolf_rowid";
+
+/**
+ * Parents before children. D1 may apply a large import in several batches, so
+ * PRAGMA defer_foreign_keys can't be relied on to cover forward references.
+ */
+async function orderByForeignKeys(db: D1Database, tables: MasterRow[]): Promise<MasterRow[]> {
+	const names = new Set(tables.map((t) => t.name));
+	const parents = new Map<string, Set<string>>();
+	for (const t of tables) {
+		const { results } = await db.prepare(`PRAGMA foreign_key_list(${quoteIdent(t.name)})`).all<{ table: string }>();
+		parents.set(t.name, new Set(results.map((r) => r.table).filter((p) => p !== t.name && names.has(p))));
+	}
+	const ordered: MasterRow[] = [];
+	const done = new Set<string>();
+	const visiting = new Set<string>();
+	const byName = new Map(tables.map((t) => [t.name, t]));
+	const visit = (name: string) => {
+		if (done.has(name) || visiting.has(name)) return; // a cycle falls back to name order
+		visiting.add(name);
+		for (const parent of parents.get(name) ?? []) visit(parent);
+		visiting.delete(name);
+		done.add(name);
+		ordered.push(byName.get(name)!);
+	};
+	for (const t of tables) visit(t.name);
+	return ordered;
+}
+
+export interface DumpResult {
+	sql: string;
+	tables: number;
+	rows: number;
+}
+
+/** The whole dump as one string (tests, small databases). Backups stream dumpLines instead. */
+export async function dumpDatabase(db: D1Database): Promise<DumpResult> {
+	const counts = { tables: 0, rows: 0 };
+	const out: string[] = [];
+	for await (const line of dumpLines(db, counts)) out.push(line);
+	return { sql: `${out.join("\n")}\n`, ...counts };
+}
+
+/** Columns that, when a table declares them, hide the real rowid from `rowid` in queries. */
+const ROWID_NAMES = new Set(["rowid", "_rowid_", "oid"]);
+
+/**
+ * The dump, one statement per item (no trailing newline), read a page of rows
+ * at a time so memory stays bounded however big the database is. `counts` is
+ * filled in as it goes (complete once the iteration ends). Tables with a
+ * rowid are paged by rowid (`WHERE rowid > ?`), which stays fast on big
+ * tables where OFFSET rescans every earlier row; WITHOUT ROWID tables use OFFSET.
+ */
+export async function* dumpLines(db: D1Database, counts: { tables: number; rows: number } = { tables: 0, rows: 0 }): AsyncGenerator<string> {
+	const { results: master } = await db
+		.prepare("SELECT type, name, tbl_name, sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY name")
+		.all<MasterRow>();
+
+	const isSearch = (row: MasterRow) =>
+		row.name.startsWith(SEARCH_PREFIX) || row.tbl_name.startsWith(SEARCH_PREFIX) || (row.sql ?? "").includes(SEARCH_PREFIX);
+
+	const tables = master.filter((r) => r.type === "table" && !isInternal(r.name) && !r.name.startsWith(SEARCH_PREFIX));
+	const searchTables = master.filter(
+		(r) => r.type === "table" && r.name.startsWith(SEARCH_PREFIX) && /^CREATE VIRTUAL TABLE/i.test(r.sql ?? ""),
+	);
+	const searchTriggers = master.filter((r) => r.type === "trigger" && r.name.startsWith(SEARCH_PREFIX));
+	const indexes = master.filter((r) => r.type === "index" && !isInternal(r.name) && !isSearch(r));
+	const triggers = master.filter((r) => r.type === "trigger" && !isSearch(r));
+
+	const ordered = await orderByForeignKeys(db, tables);
+
+	const statement = (sql: string) => `${sql.trim().replace(/;$/, "")};`;
+	counts.tables = tables.length;
+	yield "PRAGMA defer_foreign_keys=TRUE;";
+
+	for (const t of tables) yield statement((t.sql ?? "").replace(/^CREATE TABLE /i, "CREATE TABLE IF NOT EXISTS "));
+	for (const t of searchTables) yield statement(t.sql ?? "");
+	for (const t of searchTriggers) yield statement(t.sql ?? "");
+
+	for (const t of ordered) {
+		const table = quoteIdent(t.name);
+		const { results: info } = await db.prepare(`PRAGMA table_info(${table})`).all<{ name: string; pk: number }>();
+		const withRowid = !/WITHOUT\s+ROWID/i.test(t.sql ?? "");
+		// A column named rowid (or oid) shadows the real one: page that table by OFFSET.
+		const byRowid = withRowid && !info.some((c) => ROWID_NAMES.has(c.name.toLowerCase()));
+		const pk = info.filter((c) => c.pk > 0).map((c) => c.name);
+		// Rows are identified by primary key for split values; tables without one by rowid.
+		const keyColumns = pk.length || !withRowid ? pk : [ROWID];
+		// The rowid alias is never written out (rowStatements skips it); it's the paging cursor.
+		const select = byRowid || (withRowid && !pk.length) ? `rowid AS ${ROWID}, *` : "*";
+		const order = withRowid ? " ORDER BY rowid" : "";
+		let after: unknown = null;
+		let offset = 0;
+		let limit = PAGE_SIZE;
+		for (;;) {
+			let results: Record<string, unknown>[];
+			try {
+				({ results } = byRowid
+					? await db
+							.prepare(`SELECT ${select} FROM ${table}${after === null ? "" : ` WHERE rowid > ?`}${order} LIMIT ?`)
+							.bind(...(after === null ? [limit] : [after, limit]))
+							.all<Record<string, unknown>>()
+					: await db
+							.prepare(`SELECT ${select} FROM ${table}${order} LIMIT ? OFFSET ?`)
+							.bind(limit, offset)
+							.all<Record<string, unknown>>());
+			} catch (error) {
+				// D1 refuses results over 50 MB (from November 4, 2026). Ask again for fewer rows, down to one.
+				// Any other error (an outage, a bad query) goes straight up instead of being retried.
+				if (limit === 1 || !RESULT_TOO_LARGE.test(String(error))) throw error;
+				limit = Math.max(1, Math.floor(limit / 4));
+				continue;
+			}
+			let pageBytes = 0;
+			for (const row of results) {
+				if (t.name === "options" && typeof row.name === "string" && SKIP_OPTION.test(row.name)) continue;
+				const key = Object.fromEntries(keyColumns.map((c) => [c, row[c]]));
+				for (const sql of rowStatements(table, row, key)) {
+					pageBytes += sql.length;
+					yield sql;
+				}
+				counts.rows++;
+			}
+			if (results.length < limit) break;
+			offset += results.length;
+			if (byRowid) after = results[results.length - 1][ROWID];
+			// Size the next read from this one: about PAGE_TARGET_BYTES, never more than PAGE_SIZE rows.
+			limit = nextPageSize(pageBytes, results.length);
+		}
+	}
+
+	for (const i of indexes) yield statement(i.sql ?? "");
+	for (const t of triggers) yield statement(t.sql ?? "");
+}