@coywolf/emdash 0.0.0-stage → 0.35.14

package/src/backups/restore.ts

@@ -0,0 +1,209 @@
+/**
+ * Restores, through the Cloudflare API (a Worker can't create databases or
+ * time-travel through its D1 binding).
+ *
+ * Rewinding rolls the live database back, including this plugin's own
+ * settings and storage, so everything a restore depends on lives outside the
+ * database: the API token is a Worker secret, the account and database IDs are
+ * plugin options, and undo points are written to the backup bucket.
+ */
+import { PluginRouteError } from "emdash";
+
+import { stampToDate } from "./store.js";
+
+const API = "https://api.cloudflare.com/client/v4";
+/** One file per rewind, newest last: undo pops the most recent, so earlier states stay reachable. */
+const UNDO_PREFIX = "restore/undo/";
+
+export interface RestoreConfig {
+	token: string;
+	accountId: string;
+	databaseId: string;
+}
+
+export interface UndoPoint {
+	/** Bookmark of the database just before the rewind. */
+	bookmark: string;
+	rewoundAt: string;
+	rewoundTo: string;
+}
+
+async function cf<T>(config: RestoreConfig, path: string, init: RequestInit = {}): Promise<T> {
+	const res = await fetch(`${API}/accounts/${config.accountId}${path}`, {
+		...init,
+		headers: { Authorization: `Bearer ${config.token}`, "Content-Type": "application/json", ...init.headers },
+	});
+	const body = (await res.json().catch(() => ({}))) as { success?: boolean; result?: T; errors?: Array<{ message: string }> };
+	if (!res.ok || body.success === false) {
+		// Surface the API's message in the admin (plain Errors become a generic 500).
+		throw PluginRouteError.badRequest(
+			`Cloudflare API ${res.status}: ${body.errors?.map((e) => e.message).join("; ") || res.statusText}`,
+		);
+	}
+	return body.result as T;
+}
+
+const db = (config: RestoreConfig) => `/d1/database/${config.databaseId}`;
+
+export async function currentBookmark(config: RestoreConfig): Promise<string> {
+	const result = await cf<{ bookmark: string }>(config, `${db(config)}/time_travel/bookmark`);
+	return result.bookmark;
+}
+
+/** Roll the live database back to a backup's moment, saving an undo point first. */
+export async function rewind(
+	config: RestoreConfig,
+	backups: R2Bucket,
+	target: { stamp: string; bookmark?: string },
+): Promise<UndoPoint> {
+	const query = target.bookmark
+		? `bookmark=${encodeURIComponent(target.bookmark)}`
+		: `timestamp=${encodeURIComponent(stampToDate(target.stamp).toISOString())}`;
+	const before = await currentBookmark(config);
+	const undo: UndoPoint = { bookmark: before, rewoundAt: new Date().toISOString(), rewoundTo: target.stamp };
+	await backups.put(`${UNDO_PREFIX}${undo.rewoundAt}.json`, JSON.stringify(undo), {
+		httpMetadata: { contentType: "application/json" },
+	});
+	await cf(config, `${db(config)}/time_travel/restore?${query}`, { method: "POST" });
+	return undo;
+}
+
+async function latestUndoKey(backups: R2Bucket): Promise<string | null> {
+	const keys: string[] = [];
+	let cursor: string | undefined;
+	do {
+		const page = await backups.list({ prefix: UNDO_PREFIX, cursor });
+		for (const o of page.objects) keys.push(o.key);
+		cursor = page.truncated ? page.cursor : undefined;
+	} while (cursor);
+	return keys.sort().at(-1) ?? null;
+}
+
+/** The most recent rewind that hasn't been undone. */
+export async function getUndo(backups: R2Bucket): Promise<UndoPoint | null> {
+	const key = await latestUndoKey(backups);
+	const object = key ? await backups.get(key) : null;
+	return object ? ((await object.json()) as UndoPoint) : null;
+}
+
+/** Return the database to where it was before the most recent rewind. */
+export async function undoRewind(config: RestoreConfig, backups: R2Bucket): Promise<UndoPoint> {
+	const key = await latestUndoKey(backups);
+	const object = key ? await backups.get(key) : null;
+	if (!key || !object) throw PluginRouteError.badRequest("There is no rewind to undo.");
+	const undo = (await object.json()) as UndoPoint;
+	await cf(config, `${db(config)}/time_travel/restore?bookmark=${encodeURIComponent(undo.bookmark)}`, { method: "POST" });
+	await backups.delete(key);
+	return undo;
+}
+
+async function md5(bytes: Uint8Array<ArrayBuffer>): Promise<string> {
+	// MD5 is what D1's import API uses as the upload ETag (Workers support it in WebCrypto).
+	const digest = await crypto.subtle.digest("MD5", bytes);
+	return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+interface ImportStatus {
+	status?: "active" | "complete" | "error";
+	at_bookmark?: string;
+	upload_url?: string;
+	filename?: string;
+	error?: string;
+	success?: boolean;
+}
+
+export interface NewDatabaseRestore {
+	databaseId: string;
+	databaseName: string;
+	tables: number;
+	rows: number;
+	mismatches: Record<string, [number, number | undefined]>;
+}
+
+/**
+ * Import a dump into a brand-new D1 database and verify row counts. The live
+ * site keeps running on its current database until its binding is switched.
+ */
+export async function restoreToNewDatabase(
+	config: RestoreConfig,
+	databaseName: string,
+	sql: string,
+): Promise<NewDatabaseRestore> {
+	const suffix = crypto.randomUUID().slice(0, 6);
+	const created = await cf<{ uuid: string; name: string }>(config, "/d1/database", {
+		method: "POST",
+		body: JSON.stringify({ name: `${databaseName}-${suffix}` }),
+	});
+	const target: RestoreConfig = { ...config, databaseId: created.uuid };
+	try {
+		return await importAndVerify(target, created, sql);
+	} catch (error) {
+		// Don't leave a half-imported database behind.
+		await cf(config, `/d1/database/${created.uuid}`, { method: "DELETE" }).catch(() => undefined);
+		throw error;
+	}
+}
+
+async function importAndVerify(
+	target: RestoreConfig,
+	created: { uuid: string; name: string },
+	sql: string,
+): Promise<NewDatabaseRestore> {
+	const path = `${db(target)}/import`;
+
+	const bytes = new TextEncoder().encode(sql) as Uint8Array<ArrayBuffer>;
+	const etag = await md5(bytes);
+	const init = await cf<ImportStatus>(target, path, { method: "POST", body: JSON.stringify({ action: "init", etag }) });
+	if (init.upload_url) {
+		const put = await fetch(init.upload_url, { method: "PUT", body: bytes });
+		if (!put.ok) throw PluginRouteError.badRequest(`Uploading the dump failed (${put.status})`);
+	}
+	let status = await cf<ImportStatus>(target, path, {
+		method: "POST",
+		body: JSON.stringify({ action: "ingest", etag, filename: init.filename }),
+	});
+	const bookmark = status.at_bookmark;
+	for (let i = 0; status.status !== "complete" && status.status !== "error" && i < 60; i++) {
+		await new Promise((r) => setTimeout(r, 1000));
+		status = await cf<ImportStatus>(target, path, {
+			method: "POST",
+			body: JSON.stringify({ action: "poll", current_bookmark: bookmark }),
+		});
+	}
+	if (status.status !== "complete") {
+		throw PluginRouteError.badRequest(`Import ${status.status ?? "timed out"}: ${status.error ?? "no details"}`);
+	}
+
+	// Verify: rows per table in the dump vs. the new database.
+	const expected = new Map<string, number>();
+	for (const match of sql.matchAll(/^INSERT INTO "((?:[^"]|"")+)"/gm)) {
+		const table = match[1].replace(/""/g, '"');
+		expected.set(table, (expected.get(table) ?? 0) + 1);
+	}
+	const tables = [...expected.keys()];
+	const actual = new Map<string, number>();
+	for (let i = 0; i < tables.length; i += 5) {
+		// D1 allows at most 5 terms in a compound SELECT.
+		const sqlCounts = tables
+			.slice(i, i + 5)
+			.map((t) => `SELECT '${t.replace(/'/g, "''")}' AS t, count(*) AS n FROM "${t.replace(/"/g, '""')}"`)
+			.join(" UNION ALL ");
+		const result = await cf<Array<{ results: Array<{ t: string; n: number }> }>>(target, `${db(target)}/query`, {
+			method: "POST",
+			body: JSON.stringify({ sql: sqlCounts }),
+		});
+		for (const row of result[0]?.results ?? []) actual.set(row.t, row.n);
+	}
+	const mismatches: NewDatabaseRestore["mismatches"] = {};
+	for (const [table, n] of expected) if (actual.get(table) !== n) mismatches[table] = [n, actual.get(table)];
+
+	return {
+		databaseId: created.uuid,
+		databaseName: created.name,
+		tables: tables.length,
+		rows: [...expected.values()].reduce((a, b) => a + b, 0),
+		mismatches,
+	};
+}
+
+export { restoreMissingMedia } from "./store.js";

package/src/backups/store.ts

@@ -0,0 +1,409 @@
+/**
+ * Backup storage layout (shared with .github/workflows/backup.yml):
+ *
+ *   d1/<stamp>/<file>.sql.gz   database dump
+ *   d1/<stamp>/manifest.json   what the dump is and where it came from
+ *   media/current/<key>        mirror of the media bucket
+ *   media/changed/<stamp>/<key> media replaced or deleted since the previous run
+ *   uploads/current/<key>      mirror of the private form uploads bucket (when bound)
+ *   uploads/changed/<stamp>/<key> uploads replaced or deleted since the previous run
+ *
+ * The backup bucket must stay private: uploads/ holds files people sent
+ * through forms, which aren't public on the site.
+ */
+
+export interface Manifest {
+	stamp: string;
+	database?: string;
+	file: string;
+	bytes: number;
+	sha256: string;
+	/** "github-actions" (external nightly job), "scheduled" (plugin), or "admin" (Back up now). */
+	source?: string;
+	timeTravelBookmark?: string;
+	gitSha?: string;
+	tables?: number;
+	rows?: number;
+	media?: MirrorResult;
+	/** Private form uploads, when the site has an uploads bucket. */
+	uploads?: MirrorResult;
+}
+
+export interface MirrorResult {
+	copied: number;
+	preserved: number;
+	total: number;
+	pending?: number;
+}
+
+/** Folder in the backup bucket for each mirrored bucket. */
+export type MirrorPrefix = "media" | "uploads";
+
+export interface BackupEntry extends Manifest {
+	createdAt: string;
+}
+
+export const STAMP = /^\d{4}-\d{2}-\d{2}T\d{4}Z$/;
+export const FILE = /^[a-z0-9._-]+$/i;
+
+/** 2026-10-03T1802Z — sortable, safe in object keys and URLs. */
+export function makeStamp(date = new Date()): string {
+	const iso = date.toISOString();
+	return `${iso.slice(0, 13)}${iso.slice(14, 16)}Z`;
+}
+
+export function stampToDate(stamp: string): Date {
+	return new Date(`${stamp.slice(0, 13)}:${stamp.slice(13, 15)}:00Z`);
+}
+
+async function sha256(bytes: Uint8Array<ArrayBuffer>): Promise<string> {
+	const digest = await crypto.subtle.digest("SHA-256", bytes);
+	return [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("");
+}
+
+/** R2's smallest multipart part. Every part but the last must be the same size. */
+export const PART_BYTES = 5 * 1024 * 1024;
+/** Text handed to the gzip stream at a time. */
+const FEED_BYTES = 64 * 1024;
+const utf8 = new TextEncoder();
+
+/** SHA-256 over a stream of chunks: Workers' DigestStream, else (Node tests) the chunks kept and hashed at the end. */
+function digester(): { update(chunk: Uint8Array<ArrayBuffer>): Promise<void>; digest(): Promise<string> } {
+	const hex = (digest: ArrayBuffer) => [...new Uint8Array(digest)].map((b) => b.toString(16).padStart(2, "0")).join("");
+	const Stream = (crypto as unknown as { DigestStream?: new (algorithm: string) => WritableStream<Uint8Array> & { digest: Promise<ArrayBuffer> } }).DigestStream;
+	if (Stream) {
+		const stream = new Stream("SHA-256");
+		const writer = stream.getWriter();
+		return {
+			update: (chunk) => writer.write(chunk),
+			digest: async () => {
+				await writer.close();
+				return hex(await stream.digest);
+			},
+		};
+	}
+	const chunks: Uint8Array<ArrayBuffer>[] = [];
+	return {
+		update: async (chunk) => {
+			chunks.push(chunk);
+		},
+		digest: () => sha256(concat(chunks, chunks.reduce((n, c) => n + c.byteLength, 0))),
+	};
+}
+
+function concat(chunks: Uint8Array<ArrayBuffer>[], size: number): Uint8Array<ArrayBuffer> {
+	const out = new Uint8Array(size);
+	let at = 0;
+	for (const chunk of chunks) {
+		out.set(chunk, at);
+		at += chunk.byteLength;
+	}
+	return out;
+}
+
+/**
+ * Gzip the dump's statements (one per line) straight into the backup bucket:
+ * one put when it's under a part, else an R2 multipart upload in PART_BYTES
+ * parts. Memory stays around one part plus a page of rows, however big the
+ * database. Returns the stored size and SHA-256 of the gzip file.
+ */
+export async function writeDumpStream(
+	bucket: R2Bucket,
+	stamp: string,
+	file: string,
+	lines: AsyncIterable<string>,
+	partBytes = PART_BYTES,
+): Promise<{ bytes: number; sha256: string }> {
+	const key = `d1/${stamp}/${file}`;
+	const httpMetadata = { contentType: "application/gzip" };
+	const gzip = new CompressionStream("gzip");
+	const writer = gzip.writable.getWriter();
+	// Feed the compressor while the loop below drains it (each waits on the other).
+	/** An error from the dump itself (a failed query), which explains more than the stream errors it causes. */
+	let dumpError: { error: unknown } | null = null;
+	const feed = (async () => {
+		let batch = "";
+		const iterator = lines[Symbol.asyncIterator]();
+		for (;;) {
+			let next: IteratorResult<string>;
+			try {
+				next = await iterator.next();
+			} catch (error) {
+				dumpError = { error };
+				throw error;
+			}
+			if (next.done) break;
+			batch += `${next.value}\n`;
+			if (batch.length >= FEED_BYTES) {
+				await writer.write(utf8.encode(batch));
+				batch = "";
+			}
+		}
+		if (batch) await writer.write(utf8.encode(batch));
+		await writer.close();
+	})();
+	feed.catch((error) => writer.abort(error).catch(() => undefined));
+
+	const hash = digester();
+	let pending: Uint8Array<ArrayBuffer>[] = [];
+	let pendingBytes = 0;
+	let bytes = 0;
+	/** The first `size` pending bytes as one buffer; the rest stay pending. */
+	const take = (size: number): Uint8Array<ArrayBuffer> => {
+		const all = concat(pending, pendingBytes);
+		pending = size < pendingBytes ? [all.slice(size)] : [];
+		pendingBytes -= size;
+		return size < all.byteLength ? all.slice(0, size) : all;
+	};
+
+	let upload: R2MultipartUpload | null = null;
+	const parts: R2UploadedPart[] = [];
+	const reader = gzip.readable.getReader();
+	try {
+		for (;;) {
+			const { value, done } = await reader.read();
+			if (done) break;
+			const chunk = value as Uint8Array<ArrayBuffer>;
+			bytes += chunk.byteLength;
+			await hash.update(chunk);
+			pending.push(chunk);
+			pendingBytes += chunk.byteLength;
+			while (pendingBytes >= partBytes) {
+				upload ??= await bucket.createMultipartUpload(key, { httpMetadata });
+				parts.push(await upload.uploadPart(parts.length + 1, take(partBytes)));
+			}
+		}
+		await feed;
+		const rest = take(pendingBytes);
+		if (upload) {
+			if (rest.byteLength) parts.push(await upload.uploadPart(parts.length + 1, rest));
+			await upload.complete(parts);
+		} else {
+			await bucket.put(key, rest, { httpMetadata });
+		}
+	} catch (error) {
+		await upload?.abort().catch(() => undefined);
+		// Unblock the feed (it may be waiting for room in the stream) and let it finish.
+		await reader.cancel(error).catch(() => undefined);
+		await feed.catch(() => undefined);
+		throw dumpError ? (dumpError as { error: unknown }).error : error;
+	}
+	return { bytes, sha256: await hash.digest() };
+}
+
+/** Write (or rewrite) a backup's manifest for a dump already stored. */
+export async function writeManifest(
+	bucket: R2Bucket,
+	stamp: string,
+	file: string,
+	written: { bytes: number; sha256: string },
+	extra: Partial<Manifest>,
+): Promise<Manifest> {
+	const manifest: Manifest = { ...extra, stamp, file, bytes: written.bytes, sha256: written.sha256 };
+	await bucket.put(`d1/${stamp}/manifest.json`, JSON.stringify(manifest, null, 1), {
+		httpMetadata: { contentType: "application/json" },
+	});
+	return manifest;
+}
+
+
+/** Every backup with a manifest, newest first. */
+export async function listBackups(bucket: R2Bucket): Promise<BackupEntry[]> {
+	const stamps: string[] = [];
+	let cursor: string | undefined;
+	do {
+		const page = await bucket.list({ prefix: "d1/", delimiter: "/", cursor });
+		for (const prefix of page.delimitedPrefixes) {
+			const stamp = prefix.slice(3, -1);
+			if (STAMP.test(stamp)) stamps.push(stamp);
+		}
+		cursor = page.truncated ? page.cursor : undefined;
+	} while (cursor);
+
+	const entries = await Promise.all(
+		stamps.map(async (stamp) => {
+			const object = await bucket.get(`d1/${stamp}/manifest.json`);
+			if (!object) return null;
+			try {
+				const manifest = (await object.json()) as Manifest;
+				return { ...manifest, stamp, createdAt: stampToDate(stamp).toISOString() };
+			} catch {
+				return null;
+			}
+		}),
+	);
+	return entries.filter((e): e is BackupEntry => e !== null).sort((a, b) => b.stamp.localeCompare(a.stamp));
+}
+
+/**
+ * Mirror the media bucket into media/current/, moving anything replaced or
+ * deleted since the last run to media/changed/<stamp>/ (same as the nightly
+ * job's `rclone sync --backup-dir`). Compares by ETag.
+ *
+ * Copies at most `maxChanges` files per run so a large first mirror stays under
+ * the Workers subrequest limit; the rest are reported as `pending` and copied
+ * on the next run.
+ */
+export async function mirrorMedia(media: R2Bucket, backups: R2Bucket, stamp: string, maxChanges = 300): Promise<MirrorResult> {
+	return mirrorBucket(media, backups, stamp, "media", maxChanges);
+}
+
+/** mirrorMedia for any bucket, into <prefix>/current/ and <prefix>/changed/<stamp>/. */
+export async function mirrorBucket(
+	media: R2Bucket,
+	backups: R2Bucket,
+	stamp: string,
+	prefix: MirrorPrefix,
+	maxChanges = 300,
+): Promise<MirrorResult> {
+	const current = `${prefix}/current/`;
+	const source = new Map<string, string>();
+	let cursor: string | undefined;
+	do {
+		const page = await media.list({ cursor });
+		for (const object of page.objects) source.set(object.key, object.etag);
+		cursor = page.truncated ? page.cursor : undefined;
+	} while (cursor);
+
+	const mirror = new Map<string, string>();
+	do {
+		const page = await backups.list({ prefix: current, cursor });
+		for (const object of page.objects) mirror.set(object.key.slice(current.length), object.etag);
+		cursor = page.truncated ? page.cursor : undefined;
+	} while (cursor);
+
+	let copied = 0;
+	let preserved = 0;
+	const keep = async (key: string) => {
+		const old = await backups.get(`${current}${key}`);
+		if (!old) return;
+		await backups.put(`${prefix}/changed/${stamp}/${key}`, old.body, { httpMetadata: old.httpMetadata });
+		preserved++;
+	};
+
+	// Decide the changes first, then make them a few files at a time.
+	const tasks: Array<() => Promise<void>> = [];
+	let pending = 0;
+	for (const [key, etag] of source) {
+		if (mirror.get(key) === etag) continue;
+		if (tasks.length >= maxChanges) {
+			pending++;
+			continue;
+		}
+		tasks.push(async () => {
+			if (mirror.has(key)) await keep(key);
+			const object = await media.get(key);
+			if (!object) return;
+			await backups.put(`${current}${key}`, object.body, { httpMetadata: object.httpMetadata });
+			copied++;
+		});
+	}
+	for (const key of mirror.keys()) {
+		if (source.has(key)) continue;
+		if (tasks.length >= maxChanges) {
+			pending++;
+			continue;
+		}
+		tasks.push(async () => {
+			await keep(key);
+			await backups.delete(`${current}${key}`);
+		});
+	}
+	await inPool(tasks, MIRROR_CONCURRENCY);
+	return { copied, preserved, total: source.size, pending };
+}
+
+/** Files copied at once when mirroring (each copy is a get and a put, streamed). */
+export const MIRROR_CONCURRENCY = 8;
+
+/** Run tasks with at most `limit` in flight; the first failure is thrown once the others settle. */
+async function inPool(tasks: Array<() => Promise<void>>, limit: number): Promise<void> {
+	let next = 0;
+	let failure: { error: unknown } | null = null;
+	const worker = async () => {
+		while (next < tasks.length && !failure) {
+			const task = tasks[next++];
+			try {
+				await task();
+			} catch (error) {
+				failure ??= { error };
+			}
+		}
+	};
+	await Promise.all(Array.from({ length: Math.min(limit, tasks.length) }, worker));
+	if (failure) throw (failure as { error: unknown }).error;
+}
+
+/**
+ * Copy media files that are in the backup mirror but missing from the media
+ * bucket. Never overwrites. Copies at most `max` files per call so one request
+ * stays well inside Worker limits; `pending` is what's left for the next call
+ * (a coywolf.com restore trial copied 2,021 files in one request and took
+ * over five minutes).
+ */
+export async function restoreMissingMedia(
+	media: R2Bucket,
+	backups: R2Bucket,
+	max = 200,
+	prefix: MirrorPrefix = "media",
+): Promise<{ restored: number; checked: number; pending: number }> {
+	const current = `${prefix}/current/`;
+	const present = new Set<string>();
+	let cursor: string | undefined;
+	do {
+		const page = await media.list({ cursor });
+		for (const o of page.objects) present.add(o.key);
+		cursor = page.truncated ? page.cursor : undefined;
+	} while (cursor);
+
+	let restored = 0;
+	let checked = 0;
+	let pending = 0;
+	do {
+		const page = await backups.list({ prefix: current, cursor });
+		for (const o of page.objects) {
+			checked++;
+			const key = o.key.slice(current.length);
+			if (present.has(key)) continue;
+			if (restored >= max) {
+				pending++;
+				continue;
+			}
+			const object = await backups.get(o.key);
+			if (!object) continue;
+			await media.put(key, object.body, { httpMetadata: object.httpMetadata });
+			restored++;
+		}
+		cursor = page.truncated ? page.cursor : undefined;
+	} while (cursor);
+	return { restored, checked, pending };
+}
+
+/** Delete database backups and replaced media and upload copies older than `days`. Returns the number of stamps pruned. */
+export async function pruneBackups(bucket: R2Bucket, days: number): Promise<number> {
+	const cutoff = makeStamp(new Date(Date.now() - days * 86_400_000));
+	let pruned = 0;
+	for (const prefix of ["d1/", "media/changed/", "uploads/changed/"]) {
+		let cursor: string | undefined;
+		do {
+			const page = await bucket.list({ prefix, delimiter: "/", cursor });
+			for (const dir of page.delimitedPrefixes) {
+				const stamp = dir.slice(prefix.length, -1);
+				if (!STAMP.test(stamp) || stamp >= cutoff) continue;
+				await deletePrefix(bucket, dir);
+				if (prefix === "d1/") pruned++;
+			}
+			cursor = page.truncated ? page.cursor : undefined;
+		} while (cursor);
+	}
+	return pruned;
+}
+
+async function deletePrefix(bucket: R2Bucket, prefix: string) {
+	let cursor: string | undefined;
+	do {
+		const page = await bucket.list({ prefix, cursor });
+		if (page.objects.length) await bucket.delete(page.objects.map((o) => o.key));
+		cursor = page.truncated ? page.cursor : undefined;
+	} while (cursor);
+}