@jant/core 0.7.0 → 0.7.1

package/bin/lib/site-pull-media.js

@@ -12,7 +12,6 @@ import {
 } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { basename, dirname, extname, join, relative } from "node:path";
-import { unzipSync, zipSync } from "fflate";
 import { parse, stringify } from "smol-toml";
 import {
   collectMediaReferences as collectParsedMediaReferences,
@@ -20,6 +19,7 @@ import {
   rewriteMediaReferences,
 } from "./site-media-parser.js";
 import { formatFrontMatter, parseFrontMatter } from "./hugo-markdown.js";
+import { extractZipFile, writeDirectoryToZip } from "./zip-archive.js";
 
 export function getSitePathPrefix(baseUrl) {
   if (typeof baseUrl !== "string" || baseUrl.trim() === "") {
@@ -477,29 +477,6 @@ async function resolveExistingPulledPath(
   return toPulledPublicPath(pathname, sitePathPrefix);
 }
 
-async function packDirectoryToZip(rootDir) {
-  const files = {};
-  const allFiles = await walkFiles(rootDir);
-
-  for (const fullPath of allFiles) {
-    const relPath = relative(rootDir, fullPath).replace(/\\/g, "/");
-    files[relPath] = new Uint8Array(await readFile(fullPath));
-  }
-
-  return zipSync(files);
-}
-
-async function unpackZipToDirectory(zipBytes, rootDir) {
-  const files = unzipSync(zipBytes);
-  await Promise.all(
-    Object.entries(files).map(async ([relPath, bytes]) => {
-      const fullPath = join(rootDir, relPath);
-      await mkdir(dirname(fullPath), { recursive: true });
-      await writeFile(fullPath, bytes);
-    }),
-  );
-}
-
 export async function pullSiteExportDirectory(rootDir, options = {}) {
   const logger =
     typeof options.logger === "function" ? options.logger : () => {};
@@ -681,14 +658,30 @@ export async function pullSiteExportDirectory(rootDir, options = {}) {
   return stats;
 }
 
-export async function pullSiteExportZipBytes(zipBytes, options = {}) {
+/**
+ * Pull media into an export ZIP: extract it to a temporary directory, pull
+ * there, and write the result as a new archive. Nothing is held in memory
+ * beyond one media file at a time, so the archive can be any size.
+ *
+ * @param {string} inputZipPath - Export archive to read
+ * @param {string} outputZipPath - Archive to write; may be the input
+ * @param {Parameters<typeof pullSiteExportDirectory>[1]} [options]
+ * @returns {Promise<Awaited<ReturnType<typeof pullSiteExportDirectory>>>} Pull stats
+ * @example
+ * await pullSiteExportZipFile("export.zip", "export.zip", { logger });
+ */
+export async function pullSiteExportZipFile(
+  inputZipPath,
+  outputZipPath,
+  options = {},
+) {
   const tempDir = await mkdtemp(join(tmpdir(), "jant-site-pull-"));
 
   try {
-    await unpackZipToDirectory(zipBytes, tempDir);
+    await extractZipFile(inputZipPath, tempDir);
     const stats = await pullSiteExportDirectory(tempDir, options);
-    const pulledZip = await packDirectoryToZip(tempDir);
-    return { zipBytes: pulledZip, stats };
+    await writeDirectoryToZip(tempDir, outputZipPath);
+    return stats;
   } finally {
     await rm(tempDir, { recursive: true, force: true });
   }

package/bin/lib/site-selection.js

@@ -176,6 +176,15 @@ function formatSiteSummary(rows) {
     .join(", ");
 }
 
+/**
+ * How the target instance maps requests to sites.
+ *
+ * @param {{ SITE_RESOLUTION_MODE?: unknown }} [env] The process environment,
+ *   or runtime bindings built from it; defaults to `process.env`
+ * @returns {"host-based" | "single-site"} `host-based` only when set to it
+ * @example
+ * if (getCliSiteResolutionMode(process.env) === "host-based") { ... }
+ */
 export function getCliSiteResolutionMode(env = process.env) {
   return env.SITE_RESOLUTION_MODE === "host-based"
     ? "host-based"
@@ -192,7 +201,7 @@ export function getCliSiteResolutionMode(env = process.env) {
  *
  * @param {{ query(sql: string): Promise<Record<string, unknown>[]>, execute?(sql: string): Promise<void> }} queryRunner
  *   Runs SQL against the target database
- * @param {{ env?: Record<string, string | undefined>, site?: string, host?: string, pathPrefix?: string, url?: string, createIfMissing?: boolean, bootstrapSite?: { id?: string, key?: string } }} [options]
+ * @param {{ env?: { SITE_RESOLUTION_MODE?: unknown }, site?: string, host?: string, pathPrefix?: string, url?: string, createIfMissing?: boolean, bootstrapSite?: { id?: string, key?: string } }} [options]
  *   The environment, the command's site flags, and whether single-site mode
  *   may create the site shell when none exists
  * @returns {Promise<{ created: boolean, site: { id: string, key: string, status: string, createdAt: number, updatedAt: number } }>}

package/bin/lib/site-snapshot.js

@@ -5,31 +5,101 @@ export const SNAPSHOT_FORMAT = "jant-site-snapshot";
 export const SNAPSHOT_VERSION = 2;
 export const SUPPORTED_SNAPSHOT_VERSIONS = [1, SNAPSHOT_VERSION];
 
+/**
+ * Site content, in the order rows are inserted on import.
+ *
+ * Membership is what this list decides: a table absent from it is never read,
+ * so its rows leave no trace in the snapshot and `--replace` cannot clear them.
+ *
+ * The order here is kept as a foreign-key topological sort for readability, but
+ * it is not what an export follows — `sortExportTables` re-sorts by
+ * `TABLE_EXPORT_ORDER` in `sql-export.js`, and that is the list to change when
+ * the dump order has to change. `src/__tests__/snapshot-tables.test.ts` checks
+ * the sorted result for that reason.
+ */
 export const SNAPSHOT_TABLES = [
   "site_setting",
   "collection",
-  "nav_item",
-  "collection_directory_item",
+  "smart_collection",
   "post",
   "thread_collection",
+  "nav_item",
+  "collection_directory_item",
   "path_registry",
   "media",
 ];
 
+/**
+ * The same content, in the order `--replace` deletes it: children first, so no
+ * delete depends on a cascade to clean up after it. `site_setting` is absent on
+ * purpose — `buildReplaceSql` clears it by key, because a site's settings hold
+ * more than the snapshot carries.
+ */
 export const SNAPSHOT_CLEAR_TABLES = [
-  "thread_collection",
   "media",
   "path_registry",
   "collection_directory_item",
   "nav_item",
+  "thread_collection",
   "post",
+  "smart_collection",
   "collection",
 ];
 
+/**
+ * Site-scoped tables the snapshot deliberately leaves alone.
+ *
+ * Every table carrying a `site_id` has to appear here or in `SNAPSHOT_TABLES`
+ * — `src/__tests__/snapshot-tables.test.ts` fails the build otherwise, so a new
+ * table forces a decision rather than being silently dropped. The reasons fall
+ * into three groups: auth and routing shell that a content restore must
+ * preserve, credentials and external integration bindings that do not travel
+ * with content, and transient runtime bookkeeping.
+ */
+export const SNAPSHOT_EXCLUDED_TABLES = [
+  // Auth and routing shell. `--replace` restores content into an existing
+  // site; its members and domains belong to the deployment, not the content.
+  "site_member",
+  "site_domain",
+  // Credentials. Reissued per deployment, never copied between sites. The demo
+  // rebuild clears these through the internal admin route instead.
+  "api_token",
+  // External integration bindings. They point at an installation or a chat on
+  // someone else's service, so restoring them into another site would aim that
+  // site at a binding it does not own.
+  "github_app_installation",
+  "telegram_binding",
+  "telegram_pending_binding",
+  "telegram_media_group_item",
+  // Transient runtime bookkeeping, rebuilt by the runtime as it goes:
+  // in-flight uploads, and the recycle-window ledger for deleted objects.
+  "upload_session",
+  "storage_purge",
+];
+
+/**
+ * Settings the snapshot carries: the site's published identity and appearance.
+ *
+ * `site_setting` holds more than a site's content — deployment wiring, secrets,
+ * integration bindings, local UI state — so both the export and `--replace`
+ * work through this allowlist rather than the whole table. Membership is what
+ * it decides: a key absent from it is neither exported nor cleared on import.
+ *
+ * Every DB-backed key in `CONFIG_FIELDS` has to appear here or in
+ * `SNAPSHOT_EXCLUDED_SETTING_KEYS`, and
+ * `src/__tests__/snapshot-settings.test.ts` fails the build otherwise, so a new
+ * setting forces a decision instead of being silently dropped.
+ */
 export const SNAPSHOT_SETTING_KEYS = [
   "SITE_NAME",
   "SITE_DESCRIPTION",
+  // The language trio travels together. `post.language` and
+  // `post.translation_group_id` are already carried by the whole-row `post`
+  // dump, so leaving the switch behind restored a site whose posts were
+  // stamped per language while every per-language view was gone.
   "SITE_LANGUAGE",
+  "MULTILINGUAL_ENABLED",
+  "ADDITIONAL_LANGUAGES",
   "MAIN_RSS_FEED",
   "PUBLIC_API_ENABLED",
   "RSS_FEEDS_ENABLED",
@@ -48,6 +118,68 @@ export const SNAPSHOT_SETTING_KEYS = [
   "NOINDEX",
 ];
 
+/**
+ * DB-backed settings the snapshot deliberately leaves alone.
+ *
+ * `envOnly` keys are absent from both lists on purpose: they never reach
+ * `site_setting`, so there is no row to carry or clear.
+ */
+export const SNAPSHOT_EXCLUDED_SETTING_KEYS = [
+  // The operator's own dashboard locale, not the site's published language.
+  // A content restore should not relabel the dashboard of whoever runs the
+  // target site.
+  "DASHBOARD_LANGUAGE",
+  // Sizing and pacing knobs. They describe how much the target deployment
+  // serves at a time, not what it publishes, and they carry env defaults the
+  // deployment picked.
+  "ARCHIVE_DEFAULT_LAYOUT",
+  "PAGE_SIZE",
+  "SEARCH_PAGE_SIZE",
+  "ARCHIVE_PAGE_SIZE",
+  "SUMMARY_MAX_PARAGRAPHS",
+  "SUMMARY_MAX_CHARS",
+  "RSS_FEED_LIMIT",
+  "RSS_PUBLISH_DELAY_SECONDS",
+  // Code injection. `CUSTOM_CSS` is declarative and travels; these two are
+  // executable markup, and importing an archive must not be a way to run
+  // script on the importing site.
+  "CUSTOM_HEAD_HTML",
+  "CUSTOM_BODY_END_HTML",
+  // Being listed in the Jant Discover directory is the site owner's consent,
+  // and the announcement state below it belongs to the announcing instance.
+  "DISCOVER",
+  "DISCOVER_ANNOUNCE_STATE",
+  // Local UI state: which affordances this author has already been shown, and
+  // how far setup got.
+  "DISCOVERY_COMPOSE_OPEN_SHORTCUT_AT",
+  "DISCOVERY_SLASH_COMMAND_AT",
+  "ONBOARDING_STATUS",
+  // Credential. Reissued per deployment, never copied between sites.
+  "PASSWORD_RESET_TOKEN",
+  // External integration bindings, their secrets, and their sync bookkeeping.
+  // They point at a repo, an installation, or a chat on someone else's
+  // service, so restoring them would aim the target site at a binding it does
+  // not own — the same reason `github_app_installation` and `telegram_binding`
+  // are excluded tables.
+  "GITHUB_SYNC_ENABLED",
+  "GITHUB_SYNC_REPO",
+  "GITHUB_SYNC_TOKEN",
+  "GITHUB_SYNC_WEBHOOK_SECRET",
+  "GITHUB_SYNC_WEBHOOK_ID",
+  "GITHUB_SYNC_LAST_PUSH_SHA",
+  "GITHUB_SYNC_LAST_PUSH_AT",
+  "GITHUB_SYNC_PENDING",
+  "GITHUB_SYNC_PENDING_AT",
+  "GITHUB_SYNC_DIRTY",
+  "GITHUB_SYNC_LAST_ERROR",
+  "GITHUB_SYNC_AUTH_MODE",
+  "GITHUB_SYNC_APP_INSTALLATION_ID",
+  "TELEGRAM_BOT_TOKEN",
+  "TELEGRAM_BOT_ID",
+  "TELEGRAM_BOT_USERNAME",
+  "TELEGRAM_BOT_WEBHOOK_SECRET",
+];
+
 function escapeSqlString(value) {
   return String(value).replaceAll("'", "''");
 }
@@ -78,6 +210,12 @@ const SELECT_SQL_BY_TABLE = {
     WHERE "site_id" = ?1
     ORDER BY "position", "id"
   `,
+  smart_collection: `
+    SELECT *
+    FROM "smart_collection"
+    WHERE "site_id" = ?1
+    ORDER BY "created_at", "id"
+  `,
   post: `
     SELECT *
     FROM "post"
@@ -118,6 +256,14 @@ export function getSnapshotSelectSql(tableName, siteId) {
   return statement.trim().replaceAll("?1", `'${escapeSqlString(siteId)}'`);
 }
 
+/**
+ * SQL listing a site's stored media objects: originals and video posters.
+ *
+ * @param {string} siteId Site whose media rows to read
+ * @returns {string} A query returning `key` and `contentType` columns
+ * @example
+ * collectSnapshotObjects(await query(buildSnapshotStorageQuery(site.id)))
+ */
 export function buildSnapshotStorageQuery(siteId) {
   return `
     SELECT "key", "contentType"
@@ -146,6 +292,27 @@ export function buildSnapshotStorageQuery(siteId) {
   `.trim();
 }
 
+/**
+ * A storage object a snapshot carries.
+ *
+ * @typedef {object} SnapshotObject
+ * @property {string} key Storage key
+ * @property {string} contentType MIME type, or `""` when neither the row nor
+ *   the key's extension gives one
+ */
+
+/**
+ * Deduplicate the rows of `buildSnapshotStorageQuery` into storage objects.
+ *
+ * Rows with a blank key are skipped. A row without a content type takes one
+ * from the key's extension.
+ *
+ * @param {Record<string, unknown>[]} rows Rows with `key` and `contentType`
+ * @returns {SnapshotObject[]} One entry per distinct key
+ * @example
+ * collectSnapshotObjects([{ key: "media/a.png", contentType: null }])
+ * // => [{ key: "media/a.png", contentType: "image/png" }]
+ */
 export function collectSnapshotObjects(rows) {
   const objects = new Map();
 
@@ -198,6 +365,28 @@ export function buildSnapshotMeta(site, options = {}) {
   };
 }
 
+/**
+ * A snapshot's `meta.json`, as `assertSnapshotMeta` accepts it.
+ *
+ * @typedef {object} SnapshotMeta
+ * @property {typeof SNAPSHOT_FORMAT} format
+ * @property {number} version One of `SUPPORTED_SNAPSHOT_VERSIONS`
+ * @property {"sqlite" | "pg"} [dialect] Absent from snapshots that predate it
+ * @property {{ id: string, key: string }} [site] The site the snapshot was
+ *   exported from; absent from legacy snapshots
+ * @property {unknown} [tables] Listed by legacy snapshots only
+ */
+
+/**
+ * Refuse a `meta.json` this version cannot import.
+ *
+ * @param {unknown} meta Parsed `meta.json`
+ * @returns {asserts meta is SnapshotMeta}
+ * @throws {Error} When the format, version, dialect, or site is not supported
+ * @example
+ * const meta = JSON.parse(await readFile(metaPath, "utf8"));
+ * assertSnapshotMeta(meta);
+ */
 export function assertSnapshotMeta(meta) {
   if (!meta || typeof meta !== "object") {
     throw new Error("Snapshot meta.json is missing or invalid.");
@@ -274,6 +463,15 @@ export function isLegacySnapshotMeta(meta) {
   return !meta?.site || tables.includes("setting");
 }
 
+/**
+ * The site a snapshot was exported from, for creating or remapping the target.
+ *
+ * @param {SnapshotMeta} meta Snapshot meta accepted by `assertSnapshotMeta`
+ * @returns {{ id: string, key: string } | undefined} The source site, or
+ *   undefined for a legacy snapshot, which records none
+ * @example
+ * const snapshotSite = getSnapshotBootstrapSite(meta);
+ */
 export function getSnapshotBootstrapSite(meta) {
   if (isLegacySnapshotMeta(meta)) {
     return undefined;
@@ -285,6 +483,18 @@ export function getSnapshotBootstrapSite(meta) {
   };
 }
 
+/**
+ * Refuse to import a snapshot into a site other than the one it came from.
+ *
+ * Legacy snapshots record no site and pass.
+ *
+ * @param {SnapshotMeta} meta Snapshot meta accepted by `assertSnapshotMeta`
+ * @param {{ id: string }} site The import target
+ * @returns {void}
+ * @throws {Error} When the snapshot names a different site
+ * @example
+ * validateSnapshotTargetSite(meta, targetSite);
+ */
 export function validateSnapshotTargetSite(meta, site) {
   if (isLegacySnapshotMeta(meta)) {
     return;
@@ -297,6 +507,17 @@ export function validateSnapshotTargetSite(meta, site) {
   }
 }
 
+/**
+ * Replace a snapshot's source site id with the target's throughout its SQL.
+ *
+ * @param {string} sql Snapshot `db.sql`
+ * @param {string} sourceSiteId Site the snapshot was exported from; empty
+ *   leaves the SQL unchanged
+ * @param {string} targetSiteId Site the snapshot is imported into
+ * @returns {string} The rewritten SQL
+ * @example
+ * rewriteSnapshotSiteIdentifiers(dbSql, snapshotSite.id, targetSite.id)
+ */
 export function rewriteSnapshotSiteIdentifiers(
   sql,
   sourceSiteId,
@@ -311,6 +532,17 @@ export function rewriteSnapshotSiteIdentifiers(
   return sql.replaceAll(escapedSource, escapedTarget);
 }
 
+/**
+ * Move a storage key from the source site's namespace to the target's.
+ *
+ * @param {string} key Storage key as exported
+ * @param {string} sourceSiteId Site the snapshot was exported from; empty
+ *   leaves the key unchanged
+ * @param {string} targetSiteId Site the snapshot is imported into
+ * @returns {string} The key to store the object under
+ * @example
+ * remapSnapshotObjectKey(entry.key, snapshotSite.id, targetSite.id)
+ */
 export function remapSnapshotObjectKey(key, sourceSiteId, targetSiteId) {
   if (!sourceSiteId || sourceSiteId === targetSiteId) {
     return key;
@@ -325,6 +557,12 @@ export function remapSnapshotObjectKey(key, sourceSiteId, targetSiteId) {
  * export time (with forward slashes). If the snapshot was produced by a
  * different site than the import target, callers apply
  * `remapSnapshotObjectKey()` before uploading.
+ *
+ * @param {string} rootDir Snapshot directory
+ * @returns {Promise<{ key: string, filePath: string, contentType: string }[]>}
+ *   Files sorted by key; empty when there is no `objects/` directory
+ * @example
+ * const objectFiles = await enumerateSnapshotObjectFiles(snapshotDir);
  */
 export async function enumerateSnapshotObjectFiles(rootDir) {
   const objectsRoot = join(rootDir, "objects");
@@ -485,6 +723,16 @@ function splitSqlStatements(sql) {
   return statements;
 }
 
+/**
+ * Scope a legacy single-site dump to a site: global `setting` rows become
+ * `site_setting` rows, and site-owned tables gain a `site_id` column.
+ *
+ * @param {string} sql Legacy snapshot `db.sql`
+ * @param {string} siteId Site the snapshot is imported into
+ * @returns {string} Site-scoped SQL
+ * @example
+ * rewriteLegacySnapshotSql(dbSql, targetSite.id)
+ */
 export function rewriteLegacySnapshotSql(sql, siteId) {
   const rewrittenStatements = splitSqlStatements(sql).map((statement) => {
     const normalized = statement.trim();
@@ -840,6 +1088,58 @@ function parseSqlScalar(raw) {
   return trimmed;
 }
 
+/**
+ * SQL clearing a site's snapshot content before a replacing import.
+ *
+ * @param {string} siteId Site to clear
+ * @returns {string} `DELETE` statements for every `SNAPSHOT_CLEAR_TABLES`
+ *   table and the snapshot's setting keys
+ * @example
+ * await execute(`${buildReplaceSql(site.id)}\n${dbSql}`);
+ */
+/**
+ * Order `post` rows so every row comes after its Thread root and the post it
+ * replies to.
+ *
+ * The dump reads posts by creation time, and `(site_id, thread_id)` and
+ * `(site_id, reply_to_id)` are foreign keys. A post moved into a Thread keeps
+ * its own creation time, and an import restores creation times, so a reply
+ * can be older than its root; inserted in creation order it names a row that
+ * isn't there yet, and the whole import fails. Otherwise the order is kept.
+ *
+ * @param {Record<string, unknown>[]} rows - `post` rows in dump order
+ * @returns {Record<string, unknown>[]} The same rows, parents first
+ * @example
+ * orderSnapshotPostRows([reply, root]); // [root, reply]
+ */
+export function orderSnapshotPostRows(rows) {
+  const byId = new Map(rows.map((row) => [row.id, row]));
+  const placed = new Set();
+  const ordered = [];
+
+  const place = (row, visiting) => {
+    if (placed.has(row.id) || visiting.has(row.id)) return;
+    visiting.add(row.id);
+    for (const parentId of [row.thread_id, row.reply_to_id]) {
+      const parent = parentId !== row.id ? byId.get(parentId) : undefined;
+      if (parent) place(parent, visiting);
+    }
+    placed.add(row.id);
+    ordered.push(row);
+  };
+
+  for (const row of rows) place(row, new Set());
+  return ordered;
+}
+
+/**
+ * SQL a snapshot import runs before its inserts on SQLite and D1: foreign
+ * keys are checked at commit rather than per row, so a snapshot written
+ * before posts were ordered parents-first still loads. Postgres has no
+ * equivalent for its non-deferrable keys; its snapshots are ordered at export.
+ */
+export const DEFER_FOREIGN_KEYS_SQL = "PRAGMA defer_foreign_keys = ON;";
+
 export function buildReplaceSql(siteId) {
   const statements = [];
 
@@ -856,6 +1156,41 @@ export function buildReplaceSql(siteId) {
   return statements.join("\n");
 }
 
+/** The storage drivers a `media` row can name as its `provider`. */
+const MEDIA_STORAGE_PROVIDERS = ["r2", "s3", "local"];
+
+/**
+ * SQL recording a site's media under the storage an import just wrote it to.
+ *
+ * A snapshot carries each media row's `provider` from the site it was exported
+ * from, but the import uploads every object into the target's own storage. A
+ * row exported on R2 and imported into S3 would still say `r2`, and everything
+ * that reads `provider` as where the bytes live goes wrong: the public URL looks
+ * for `R2_PUBLIC_URL`, replacing the avatar or favicon misses the existing row
+ * and inserts a second one for the same key, and a trashed object is never
+ * purged because the sweep only handles the active driver. Run after the
+ * snapshot's inserts, while every media row of the site is one the snapshot
+ * brought.
+ *
+ * @param {string} siteId - Site whose media was imported
+ * @param {string} provider - Storage driver the import uploaded through
+ * @returns {string} An `UPDATE` statement for the site's media rows
+ * @throws {Error} When `provider` is not one of r2, s3, or local
+ * @example
+ * await execute(
+ *   `${buildReplaceSql(site.id)}\n${dbSql}\n${buildMediaProviderSql(site.id, "s3")}`,
+ * );
+ */
+export function buildMediaProviderSql(siteId, provider) {
+  if (!MEDIA_STORAGE_PROVIDERS.includes(provider)) {
+    throw new Error(
+      `Snapshot import cannot record media under storage driver "${provider}". Expected one of: ${MEDIA_STORAGE_PROVIDERS.join(", ")}.`,
+    );
+  }
+
+  return `UPDATE "media" SET "provider" = '${provider}' WHERE "site_id" = '${escapeSqlString(siteId)}';`;
+}
+
 export function normalizeD1Sql(sql) {
   return sql
     .replace(/^\s*BEGIN(?:\s+TRANSACTION)?\s*;\s*$/gim, "")

package/bin/lib/sql-export.js

@@ -1,4 +1,17 @@
-const TABLE_EXPORT_ORDER = [
+/**
+ * The order rows are dumped, and so the order they are replayed on import.
+ *
+ * It is a foreign-key topological sort: a table follows everything it points
+ * at. `sortExportTables` applies it to whatever table list it is handed, so
+ * this — not the caller's list — is what decides an export's order.
+ *
+ * A table missing from here is not an ordering no-op. Unknown names sort after
+ * every known one, so a new child table lands at the end, behind its own
+ * parents, and the dump only fails once there is a row to violate the
+ * constraint. `snapshot-tables.test.ts` checks the sorted result, not the
+ * caller's list, for that reason.
+ */
+export const TABLE_EXPORT_ORDER = [
   "site",
   "user",
   "account",
@@ -8,11 +21,15 @@ const TABLE_EXPORT_ORDER = [
   "site_setting",
   "site_member",
   "collection",
-  "nav_item",
-  "collection_directory_item",
+  // Before the three tables that hold a key into it: `nav_item`,
+  // `collection_directory_item` and `path_registry`.
+  "smart_collection",
   "api_token",
   "post",
   "thread_collection",
+  // After `post`: a `page` nav item carries a `post_id`.
+  "nav_item",
+  "collection_directory_item",
   "path_registry",
   "media",
 ];
@@ -167,8 +184,48 @@ export async function getTableColumns(
     .map((row) => String(row.name));
 }
 
+/**
+ * Read an ordered SELECT a page at a time.
+ *
+ * `LIMIT … OFFSET …` is appended to the statement, so it must end in an
+ * ORDER BY with a unique tiebreaker. Pages are separate reads: rows written
+ * between two of them can be skipped or repeated, which is why a snapshot
+ * should be taken of a site nobody is writing to.
+ *
+ * @param {{ query(sql: string): Promise<Record<string, unknown>[]> }} queryRunner
+ * @param {string} selectSql - An ordered SELECT with no LIMIT
+ * @param {number} pageSize - Rows per read
+ * @returns {Promise<Record<string, unknown>[]>} Every row, in order
+ * @example
+ * await queryAllPages(runner, 'SELECT * FROM "post" ORDER BY rowid', 200);
+ */
+async function queryAllPages(queryRunner, selectSql, pageSize) {
+  const base = selectSql.trim();
+  const rows = [];
+  for (let offset = 0; ; offset += pageSize) {
+    const page = await queryRunner.query(
+      `${base} LIMIT ${pageSize} OFFSET ${offset}`,
+    );
+    rows.push(...page);
+    if (page.length < pageSize) return rows;
+  }
+}
+
+/**
+ * Dump tables as INSERT statements.
+ *
+ * `options.pageSize` reads each table in pages. D1 needs it: every query runs
+ * through `wrangler d1 execute --json`, and one SELECT of a real site's
+ * `post` table outgrew the output the CLI can buffer.
+ * `options.orderRowsByTable` reorders a table's rows before they are written,
+ * for inserts that must follow their own foreign keys.
+ */
 export async function dumpDatabaseToSql(queryRunner, options) {
   const dialect = options.dialect ?? "sqlite";
+  const pageSize =
+    Number.isInteger(options.pageSize) && options.pageSize > 0
+      ? options.pageSize
+      : null;
   const onProgress =
     typeof options.onProgress === "function" ? options.onProgress : null;
   const configuredTables = Array.isArray(options.tables)
@@ -194,8 +251,14 @@ export async function dumpDatabaseToSql(queryRunner, options) {
 
     const selectSql =
       options.selectSqlByTable?.[tableName] ||
-      `SELECT * FROM ${quoteIdentifier(tableName)}`;
-    const rows = await queryRunner.query(selectSql);
+      (pageSize && dialect === "sqlite"
+        ? `SELECT * FROM ${quoteIdentifier(tableName)} ORDER BY rowid`
+        : `SELECT * FROM ${quoteIdentifier(tableName)}`);
+    const readRows = pageSize
+      ? await queryAllPages(queryRunner, selectSql, pageSize)
+      : await queryRunner.query(selectSql);
+    const orderRows = options.orderRowsByTable?.[tableName];
+    const rows = orderRows ? orderRows(readRows) : readRows;
     if (rows.length === 0) {
       continue;
     }

package/bin/lib/wrangler-cli.js

@@ -48,12 +48,20 @@ function resolveWranglerBin(cwd = process.cwd()) {
   return resolve(dirname(manifestPath), binEntry);
 }
 
+/**
+ * The most output a Wrangler call may print before the CLI gives up on it.
+ * `d1 execute --json` prints whole result sets, and Node's default of 1 MiB
+ * overflowed on one page of a real site's posts.
+ */
+export const WRANGLER_MAX_BUFFER = 64 * 1024 * 1024;
+
 export function runLocalWrangler(args, options = {}) {
   const {
     cwd = process.cwd(),
     encoding = "utf-8",
     env = process.env,
     stdio = "pipe",
+    maxBuffer = WRANGLER_MAX_BUFFER,
     ...execOptions
   } = options;
 
@@ -63,5 +71,6 @@ export function runLocalWrangler(args, options = {}) {
     encoding,
     env,
     stdio,
+    maxBuffer,
   });
 }