@jant/core 0.3.46 → 0.3.47

package/bin/lib/site-snapshot.js

@@ -1,9 +1,8 @@
-import { createHash } from "node:crypto";
-import { readFile } from "node:fs/promises";
+import { readdir } from "node:fs/promises";
+import { join, relative } from "node:path";
 
 export const SNAPSHOT_FORMAT = "jant-site-snapshot";
 export const SNAPSHOT_VERSION = 1;
-export const SNAPSHOT_SCOPE = "content";
 
 export const SNAPSHOT_TABLES = [
   "site_setting",
@@ -47,11 +46,6 @@ export const SNAPSHOT_SETTING_KEYS = [
   "NOINDEX",
 ];
 
-export const SNAPSHOT_STORAGE_SETTING_KEYS = [
-  "SITE_AVATAR",
-  "SITE_FAVICON_APPLE_TOUCH",
-];
-
 function escapeSqlString(value) {
   return String(value).replaceAll("'", "''");
 }
@@ -143,16 +137,6 @@ export function buildSnapshotStorageQuery(siteId) {
       WHERE "poster_key" IS NOT NULL
         AND "site_id" = '${escapeSqlString(siteId)}'
         AND trim("poster_key") <> ''
-
-      UNION ALL
-
-      SELECT
-        "value" AS "key",
-        NULL AS "contentType"
-      FROM "site_setting"
-      WHERE "key" IN (${quoteList(SNAPSHOT_STORAGE_SETTING_KEYS)})
-        AND "site_id" = '${escapeSqlString(siteId)}'
-        AND trim("value") <> ''
     )
     WHERE "key" IS NOT NULL
       AND trim("key") <> ''
@@ -191,19 +175,24 @@ export function snapshotObjectPath(key) {
   return `objects/${key}`.replace(/\\/g, "/");
 }
 
-export function buildSnapshotMeta(source, site) {
+export const SNAPSHOT_DIALECTS = ["sqlite", "pg"];
+
+export function buildSnapshotMeta(site, options = {}) {
+  const dialect = options.dialect;
+  if (dialect && !SNAPSHOT_DIALECTS.includes(dialect)) {
+    throw new Error(
+      `Unsupported snapshot dialect: ${dialect}. Expected one of ${SNAPSHOT_DIALECTS.join(", ")}.`,
+    );
+  }
+
   return {
     format: SNAPSHOT_FORMAT,
     version: SNAPSHOT_VERSION,
-    scope: SNAPSHOT_SCOPE,
-    createdAt: new Date().toISOString(),
-    source,
+    ...(dialect ? { dialect } : {}),
     site: {
       id: site.id,
       key: site.key,
     },
-    tables: SNAPSHOT_TABLES,
-    settingKeys: SNAPSHOT_SETTING_KEYS,
   };
 }
 
@@ -224,9 +213,12 @@ export function assertSnapshotMeta(meta) {
     );
   }
 
-  if (meta.scope !== SNAPSHOT_SCOPE) {
+  if (
+    meta.dialect !== undefined &&
+    !SNAPSHOT_DIALECTS.includes(meta.dialect)
+  ) {
     throw new Error(
-      `Unsupported snapshot scope: expected ${SNAPSHOT_SCOPE}, got ${String(meta.scope)}`,
+      `Snapshot meta has unsupported dialect "${String(meta.dialect)}". Expected one of ${SNAPSHOT_DIALECTS.join(", ")}.`,
     );
   }
 
@@ -241,20 +233,39 @@ export function assertSnapshotMeta(meta) {
   }
 }
 
-export function assertSnapshotManifest(manifest) {
-  if (!manifest || typeof manifest !== "object") {
-    throw new Error("Snapshot storage-manifest.json is missing or invalid.");
-  }
+/**
+ * Read the snapshot's source dialect, if recorded.
+ *
+ * Older snapshots predate the `dialect` field — those return `undefined` and
+ * the caller decides whether to skip the check or refuse with a clear error.
+ */
+export function getSnapshotDialect(meta) {
+  return SNAPSHOT_DIALECTS.includes(meta?.dialect) ? meta.dialect : undefined;
+}
 
-  if (manifest.version !== SNAPSHOT_VERSION) {
-    throw new Error(
-      `Unsupported storage manifest version: expected ${SNAPSHOT_VERSION}, got ${String(manifest.version)}`,
-    );
+/**
+ * Refuse to apply a snapshot whose source dialect doesn't match the target.
+ *
+ * Cross-dialect db.sql is not safe to replay: SQLite and Postgres differ on
+ * BLOB literals (`X'...'` vs `'\x...'`), boolean encoding (`0/1` vs `t/f`),
+ * `tsvector`/`generated` columns, identifier quoting edge cases, etc. Better
+ * to fail at the start of import than mid-way with a cryptic SQL error.
+ */
+export function assertSnapshotDialectMatches(meta, targetDialect) {
+  const sourceDialect = getSnapshotDialect(meta);
+  if (!sourceDialect) {
+    return;
   }
 
-  if (!Array.isArray(manifest.objects)) {
+  if (sourceDialect !== targetDialect) {
     throw new Error(
-      "Snapshot storage-manifest.json must contain an objects array.",
+      [
+        `Snapshot dialect mismatch: source is ${sourceDialect}, target is ${targetDialect}.`,
+        "Snapshot db.sql is dialect-specific (BLOB literals, generated columns, FTS, etc.)",
+        "and cannot be replayed across SQLite and Postgres safely.",
+        "Use `jant site export <url>` (HTTP, dialect-neutral) to move content between",
+        "different DB engines.",
+      ].join("\n"),
     );
   }
 }
@@ -301,22 +312,55 @@ export function rewriteSnapshotSiteIdentifiers(
   return sql.replaceAll(escapedSource, escapedTarget);
 }
 
-export function remapSnapshotManifestObjects(
-  manifest,
-  sourceSiteId,
-  targetSiteId,
-) {
+export function remapSnapshotObjectKey(key, sourceSiteId, targetSiteId) {
   if (!sourceSiteId || sourceSiteId === targetSiteId) {
-    return manifest;
+    return key;
   }
+  return String(key).replaceAll(sourceSiteId, targetSiteId);
+}
 
-  return {
-    ...manifest,
-    objects: manifest.objects.map((object) => ({
-      ...object,
-      key: String(object.key).replaceAll(sourceSiteId, targetSiteId),
-    })),
-  };
+/**
+ * Walks `<rootDir>/objects/` recursively and returns one entry per file.
+ *
+ * The relative path inside `objects/` is the storage key as it existed at
+ * export time (with forward slashes). If the snapshot was produced by a
+ * different site than the import target, callers apply
+ * `remapSnapshotObjectKey()` before uploading.
+ */
+export async function enumerateSnapshotObjectFiles(rootDir) {
+  const objectsRoot = join(rootDir, "objects");
+  const entries = [];
+
+  async function walk(dir) {
+    let items;
+    try {
+      items = await readdir(dir, { withFileTypes: true });
+    } catch (error) {
+      if (error && error.code === "ENOENT") {
+        return;
+      }
+      throw error;
+    }
+
+    for (const item of items) {
+      const fullPath = join(dir, item.name);
+      if (item.isDirectory()) {
+        await walk(fullPath);
+        continue;
+      }
+
+      const key = relative(objectsRoot, fullPath).replace(/\\/g, "/");
+      entries.push({
+        key,
+        filePath: fullPath,
+        contentType: guessContentTypeFromKey(key),
+      });
+    }
+  }
+
+  await walk(objectsRoot);
+  entries.sort((a, b) => a.key.localeCompare(b.key));
+  return entries;
 }
 
 function prependSiteIdInsert(sql, tableName, siteId) {
@@ -407,6 +451,98 @@ export function rewriteLegacySnapshotSql(sql, siteId) {
   return `${rewrittenStatements.join(";\n")};\n`;
 }
 
+/**
+ * Pull the storage_key + poster_key values referenced by every media INSERT
+ * inside a snapshot's `db.sql`.
+ *
+ * The dump format is controlled by `dumpDatabaseToSql`, which produces
+ * `INSERT INTO "media" (col, ...) VALUES (val, ...);` statements with single
+ * quoted string literals. We use the SQL-aware splitter to chunk the dump,
+ * then parse each media INSERT via the column list. This is the import-side
+ * "what should be on storage after this snapshot lands" question, used by
+ * the preflight check that runs before db.sql is applied.
+ */
+export function extractMediaStorageKeysFromDumpSql(sql, sourceSiteId) {
+  const uncommented = sql
+    .split("\n")
+    .filter((line) => !line.trimStart().startsWith("--"))
+    .join("\n");
+  const statements = splitSqlStatements(uncommented);
+  const keys = new Set();
+
+  for (const statement of statements) {
+    const match = statement.match(
+      /INSERT\s+INTO\s+"?media"?\s*\(([^)]+)\)\s*VALUES\s*\(([\s\S]+)\)\s*;?\s*$/i,
+    );
+    if (!match) continue;
+
+    const colNames = match[1]
+      .split(",")
+      .map((col) => col.trim().replace(/^"|"$/g, ""));
+    const values = parseSqlValueList(match[2]);
+    if (values.length !== colNames.length) continue;
+
+    if (sourceSiteId) {
+      const siteIdIdx = colNames.indexOf("site_id");
+      if (siteIdIdx >= 0) {
+        const siteIdVal = parseSqlScalar(values[siteIdIdx]);
+        if (siteIdVal !== sourceSiteId) continue;
+      }
+    }
+
+    for (const col of ["storage_key", "poster_key"]) {
+      const idx = colNames.indexOf(col);
+      if (idx < 0) continue;
+      const value = parseSqlScalar(values[idx]);
+      if (typeof value === "string" && value.trim() !== "") {
+        keys.add(value);
+      }
+    }
+  }
+
+  return keys;
+}
+
+function parseSqlValueList(raw) {
+  const values = [];
+  let current = "";
+  let inString = false;
+
+  for (let index = 0; index < raw.length; index += 1) {
+    const char = raw[index];
+    if (char === "'") {
+      current += char;
+      if (inString && raw[index + 1] === "'") {
+        current += "'";
+        index += 1;
+        continue;
+      }
+      inString = !inString;
+      continue;
+    }
+    if (char === "," && !inString) {
+      values.push(current.trim());
+      current = "";
+      continue;
+    }
+    current += char;
+  }
+
+  const tail = current.trim();
+  if (tail) values.push(tail);
+  return values;
+}
+
+function parseSqlScalar(raw) {
+  if (raw === undefined) return null;
+  const trimmed = raw.trim();
+  if (trimmed === "" || /^null$/i.test(trimmed)) return null;
+  if (trimmed.startsWith("'") && trimmed.endsWith("'")) {
+    return trimmed.slice(1, -1).replaceAll("''", "'");
+  }
+  return trimmed;
+}
+
 export function buildReplaceSql(siteId) {
   const statements = [];
 
@@ -431,11 +567,6 @@ export function normalizeD1Sql(sql) {
     .trim();
 }
 
-export async function sha256File(filePath) {
-  const bytes = await readFile(filePath);
-  return createHash("sha256").update(bytes).digest("hex");
-}
-
 export function guessContentTypeFromKey(key) {
   const normalized = String(key).toLowerCase();
 

package/bin/lib/sql-export.js

@@ -136,29 +136,41 @@ export async function getTableColumns(
   dialect = "sqlite",
 ) {
   if (dialect === "pg") {
+    // Skip GENERATED ALWAYS columns (e.g. post.search_text, post.search_document):
+    // Postgres rejects any explicit value — even `DEFAULT` — on those, so they
+    // must not appear in the INSERT column list. The target instance recomputes
+    // them from the source columns when the row is inserted.
     const rows = await queryRunner.query(`
       SELECT column_name AS name
       FROM information_schema.columns
       WHERE table_schema = 'public'
         AND table_name = ${sqlValue(tableName)}
+        AND is_generated = 'NEVER'
       ORDER BY ordinal_position
     `);
 
     return rows.map((row) => String(row.name));
   }
 
+  // SQLite doesn't currently use generated columns in this codebase, but
+  // table_xinfo (a strict superset of table_info) exposes the `hidden`
+  // flag (2 = VIRTUAL generated, 3 = STORED generated) so we filter those
+  // out defensively if any are introduced later.
   const rows = await queryRunner.query(
-    `PRAGMA table_info(${quoteIdentifier(tableName)})`,
+    `PRAGMA table_xinfo(${quoteIdentifier(tableName)})`,
   );
 
   return rows
     .slice()
     .sort((left, right) => Number(left.cid) - Number(right.cid))
+    .filter((row) => Number(row.hidden) !== 2 && Number(row.hidden) !== 3)
     .map((row) => String(row.name));
 }
 
 export async function dumpDatabaseToSql(queryRunner, options) {
   const dialect = options.dialect ?? "sqlite";
+  const onProgress =
+    typeof options.onProgress === "function" ? options.onProgress : null;
   const configuredTables = Array.isArray(options.tables)
     ? sortExportTables(options.tables)
     : null;
@@ -169,7 +181,12 @@ export async function dumpDatabaseToSql(queryRunner, options) {
   sql += `-- Exported: ${timestamp}\n`;
   sql += `-- Source: ${options.source}\n\n`;
 
-  for (const tableName of tables) {
+  for (const [tableIndex, tableName] of tables.entries()) {
+    onProgress?.({
+      index: tableIndex + 1,
+      total: tables.length,
+      table: tableName,
+    });
     const columnNames = await getTableColumns(queryRunner, tableName, dialect);
     if (columnNames.length === 0) {
       continue;