@jant/core 0.3.46 → 0.3.47

package/bin/commands/site/snapshot/import.js

@@ -16,24 +16,25 @@ import { loadNodeRuntime } from "../../../lib/load-node-runtime.js";
 import { openNodeDatabase } from "../../../lib/node-database.js";
 import { deleteR2Object, uploadR2Object } from "../../../lib/r2-query.js";
 import {
-  assertSnapshotManifest,
+  assertSnapshotDialectMatches,
   assertSnapshotMeta,
   buildReplaceSql,
   buildSnapshotStorageQuery,
   collectSnapshotObjects,
+  enumerateSnapshotObjectFiles,
+  extractMediaStorageKeysFromDumpSql,
   getSnapshotBootstrapSite,
   normalizeD1Sql,
-  remapSnapshotManifestObjects,
+  remapSnapshotObjectKey,
   rewriteLegacySnapshotSql,
   rewriteSnapshotSiteIdentifiers,
-  sha256File,
   validateSnapshotTargetSite,
 } from "../../../lib/site-snapshot.js";
 import {
   getCliSiteResolutionMode,
   resolveCliSite,
 } from "../../../lib/site-selection.js";
-import { resolveCliRuntime } from "../../../lib/runtime-target.js";
+import { bootstrapCliRuntime } from "../../../lib/runtime-target.js";
 
 function isZipPath(filePath) {
   return filePath.toLowerCase().endsWith(".zip");
@@ -52,10 +53,15 @@ function createWranglerOptions(values) {
 
 async function createNodeImportContext() {
   const nodeDatabase = await openNodeDatabase(process.env);
-  const { createNodeCliRuntime } = await loadNodeRuntime();
-  const runtime = await createNodeCliRuntime(nodeDatabase.bindings);
+  // Only need the storage driver — `createNodeCliRuntime` would also resolve
+  // the current site, which (a) is redundant with the bin-level resolveCliSite
+  // call below and (b) prints a generic "/setup first" error when the
+  // snapshot's own error path is more informative.
+  const { createStorageDriver } = await loadNodeRuntime();
+  const storage = createStorageDriver(nodeDatabase.bindings);
 
   return {
+    dialect: nodeDatabase.database.dialect,
     async close() {
       await nodeDatabase.close();
     },
@@ -66,20 +72,20 @@ async function createNodeImportContext() {
       await nodeDatabase.execute(sql);
     },
     async uploadObject(key, filePath, contentType) {
-      if (!runtime.storage) {
+      if (!storage) {
         throw new Error("Snapshot import requires configured storage.");
       }
 
       const bytes = new Uint8Array(await readFile(filePath));
-      await runtime.storage.put(key, bytes, {
+      await storage.put(key, bytes, {
         contentType: contentType || undefined,
       });
     },
     async deleteObject(key) {
-      if (!runtime.storage) {
+      if (!storage) {
         return;
       }
-      await runtime.storage.delete(key);
+      await storage.delete(key);
     },
   };
 }
@@ -88,6 +94,7 @@ function createD1ImportContext(runtime, values) {
   const wranglerOptions = createWranglerOptions(values);
 
   return {
+    dialect: "sqlite",
     async close() {},
     async query(sql) {
       return queryD1(sql, runtime, wranglerOptions);
@@ -155,23 +162,6 @@ async function readSnapshotJson(rootDir, filename) {
   return JSON.parse(await readFile(absolutePath, "utf-8"));
 }
 
-async function validateManifestObjects(rootDir, manifest) {
-  for (const object of manifest.objects) {
-    const absolutePath = join(rootDir, object.file);
-    const fileStat = await stat(absolutePath).catch(() => null);
-    if (!fileStat?.isFile()) {
-      throw new Error(`Snapshot object file is missing: ${object.file}`);
-    }
-
-    const actualHash = await sha256File(absolutePath);
-    if (actualHash !== object.sha256) {
-      throw new Error(
-        `Snapshot object checksum mismatch for ${object.key}: expected ${object.sha256}, got ${actualHash}`,
-      );
-    }
-  }
-}
-
 export async function run(argv) {
   const { values } = parseArgs({
     args: argv,
@@ -184,12 +174,14 @@ export async function run(argv) {
       host: { type: "string" },
       help: { type: "boolean", short: "h" },
       local: { type: "boolean", default: false },
+      node: { type: "boolean", default: false },
       path: { type: "string", default: "." },
       "path-prefix": { type: "string" },
       "persist-to": { type: "string" },
       remote: { type: "boolean", default: false },
       replace: { type: "boolean", default: false },
       "remap-site": { type: "boolean", default: false },
+      "allow-missing-objects": { type: "boolean", default: false },
       site: { type: "string" },
       url: { type: "string" },
     },
@@ -197,7 +189,7 @@ export async function run(argv) {
 
   if (values.help) {
     console.log(
-      "Usage: jant site snapshot import --path <dir|zip> --replace [--local | --remote]",
+      "Usage: jant site snapshot import --path <dir|zip> --replace [--local | --remote | --node]",
     );
     console.log("");
     console.log(
@@ -219,6 +211,9 @@ export async function run(argv) {
       "  --local                 Force local D1 instead of DATABASE_URL",
     );
     console.log("  --remote                Import into remote D1");
+    console.log(
+      "  --node                  Force Node runtime even if DATABASE_URL is unset",
+    );
     console.log(
       "  --config                Wrangler config file (default: wrangler.toml)",
     );
@@ -236,6 +231,27 @@ export async function run(argv) {
     console.log(
       "  --remap-site            Rewrite snapshot site_id and storage keys to the resolved target site",
     );
+    console.log(
+      "  --allow-missing-objects Continue importing even when objects/ is missing files referenced by db.sql.",
+    );
+    console.log(
+      "                          Use this when the target storage already has those keys (e.g. a snapshot",
+    );
+    console.log(
+      "                          exported with --skip-objects between sites that share an R2 bucket).",
+    );
+    console.log(
+      "                          Without this flag, import aborts before applying db.sql and prints the",
+    );
+    console.log("                          missing key list.");
+    console.log("");
+    console.log(
+      "`.env.node` next to your project (or in packages/core/) is auto-loaded.",
+    );
+    console.log(
+      "If DATABASE_URL or DATA_DIR is then set and no runtime flag is passed,",
+    );
+    console.log("this command uses the Node database runtime.");
     console.log("");
     console.log(
       "In single-site mode, snapshot imports automatically remap to the only initialized site.",
@@ -253,7 +269,7 @@ export async function run(argv) {
     );
   }
 
-  const runtime = resolveCliRuntime(values);
+  const { runtime } = bootstrapCliRuntime(values);
   const inputPath = resolve(process.cwd(), values.path);
   const materialized = await materializeSnapshotInput(inputPath);
   const context =
@@ -263,16 +279,10 @@ export async function run(argv) {
 
   try {
     const meta = await readSnapshotJson(materialized.rootDir, "meta.json");
-    const rawManifest = await readSnapshotJson(
-      materialized.rootDir,
-      "storage-manifest.json",
-    );
     assertSnapshotMeta(meta);
+    assertSnapshotDialectMatches(meta, context.dialect);
     const explicitRemap = values["remap-site"] === true;
     const snapshotSite = getSnapshotBootstrapSite(meta);
-    const originalManifest = rawManifest;
-    assertSnapshotManifest(originalManifest);
-    await validateManifestObjects(materialized.rootDir, originalManifest);
     const resolutionMode = getCliSiteResolutionMode(process.env);
 
     const { site: targetSite } = await resolveCliSite(context, {
@@ -307,17 +317,63 @@ export async function run(argv) {
       );
     }
 
-    const manifest = shouldRemapSite
-      ? remapSnapshotManifestObjects(
-          originalManifest,
-          snapshotSite?.id ?? "",
-          targetSite.id,
-        )
-      : originalManifest;
+    const sourceSiteId = shouldRemapSite ? (snapshotSite?.id ?? "") : "";
+    const objectFiles = await enumerateSnapshotObjectFiles(
+      materialized.rootDir,
+    );
+    const snapshotObjects = objectFiles.map((entry) => ({
+      filePath: entry.filePath,
+      contentType: entry.contentType,
+      key: shouldRemapSite
+        ? remapSnapshotObjectKey(entry.key, sourceSiteId, targetSite.id)
+        : entry.key,
+    }));
+    const snapshotKeys = new Set(snapshotObjects.map((object) => object.key));
+
+    const rawDbSql = await readFile(
+      join(materialized.rootDir, "db.sql"),
+      "utf-8",
+    );
 
-    const snapshotKeys = new Set(
-      manifest.objects.map((object) => String(object.key)),
+    // Preflight: every storage_key/poster_key referenced by db.sql must have
+    // a corresponding file in objects/, or we'll end up with broken media
+    // unless the target storage already has it. Default to abort; let the
+    // user override with --allow-missing-objects when they know the files
+    // already live in the target bucket (typical --skip-objects flow).
+    const sourceKeysInObjects = new Set(objectFiles.map((entry) => entry.key));
+    const expectedSourceKeys = extractMediaStorageKeysFromDumpSql(
+      rawDbSql,
+      snapshotSite?.id ?? "",
     );
+    const missingFromObjects = [...expectedSourceKeys]
+      .filter((key) => !sourceKeysInObjects.has(key))
+      .sort();
+
+    if (missingFromObjects.length > 0) {
+      const display = missingFromObjects.slice(0, 10);
+      const remainder = missingFromObjects.length - display.length;
+      console.warn(
+        `\n${missingFromObjects.length} object(s) referenced by db.sql are missing from the snapshot's objects/:`,
+      );
+      for (const key of display) {
+        console.warn(`  ${key}`);
+      }
+      if (remainder > 0) {
+        console.warn(`  …and ${remainder} more`);
+      }
+
+      if (!values["allow-missing-objects"]) {
+        throw new Error(
+          "Snapshot is missing storage objects referenced by db.sql. " +
+            "Pass --allow-missing-objects to import anyway (only safe when the target storage already has these keys).",
+        );
+      }
+
+      console.warn(
+        "Continuing with --allow-missing-objects; the target storage must already have these keys or media references will 404.\n",
+      );
+    }
+
     const currentObjectRows = await context.query(
       buildSnapshotStorageQuery(targetSite.id),
     );
@@ -325,18 +381,14 @@ export async function run(argv) {
       collectSnapshotObjects(currentObjectRows).map((object) => object.key),
     );
 
-    for (const object of manifest.objects) {
+    for (const object of snapshotObjects) {
       await context.uploadObject(
         object.key,
-        join(materialized.rootDir, object.file),
-        typeof object.contentType === "string" ? object.contentType : "",
+        object.filePath,
+        object.contentType || "",
       );
     }
 
-    const rawDbSql = await readFile(
-      join(materialized.rootDir, "db.sql"),
-      "utf-8",
-    );
     const dbSql = snapshotSite
       ? shouldRemapSite
         ? rewriteSnapshotSiteIdentifiers(

package/bin/lib/node-env.js

@@ -0,0 +1,100 @@
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+function stripSurroundingQuotes(value) {
+  if (value.length < 2) return value;
+  const first = value[0];
+  const last = value[value.length - 1];
+  if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+    return value.slice(1, -1);
+  }
+  return value;
+}
+
+function fileExists(envPath) {
+  try {
+    readFileSync(envPath, "utf8");
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Locate `.env.node` for CLI auto-load. Searches, in order:
+ *   1. `<cwd>/.env.node`             — user's site directory
+ *   2. `<bin>/../../.env.node`        — `packages/core/.env.node` (in-repo dev)
+ *
+ * Returns the first existing path, or `null` if none is found.
+ */
+export function findNodeEnvPath(cwd = process.cwd()) {
+  const candidates = [
+    resolve(cwd, ".env.node"),
+    resolve(dirname(fileURLToPath(import.meta.url)), "../../.env.node"),
+  ];
+
+  for (const candidate of candidates) {
+    if (fileExists(candidate)) {
+      return candidate;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Parse the .env.node file and assign keys into `env`. Existing values in
+ * `env` are preserved (already-exported shell vars win over file values).
+ *
+ * Returns a result object useful for debug logging:
+ *   { envPath, found, assignedKeys, skippedKeys }
+ */
+export function loadNodeEnvFile(envPath, env = process.env) {
+  const result = {
+    envPath,
+    found: false,
+    assignedKeys: [],
+    skippedKeys: [],
+  };
+
+  let content;
+  try {
+    content = readFileSync(envPath, "utf8");
+  } catch {
+    return result;
+  }
+
+  result.found = true;
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const eqIdx = trimmed.indexOf("=");
+    if (eqIdx < 1) continue;
+    const key = trimmed.slice(0, eqIdx).trim();
+    const value = stripSurroundingQuotes(trimmed.slice(eqIdx + 1).trim());
+    if (key in env) {
+      result.skippedKeys.push(key);
+      continue;
+    }
+    env[key] = value;
+    result.assignedKeys.push(key);
+  }
+
+  return result;
+}
+
+/**
+ * Auto-locate and load `.env.node` for any DB-touching CLI command.
+ *
+ * Always called before `resolveCliRuntime()`, so DATABASE_URL / DATA_DIR
+ * defined in `.env.node` make `--node` (or auto-detect) work without
+ * requiring the user to source the file manually.
+ */
+export function autoloadNodeEnv(env = process.env) {
+  const envPath = findNodeEnvPath();
+  if (!envPath) {
+    return { envPath: null, found: false, assignedKeys: [], skippedKeys: [] };
+  }
+  return loadNodeEnvFile(envPath, env);
+}

package/bin/lib/runtime-target.js

@@ -1,3 +1,6 @@
+import { autoloadNodeEnv } from "./node-env.js";
+import { resolveDatabaseDialect } from "./node-sqlite.js";
+
 export function resolveCliRuntime(values, env = process.env) {
   const flags = [values.local, values.remote, values.node].filter(Boolean);
   if (flags.length > 1) {
@@ -36,3 +39,64 @@ export function getCliRuntimeLabel(runtime) {
       return "Node database";
   }
 }
+
+function describeNodeTarget(env) {
+  const databaseUrl = env.DATABASE_URL;
+  if (typeof databaseUrl === "string" && databaseUrl.length > 0) {
+    const dialect = resolveDatabaseDialect(databaseUrl);
+    if (dialect === "sqlite") {
+      return `sqlite ${databaseUrl}`;
+    }
+
+    try {
+      const parsed = new URL(databaseUrl);
+      const protocol = parsed.protocol.replace(/:$/, "");
+      const host = parsed.hostname || "?";
+      const port = parsed.port ? `:${parsed.port}` : "";
+      const database = parsed.pathname.replace(/^\/+/, "") || "?";
+      return `${protocol} ${host}${port}/${database}`;
+    } catch {
+      return "<invalid DATABASE_URL>";
+    }
+  }
+
+  if (typeof env.DATA_DIR === "string" && env.DATA_DIR.length > 0) {
+    return `DATA_DIR=${env.DATA_DIR}`;
+  }
+
+  return "<unset>";
+}
+
+export function formatRuntimeBanner(runtime, env = process.env) {
+  switch (runtime) {
+    case "node":
+      return `[jant] target = node (${describeNodeTarget(env)})`;
+    case "d1-remote":
+      return "[jant] target = remote D1 (wrangler)";
+    case "d1-local":
+    default:
+      return "[jant] target = local D1 (wrangler)";
+  }
+}
+
+/**
+ * One-call helper for DB-touching CLI commands:
+ *   1. Auto-load `.env.node` (so DATABASE_URL/DATA_DIR work without sourcing).
+ *   2. Resolve the runtime from flags and env.
+ *   3. Print a one-line banner so the user immediately sees which target
+ *      was picked, instead of finding out minutes later when something fails.
+ *
+ * Pass `{ silent: true }` to suppress the banner (e.g. in tests or when the
+ * caller wants to print its own header first).
+ */
+export function bootstrapCliRuntime(values, options = {}) {
+  const env = options.env ?? process.env;
+  const envLoad = autoloadNodeEnv(env);
+  const runtime = resolveCliRuntime(values, env);
+
+  if (!options.silent) {
+    console.log(formatRuntimeBanner(runtime, env));
+  }
+
+  return { runtime, envLoad };
+}