@jant/core 0.3.45 → 0.3.47

package/bin/commands/site/snapshot/export.js

@@ -5,7 +5,6 @@ import {
   readFile,
   readdir,
   rm,
-  stat,
   writeFile,
 } from "node:fs/promises";
 import { tmpdir } from "node:os";
@@ -24,15 +23,14 @@ import {
   buildSnapshotStorageQuery,
   collectSnapshotObjects,
   getSnapshotSelectSql,
-  sha256File,
   SNAPSHOT_TABLES,
   snapshotObjectPath,
 } from "../../../lib/site-snapshot.js";
 import { resolveCliSite } from "../../../lib/site-selection.js";
 import { dumpDatabaseToSql } from "../../../lib/sql-export.js";
 import {
+  bootstrapCliRuntime,
   getCliRuntimeLabel,
-  resolveCliRuntime,
 } from "../../../lib/runtime-target.js";
 import { resolveWranglerVarString } from "../../../lib/wrangler-config.js";
 
@@ -99,8 +97,12 @@ async function assertWritableOutput(outputPath, force) {
 
 async function createNodeExportContext() {
   const nodeDatabase = await openNodeDatabase(process.env);
-  const { createNodeCliRuntime } = await loadNodeRuntime();
-  const runtime = await createNodeCliRuntime(nodeDatabase.bindings);
+  // Only need the storage driver — `createNodeCliRuntime` would also resolve
+  // the current site, which (a) is redundant with the bin-level resolveCliSite
+  // call below and (b) prints a generic "/setup first" error when the
+  // snapshot's own error path is more informative.
+  const { createStorageDriver } = await loadNodeRuntime();
+  const storage = createStorageDriver(nodeDatabase.bindings);
 
   return {
     dialect: nodeDatabase.database.dialect,
@@ -111,11 +113,11 @@ async function createNodeExportContext() {
       return nodeDatabase.query(sql);
     },
     async downloadObject(key, filePath) {
-      if (!runtime.storage) {
+      if (!storage) {
         throw new Error("Snapshot export requires configured storage.");
       }
 
-      const object = await runtime.storage.get(key);
+      const object = await storage.get(key);
       if (!object?.body) {
         throw new Error(`Storage object not found: ${key}`);
       }
@@ -178,6 +180,7 @@ export async function run(argv) {
       host: { type: "string" },
       help: { type: "boolean", short: "h" },
       local: { type: "boolean", default: false },
+      node: { type: "boolean", default: false },
       output: {
         type: "string",
         short: "o",
@@ -187,13 +190,14 @@ export async function run(argv) {
       "persist-to": { type: "string" },
       remote: { type: "boolean", default: false },
       site: { type: "string" },
+      "skip-objects": { type: "boolean", default: false },
       url: { type: "string" },
     },
   });
 
   if (values.help) {
     console.log(
-      "Usage: jant site snapshot export [--local | --remote] [--output <dir|zip>]",
+      "Usage: jant site snapshot export [--local | --remote | --node] [--output <dir|zip>]",
     );
     console.log("");
     console.log(
@@ -205,6 +209,9 @@ export async function run(argv) {
       "  --local                 Force local D1 instead of DATABASE_URL",
     );
     console.log("  --remote                Export from remote D1");
+    console.log(
+      "  --node                  Force Node runtime even if DATABASE_URL is unset",
+    );
     console.log(
       "  --output, -o           Output directory or .zip file (default: jant-site-snapshot)",
     );
@@ -227,14 +234,33 @@ export async function run(argv) {
     console.log(
       "  --persist-to            Local D1/R2 state directory override",
     );
+    console.log(
+      "  --skip-objects          Skip downloading storage objects. The archive only contains meta.json and db.sql.",
+    );
+    console.log(
+      "                          Only safe when the import target's storage already has the same keys",
+    );
+    console.log(
+      "                          (e.g. moving between Workers that share an R2 bucket). Otherwise the",
+    );
+    console.log(
+      "                          imported site will be missing media — pair with `--allow-missing-objects`",
+    );
+    console.log("                          on import.");
     console.log("");
     console.log(
-      "If DATABASE_URL or DATA_DIR is set and no runtime flag is passed, this command uses the Node database runtime and the configured storage driver.",
+      "`.env.node` next to your project (or in packages/core/) is auto-loaded.",
+    );
+    console.log(
+      "If DATABASE_URL or DATA_DIR is then set and no runtime flag is passed,",
+    );
+    console.log(
+      "this command uses the Node database runtime and configured storage driver.",
     );
     process.exit(0);
   }
 
-  const runtime = resolveCliRuntime(values);
+  const { runtime } = bootstrapCliRuntime(values);
   const outputPath = resolve(process.cwd(), values.output);
   const shouldZip = isZipPath(outputPath);
   const scratchDir = shouldZip
@@ -259,6 +285,7 @@ export async function run(argv) {
       url: values.url,
     });
 
+    console.log(`Dumping database (${SNAPSHOT_TABLES.length} tables)...`);
     const dbSql = await dumpDatabaseToSql(
       {
         query(sql) {
@@ -275,55 +302,41 @@ export async function run(argv) {
             getSnapshotSelectSql(tableName, site.id),
           ]),
         ),
+        onProgress: ({ index, total, table }) => {
+          console.log(`  [${index}/${total}] ${table}`);
+        },
       },
     );
 
+    console.log("Listing storage objects...");
     const objectRows = await context.query(buildSnapshotStorageQuery(site.id));
     const objects = collectSnapshotObjects(objectRows);
-    const manifestObjects = [];
 
     await writeFile(join(scratchDir, "db.sql"), dbSql);
 
-    if (objects.length > 0) {
-      console.log(`Downloading ${objects.length} referenced object(s)...`);
-    }
+    if (values["skip-objects"]) {
+      if (objects.length > 0) {
+        console.log(
+          `--skip-objects: leaving ${objects.length} referenced object(s) out of the archive.`,
+        );
+      }
+    } else {
+      if (objects.length > 0) {
+        console.log(`Downloading ${objects.length} referenced object(s)...`);
+      }
 
-    for (const [index, object] of objects.entries()) {
-      const relativeObjectPath = snapshotObjectPath(object.key);
-      const absoluteObjectPath = join(scratchDir, relativeObjectPath);
-      console.log(`[${index + 1}/${objects.length}] ${object.key}`);
-      await context.downloadObject(object.key, absoluteObjectPath);
-      const fileStat = await stat(absoluteObjectPath);
-      manifestObjects.push({
-        key: object.key,
-        file: relativeObjectPath,
-        contentType: object.contentType || undefined,
-        size: fileStat.size,
-        sha256: await sha256File(absoluteObjectPath),
-      });
+      for (const [index, object] of objects.entries()) {
+        const relativeObjectPath = snapshotObjectPath(object.key);
+        const absoluteObjectPath = join(scratchDir, relativeObjectPath);
+        console.log(`[${index + 1}/${objects.length}] ${object.key}`);
+        await context.downloadObject(object.key, absoluteObjectPath);
+      }
     }
 
     await writeFile(
       join(scratchDir, "meta.json"),
       JSON.stringify(
-        buildSnapshotMeta(
-          {
-            runtime,
-            label: getCliRuntimeLabel(runtime),
-          },
-          site,
-        ),
-        null,
-        2,
-      ) + "\n",
-    );
-    await writeFile(
-      join(scratchDir, "storage-manifest.json"),
-      JSON.stringify(
-        {
-          version: 1,
-          objects: manifestObjects,
-        },
+        buildSnapshotMeta(site, { dialect: context.dialect }),
         null,
         2,
       ) + "\n",

package/bin/commands/site/snapshot/import.js

@@ -16,24 +16,25 @@ import { loadNodeRuntime } from "../../../lib/load-node-runtime.js";
 import { openNodeDatabase } from "../../../lib/node-database.js";
 import { deleteR2Object, uploadR2Object } from "../../../lib/r2-query.js";
 import {
-  assertSnapshotManifest,
+  assertSnapshotDialectMatches,
   assertSnapshotMeta,
   buildReplaceSql,
   buildSnapshotStorageQuery,
   collectSnapshotObjects,
+  enumerateSnapshotObjectFiles,
+  extractMediaStorageKeysFromDumpSql,
   getSnapshotBootstrapSite,
   normalizeD1Sql,
-  remapSnapshotManifestObjects,
+  remapSnapshotObjectKey,
   rewriteLegacySnapshotSql,
   rewriteSnapshotSiteIdentifiers,
-  sha256File,
   validateSnapshotTargetSite,
 } from "../../../lib/site-snapshot.js";
 import {
   getCliSiteResolutionMode,
   resolveCliSite,
 } from "../../../lib/site-selection.js";
-import { resolveCliRuntime } from "../../../lib/runtime-target.js";
+import { bootstrapCliRuntime } from "../../../lib/runtime-target.js";
 
 function isZipPath(filePath) {
   return filePath.toLowerCase().endsWith(".zip");
@@ -52,10 +53,15 @@ function createWranglerOptions(values) {
 
 async function createNodeImportContext() {
   const nodeDatabase = await openNodeDatabase(process.env);
-  const { createNodeCliRuntime } = await loadNodeRuntime();
-  const runtime = await createNodeCliRuntime(nodeDatabase.bindings);
+  // Only need the storage driver — `createNodeCliRuntime` would also resolve
+  // the current site, which (a) is redundant with the bin-level resolveCliSite
+  // call below and (b) prints a generic "/setup first" error when the
+  // snapshot's own error path is more informative.
+  const { createStorageDriver } = await loadNodeRuntime();
+  const storage = createStorageDriver(nodeDatabase.bindings);
 
   return {
+    dialect: nodeDatabase.database.dialect,
     async close() {
       await nodeDatabase.close();
     },
@@ -66,20 +72,20 @@ async function createNodeImportContext() {
       await nodeDatabase.execute(sql);
     },
     async uploadObject(key, filePath, contentType) {
-      if (!runtime.storage) {
+      if (!storage) {
         throw new Error("Snapshot import requires configured storage.");
       }
 
       const bytes = new Uint8Array(await readFile(filePath));
-      await runtime.storage.put(key, bytes, {
+      await storage.put(key, bytes, {
         contentType: contentType || undefined,
       });
     },
     async deleteObject(key) {
-      if (!runtime.storage) {
+      if (!storage) {
         return;
       }
-      await runtime.storage.delete(key);
+      await storage.delete(key);
     },
   };
 }
@@ -88,6 +94,7 @@ function createD1ImportContext(runtime, values) {
   const wranglerOptions = createWranglerOptions(values);
 
   return {
+    dialect: "sqlite",
     async close() {},
     async query(sql) {
       return queryD1(sql, runtime, wranglerOptions);
@@ -155,23 +162,6 @@ async function readSnapshotJson(rootDir, filename) {
   return JSON.parse(await readFile(absolutePath, "utf-8"));
 }
 
-async function validateManifestObjects(rootDir, manifest) {
-  for (const object of manifest.objects) {
-    const absolutePath = join(rootDir, object.file);
-    const fileStat = await stat(absolutePath).catch(() => null);
-    if (!fileStat?.isFile()) {
-      throw new Error(`Snapshot object file is missing: ${object.file}`);
-    }
-
-    const actualHash = await sha256File(absolutePath);
-    if (actualHash !== object.sha256) {
-      throw new Error(
-        `Snapshot object checksum mismatch for ${object.key}: expected ${object.sha256}, got ${actualHash}`,
-      );
-    }
-  }
-}
-
 export async function run(argv) {
   const { values } = parseArgs({
     args: argv,
@@ -184,12 +174,14 @@ export async function run(argv) {
       host: { type: "string" },
       help: { type: "boolean", short: "h" },
       local: { type: "boolean", default: false },
+      node: { type: "boolean", default: false },
       path: { type: "string", default: "." },
       "path-prefix": { type: "string" },
       "persist-to": { type: "string" },
       remote: { type: "boolean", default: false },
       replace: { type: "boolean", default: false },
       "remap-site": { type: "boolean", default: false },
+      "allow-missing-objects": { type: "boolean", default: false },
       site: { type: "string" },
       url: { type: "string" },
     },
@@ -197,7 +189,7 @@ export async function run(argv) {
 
   if (values.help) {
     console.log(
-      "Usage: jant site snapshot import --path <dir|zip> --replace [--local | --remote]",
+      "Usage: jant site snapshot import --path <dir|zip> --replace [--local | --remote | --node]",
     );
     console.log("");
     console.log(
@@ -219,6 +211,9 @@ export async function run(argv) {
       "  --local                 Force local D1 instead of DATABASE_URL",
     );
     console.log("  --remote                Import into remote D1");
+    console.log(
+      "  --node                  Force Node runtime even if DATABASE_URL is unset",
+    );
     console.log(
       "  --config                Wrangler config file (default: wrangler.toml)",
     );
@@ -236,6 +231,27 @@ export async function run(argv) {
     console.log(
       "  --remap-site            Rewrite snapshot site_id and storage keys to the resolved target site",
     );
+    console.log(
+      "  --allow-missing-objects Continue importing even when objects/ is missing files referenced by db.sql.",
+    );
+    console.log(
+      "                          Use this when the target storage already has those keys (e.g. a snapshot",
+    );
+    console.log(
+      "                          exported with --skip-objects between sites that share an R2 bucket).",
+    );
+    console.log(
+      "                          Without this flag, import aborts before applying db.sql and prints the",
+    );
+    console.log("                          missing key list.");
+    console.log("");
+    console.log(
+      "`.env.node` next to your project (or in packages/core/) is auto-loaded.",
+    );
+    console.log(
+      "If DATABASE_URL or DATA_DIR is then set and no runtime flag is passed,",
+    );
+    console.log("this command uses the Node database runtime.");
     console.log("");
     console.log(
       "In single-site mode, snapshot imports automatically remap to the only initialized site.",
@@ -253,7 +269,7 @@ export async function run(argv) {
     );
   }
 
-  const runtime = resolveCliRuntime(values);
+  const { runtime } = bootstrapCliRuntime(values);
   const inputPath = resolve(process.cwd(), values.path);
   const materialized = await materializeSnapshotInput(inputPath);
   const context =
@@ -263,16 +279,10 @@ export async function run(argv) {
 
   try {
     const meta = await readSnapshotJson(materialized.rootDir, "meta.json");
-    const rawManifest = await readSnapshotJson(
-      materialized.rootDir,
-      "storage-manifest.json",
-    );
     assertSnapshotMeta(meta);
+    assertSnapshotDialectMatches(meta, context.dialect);
     const explicitRemap = values["remap-site"] === true;
     const snapshotSite = getSnapshotBootstrapSite(meta);
-    const originalManifest = rawManifest;
-    assertSnapshotManifest(originalManifest);
-    await validateManifestObjects(materialized.rootDir, originalManifest);
     const resolutionMode = getCliSiteResolutionMode(process.env);
 
     const { site: targetSite } = await resolveCliSite(context, {
@@ -307,17 +317,63 @@ export async function run(argv) {
       );
     }
 
-    const manifest = shouldRemapSite
-      ? remapSnapshotManifestObjects(
-          originalManifest,
-          snapshotSite?.id ?? "",
-          targetSite.id,
-        )
-      : originalManifest;
+    const sourceSiteId = shouldRemapSite ? (snapshotSite?.id ?? "") : "";
+    const objectFiles = await enumerateSnapshotObjectFiles(
+      materialized.rootDir,
+    );
+    const snapshotObjects = objectFiles.map((entry) => ({
+      filePath: entry.filePath,
+      contentType: entry.contentType,
+      key: shouldRemapSite
+        ? remapSnapshotObjectKey(entry.key, sourceSiteId, targetSite.id)
+        : entry.key,
+    }));
+    const snapshotKeys = new Set(snapshotObjects.map((object) => object.key));
+
+    const rawDbSql = await readFile(
+      join(materialized.rootDir, "db.sql"),
+      "utf-8",
+    );
 
-    const snapshotKeys = new Set(
-      manifest.objects.map((object) => String(object.key)),
+    // Preflight: every storage_key/poster_key referenced by db.sql must have
+    // a corresponding file in objects/, or we'll end up with broken media
+    // unless the target storage already has it. Default to abort; let the
+    // user override with --allow-missing-objects when they know the files
+    // already live in the target bucket (typical --skip-objects flow).
+    const sourceKeysInObjects = new Set(objectFiles.map((entry) => entry.key));
+    const expectedSourceKeys = extractMediaStorageKeysFromDumpSql(
+      rawDbSql,
+      snapshotSite?.id ?? "",
     );
+    const missingFromObjects = [...expectedSourceKeys]
+      .filter((key) => !sourceKeysInObjects.has(key))
+      .sort();
+
+    if (missingFromObjects.length > 0) {
+      const display = missingFromObjects.slice(0, 10);
+      const remainder = missingFromObjects.length - display.length;
+      console.warn(
+        `\n${missingFromObjects.length} object(s) referenced by db.sql are missing from the snapshot's objects/:`,
+      );
+      for (const key of display) {
+        console.warn(`  ${key}`);
+      }
+      if (remainder > 0) {
+        console.warn(`  …and ${remainder} more`);
+      }
+
+      if (!values["allow-missing-objects"]) {
+        throw new Error(
+          "Snapshot is missing storage objects referenced by db.sql. " +
+            "Pass --allow-missing-objects to import anyway (only safe when the target storage already has these keys).",
+        );
+      }
+
+      console.warn(
+        "Continuing with --allow-missing-objects; the target storage must already have these keys or media references will 404.\n",
+      );
+    }
+
     const currentObjectRows = await context.query(
       buildSnapshotStorageQuery(targetSite.id),
     );
@@ -325,18 +381,14 @@ export async function run(argv) {
       collectSnapshotObjects(currentObjectRows).map((object) => object.key),
     );
 
-    for (const object of manifest.objects) {
+    for (const object of snapshotObjects) {
       await context.uploadObject(
         object.key,
-        join(materialized.rootDir, object.file),
-        typeof object.contentType === "string" ? object.contentType : "",
+        object.filePath,
+        object.contentType || "",
       );
     }
 
-    const rawDbSql = await readFile(
-      join(materialized.rootDir, "db.sql"),
-      "utf-8",
-    );
     const dbSql = snapshotSite
       ? shouldRemapSite
         ? rewriteSnapshotSiteIdentifiers(

package/bin/lib/node-env.js

@@ -0,0 +1,100 @@
+import { readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+function stripSurroundingQuotes(value) {
+  if (value.length < 2) return value;
+  const first = value[0];
+  const last = value[value.length - 1];
+  if ((first === '"' && last === '"') || (first === "'" && last === "'")) {
+    return value.slice(1, -1);
+  }
+  return value;
+}
+
+function fileExists(envPath) {
+  try {
+    readFileSync(envPath, "utf8");
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Locate `.env.node` for CLI auto-load. Searches, in order:
+ *   1. `<cwd>/.env.node`             — user's site directory
+ *   2. `<bin>/../../.env.node`        — `packages/core/.env.node` (in-repo dev)
+ *
+ * Returns the first existing path, or `null` if none is found.
+ */
+export function findNodeEnvPath(cwd = process.cwd()) {
+  const candidates = [
+    resolve(cwd, ".env.node"),
+    resolve(dirname(fileURLToPath(import.meta.url)), "../../.env.node"),
+  ];
+
+  for (const candidate of candidates) {
+    if (fileExists(candidate)) {
+      return candidate;
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Parse the .env.node file and assign keys into `env`. Existing values in
+ * `env` are preserved (already-exported shell vars win over file values).
+ *
+ * Returns a result object useful for debug logging:
+ *   { envPath, found, assignedKeys, skippedKeys }
+ */
+export function loadNodeEnvFile(envPath, env = process.env) {
+  const result = {
+    envPath,
+    found: false,
+    assignedKeys: [],
+    skippedKeys: [],
+  };
+
+  let content;
+  try {
+    content = readFileSync(envPath, "utf8");
+  } catch {
+    return result;
+  }
+
+  result.found = true;
+  for (const line of content.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed || trimmed.startsWith("#")) continue;
+    const eqIdx = trimmed.indexOf("=");
+    if (eqIdx < 1) continue;
+    const key = trimmed.slice(0, eqIdx).trim();
+    const value = stripSurroundingQuotes(trimmed.slice(eqIdx + 1).trim());
+    if (key in env) {
+      result.skippedKeys.push(key);
+      continue;
+    }
+    env[key] = value;
+    result.assignedKeys.push(key);
+  }
+
+  return result;
+}
+
+/**
+ * Auto-locate and load `.env.node` for any DB-touching CLI command.
+ *
+ * Always called before `resolveCliRuntime()`, so DATABASE_URL / DATA_DIR
+ * defined in `.env.node` make `--node` (or auto-detect) work without
+ * requiring the user to source the file manually.
+ */
+export function autoloadNodeEnv(env = process.env) {
+  const envPath = findNodeEnvPath();
+  if (!envPath) {
+    return { envPath: null, found: false, assignedKeys: [], skippedKeys: [] };
+  }
+  return loadNodeEnvFile(envPath, env);
+}

package/bin/lib/runtime-target.js

@@ -1,3 +1,6 @@
+import { autoloadNodeEnv } from "./node-env.js";
+import { resolveDatabaseDialect } from "./node-sqlite.js";
+
 export function resolveCliRuntime(values, env = process.env) {
   const flags = [values.local, values.remote, values.node].filter(Boolean);
   if (flags.length > 1) {
@@ -36,3 +39,64 @@ export function getCliRuntimeLabel(runtime) {
       return "Node database";
   }
 }
+
+function describeNodeTarget(env) {
+  const databaseUrl = env.DATABASE_URL;
+  if (typeof databaseUrl === "string" && databaseUrl.length > 0) {
+    const dialect = resolveDatabaseDialect(databaseUrl);
+    if (dialect === "sqlite") {
+      return `sqlite ${databaseUrl}`;
+    }
+
+    try {
+      const parsed = new URL(databaseUrl);
+      const protocol = parsed.protocol.replace(/:$/, "");
+      const host = parsed.hostname || "?";
+      const port = parsed.port ? `:${parsed.port}` : "";
+      const database = parsed.pathname.replace(/^\/+/, "") || "?";
+      return `${protocol} ${host}${port}/${database}`;
+    } catch {
+      return "<invalid DATABASE_URL>";
+    }
+  }
+
+  if (typeof env.DATA_DIR === "string" && env.DATA_DIR.length > 0) {
+    return `DATA_DIR=${env.DATA_DIR}`;
+  }
+
+  return "<unset>";
+}
+
+export function formatRuntimeBanner(runtime, env = process.env) {
+  switch (runtime) {
+    case "node":
+      return `[jant] target = node (${describeNodeTarget(env)})`;
+    case "d1-remote":
+      return "[jant] target = remote D1 (wrangler)";
+    case "d1-local":
+    default:
+      return "[jant] target = local D1 (wrangler)";
+  }
+}
+
+/**
+ * One-call helper for DB-touching CLI commands:
+ *   1. Auto-load `.env.node` (so DATABASE_URL/DATA_DIR work without sourcing).
+ *   2. Resolve the runtime from flags and env.
+ *   3. Print a one-line banner so the user immediately sees which target
+ *      was picked, instead of finding out minutes later when something fails.
+ *
+ * Pass `{ silent: true }` to suppress the banner (e.g. in tests or when the
+ * caller wants to print its own header first).
+ */
+export function bootstrapCliRuntime(values, options = {}) {
+  const env = options.env ?? process.env;
+  const envLoad = autoloadNodeEnv(env);
+  const runtime = resolveCliRuntime(values, env);
+
+  if (!options.silent) {
+    console.log(formatRuntimeBanner(runtime, env));
+  }
+
+  return { runtime, envLoad };
+}