@jant/core 0.6.16 → 0.7.1

package/bin/commands/reset-password.js

@@ -7,7 +7,7 @@ import {
   bootstrapCliRuntime,
   getCliRuntimeLabel,
 } from "../lib/runtime-target.js";
-import { resolveCliSite } from "../lib/site-selection.js";
+import { parseCliSiteSelector, resolveCliSite } from "../lib/site-selection.js";
 
 export async function run(argv) {
   const { values } = parseArgs({
@@ -29,7 +29,9 @@ export async function run(argv) {
   });
 
   if (values.help) {
-    console.log("Usage: jant reset-password [--local | --remote | --node]");
+    console.log(
+      "Usage: jant reset-password [--local | --remote | --node] [--site <key|id> | --host <host> | --url <url>]",
+    );
     console.log("");
     console.log("Generate a password reset token (expires in 15 minutes).");
     console.log("");
@@ -37,7 +39,7 @@ export async function run(argv) {
     console.log("  --local   Force local D1 instead of DATABASE_URL");
     console.log("  --remote  Run against remote D1 database (default: local)");
     console.log("  --node    Force Node runtime even if DATABASE_URL is unset");
-    console.log("  --site    Target site id");
+    console.log("  --site    Target site key or id");
     console.log("  --host    Target site host");
     console.log("  --url     Target site URL");
     console.log("  --path-prefix Path prefix used with --host");
@@ -53,10 +55,20 @@ export async function run(argv) {
       "If DATABASE_URL or DATA_DIR is then set and no runtime flag is passed,",
     );
     console.log("this command uses the Node database runtime.");
+    console.log("");
+    console.log(
+      "With SITE_RESOLUTION_MODE=host-based, pass one of --site, --host, or --url.",
+    );
     process.exit(0);
   }
 
   const { runtime } = bootstrapCliRuntime(values);
+  const siteFlags = {
+    host: values.host,
+    pathPrefix: values["path-prefix"],
+    site: values.site,
+    url: values.url,
+  };
 
   const token = randomBytes(32).toString("hex");
   const hash = createHash("sha256").update(token).digest("hex");
@@ -65,11 +77,15 @@ export async function run(argv) {
   const timestamp = Math.floor(Date.now() / 1000);
 
   if (runtime === "node") {
+    const siteSelector = parseCliSiteSelector(siteFlags);
     const { createNodeCliRuntime } = await loadNodeRuntime();
     const nodeDatabase = await openNodeDatabase(process.env);
 
     try {
-      const nodeRuntime = await createNodeCliRuntime(nodeDatabase.bindings);
+      const nodeRuntime = await createNodeCliRuntime(
+        nodeDatabase.bindings,
+        siteSelector,
+      );
       await nodeRuntime.services.settings.set("PASSWORD_RESET_TOKEN", value);
     } finally {
       await nodeDatabase.close();
@@ -95,10 +111,7 @@ export async function run(argv) {
       },
       {
         env: process.env,
-        host: values.host,
-        pathPrefix: values["path-prefix"],
-        site: values.site,
-        url: values.url,
+        ...siteFlags,
       },
     );
 

package/bin/commands/setup.js

@@ -0,0 +1,136 @@
+import { parseArgs } from "node:util";
+import { loadNodeRuntime } from "../lib/load-node-runtime.js";
+import { openNodeDatabase } from "../lib/node-database.js";
+import { bootstrapCliRuntime } from "../lib/runtime-target.js";
+
+function printHelp() {
+  console.log(
+    "Usage: jant setup --email <address> --password-stdin [--site-name <name>] [--language <tag>] [--time-zone <zone>] [--site-id <id>]",
+  );
+  console.log("");
+  console.log(
+    "Set up a self-hosted install without the browser: the owner account, the site, and onboarding.",
+  );
+  console.log("Does the same as the two /setup screens.");
+  console.log("");
+  console.log("Options:");
+  console.log("  --email           The owner's sign-in address");
+  console.log(
+    "  --password-stdin  Read the owner's password from standard input",
+  );
+  console.log(
+    "  --site-name       The site's name (default: the built-in fallback name)",
+  );
+  console.log(
+    "  --language        Language the site publishes in, as a BCP 47 tag (default: en)",
+  );
+  console.log("  --time-zone       The site's IANA time zone (default: UTC)");
+  console.log(
+    "  --site-id         Create the site with this id instead of a random one",
+  );
+  console.log(
+    "  --node            Force Node runtime even if DATABASE_URL is unset",
+  );
+  console.log("");
+  console.log("Safe to run on every start. An install that has finished setup");
+  console.log(
+    "is left unchanged, so a password changed since then is not reset.",
+  );
+  console.log("");
+  console.log("Example:");
+  console.log(
+    `  printf '%s' "$OWNER_PASSWORD" | jant setup --email owner@example.com --password-stdin`,
+  );
+  console.log("");
+  console.log(
+    "Node runtime and SITE_RESOLUTION_MODE=single-site only. `.env.node` next to",
+  );
+  console.log("your project (or in packages/core/) is auto-loaded.");
+}
+
+/**
+ * Read a secret piped to the command, the way `docker login --password-stdin`
+ * does: everything on the stream, minus the one line break `echo` adds.
+ *
+ * @param stream - Where the secret arrives
+ * @returns The secret
+ */
+async function readPasswordFromStdin(stream) {
+  if (stream.isTTY) {
+    throw new Error(
+      "--password-stdin reads the password from a pipe. Example: printf '%s' \"$OWNER_PASSWORD\" | jant setup --email <address> --password-stdin",
+    );
+  }
+
+  const chunks = [];
+  for await (const chunk of stream) {
+    chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
+  }
+  return Buffer.concat(chunks)
+    .toString("utf8")
+    .replace(/\r?\n$/, "");
+}
+
+const OUTCOME_MESSAGES = {
+  created: (siteId) => `Set up site ${siteId}.`,
+  resumed: (siteId) =>
+    `Finished setting up site ${siteId}, which an earlier run had started.`,
+  "already-set-up": (siteId) =>
+    `Site ${siteId} was already set up. Nothing changed.`,
+};
+
+export async function run(argv, io = { stdin: process.stdin }) {
+  const { values } = parseArgs({
+    args: argv,
+    options: {
+      email: { type: "string" },
+      help: { type: "boolean", short: "h" },
+      language: { type: "string" },
+      node: { type: "boolean", default: false },
+      "password-stdin": { type: "boolean", default: false },
+      "site-id": { type: "string" },
+      "site-name": { type: "string" },
+      "time-zone": { type: "string" },
+    },
+  });
+
+  if (values.help) {
+    printHelp();
+    process.exit(0);
+  }
+
+  if (!values.email) {
+    throw new Error("Pass the owner's sign-in address with --email.");
+  }
+  if (!values["password-stdin"]) {
+    throw new Error(
+      "Pass --password-stdin and pipe the owner's password in. The password is never read from an argument, where it would show up in the process list.",
+    );
+  }
+
+  const { runtime } = bootstrapCliRuntime(values);
+  if (runtime !== "node") {
+    throw new Error(
+      "jant setup runs against a Node database. Set DATABASE_URL or DATA_DIR, or pass --node.",
+    );
+  }
+
+  const password = await readPasswordFromStdin(io.stdin);
+  const { setUpNodeInstance } = await loadNodeRuntime();
+  const nodeDatabase = await openNodeDatabase(process.env);
+
+  try {
+    const result = await setUpNodeInstance(nodeDatabase.bindings, {
+      email: values.email,
+      password,
+      siteId: values["site-id"],
+      siteName: values["site-name"],
+      siteLanguage: values.language,
+      timeZone: values["time-zone"],
+    });
+    console.log(OUTCOME_MESSAGES[result.outcome](result.siteId));
+    return result;
+  } finally {
+    await nodeDatabase.close();
+  }
+}

package/bin/commands/site/export.js

@@ -1,14 +1,26 @@
-import { mkdirSync, readdirSync, writeFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
+import { createWriteStream, mkdirSync, readdirSync } from "node:fs";
+import { mkdir, mkdtemp, rename, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
 import { parseArgs } from "node:util";
-import { unzipSync } from "fflate";
 import {
   CLI_API_TOKEN_ENV_VAR,
   getCliApiToken,
 } from "../../lib/cli-api-token.js";
-import { pullSiteExportZipBytes } from "../../lib/site-pull-media.js";
+import { pullSiteExportDirectory } from "../../lib/site-pull-media.js";
+import { extractZipFile, writeDirectoryToZip } from "../../lib/zip-archive.js";
 
-async function exportRemoteSite(url, token) {
+/**
+ * Stream the site's export archive to a file. The site streams it too, and a
+ * site that bundles its media answers with gigabytes.
+ *
+ * @param {string} url - Site URL
+ * @param {string} token - API token
+ * @param {string} destination - File to write
+ */
+async function downloadSiteArchive(url, token, destination) {
   const response = await fetch(`${url.replace(/\/$/, "")}/api/export/hugo`, {
     method: "POST",
     headers: {
@@ -16,12 +28,16 @@ async function exportRemoteSite(url, token) {
     },
   });
 
-  if (!response.ok) {
+  if (!response.ok || !response.body) {
     const text = await response.text();
     throw new Error(`HTTP ${response.status}: ${text}`);
   }
 
-  return new Uint8Array(await response.arrayBuffer());
+  await mkdir(dirname(destination), { recursive: true });
+  await pipeline(
+    Readable.fromWeb(response.body),
+    createWriteStream(destination),
+  );
 }
 
 function describeProgressUrl(value) {
@@ -161,22 +177,8 @@ export async function run(argv) {
     process.exit(1);
   }
 
-  console.log(`Exporting site from ${url}...`);
-
-  const zipBytes = await exportRemoteSite(url, token);
-  let zip = zipBytes;
-  let pullStats = null;
-
-  if (pullMedia) {
-    console.log("Preparing pull-media export ZIP...");
-    const pulled = await pullSiteExportZipBytes(zip, {
-      assetLoader: null,
-      logger: logPullProgress,
-    });
-    zip = pulled.zipBytes;
-    pullStats = pulled.stats;
-  }
-
+  // Fail before the download, not after it: pulling a real site's media
+  // takes minutes.
   if (outputDirectory) {
     let existingEntries = [];
     try {
@@ -194,20 +196,55 @@ export async function run(argv) {
       );
       process.exit(1);
     }
+  }
 
-    console.log(`Writing export directory ${values.directory}...`);
-    const files = unzipSync(zip);
-    for (const [relativePath, bytes] of Object.entries(files)) {
-      const fullPath = resolve(outputDirectory, relativePath);
-      mkdirSync(dirname(fullPath), { recursive: true });
-      writeFileSync(fullPath, Buffer.from(bytes));
-    }
-    console.log(`Exported site from ${url} to ${values.directory}`);
-    console.log(`Preview with: cd ${values.directory} && hugo serve`);
-  } else {
+  console.log(`Exporting site from ${url}...`);
+
+  // Nothing to add: the site's archive is the export.
+  if (!pullMedia && !outputDirectory) {
     console.log(`Writing ${values.output}...`);
-    writeFileSync(output, Buffer.from(zip));
+    const partial = `${output}.partial`;
+    try {
+      await downloadSiteArchive(url, token, partial);
+      await rename(partial, output);
+    } catch (error) {
+      await rm(partial, { force: true });
+      throw error;
+    }
     console.log(`Exported site from ${url} to ${values.output}`);
+    return;
+  }
+
+  // Unpacked to disk and media downloaded next to it, so the export never
+  // holds more than one media file in memory.
+  const tempDir = await mkdtemp(join(tmpdir(), "jant-site-export-"));
+  const workDir = outputDirectory ?? join(tempDir, "site");
+  let pullStats = null;
+
+  try {
+    const archivePath = join(tempDir, "site.zip");
+    await downloadSiteArchive(url, token, archivePath);
+    await extractZipFile(archivePath, workDir);
+    await rm(archivePath, { force: true });
+
+    if (pullMedia) {
+      console.log("Pulling media...");
+      pullStats = await pullSiteExportDirectory(workDir, {
+        assetLoader: null,
+        logger: logPullProgress,
+      });
+    }
+
+    if (outputDirectory) {
+      console.log(`Exported site from ${url} to ${values.directory}`);
+      console.log(`Preview with: cd ${values.directory} && hugo serve`);
+    } else {
+      console.log(`Writing ${values.output}...`);
+      await writeDirectoryToZip(workDir, output);
+      console.log(`Exported site from ${url} to ${values.output}`);
+    }
+  } finally {
+    await rm(tempDir, { recursive: true, force: true });
   }
 
   if (pullStats) {

package/bin/commands/site/pull-media.js

@@ -1,10 +1,9 @@
-import { readFileSync, writeFileSync } from "node:fs";
 import { stat } from "node:fs/promises";
 import { resolve } from "node:path";
 import { parseArgs } from "node:util";
 import {
   pullSiteExportDirectory,
-  pullSiteExportZipBytes,
+  pullSiteExportZipFile,
 } from "../../lib/site-pull-media.js";
 
 function describeProgressUrl(value) {
@@ -102,13 +101,10 @@ export async function run(argv) {
   }
 
   const outputPath = resolve(process.cwd(), values.output || values.path);
-  const inputBytes = new Uint8Array(readFileSync(inputPath));
   console.log(`Pulling media in ${values.path}...`);
-  const { zipBytes, stats } = await pullSiteExportZipBytes(inputBytes, {
+  const stats = await pullSiteExportZipFile(inputPath, outputPath, {
     logger: logPullProgress,
   });
-  console.log(`Writing ${values.output || values.path}...`);
-  writeFileSync(outputPath, Buffer.from(zipBytes));
   console.log(
     `Pulled media in ${values.path} -> ${values.output || values.path}`,
   );

package/bin/commands/site/snapshot/export.js

@@ -1,17 +1,12 @@
-import { existsSync } from "node:fs";
-import {
-  mkdir,
-  mkdtemp,
-  readFile,
-  readdir,
-  rm,
-  writeFile,
-} from "node:fs/promises";
+import { createWriteStream, existsSync } from "node:fs";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { dirname, join, relative, resolve } from "node:path";
+import { dirname, join, resolve } from "node:path";
 import { parseArgs } from "node:util";
-import { zipSync } from "fflate";
-import { queryD1 } from "../../../lib/d1-query.js";
+import { writeDirectoryToZip } from "../../../lib/zip-archive.js";
+import { D1_DUMP_PAGE_SIZE, queryD1 } from "../../../lib/d1-query.js";
 import { loadNodeRuntime } from "../../../lib/load-node-runtime.js";
 import { openNodeDatabase } from "../../../lib/node-database.js";
 import {
@@ -23,6 +18,7 @@ import {
   buildSnapshotStorageQuery,
   collectSnapshotObjects,
   getSnapshotSelectSql,
+  orderSnapshotPostRows,
   SNAPSHOT_TABLES,
   snapshotObjectPath,
 } from "../../../lib/site-snapshot.js";
@@ -38,49 +34,6 @@ function isZipPath(filePath) {
   return filePath.toLowerCase().endsWith(".zip");
 }
 
-async function readStorageBody(body) {
-  const reader = body.getReader();
-  const chunks = [];
-  let totalLength = 0;
-
-  for (;;) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    chunks.push(value);
-    totalLength += value.length;
-  }
-
-  const bytes = new Uint8Array(totalLength);
-  let offset = 0;
-  for (const chunk of chunks) {
-    bytes.set(chunk, offset);
-    offset += chunk.length;
-  }
-
-  return bytes;
-}
-
-async function readDirectoryEntries(rootDir) {
-  const entries = {};
-
-  async function walk(dir) {
-    const items = await readdir(dir, { withFileTypes: true });
-    for (const item of items) {
-      const fullPath = join(dir, item.name);
-      if (item.isDirectory()) {
-        await walk(fullPath);
-        continue;
-      }
-
-      const relativePath = relative(rootDir, fullPath).replace(/\\/g, "/");
-      entries[relativePath] = new Uint8Array(await readFile(fullPath));
-    }
-  }
-
-  await walk(rootDir);
-  return entries;
-}
-
 async function assertWritableOutput(outputPath, force) {
   if (!existsSync(outputPath)) {
     return;
@@ -123,7 +76,12 @@ async function createNodeExportContext() {
       }
 
       await mkdir(dirname(filePath), { recursive: true });
-      await writeFile(filePath, await readStorageBody(object.body));
+      // Straight to disk: a snapshot holds every media file, and collecting
+      // each one in memory first ran a 3 GB export to a 2 GB peak.
+      await pipeline(
+        Readable.fromWeb(object.body),
+        createWriteStream(filePath),
+      );
     },
   };
 }
@@ -215,7 +173,7 @@ export async function run(argv) {
     console.log(
       "  --output, -o           Output directory or .zip file (default: jant-site-snapshot)",
     );
-    console.log("  --site                  Target site id");
+    console.log("  --site                  Target site key or id");
     console.log("  --host                  Target site host");
     console.log("  --url                   Target site URL");
     console.log("  --path-prefix           Path prefix used with --host");
@@ -295,6 +253,9 @@ export async function run(argv) {
       {
         dialect: context.dialect,
         source: runtime,
+        // D1 answers through Wrangler's buffered output; read it in pages.
+        pageSize: runtime === "node" ? undefined : D1_DUMP_PAGE_SIZE,
+        orderRowsByTable: { post: orderSnapshotPostRows },
         tables: SNAPSHOT_TABLES,
         selectSqlByTable: Object.fromEntries(
           SNAPSHOT_TABLES.map((tableName) => [
@@ -344,10 +305,7 @@ export async function run(argv) {
 
     if (shouldZip) {
       await mkdir(dirname(outputPath), { recursive: true });
-      const zipped = zipSync(await readDirectoryEntries(scratchDir), {
-        level: 6,
-      });
-      await writeFile(outputPath, zipped);
+      await writeDirectoryToZip(scratchDir, outputPath);
       if (process.env.SNAPSHOT_SUPPRESS_SUCCESS_LOG !== "true") {
         console.log(
           `Exported ${getCliRuntimeLabel(runtime)} snapshot to ${values.output}`,

package/bin/commands/site/snapshot/import.js

@@ -1,16 +1,10 @@
-import { existsSync } from "node:fs";
-import {
-  mkdir,
-  mkdtemp,
-  readFile,
-  rm,
-  stat,
-  writeFile,
-} from "node:fs/promises";
+import { createReadStream, existsSync } from "node:fs";
+import { Readable } from "node:stream";
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { dirname, join, resolve } from "node:path";
+import { join, resolve } from "node:path";
 import { parseArgs } from "node:util";
-import { unzipSync } from "fflate";
+import { extractZipFile } from "../../../lib/zip-archive.js";
 import { executeD1, queryD1 } from "../../../lib/d1-query.js";
 import { loadNodeRuntime } from "../../../lib/load-node-runtime.js";
 import { openNodeDatabase } from "../../../lib/node-database.js";
@@ -18,6 +12,8 @@ import { deleteR2Object, uploadR2Object } from "../../../lib/r2-query.js";
 import {
   assertSnapshotDialectMatches,
   assertSnapshotMeta,
+  buildMediaProviderSql,
+  DEFER_FOREIGN_KEYS_SQL,
   buildReplaceSql,
   buildSnapshotStorageQuery,
   collectSnapshotObjects,
@@ -58,11 +54,13 @@ async function createNodeImportContext() {
   // the current site, which (a) is redundant with the bin-level resolveCliSite
   // call below and (b) prints a generic "/setup first" error when the
   // snapshot's own error path is more informative.
-  const { createStorageDriver } = await loadNodeRuntime();
+  const { createStorageDriver, getConfiguredStorageDriver } =
+    await loadNodeRuntime();
   const storage = createStorageDriver(nodeDatabase.bindings);
 
   return {
     dialect: nodeDatabase.database.dialect,
+    storageProvider: getConfiguredStorageDriver(nodeDatabase.bindings),
     async close() {
       await nodeDatabase.close();
     },
@@ -77,8 +75,7 @@ async function createNodeImportContext() {
         throw new Error("Snapshot import requires configured storage.");
       }
 
-      const bytes = new Uint8Array(await readFile(filePath));
-      await storage.put(key, bytes, {
+      await storage.put(key, Readable.toWeb(createReadStream(filePath)), {
         contentType: contentType || undefined,
       });
     },
@@ -96,6 +93,8 @@ function createD1ImportContext(runtime, values) {
 
   return {
     dialect: "sqlite",
+    // Objects go up through the wrangler R2 binding below.
+    storageProvider: "r2",
     async close() {},
     async query(sql) {
       return queryD1(sql, runtime, wranglerOptions);
@@ -141,14 +140,7 @@ async function materializeSnapshotInput(inputPath) {
   }
 
   const outputDir = await mkdtemp(join(tmpdir(), "jant-site-snapshot-import-"));
-  const bytes = new Uint8Array(await readFile(inputPath));
-  const files = unzipSync(bytes);
-
-  for (const [relativePath, data] of Object.entries(files)) {
-    const absolutePath = join(outputDir, relativePath);
-    await mkdir(dirname(absolutePath), { recursive: true });
-    await writeFile(absolutePath, data);
-  }
+  await extractZipFile(inputPath, outputDir);
 
   return {
     cleanup: async () => {
@@ -204,7 +196,7 @@ export async function run(argv) {
     console.log(
       "  --replace               Replace the current content scope before importing",
     );
-    console.log("  --site                  Target site id");
+    console.log("  --site                  Target site key or id");
     console.log("  --host                  Target site host");
     console.log("  --url                   Target site URL");
     console.log("  --path-prefix           Path prefix used with --host");
@@ -402,7 +394,14 @@ export async function run(argv) {
       );
     }
 
-    await context.execute(`${buildReplaceSql(targetSite.id)}\n${dbSql}`);
+    await context.execute(
+      [
+        ...(context.dialect === "postgres" ? [] : [DEFER_FOREIGN_KEYS_SQL]),
+        buildReplaceSql(targetSite.id),
+        dbSql,
+        buildMediaProviderSql(targetSite.id, context.storageProvider),
+      ].join("\n"),
+    );
 
     const keysToDelete = [...currentKeys].filter(
       (key) => !snapshotKeys.has(key),