@jant/core 0.7.0 → 0.7.1

package/bin/commands/setup.js

@@ -0,0 +1,136 @@
+import { parseArgs } from "node:util";
+import { loadNodeRuntime } from "../lib/load-node-runtime.js";
+import { openNodeDatabase } from "../lib/node-database.js";
+import { bootstrapCliRuntime } from "../lib/runtime-target.js";
+
+function printHelp() {
+  console.log(
+    "Usage: jant setup --email <address> --password-stdin [--site-name <name>] [--language <tag>] [--time-zone <zone>] [--site-id <id>]",
+  );
+  console.log("");
+  console.log(
+    "Set up a self-hosted install without the browser: the owner account, the site, and onboarding.",
+  );
+  console.log("Does the same as the two /setup screens.");
+  console.log("");
+  console.log("Options:");
+  console.log("  --email           The owner's sign-in address");
+  console.log(
+    "  --password-stdin  Read the owner's password from standard input",
+  );
+  console.log(
+    "  --site-name       The site's name (default: the built-in fallback name)",
+  );
+  console.log(
+    "  --language        Language the site publishes in, as a BCP 47 tag (default: en)",
+  );
+  console.log("  --time-zone       The site's IANA time zone (default: UTC)");
+  console.log(
+    "  --site-id         Create the site with this id instead of a random one",
+  );
+  console.log(
+    "  --node            Force Node runtime even if DATABASE_URL is unset",
+  );
+  console.log("");
+  console.log("Safe to run on every start. An install that has finished setup");
+  console.log(
+    "is left unchanged, so a password changed since then is not reset.",
+  );
+  console.log("");
+  console.log("Example:");
+  console.log(
+    `  printf '%s' "$OWNER_PASSWORD" | jant setup --email owner@example.com --password-stdin`,
+  );
+  console.log("");
+  console.log(
+    "Node runtime and SITE_RESOLUTION_MODE=single-site only. `.env.node` next to",
+  );
+  console.log("your project (or in packages/core/) is auto-loaded.");
+}
+
+/**
+ * Read a secret piped to the command, the way `docker login --password-stdin`
+ * does: everything on the stream, minus the one line break `echo` adds.
+ *
+ * @param stream - Where the secret arrives
+ * @returns The secret
+ */
+async function readPasswordFromStdin(stream) {
+  if (stream.isTTY) {
+    throw new Error(
+      "--password-stdin reads the password from a pipe. Example: printf '%s' \"$OWNER_PASSWORD\" | jant setup --email <address> --password-stdin",
+    );
+  }
+
+  const chunks = [];
+  for await (const chunk of stream) {
+    chunks.push(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
+  }
+  return Buffer.concat(chunks)
+    .toString("utf8")
+    .replace(/\r?\n$/, "");
+}
+
+const OUTCOME_MESSAGES = {
+  created: (siteId) => `Set up site ${siteId}.`,
+  resumed: (siteId) =>
+    `Finished setting up site ${siteId}, which an earlier run had started.`,
+  "already-set-up": (siteId) =>
+    `Site ${siteId} was already set up. Nothing changed.`,
+};
+
+export async function run(argv, io = { stdin: process.stdin }) {
+  const { values } = parseArgs({
+    args: argv,
+    options: {
+      email: { type: "string" },
+      help: { type: "boolean", short: "h" },
+      language: { type: "string" },
+      node: { type: "boolean", default: false },
+      "password-stdin": { type: "boolean", default: false },
+      "site-id": { type: "string" },
+      "site-name": { type: "string" },
+      "time-zone": { type: "string" },
+    },
+  });
+
+  if (values.help) {
+    printHelp();
+    process.exit(0);
+  }
+
+  if (!values.email) {
+    throw new Error("Pass the owner's sign-in address with --email.");
+  }
+  if (!values["password-stdin"]) {
+    throw new Error(
+      "Pass --password-stdin and pipe the owner's password in. The password is never read from an argument, where it would show up in the process list.",
+    );
+  }
+
+  const { runtime } = bootstrapCliRuntime(values);
+  if (runtime !== "node") {
+    throw new Error(
+      "jant setup runs against a Node database. Set DATABASE_URL or DATA_DIR, or pass --node.",
+    );
+  }
+
+  const password = await readPasswordFromStdin(io.stdin);
+  const { setUpNodeInstance } = await loadNodeRuntime();
+  const nodeDatabase = await openNodeDatabase(process.env);
+
+  try {
+    const result = await setUpNodeInstance(nodeDatabase.bindings, {
+      email: values.email,
+      password,
+      siteId: values["site-id"],
+      siteName: values["site-name"],
+      siteLanguage: values.language,
+      timeZone: values["time-zone"],
+    });
+    console.log(OUTCOME_MESSAGES[result.outcome](result.siteId));
+    return result;
+  } finally {
+    await nodeDatabase.close();
+  }
+}

package/bin/commands/site/export.js

@@ -1,14 +1,26 @@
-import { mkdirSync, readdirSync, writeFileSync } from "node:fs";
-import { dirname, resolve } from "node:path";
+import { createWriteStream, mkdirSync, readdirSync } from "node:fs";
+import { mkdir, mkdtemp, rename, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { dirname, join, resolve } from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
 import { parseArgs } from "node:util";
-import { unzipSync } from "fflate";
 import {
   CLI_API_TOKEN_ENV_VAR,
   getCliApiToken,
 } from "../../lib/cli-api-token.js";
-import { pullSiteExportZipBytes } from "../../lib/site-pull-media.js";
+import { pullSiteExportDirectory } from "../../lib/site-pull-media.js";
+import { extractZipFile, writeDirectoryToZip } from "../../lib/zip-archive.js";
 
-async function exportRemoteSite(url, token) {
+/**
+ * Stream the site's export archive to a file. The site streams it too, and a
+ * site that bundles its media answers with gigabytes.
+ *
+ * @param {string} url - Site URL
+ * @param {string} token - API token
+ * @param {string} destination - File to write
+ */
+async function downloadSiteArchive(url, token, destination) {
   const response = await fetch(`${url.replace(/\/$/, "")}/api/export/hugo`, {
     method: "POST",
     headers: {
@@ -16,12 +28,16 @@ async function exportRemoteSite(url, token) {
     },
   });
 
-  if (!response.ok) {
+  if (!response.ok || !response.body) {
     const text = await response.text();
     throw new Error(`HTTP ${response.status}: ${text}`);
   }
 
-  return new Uint8Array(await response.arrayBuffer());
+  await mkdir(dirname(destination), { recursive: true });
+  await pipeline(
+    Readable.fromWeb(response.body),
+    createWriteStream(destination),
+  );
 }
 
 function describeProgressUrl(value) {
@@ -161,22 +177,8 @@ export async function run(argv) {
     process.exit(1);
   }
 
-  console.log(`Exporting site from ${url}...`);
-
-  const zipBytes = await exportRemoteSite(url, token);
-  let zip = zipBytes;
-  let pullStats = null;
-
-  if (pullMedia) {
-    console.log("Preparing pull-media export ZIP...");
-    const pulled = await pullSiteExportZipBytes(zip, {
-      assetLoader: null,
-      logger: logPullProgress,
-    });
-    zip = pulled.zipBytes;
-    pullStats = pulled.stats;
-  }
-
+  // Fail before the download, not after it: pulling a real site's media
+  // takes minutes.
   if (outputDirectory) {
     let existingEntries = [];
     try {
@@ -194,20 +196,55 @@ export async function run(argv) {
       );
       process.exit(1);
     }
+  }
 
-    console.log(`Writing export directory ${values.directory}...`);
-    const files = unzipSync(zip);
-    for (const [relativePath, bytes] of Object.entries(files)) {
-      const fullPath = resolve(outputDirectory, relativePath);
-      mkdirSync(dirname(fullPath), { recursive: true });
-      writeFileSync(fullPath, Buffer.from(bytes));
-    }
-    console.log(`Exported site from ${url} to ${values.directory}`);
-    console.log(`Preview with: cd ${values.directory} && hugo serve`);
-  } else {
+  console.log(`Exporting site from ${url}...`);
+
+  // Nothing to add: the site's archive is the export.
+  if (!pullMedia && !outputDirectory) {
     console.log(`Writing ${values.output}...`);
-    writeFileSync(output, Buffer.from(zip));
+    const partial = `${output}.partial`;
+    try {
+      await downloadSiteArchive(url, token, partial);
+      await rename(partial, output);
+    } catch (error) {
+      await rm(partial, { force: true });
+      throw error;
+    }
     console.log(`Exported site from ${url} to ${values.output}`);
+    return;
+  }
+
+  // Unpacked to disk and media downloaded next to it, so the export never
+  // holds more than one media file in memory.
+  const tempDir = await mkdtemp(join(tmpdir(), "jant-site-export-"));
+  const workDir = outputDirectory ?? join(tempDir, "site");
+  let pullStats = null;
+
+  try {
+    const archivePath = join(tempDir, "site.zip");
+    await downloadSiteArchive(url, token, archivePath);
+    await extractZipFile(archivePath, workDir);
+    await rm(archivePath, { force: true });
+
+    if (pullMedia) {
+      console.log("Pulling media...");
+      pullStats = await pullSiteExportDirectory(workDir, {
+        assetLoader: null,
+        logger: logPullProgress,
+      });
+    }
+
+    if (outputDirectory) {
+      console.log(`Exported site from ${url} to ${values.directory}`);
+      console.log(`Preview with: cd ${values.directory} && hugo serve`);
+    } else {
+      console.log(`Writing ${values.output}...`);
+      await writeDirectoryToZip(workDir, output);
+      console.log(`Exported site from ${url} to ${values.output}`);
+    }
+  } finally {
+    await rm(tempDir, { recursive: true, force: true });
   }
 
   if (pullStats) {

package/bin/commands/site/pull-media.js

@@ -1,10 +1,9 @@
-import { readFileSync, writeFileSync } from "node:fs";
 import { stat } from "node:fs/promises";
 import { resolve } from "node:path";
 import { parseArgs } from "node:util";
 import {
   pullSiteExportDirectory,
-  pullSiteExportZipBytes,
+  pullSiteExportZipFile,
 } from "../../lib/site-pull-media.js";
 
 function describeProgressUrl(value) {
@@ -102,13 +101,10 @@ export async function run(argv) {
   }
 
   const outputPath = resolve(process.cwd(), values.output || values.path);
-  const inputBytes = new Uint8Array(readFileSync(inputPath));
   console.log(`Pulling media in ${values.path}...`);
-  const { zipBytes, stats } = await pullSiteExportZipBytes(inputBytes, {
+  const stats = await pullSiteExportZipFile(inputPath, outputPath, {
     logger: logPullProgress,
   });
-  console.log(`Writing ${values.output || values.path}...`);
-  writeFileSync(outputPath, Buffer.from(zipBytes));
   console.log(
     `Pulled media in ${values.path} -> ${values.output || values.path}`,
   );

package/bin/commands/site/snapshot/export.js

@@ -1,17 +1,12 @@
-import { existsSync } from "node:fs";
-import {
-  mkdir,
-  mkdtemp,
-  readFile,
-  readdir,
-  rm,
-  writeFile,
-} from "node:fs/promises";
+import { createWriteStream, existsSync } from "node:fs";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { dirname, join, relative, resolve } from "node:path";
+import { dirname, join, resolve } from "node:path";
 import { parseArgs } from "node:util";
-import { zipSync } from "fflate";
-import { queryD1 } from "../../../lib/d1-query.js";
+import { writeDirectoryToZip } from "../../../lib/zip-archive.js";
+import { D1_DUMP_PAGE_SIZE, queryD1 } from "../../../lib/d1-query.js";
 import { loadNodeRuntime } from "../../../lib/load-node-runtime.js";
 import { openNodeDatabase } from "../../../lib/node-database.js";
 import {
@@ -23,6 +18,7 @@ import {
   buildSnapshotStorageQuery,
   collectSnapshotObjects,
   getSnapshotSelectSql,
+  orderSnapshotPostRows,
   SNAPSHOT_TABLES,
   snapshotObjectPath,
 } from "../../../lib/site-snapshot.js";
@@ -38,49 +34,6 @@ function isZipPath(filePath) {
   return filePath.toLowerCase().endsWith(".zip");
 }
 
-async function readStorageBody(body) {
-  const reader = body.getReader();
-  const chunks = [];
-  let totalLength = 0;
-
-  for (;;) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    chunks.push(value);
-    totalLength += value.length;
-  }
-
-  const bytes = new Uint8Array(totalLength);
-  let offset = 0;
-  for (const chunk of chunks) {
-    bytes.set(chunk, offset);
-    offset += chunk.length;
-  }
-
-  return bytes;
-}
-
-async function readDirectoryEntries(rootDir) {
-  const entries = {};
-
-  async function walk(dir) {
-    const items = await readdir(dir, { withFileTypes: true });
-    for (const item of items) {
-      const fullPath = join(dir, item.name);
-      if (item.isDirectory()) {
-        await walk(fullPath);
-        continue;
-      }
-
-      const relativePath = relative(rootDir, fullPath).replace(/\\/g, "/");
-      entries[relativePath] = new Uint8Array(await readFile(fullPath));
-    }
-  }
-
-  await walk(rootDir);
-  return entries;
-}
-
 async function assertWritableOutput(outputPath, force) {
   if (!existsSync(outputPath)) {
     return;
@@ -123,7 +76,12 @@ async function createNodeExportContext() {
       }
 
       await mkdir(dirname(filePath), { recursive: true });
-      await writeFile(filePath, await readStorageBody(object.body));
+      // Straight to disk: a snapshot holds every media file, and collecting
+      // each one in memory first ran a 3 GB export to a 2 GB peak.
+      await pipeline(
+        Readable.fromWeb(object.body),
+        createWriteStream(filePath),
+      );
     },
   };
 }
@@ -295,6 +253,9 @@ export async function run(argv) {
       {
         dialect: context.dialect,
         source: runtime,
+        // D1 answers through Wrangler's buffered output; read it in pages.
+        pageSize: runtime === "node" ? undefined : D1_DUMP_PAGE_SIZE,
+        orderRowsByTable: { post: orderSnapshotPostRows },
         tables: SNAPSHOT_TABLES,
         selectSqlByTable: Object.fromEntries(
           SNAPSHOT_TABLES.map((tableName) => [
@@ -344,10 +305,7 @@ export async function run(argv) {
 
     if (shouldZip) {
       await mkdir(dirname(outputPath), { recursive: true });
-      const zipped = zipSync(await readDirectoryEntries(scratchDir), {
-        level: 6,
-      });
-      await writeFile(outputPath, zipped);
+      await writeDirectoryToZip(scratchDir, outputPath);
       if (process.env.SNAPSHOT_SUPPRESS_SUCCESS_LOG !== "true") {
         console.log(
           `Exported ${getCliRuntimeLabel(runtime)} snapshot to ${values.output}`,

package/bin/commands/site/snapshot/import.js

@@ -1,16 +1,10 @@
-import { existsSync } from "node:fs";
-import {
-  mkdir,
-  mkdtemp,
-  readFile,
-  rm,
-  stat,
-  writeFile,
-} from "node:fs/promises";
+import { createReadStream, existsSync } from "node:fs";
+import { Readable } from "node:stream";
+import { mkdtemp, readFile, rm, stat } from "node:fs/promises";
 import { tmpdir } from "node:os";
-import { dirname, join, resolve } from "node:path";
+import { join, resolve } from "node:path";
 import { parseArgs } from "node:util";
-import { unzipSync } from "fflate";
+import { extractZipFile } from "../../../lib/zip-archive.js";
 import { executeD1, queryD1 } from "../../../lib/d1-query.js";
 import { loadNodeRuntime } from "../../../lib/load-node-runtime.js";
 import { openNodeDatabase } from "../../../lib/node-database.js";
@@ -18,6 +12,8 @@ import { deleteR2Object, uploadR2Object } from "../../../lib/r2-query.js";
 import {
   assertSnapshotDialectMatches,
   assertSnapshotMeta,
+  buildMediaProviderSql,
+  DEFER_FOREIGN_KEYS_SQL,
   buildReplaceSql,
   buildSnapshotStorageQuery,
   collectSnapshotObjects,
@@ -58,11 +54,13 @@ async function createNodeImportContext() {
   // the current site, which (a) is redundant with the bin-level resolveCliSite
   // call below and (b) prints a generic "/setup first" error when the
   // snapshot's own error path is more informative.
-  const { createStorageDriver } = await loadNodeRuntime();
+  const { createStorageDriver, getConfiguredStorageDriver } =
+    await loadNodeRuntime();
   const storage = createStorageDriver(nodeDatabase.bindings);
 
   return {
     dialect: nodeDatabase.database.dialect,
+    storageProvider: getConfiguredStorageDriver(nodeDatabase.bindings),
     async close() {
       await nodeDatabase.close();
     },
@@ -77,8 +75,7 @@ async function createNodeImportContext() {
         throw new Error("Snapshot import requires configured storage.");
       }
 
-      const bytes = new Uint8Array(await readFile(filePath));
-      await storage.put(key, bytes, {
+      await storage.put(key, Readable.toWeb(createReadStream(filePath)), {
         contentType: contentType || undefined,
       });
     },
@@ -96,6 +93,8 @@ function createD1ImportContext(runtime, values) {
 
   return {
     dialect: "sqlite",
+    // Objects go up through the wrangler R2 binding below.
+    storageProvider: "r2",
     async close() {},
     async query(sql) {
       return queryD1(sql, runtime, wranglerOptions);
@@ -141,14 +140,7 @@ async function materializeSnapshotInput(inputPath) {
   }
 
   const outputDir = await mkdtemp(join(tmpdir(), "jant-site-snapshot-import-"));
-  const bytes = new Uint8Array(await readFile(inputPath));
-  const files = unzipSync(bytes);
-
-  for (const [relativePath, data] of Object.entries(files)) {
-    const absolutePath = join(outputDir, relativePath);
-    await mkdir(dirname(absolutePath), { recursive: true });
-    await writeFile(absolutePath, data);
-  }
+  await extractZipFile(inputPath, outputDir);
 
   return {
     cleanup: async () => {
@@ -402,7 +394,14 @@ export async function run(argv) {
       );
     }
 
-    await context.execute(`${buildReplaceSql(targetSite.id)}\n${dbSql}`);
+    await context.execute(
+      [
+        ...(context.dialect === "postgres" ? [] : [DEFER_FOREIGN_KEYS_SQL]),
+        buildReplaceSql(targetSite.id),
+        dbSql,
+        buildMediaProviderSql(targetSite.id, context.storageProvider),
+      ].join("\n"),
+    );
 
     const keysToDelete = [...currentKeys].filter(
       (key) => !snapshotKeys.has(key),

package/bin/lib/d1-query.js

@@ -1,9 +1,23 @@
-import { runLocalWrangler } from "./wrangler-cli.js";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { WRANGLER_MAX_BUFFER, runLocalWrangler } from "./wrangler-cli.js";
 import { extractWranglerJson } from "./wrangler-json.js";
 
 const DEFAULT_RETRY_ATTEMPTS = 4;
 const DEFAULT_RETRY_DELAY_MS = 500;
 
+/**
+ * Byte ceiling for a statement batch sent as one `--command=` argument.
+ *
+ * Linux caps a single argv entry at 128 KiB (`MAX_ARG_STRLEN`) and fails the
+ * spawn with `E2BIG`; macOS has no per-argument cap, so an oversized batch
+ * works on a developer's Mac and dies in CI and in the demo rebuild, which is
+ * exactly how it first surfaced — a snapshot import whose `db.sql` had grown
+ * past the limit. Anything at or above this goes through `--file` instead.
+ */
+const MAX_INLINE_COMMAND_BYTES = 96 * 1024;
+
 function getD1Flag(runtime) {
   return runtime === "d1-remote" ? "--remote" : "--local";
 }
@@ -62,6 +76,46 @@ function sleepSync(ms) {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
 
+const WRANGLER_OUTPUT_EXCERPT_LENGTH = 2000;
+
+/**
+ * Rows per read when dumping a D1 table: small enough that a page of long
+ * posts stays well inside {@link WRANGLER_MAX_BUFFER}.
+ */
+export const D1_DUMP_PAGE_SIZE = 200;
+
+/**
+ * Say why a Wrangler call failed without echoing its whole output: on a
+ * failed `d1 execute` that output can be a site's rows, hundreds of KB of it.
+ *
+ * @param {string[]} args - The Wrangler arguments
+ * @param {Error & { code?: string, stdout?: string, stderr?: string }} error
+ * @returns {string} A message naming the command and the cause
+ * @example
+ * describeWranglerFailure(["d1", "execute", "DB"], enobufsError);
+ * // "`wrangler d1 execute DB` printed more than 67108864 bytes, …"
+ */
+export function describeWranglerFailure(args, error) {
+  const command = `wrangler ${args.slice(0, 3).join(" ")}`;
+  if (error?.code === "ENOBUFS") {
+    return `\`${command}\` printed more than ${WRANGLER_MAX_BUFFER} bytes, the most the CLI buffers.`;
+  }
+
+  const stderr = String(error?.stderr ?? "").trim();
+  if (stderr) return `\`${command}\` failed: ${stderr}`;
+
+  const stdout = String(error?.stdout ?? "").trim();
+  if (stdout) {
+    const excerpt =
+      stdout.length > WRANGLER_OUTPUT_EXCERPT_LENGTH
+        ? `${stdout.slice(0, WRANGLER_OUTPUT_EXCERPT_LENGTH)}… (${stdout.length} characters)`
+        : stdout;
+    return `\`${command}\` failed: ${excerpt}`;
+  }
+
+  return `\`${command}\` failed: ${error?.message ?? "unknown error"}`;
+}
+
 export function isRetryableWranglerD1Failure(output, error) {
   const combined = `${output ?? ""}\n${error?.message ?? ""}`.toLowerCase();
   return [
@@ -123,12 +177,43 @@ function runWrangler(args, options = {}) {
         throw new Error(`Wrangler error: ${wranglerError}`);
       }
 
-      throw new Error(output || error.message, { cause: error });
+      throw new Error(describeWranglerFailure(args, error), { cause: error });
     }
   }
 }
 
+/**
+ * Whether a statement batch is too large to travel as one CLI argument.
+ *
+ * @param {string} sql - SQL batch about to be sent to Wrangler
+ * @returns {boolean} true when it has to go through a file instead
+ * @example
+ * ```js
+ * exceedsInlineCommandLimit("SELECT 1;"); // false
+ * ```
+ */
+export function exceedsInlineCommandLimit(sql) {
+  return Buffer.byteLength(sql, "utf8") >= MAX_INLINE_COMMAND_BYTES;
+}
+
+function withTemporarySqlFile(sql, run) {
+  const directory = mkdtempSync(join(tmpdir(), "jant-d1-"));
+  try {
+    const filePath = join(directory, "batch.sql");
+    writeFileSync(filePath, sql);
+    return run(filePath);
+  } finally {
+    rmSync(directory, { recursive: true, force: true });
+  }
+}
+
 export function executeD1(sql, runtime, options = {}) {
+  if (exceedsInlineCommandLimit(sql)) {
+    return withTemporarySqlFile(sql, (filePath) =>
+      executeD1File(filePath, runtime, options),
+    );
+  }
+
   const args = appendWranglerContext(
     [
       "d1",

package/bin/lib/hugo-markdown.js

@@ -20,6 +20,7 @@ const FRONT_MATTER_KEY_ORDER = [
   "type",
   "draft",
   "aliases",
+  "feed_id",
   "build",
 
   // Post payload
@@ -27,12 +28,15 @@ const FRONT_MATTER_KEY_ORDER = [
   "status",
   "visibility",
   "summary_text",
+  "truncated",
   "link_url",
   "source_name",
   "source_url",
   "quote_text",
   "rating",
   "featured_at",
+  "featured_post_ids",
+  "featured_sort_at",
   "pinned_at",
 
   // Bookkeeping / attachments