@jant/core 0.6.16 → 0.7.1

package/bin/lib/d1-query.js

@@ -1,8 +1,23 @@
-import { runLocalWrangler } from "./wrangler-cli.js";
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { WRANGLER_MAX_BUFFER, runLocalWrangler } from "./wrangler-cli.js";
+import { extractWranglerJson } from "./wrangler-json.js";
 
 const DEFAULT_RETRY_ATTEMPTS = 4;
 const DEFAULT_RETRY_DELAY_MS = 500;
 
+/**
+ * Byte ceiling for a statement batch sent as one `--command=` argument.
+ *
+ * Linux caps a single argv entry at 128 KiB (`MAX_ARG_STRLEN`) and fails the
+ * spawn with `E2BIG`; macOS has no per-argument cap, so an oversized batch
+ * works on a developer's Mac and dies in CI and in the demo rebuild, which is
+ * exactly how it first surfaced — a snapshot import whose `db.sql` had grown
+ * past the limit. Anything at or above this goes through `--file` instead.
+ */
+const MAX_INLINE_COMMAND_BYTES = 96 * 1024;
+
 function getD1Flag(runtime) {
   return runtime === "d1-remote" ? "--remote" : "--local";
 }
@@ -35,7 +50,7 @@ export function parseWranglerError(output) {
   }
 
   try {
-    const parsed = JSON.parse(output.trim());
+    const parsed = JSON.parse(extractWranglerJson(output).trim());
     const error = Array.isArray(parsed) ? parsed[0]?.error : parsed?.error;
     if (!error?.text) {
       return undefined;
@@ -61,6 +76,46 @@ function sleepSync(ms) {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
 
+const WRANGLER_OUTPUT_EXCERPT_LENGTH = 2000;
+
+/**
+ * Rows per read when dumping a D1 table: small enough that a page of long
+ * posts stays well inside {@link WRANGLER_MAX_BUFFER}.
+ */
+export const D1_DUMP_PAGE_SIZE = 200;
+
+/**
+ * Say why a Wrangler call failed without echoing its whole output: on a
+ * failed `d1 execute` that output can be a site's rows, hundreds of KB of it.
+ *
+ * @param {string[]} args - The Wrangler arguments
+ * @param {Error & { code?: string, stdout?: string, stderr?: string }} error
+ * @returns {string} A message naming the command and the cause
+ * @example
+ * describeWranglerFailure(["d1", "execute", "DB"], enobufsError);
+ * // "`wrangler d1 execute DB` printed more than 67108864 bytes, …"
+ */
+export function describeWranglerFailure(args, error) {
+  const command = `wrangler ${args.slice(0, 3).join(" ")}`;
+  if (error?.code === "ENOBUFS") {
+    return `\`${command}\` printed more than ${WRANGLER_MAX_BUFFER} bytes, the most the CLI buffers.`;
+  }
+
+  const stderr = String(error?.stderr ?? "").trim();
+  if (stderr) return `\`${command}\` failed: ${stderr}`;
+
+  const stdout = String(error?.stdout ?? "").trim();
+  if (stdout) {
+    const excerpt =
+      stdout.length > WRANGLER_OUTPUT_EXCERPT_LENGTH
+        ? `${stdout.slice(0, WRANGLER_OUTPUT_EXCERPT_LENGTH)}… (${stdout.length} characters)`
+        : stdout;
+    return `\`${command}\` failed: ${excerpt}`;
+  }
+
+  return `\`${command}\` failed: ${error?.message ?? "unknown error"}`;
+}
+
 export function isRetryableWranglerD1Failure(output, error) {
   const combined = `${output ?? ""}\n${error?.message ?? ""}`.toLowerCase();
   return [
@@ -75,18 +130,6 @@ export function isRetryableWranglerD1Failure(output, error) {
   ].some((fragment) => combined.includes(fragment));
 }
 
-/**
- * Strip non-JSON preamble that Wrangler sometimes writes to stdout
- * (e.g. "Proxy environment variables detected…") before the actual JSON payload.
- */
-function extractJson(raw) {
-  const idx = raw.indexOf("[");
-  const idx2 = raw.indexOf("{");
-  if (idx === -1 && idx2 === -1) return raw;
-  const start = idx === -1 ? idx2 : idx2 === -1 ? idx : Math.min(idx, idx2);
-  return raw.slice(start);
-}
-
 function getWranglerExecutionOptions(options) {
   const executionOptions = { ...options };
   for (const key of [
@@ -134,12 +177,43 @@ function runWrangler(args, options = {}) {
         throw new Error(`Wrangler error: ${wranglerError}`);
       }
 
-      throw new Error(output || error.message, { cause: error });
+      throw new Error(describeWranglerFailure(args, error), { cause: error });
     }
   }
 }
 
+/**
+ * Whether a statement batch is too large to travel as one CLI argument.
+ *
+ * @param {string} sql - SQL batch about to be sent to Wrangler
+ * @returns {boolean} true when it has to go through a file instead
+ * @example
+ * ```js
+ * exceedsInlineCommandLimit("SELECT 1;"); // false
+ * ```
+ */
+export function exceedsInlineCommandLimit(sql) {
+  return Buffer.byteLength(sql, "utf8") >= MAX_INLINE_COMMAND_BYTES;
+}
+
+function withTemporarySqlFile(sql, run) {
+  const directory = mkdtempSync(join(tmpdir(), "jant-d1-"));
+  try {
+    const filePath = join(directory, "batch.sql");
+    writeFileSync(filePath, sql);
+    return run(filePath);
+  } finally {
+    rmSync(directory, { recursive: true, force: true });
+  }
+}
+
 export function executeD1(sql, runtime, options = {}) {
+  if (exceedsInlineCommandLimit(sql)) {
+    return withTemporarySqlFile(sql, (filePath) =>
+      executeD1File(filePath, runtime, options),
+    );
+  }
+
   const args = appendWranglerContext(
     [
       "d1",
@@ -153,7 +227,7 @@ export function executeD1(sql, runtime, options = {}) {
 
   if (options.quiet) {
     const output = runWrangler([...args, "--json"], options);
-    const parsed = JSON.parse(extractJson(output));
+    const parsed = JSON.parse(extractWranglerJson(output));
     const statements = Array.isArray(parsed) ? parsed : [parsed];
 
     for (const statement of statements) {
@@ -183,7 +257,7 @@ export function queryD1(sql, runtime, options = {}) {
     ),
     options,
   );
-  const parsed = JSON.parse(extractJson(output));
+  const parsed = JSON.parse(extractWranglerJson(output));
   const statement = Array.isArray(parsed) ? parsed[0] : parsed;
 
   if (statement?.error?.text) {
@@ -208,7 +282,7 @@ export function executeD1File(filePath, runtime, options = {}) {
 
   if (options.quiet) {
     const output = runWrangler([...args, "--json"], options);
-    const parsed = JSON.parse(extractJson(output));
+    const parsed = JSON.parse(extractWranglerJson(output));
     const statements = Array.isArray(parsed) ? parsed : [parsed];
 
     for (const statement of statements) {

package/bin/lib/hugo-markdown.js

@@ -20,6 +20,7 @@ const FRONT_MATTER_KEY_ORDER = [
   "type",
   "draft",
   "aliases",
+  "feed_id",
   "build",
 
   // Post payload
@@ -27,12 +28,15 @@ const FRONT_MATTER_KEY_ORDER = [
   "status",
   "visibility",
   "summary_text",
+  "truncated",
   "link_url",
   "source_name",
   "source_url",
   "quote_text",
   "rating",
   "featured_at",
+  "featured_post_ids",
+  "featured_sort_at",
   "pinned_at",
 
   // Bookkeeping / attachments

package/bin/lib/node-env.js

@@ -22,13 +22,46 @@ function fileExists(envPath) {
 }
 
 /**
- * Locate `.env.node` for CLI auto-load. Searches, in order:
+ * Locate `.env.node` for CLI auto-load.
+ *
+ * `JANT_ENV_FILE` overrides the search whenever it is set: a path names the
+ * file to load, and an empty value means "load nothing". The empty form is
+ * what keeps a developer's `packages/core/.env.node` out of the test suite —
+ * see the `env` block in `vitest.config.ts`.
+ *
+ * Without the override, the search order is:
  *   1. `<cwd>/.env.node`             — user's site directory
  *   2. `<bin>/../../.env.node`        — `packages/core/.env.node` (in-repo dev)
  *
- * Returns the first existing path, or `null` if none is found.
+ * @param {string} [cwd] - Directory searched first, and the base a relative
+ *   `JANT_ENV_FILE` resolves against. Defaults to `process.cwd()`.
+ * @param {Record<string, string | undefined>} [env] - Environment to read
+ *   `JANT_ENV_FILE` from. Defaults to `process.env`.
+ * @returns {string | null} The env file to load, or `null` when there is none.
+ * @throws {Error} When `JANT_ENV_FILE` names a file that does not exist.
+ * @example
+ * findNodeEnvPath(process.cwd(), { JANT_ENV_FILE: "" }); // null
+ * findNodeEnvPath(process.cwd(), { JANT_ENV_FILE: "./staging.env" });
  */
-export function findNodeEnvPath(cwd = process.cwd()) {
+export function findNodeEnvPath(cwd = process.cwd(), env = process.env) {
+  const override = env.JANT_ENV_FILE;
+  if (override !== undefined) {
+    const trimmed = override.trim();
+    if (trimmed === "") {
+      return null;
+    }
+
+    const overridePath = resolve(cwd, trimmed);
+    if (!fileExists(overridePath)) {
+      throw new Error(
+        `JANT_ENV_FILE points at "${override}", which does not exist. ` +
+          "Set it to a readable env file, or to an empty value to skip the env file.",
+      );
+    }
+
+    return overridePath;
+  }
+
   const candidates = [
     resolve(cwd, ".env.node"),
     resolve(dirname(fileURLToPath(import.meta.url)), "../../.env.node"),
@@ -90,9 +123,12 @@ export function loadNodeEnvFile(envPath, env = process.env) {
  * Always called before `resolveCliRuntime()`, so DATABASE_URL / DATA_DIR
  * defined in `.env.node` make `--node` (or auto-detect) work without
  * requiring the user to source the file manually.
+ *
+ * Honours `JANT_ENV_FILE` through {@link findNodeEnvPath}, so a caller that
+ * needs a fully controlled environment can opt out of the file entirely.
  */
 export function autoloadNodeEnv(env = process.env) {
-  const envPath = findNodeEnvPath();
+  const envPath = findNodeEnvPath(process.cwd(), env);
   if (!envPath) {
     return { envPath: null, found: false, assignedKeys: [], skippedKeys: [] };
   }

package/bin/lib/site-pull-media.js

@@ -12,7 +12,6 @@ import {
 } from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { basename, dirname, extname, join, relative } from "node:path";
-import { unzipSync, zipSync } from "fflate";
 import { parse, stringify } from "smol-toml";
 import {
   collectMediaReferences as collectParsedMediaReferences,
@@ -20,6 +19,7 @@ import {
   rewriteMediaReferences,
 } from "./site-media-parser.js";
 import { formatFrontMatter, parseFrontMatter } from "./hugo-markdown.js";
+import { extractZipFile, writeDirectoryToZip } from "./zip-archive.js";
 
 export function getSitePathPrefix(baseUrl) {
   if (typeof baseUrl !== "string" || baseUrl.trim() === "") {
@@ -477,29 +477,6 @@ async function resolveExistingPulledPath(
   return toPulledPublicPath(pathname, sitePathPrefix);
 }
 
-async function packDirectoryToZip(rootDir) {
-  const files = {};
-  const allFiles = await walkFiles(rootDir);
-
-  for (const fullPath of allFiles) {
-    const relPath = relative(rootDir, fullPath).replace(/\\/g, "/");
-    files[relPath] = new Uint8Array(await readFile(fullPath));
-  }
-
-  return zipSync(files);
-}
-
-async function unpackZipToDirectory(zipBytes, rootDir) {
-  const files = unzipSync(zipBytes);
-  await Promise.all(
-    Object.entries(files).map(async ([relPath, bytes]) => {
-      const fullPath = join(rootDir, relPath);
-      await mkdir(dirname(fullPath), { recursive: true });
-      await writeFile(fullPath, bytes);
-    }),
-  );
-}
-
 export async function pullSiteExportDirectory(rootDir, options = {}) {
   const logger =
     typeof options.logger === "function" ? options.logger : () => {};
@@ -681,14 +658,30 @@ export async function pullSiteExportDirectory(rootDir, options = {}) {
   return stats;
 }
 
-export async function pullSiteExportZipBytes(zipBytes, options = {}) {
+/**
+ * Pull media into an export ZIP: extract it to a temporary directory, pull
+ * there, and write the result as a new archive. Nothing is held in memory
+ * beyond one media file at a time, so the archive can be any size.
+ *
+ * @param {string} inputZipPath - Export archive to read
+ * @param {string} outputZipPath - Archive to write; may be the input
+ * @param {Parameters<typeof pullSiteExportDirectory>[1]} [options]
+ * @returns {Promise<Awaited<ReturnType<typeof pullSiteExportDirectory>>>} Pull stats
+ * @example
+ * await pullSiteExportZipFile("export.zip", "export.zip", { logger });
+ */
+export async function pullSiteExportZipFile(
+  inputZipPath,
+  outputZipPath,
+  options = {},
+) {
   const tempDir = await mkdtemp(join(tmpdir(), "jant-site-pull-"));
 
   try {
-    await unpackZipToDirectory(zipBytes, tempDir);
+    await extractZipFile(inputZipPath, tempDir);
     const stats = await pullSiteExportDirectory(tempDir, options);
-    const pulledZip = await packDirectoryToZip(tempDir);
-    return { zipBytes: pulledZip, stats };
+    await writeDirectoryToZip(tempDir, outputZipPath);
+    return stats;
   } finally {
     await rm(tempDir, { recursive: true, force: true });
   }

package/bin/lib/site-selection.js

@@ -39,16 +39,45 @@ function normalizePathPrefix(value) {
   return normalized || null;
 }
 
-function resolveSelector(options = {}) {
-  if (typeof options.site === "string" && options.site.trim()) {
-    return {
-      kind: "site",
-      siteId: options.site.trim(),
-    };
+/**
+ * Printed when a host-based command names no site. Kept word for word with
+ * `resolveCliSite` in `src/runtime/site.ts`, which the Node path uses.
+ */
+const HOST_BASED_SITE_REQUIRED_MESSAGE =
+  "host-based mode needs a target site. Pass --site <key|id>, --host <host>, or --url <url>.";
+
+function getTrimmedOption(value) {
+  return typeof value === "string" && value.trim() ? value.trim() : null;
+}
+
+/**
+ * Turn the site flags a command parsed into the selector both resolvers take:
+ * this file's raw-SQL `resolveCliSite` and `createNodeCliRuntime`.
+ *
+ * @param {{ site?: string, host?: string, pathPrefix?: string, url?: string }} [options]
+ *   The `--site`, `--host`, `--path-prefix` and `--url` values
+ * @returns {{ kind: "site", idOrKey: string } | { kind: "host", host: string, pathPrefix: string | null } | null}
+ *   The selector, or null when no site flag was passed
+ * @throws {Error} When more than one of `--site`, `--host` and `--url` is set
+ * @example
+ * parseCliSiteSelector({ url: "https://example.com/blog/" });
+ * // { kind: "host", host: "example.com", pathPrefix: "/blog" }
+ */
+export function parseCliSiteSelector(options = {}) {
+  const site = getTrimmedOption(options.site);
+  const host = getTrimmedOption(options.host);
+  const url = getTrimmedOption(options.url);
+
+  if ([site, host, url].filter(Boolean).length > 1) {
+    throw new Error("Choose only one of --site, --host, or --url.");
+  }
+
+  if (site) {
+    return { kind: "site", idOrKey: site };
   }
 
-  if (typeof options.url === "string" && options.url.trim()) {
-    const parsed = new URL(options.url.trim());
+  if (url) {
+    const parsed = new URL(url);
     return {
       kind: "host",
       host: parsed.host,
@@ -56,10 +85,10 @@ function resolveSelector(options = {}) {
     };
   }
 
-  if (typeof options.host === "string" && options.host.trim()) {
+  if (host) {
     return {
       kind: "host",
-      host: options.host.trim(),
+      host,
       pathPrefix: normalizePathPrefix(options.pathPrefix),
     };
   }
@@ -67,21 +96,26 @@ function resolveSelector(options = {}) {
   return null;
 }
 
-async function resolveSiteById(queryRunner, siteId) {
+// Mirrors `SiteService.getByIdOrKey`: an id match wins over a key match.
+async function resolveSiteByIdOrKey(queryRunner, idOrKey) {
+  const escaped = escapeSqlString(idOrKey);
   const rows = await queryRunner.query(`
     SELECT "id", "key", "status", "created_at", "updated_at"
     FROM "site"
-    WHERE "id" = '${escapeSqlString(siteId)}'
-    LIMIT 1
+    WHERE "id" = '${escaped}' OR "key" = '${escaped}'
+    LIMIT 2
   `);
 
-  if (rows.length === 0) {
-    throw new Error(`No site found for --site ${siteId}.`);
+  const row =
+    rows.find((candidate) => getOptionalString(candidate, "id") === idOrKey) ??
+    rows[0];
+  if (!row) {
+    throw new Error(`No site found for --site ${idOrKey}.`);
   }
 
   return {
     created: false,
-    site: toSite(rows[0]),
+    site: toSite(row),
   };
 }
 
@@ -142,24 +176,63 @@ function formatSiteSummary(rows) {
     .join(", ");
 }
 
+/**
+ * How the target instance maps requests to sites.
+ *
+ * @param {{ SITE_RESOLUTION_MODE?: unknown }} [env] The process environment,
+ *   or runtime bindings built from it; defaults to `process.env`
+ * @returns {"host-based" | "single-site"} `host-based` only when set to it
+ * @example
+ * if (getCliSiteResolutionMode(process.env) === "host-based") { ... }
+ */
 export function getCliSiteResolutionMode(env = process.env) {
   return env.SITE_RESOLUTION_MODE === "host-based"
     ? "host-based"
     : "single-site";
 }
 
+/**
+ * Resolve the site a command acts on with raw SQL, for runtimes that cannot
+ * run the site service (D1 through Wrangler, snapshot tooling).
+ *
+ * A site flag wins in either mode. Without one, single-site mode uses the
+ * instance's one site, and host-based mode fails before querying: a hosted
+ * database holds every tenant, so there is no default to fall back to.
+ *
+ * @param {{ query(sql: string): Promise<Record<string, unknown>[]>, execute?(sql: string): Promise<void> }} queryRunner
+ *   Runs SQL against the target database
+ * @param {{ env?: { SITE_RESOLUTION_MODE?: unknown }, site?: string, host?: string, pathPrefix?: string, url?: string, createIfMissing?: boolean, bootstrapSite?: { id?: string, key?: string } }} [options]
+ *   The environment, the command's site flags, and whether single-site mode
+ *   may create the site shell when none exists
+ * @returns {Promise<{ created: boolean, site: { id: string, key: string, status: string, createdAt: number, updatedAt: number } }>}
+ *   The site, and whether this call created it
+ * @throws {Error} When the flags match no site, when host-based mode has none,
+ *   or when single-site mode finds zero sites it may not create or several
+ * @example
+ * const { site } = await resolveCliSite(context, {
+ *   env: process.env,
+ *   site: values.site,
+ *   host: values.host,
+ *   pathPrefix: values["path-prefix"],
+ *   url: values.url,
+ * });
+ */
 export async function resolveCliSite(queryRunner, options = {}) {
   const resolutionMode = getCliSiteResolutionMode(options.env);
-  const selector = resolveSelector(options);
+  const selector = parseCliSiteSelector(options);
 
   if (selector?.kind === "site") {
-    return resolveSiteById(queryRunner, selector.siteId);
+    return resolveSiteByIdOrKey(queryRunner, selector.idOrKey);
   }
 
   if (selector?.kind === "host") {
     return resolveSiteByHost(queryRunner, selector.host, selector.pathPrefix);
   }
 
+  if (resolutionMode === "host-based") {
+    throw new Error(HOST_BASED_SITE_REQUIRED_MESSAGE);
+  }
+
   const rows = await queryRunner.query(`
     SELECT "id", "key", "status", "created_at", "updated_at"
     FROM "site"
@@ -168,12 +241,6 @@ export async function resolveCliSite(queryRunner, options = {}) {
   `);
 
   if (rows.length === 0) {
-    if (resolutionMode !== "single-site") {
-      throw new Error(
-        "No site configured for this instance. Create a site before using host-based mode.",
-      );
-    }
-
     if (!options.createIfMissing || typeof queryRunner.execute !== "function") {
       throw new Error(
         "single-site mode requires an initialized site. Complete /setup first or run a command that can bootstrap the site shell.",
@@ -208,11 +275,9 @@ export async function resolveCliSite(queryRunner, options = {}) {
   }
 
   if (rows.length > 1) {
-    const message =
-      resolutionMode === "single-site"
-        ? `single-site mode found multiple sites in the database: ${formatSiteSummary(rows)}. Restore SITE_RESOLUTION_MODE=host-based for this database, or remove the extra sites before restarting in single-site mode.`
-        : "host-based mode requires --site, --host, or --url when the database contains multiple sites.";
-    throw new Error(message);
+    throw new Error(
+      `single-site mode found multiple sites in the database: ${formatSiteSummary(rows)}. Restore SITE_RESOLUTION_MODE=host-based for this database, or remove the extra sites before restarting in single-site mode.`,
+    );
   }
 
   return {