@opentf/web-cli 1.0.0 → 1.1.0

package/src/dev.js

@@ -1,12 +1,23 @@
-// `otfw dev` — the CSR dev server.
+// `otfw dev` — the CSR dev server (on-demand / lazy-route).
 //
-// Drives Rolldown in watch mode (the `otfwc` compiler as a transform plugin) and
-// serves the project's index.html with WebSocket live-reload on rebuild. Tailwind
-// stylesheets are compiled on the fly (see ./tailwind.js). The project root is the
-// current working directory, like `vite` / `next dev`.
+// Unlike a single eager bundle, this serves modules as the browser asks for them:
+//
+//   • /@fw.js          — the runtime (`@opentf/web`), bundled once and shared by
+//                        every chunk through an import map (so the router and signal
+//                        registry are one instance).
+//   • /bundle.js       — the app entry: `mountApp` + a route table whose loaders are
+//                        `() => import("/__route/<id>.js")`. The route modules are
+//                        *not* in this bundle.
+//   • /__route/<id>.js — one route (page or layout) compiled on first navigation,
+//                        with `@opentf/web` external, then cached in memory.
+//
+// So startup compiles the entry + runtime only; each route's cost is paid the first
+// time it's visited. The module graph (`otfwc graph`) drives HMR: on a file change
+// it tells us exactly which cached chunks to drop before a reload. Tailwind/CSS and
+// `public/` assets are served as before. Project root = cwd, like `vite`/`next dev`.
 
-import { watch } from "rolldown";
-import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { rolldown } from "rolldown";
+import { existsSync, mkdirSync, readFileSync, statSync, watch, writeFileSync } from "node:fs";
 import { join } from "node:path";
 
 import { compileCss, usesTailwind } from "./tailwind.js";
@@ -17,14 +28,18 @@ import {
   cssPlugin,
   discoverPages,
   entrySource,
+  injectBeforeBody,
+  loadConfig,
+  loadDocsPlugins,
   loadProject,
+  moduleGraph,
   otfwPlugin,
+  readHtmlShell,
 } from "./shared.js";
 
 // Resolve the start port. An explicit `--port <n>` / `-p <n>` / `--port=<n>` is
 // honored exactly (fail fast if it's busy); with no flag we default to 3000 and
-// scan upward for a free port. (No PORT env: this is a dev tool, and OpenTF's
-// production output is static — there's no long-running server to take PORT.)
+// scan upward for a free port.
 function resolvePort() {
   const argv = process.argv.slice(3); // args after `dev`
   for (let i = 0; i < argv.length; i++) {
@@ -61,7 +76,15 @@ function serve(start, explicit, options) {
   process.exit(1);
 }
 
+// A route file ↔ its `/__route/<id>.js` URL. The id is the file path, base64url so
+// it survives a URL path segment; decoding recovers the absolute path to compile.
+const ROUTE_PREFIX = "/__route/";
+const toRouteUrl = (file) => `${ROUTE_PREFIX}${Buffer.from(file).toString("base64url")}.js`;
+const fromRouteUrl = (pathname) =>
+  Buffer.from(pathname.slice(ROUTE_PREFIX.length, -".js".length), "base64url").toString("utf8");
+
 export async function runDev() {
+  const bootStart = Date.now();
   const { root, appDir, webEntry, otfwc, exclude } = loadProject();
   const { port: startPort, explicit: explicitPort } = resolvePort();
   if (!Number.isInteger(startPort) || startPort < 1 || startPort > 65535) {
@@ -75,105 +98,195 @@ export async function runDev() {
     process.exit(1);
   }
 
+  const config = await loadConfig(root);
+  const docsPlugins = await loadDocsPlugins(root, appDir, config, exclude);
+
   const devDir = join(root, ".dev");
-  mkdirSync(join(devDir, "csr"), { recursive: true });
-  const entry = join(devDir, "entry.js");
-  writeFileSync(entry, entrySource(pages, appDir));
+  mkdirSync(devDir, { recursive: true });
+  const entryFile = join(devDir, "entry.js");
+  writeFileSync(entryFile, entrySource(pages, appDir, toRouteUrl, config?.i18n, config?.nav));
 
-  // WebSocket HMR: clients on the "hmr" topic get JSON messages — a successful
-  // rebuild sends { type: "reload" }; a compile/build failure sends
-  // { type: "error" } so the injected overlay shows it over the last good page.
   let server;
   const publish = (msg) => server?.publish("hmr", JSON.stringify(msg));
-  // Per-module compile diagnostics (keyed by id); a clean build clears them.
   const compileErrors = new Map();
 
-  const watcher = watch({
-    input: entry,
-    resolve: { alias: { "@opentf/web": webEntry }, extensions: EXTENSIONS },
-    plugins: [
-      otfwPlugin(otfwc, {
-        onResult: (id, err) => (err ? compileErrors.set(id, err) : compileErrors.delete(id)),
-      }),
-      cssPlugin(),
-    ],
-    output: {
-      dir: join(devDir, "csr"),
-      format: "esm",
-      entryFileNames: "bundle.js",
-    },
+  // One persistent compiler (one `otfwc serve` child) shared by every build below.
+  const otfw = otfwPlugin(otfwc, {
+    onResult: (id, err) => (err ? compileErrors.set(id, err) : compileErrors.delete(id)),
   });
-  watcher.on("event", (e) => {
-    if (e.code === "BUNDLE_END") {
-      e.result?.close?.();
-      if (compileErrors.size > 0) {
-        const [id, message] = [...compileErrors][0];
-        console.error(`✗ ${compileErrors.size} compile error(s)`);
-        publish({ type: "error", kind: "compile", id, message });
-      } else {
-        console.log(`✓ bundled in ${e.duration}ms`);
-        publish({ type: "reload" });
+  const css = cssPlugin();
+  const plugins = [...docsPlugins, otfw, css];
+
+  // Bundle `input` to a single ESM string in memory (no disk). `external` ids are
+  // left as bare imports (resolved by the browser via the import map / route URLs).
+  // `alias` lets the runtime build resolve its own `@opentf/web` self-imports.
+  async function bundle({ input, external, alias }) {
+    const b = await rolldown({
+      input,
+      resolve: { alias: alias || {}, extensions: EXTENSIONS },
+      external,
+      plugins,
+    });
+    try {
+      const { output } = await b.generate({ format: "esm", codeSplitting: false });
+      return output[0].code;
+    } finally {
+      await b.close();
+    }
+  }
+
+  // The shared runtime, bundled once. Its components compile to `import … from
+  // "@opentf/web"`; aliasing that back to the entry keeps them inside this bundle.
+  async function buildFramework() {
+    return bundle({ input: webEntry, alias: { "@opentf/web": webEntry } });
+  }
+  // The app entry: route loaders point at `/__route/…` (external — fetched lazily)
+  // and `@opentf/web` is external (→ import map → /@fw.js).
+  async function buildEntry() {
+    return bundle({
+      input: entryFile,
+      external: (id) => id === "@opentf/web" || id.startsWith(ROUTE_PREFIX),
+    });
+  }
+  // One route module (page or layout) + its components/web-docs, runtime external.
+  async function buildRoute(file) {
+    return bundle({ input: file, external: ["@opentf/web"] });
+  }
+
+  // Everything is built on first request and cached (Vite-style), so startup does no
+  // compilation at all. `null` means "needs (re)building".
+  let fwCode = null;
+  let entryCode = null;
+  const routeCache = new Map(); // route file → compiled chunk
+
+  // A build error becomes a thrown-on-load stub so the overlay shows it in place.
+  const errorStub = (id, msg) =>
+    `throw new Error(${JSON.stringify(`Compile error in ${id}\n\n${msg}`)});`;
+
+  // Compile `file` (framework / entry / route) on demand, surfacing a compile error
+  // as a thrown-on-load stub. `build` is the matching builder.
+  async function compileOnce(file, build) {
+    try {
+      const code = await build(file);
+      compileErrors.delete(file);
+      return code;
+    } catch (e) {
+      const msg = e?.message ?? String(e);
+      compileErrors.set(file, msg);
+      publish({ type: "error", kind: "compile", id: file, message: msg });
+      return errorStub(file, msg);
+    }
+  }
+  const serveFramework = async () => (fwCode ??= await compileOnce(webEntry, buildFramework));
+  const serveEntry = async () => (entryCode ??= await compileOnce(entryFile, buildEntry));
+  async function serveRoute(file) {
+    if (!routeCache.has(file)) routeCache.set(file, await compileOnce(file, buildRoute));
+    return routeCache.get(file);
+  }
+
+  // The module graph powers precise invalidation; build it in the background so it
+  // never blocks startup. Until it's ready, a change clears every cache (safe).
+  let graph = { has: () => false, affected: () => new Set() };
+  const refreshGraph = () =>
+    moduleGraph(otfwc, webEntry, [...pages, webEntry]).then((g) => (graph = g));
+  refreshGraph();
+
+  // On a change, drop only the cached chunks the edited file reaches (its graph
+  // dependents); a reload then rebuilds them on demand. An unknown file (e.g. a
+  // workspace package we treat as external) clears everything to be safe.
+  function onChange(file) {
+    refreshGraph();
+    const hit = graph.has(file) ? graph.affected(file) : null;
+    if (!hit) {
+      fwCode = entryCode = null;
+      routeCache.clear();
+    } else {
+      if (hit.has(webEntry)) fwCode = null;
+      if (file === entryFile || pages.includes(file)) entryCode = null;
+      for (const f of [...routeCache.keys()]) if (hit.has(f)) routeCache.delete(f);
+    }
+    if (compileErrors.size > 0) {
+      const [id, message] = [...compileErrors][0];
+      publish({ type: "error", kind: "compile", id, message });
+    } else {
+      publish({ type: "reload" });
+    }
+  }
+
+  // Watch app sources; a new page/layout file regenerates the route table.
+  const watcher = watch(appDir, { recursive: true }, (_evt, name) => {
+    if (!name) return;
+    const file = join(appDir, name);
+    if (/\.(mdx|md|[jt]sx|css)$/.test(name)) {
+      if (/^(page|layout|404)\.(mdx|md|[jt]sx)$/.test(name.split("/").pop()) && !pages.includes(file)) {
+        const fresh = discoverPages(appDir, exclude);
+        pages.length = 0;
+        pages.push(...fresh);
+        writeFileSync(entryFile, entrySource(pages, appDir, toRouteUrl, config?.i18n, config?.nav));
       }
-    } else if (e.code === "ERROR") {
-      const message = e.error?.message ?? String(e.error);
-      console.error("✗ build error:\n", message);
-      publish({ type: "error", kind: "build", message, stack: e.error?.stack });
+      onChange(file);
     }
   });
+  // Ctrl+C is the normal way to stop a dev server, so treat it as a clean shutdown
+  // (exit 0) rather than the conventional 130 — otherwise `bun run dev` reports it as
+  // a failure. The compiler child is torn down by its own `exit` hook.
+  const shutdown = () => {
+    try {
+      watcher.close();
+    } catch {}
+    process.exit(0);
+  };
+  process.once("SIGINT", shutdown);
+  process.once("SIGTERM", shutdown);
+  process.once("exit", () => {
+    try {
+      watcher.close();
+    } catch {}
+  });
 
-  const indexPath = join(root, "index.html");
-
-  // Injected into the served HTML: our bundle + the dev error overlay / reload
-  // client (reconnects and reloads once the server is back after a restart).
+  // Import map (so `@opentf/web` resolves to the shared runtime chunk) + entry + the
+  // dev overlay / reload client. The import map must precede the module script.
   const injected =
+    `<script type="importmap">{"imports":{"@opentf/web":"/@fw.js"}}</script>\n` +
     `<script type="module" src="/bundle.js"></script>\n` +
     `<script>${overlayClient}</script>\n`;
 
-  // Use the project's index.html as the shell, stripping any module entry script
-  // (the app would be double-loaded) and injecting our bundle + reload client.
-  function buildHtml() {
-    let html;
-    if (existsSync(indexPath)) {
-      html = readFileSync(indexPath, "utf8").replace(
-        /<script\s+type=["']module["'][^>]*src=[^>]*>\s*<\/script>\s*/gi,
-        "",
-      );
-    } else {
-      html = `<!doctype html><html lang="en"><head>
-<meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">
-<title>OTF Web</title></head><body><div id="app"></div></body></html>`;
-    }
-    return html.includes("</body>")
-      ? html.replace("</body>", `${injected}</body>`)
-      : html + injected;
-  }
+  const buildHtml = () => injectBeforeBody(readHtmlShell(root), injected);
 
-  // Serve a static file from the project root (no traversal outside it). Tailwind
-  // entry stylesheets are compiled on request.
   async function serveStatic(pathname) {
-    const file = join(root, pathname);
-    if (!file.startsWith(root) || !existsSync(file)) return null;
+    // Only serve regular files — a request whose path is a directory (e.g. a route
+    // like `/blog` that also exists as `public/blog/`) must fall through to the SPA
+    // shell, not try to read the directory.
+    const isFile = (f) => existsSync(f) && statSync(f).isFile();
+    let file = join(root, pathname);
+    if (!file.startsWith(root)) return null;
+    if (!isFile(file)) {
+      const fromPublic = join(root, "public", pathname);
+      if (!fromPublic.startsWith(join(root, "public") + "/") || !isFile(fromPublic)) return null;
+      file = fromPublic;
+    }
     const ext = pathname.split(".").pop();
     if (ext === "css") {
       const source = readFileSync(file, "utf8");
-      const css = usesTailwind(source)
+      const out = usesTailwind(source)
         ? await compileCss(file, source, root).catch((err) => {
             console.error(`✗ tailwind failed for ${pathname}:\n${err?.message ?? err}`);
             return source;
           })
         : source;
-      return new Response(css, { headers: { "content-type": "text/css" } });
+      return new Response(out, { headers: { "content-type": "text/css" } });
     }
     return new Response(readFileSync(file), {
       headers: { "content-type": MIME[ext] ?? "application/octet-stream" },
     });
   }
 
+  const js = (code) => new Response(code, { headers: { "content-type": "text/javascript" } });
+
   server = serve(startPort, explicitPort, {
     websocket: {
       open: (ws) => {
         ws.subscribe("hmr");
-        // A client connecting while a compile error is outstanding sees it now.
         if (compileErrors.size > 0) {
           const [id, message] = [...compileErrors][0];
           ws.send(JSON.stringify({ type: "error", kind: "compile", id, message }));
@@ -183,38 +296,23 @@ export async function runDev() {
     async fetch(req, srv) {
       const { pathname } = new URL(req.url);
       if (pathname === "/__hmr") {
-        return srv.upgrade(req)
-          ? undefined
-          : new Response("upgrade failed", { status: 400 });
+        return srv.upgrade(req) ? undefined : new Response("upgrade failed", { status: 400 });
       }
-      // Built output: the entry bundle and code-split chunks live in .dev/csr.
-      if (pathname.endsWith(".js")) {
-        const built = join(devDir, "csr", pathname);
-        if (built.startsWith(join(devDir, "csr") + "/") && existsSync(built)) {
-          return new Response(readFileSync(built), {
-            headers: { "content-type": "text/javascript" },
-          });
-        }
-        if (pathname === "/bundle.js") {
-          return new Response("// building…", {
-            headers: { "content-type": "text/javascript" },
-          });
-        }
+      if (pathname === "/@fw.js") return js(await serveFramework());
+      if (pathname === "/bundle.js") return js(await serveEntry());
+      if (pathname.startsWith(ROUTE_PREFIX) && pathname.endsWith(".js")) {
+        return js(await serveRoute(fromRouteUrl(pathname)));
       }
-      // Static assets referenced by index.html (css, public/, etc).
       if (pathname !== "/") {
         const asset = await serveStatic(pathname);
         if (asset) return asset;
-        if (/\.[a-z0-9]+$/i.test(pathname)) {
-          return new Response("not found", { status: 404 });
-        }
+        if (/\.[a-z0-9]+$/i.test(pathname)) return new Response("not found", { status: 404 });
       }
-      return new Response(buildHtml(), {
-        headers: { "content-type": "text/html" },
-      });
+      return new Response(buildHtml(), { headers: { "content-type": "text/html" } });
     },
   });
 
   console.log(`\n  OTF Web dev server`);
-  console.log(`  → http://localhost:${server.port}  (${pages.length} routes)\n`);
+  console.log(`  → http://localhost:${server.port}  (${pages.length} routes, on-demand)`);
+  console.log(`  ✓ ready in ${Date.now() - bootStart}ms — routes compile on first visit\n`);
 }

package/src/prerender.js

@@ -3,86 +3,137 @@
 // render each route to a static HTML file. No DOM — the SSG output is pure string
 // concatenation, so no effects/lifecycle run.
 
-import { build } from "rolldown";
-import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import { dirname, join } from "node:path";
-import { pathToFileURL } from "node:url";
 
-import { EXTENSIONS, cssPlugin, otfwPlugin } from "./shared.js";
+import { buildServerBundle, injectHead, injectMarkup, withHtmlLang } from "./shared.js";
 
-// Generated server entry: eager-import every page module (so registerRoutes sees
-// real namespaces, enabling getStaticPaths) and re-export the render API.
-function serverEntrySource(pages) {
-  const imports = pages.map((p, i) => `import * as p${i} from ${JSON.stringify(p)};`).join("\n");
-  const map = pages.map((p, i) => `  [${JSON.stringify(p)}]: p${i},`).join("\n");
-  return (
-    `${imports}\n` +
-    `import { registerRoutes } from "@opentf/web";\n` +
-    `export { renderToString, collectRoutePaths } from "@opentf/web/server";\n` +
-    `registerRoutes({\n${map}\n});\n`
-  );
+// "/" → dist/index.html, "/post/1" → dist/post/1/index.html, "/fr/about" → dist/fr/about/index.html.
+function htmlPathFor(outDir, route) {
+  return route === "/" ? join(outDir, "index.html") : join(outDir, route, "index.html");
 }
 
-// Inject pre-rendered markup into the shell's #app container.
-function injectMarkup(shellHtml, markup) {
-  return shellHtml.replace(/(<div id="app"[^>]*>)\s*(<\/div>)/, `$1${markup}$2`);
+// Prefix a locale-agnostic route with `locale` (bare for the default) — the URL form
+// that both selects the output file and, passed to renderRoute, pins router.locale.
+function localizeFor(path, locale, defaultLocale) {
+  if (!locale || locale === defaultLocale) return path;
+  return path === "/" ? `/${locale}` : `/${locale}${path}`;
 }
 
-// "/" → dist/index.html, "/post/1" → dist/post/1/index.html.
-function htmlPathFor(outDir, route) {
-  return route === "/" ? join(outDir, "index.html") : join(outDir, route, "index.html");
+// `rel="alternate" hreflang` descriptors for a route across all locales (+ x-default),
+// merged into metadata.links so renderHead emits them (docs/I18N.md §6).
+function alternatesFor(path, locales, defaultLocale) {
+  const links = locales.map((l) => ({
+    rel: "alternate",
+    hreflang: l,
+    href: localizeFor(path, l, defaultLocale),
+  }));
+  links.push({ rel: "alternate", hreflang: "x-default", href: localizeFor(path, defaultLocale, defaultLocale) });
+  return links;
+}
+
+// Absolute URL for sitemap/robots; "" if no base URL configured.
+function absoluteUrl(baseUrl, path) {
+  if (!baseUrl) return "";
+  return baseUrl.replace(/\/+$/, "") + (path === "/" ? "/" : path);
+}
+
+// Emit sitemap.xml (absolute <loc> per rendered path) — requires a base URL. A
+// project-supplied public/sitemap.xml takes precedence (it overwrites ours on copy).
+function writeSitemap(outDir, publicDir, baseUrl, paths) {
+  if (existsSync(join(publicDir, "sitemap.xml"))) return false;
+  if (!baseUrl) {
+    console.warn("⚠ sitemap.xml skipped: no site URL (pass --base-url or set otfw.config)");
+    return false;
+  }
+  const urls = paths
+    .map((p) => `  <url><loc>${escapeXml(absoluteUrl(baseUrl, p))}</loc></url>`)
+    .join("\n");
+  const xml =
+    `<?xml version="1.0" encoding="UTF-8"?>\n` +
+    `<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>\n`;
+  writeFileSync(join(outDir, "sitemap.xml"), xml);
+  return true;
+}
+
+// Emit a permissive robots.txt referencing the sitemap (honor a public/ override).
+function writeRobots(outDir, publicDir, baseUrl, hasSitemap) {
+  if (existsSync(join(publicDir, "robots.txt"))) return false;
+  let body = "User-agent: *\nAllow: /\n";
+  if (baseUrl && hasSitemap) body += `Sitemap: ${absoluteUrl(baseUrl, "/sitemap.xml")}\n`;
+  writeFileSync(join(outDir, "robots.txt"), body);
+  return true;
+}
+
+function escapeXml(s) {
+  return String(s).replace(/[&<>'"]/g, (c) =>
+    c === "&" ? "&amp;" : c === "<" ? "&lt;" : c === ">" ? "&gt;" : c === "'" ? "&apos;" : "&quot;",
+  );
 }
 
 /**
  * Pre-render the app to static HTML files under `outDir`. Returns
  * `{ count, skipped, failed }`.
  */
-export async function runPrerender({ root, pages, webEntry, otfwc, shellHtml, outDir }) {
-  const tmp = join(root, ".otfw-ssg");
-  mkdirSync(tmp, { recursive: true });
-  const entry = join(tmp, "ssg-entry.js");
-  writeFileSync(entry, serverEntrySource(pages));
-
-  const serverApi = join(dirname(webEntry), "server", "index.js");
-  await build({
-    input: entry,
-    resolve: {
-      alias: { "@opentf/web/server": serverApi, "@opentf/web": webEntry },
-      extensions: EXTENSIONS,
-    },
-    plugins: [otfwPlugin(otfwc, { failOnError: true, target: "ssg" }), cssPlugin()],
-    output: { dir: join(tmp, "out"), format: "esm", entryFileNames: "server.js" },
-  });
+export async function runPrerender({ root, pages, webEntry, otfwc, shellHtml, outDir, baseUrl = "", docsPlugins = [], lastUpdated = {}, i18n = null, onCompile, onRender }) {
+  const { mod, cleanup } = await buildServerBundle({ root, pages, webEntry, otfwc, docsPlugins, i18n, onCompile });
 
-  // The runtime defines `class … extends HTMLElement` at load (for CSR custom
-  // elements). SSG never instantiates them, but the base class must exist so the
-  // class definitions evaluate. A bare stub suffices — no DOM (customElements
-  // stays undefined, so the elements self-register only in the browser).
-  globalThis.HTMLElement ??= class {};
-  const mod = await import(pathToFileURL(join(tmp, "out", "server.js")).href);
+  // i18n (docs/I18N.md §6): pre-render each route once per locale. The default
+  // locale is emitted bare (dist/<route>/index.html), others under their prefix
+  // (dist/<locale>/<route>/index.html). A single `[null]` means i18n is off — the
+  // loop collapses to the original one-render-per-route behavior.
+  const i18nOn = !!(i18n && Array.isArray(i18n.locales) && i18n.locales.length);
+  const locales = i18nOn ? i18n.locales : [null];
+  const defaultLocale = i18nOn ? i18n.defaultLocale : null;
 
   const { paths, skipped } = await mod.collectRoutePaths();
   const failed = [];
-  for (const route of paths) {
-    try {
-      const markup = (await mod.renderToString(route)) ?? "";
-      const file = htmlPathFor(outDir, route);
-      mkdirSync(dirname(file), { recursive: true });
-      writeFileSync(file, injectMarkup(shellHtml, markup));
-    } catch (e) {
-      failed.push(route);
-      console.error(`✗ pre-render failed for ${route}: ${e?.message ?? e}`);
+  const rendered = []; // concrete paths that produced an HTML file (for the sitemap)
+  let renderedCount = 0;
+  for (const { path, params } of paths) {
+    onRender?.(++renderedCount, paths.length);
+    for (const locale of locales) {
+      const urlPath = localizeFor(path, locale, defaultLocale);
+      try {
+        // Passing the localized URL pins router.locale (resolveLocale) and still
+        // matches the locale-agnostic route table, so `t()` renders in `locale`.
+        const { html, metadata } = (await mod.renderRoute(urlPath, params)) ?? { html: "", metadata: {} };
+        const meta = i18nOn
+          ? { ...metadata, links: [...(metadata.links || []), ...alternatesFor(path, locales, defaultLocale)] }
+          : metadata;
+        let head = mod.renderHead(meta, { path: urlPath, baseUrl });
+        // SEO: expose the page's last-updated time (git/frontmatter) as Open Graph's
+        // article:modified_time so crawlers see when the content actually changed.
+        const iso = lastUpdated[path];
+        if (iso) head += `\n<meta property="article:modified_time" content="${escapeXml(iso)}">`;
+        const file = htmlPathFor(outDir, urlPath);
+        mkdirSync(dirname(file), { recursive: true });
+        writeFileSync(file, injectMarkup(injectHead(withHtmlLang(shellHtml, locale), head), html));
+        rendered.push(urlPath);
+      } catch (e) {
+        failed.push(urlPath);
+        console.error(`✗ pre-render failed for ${urlPath}: ${e?.message ?? e}`);
+      }
     }
   }
 
-  // 404: any unmatched path resolves to the registered 404 page.
+  // 404: any unmatched path resolves to the registered 404 page. Mark noindex so
+  // crawlers don't index the error page, and keep it out of the sitemap.
   try {
-    const notFound = await mod.renderToString("/__otfw_404__");
-    if (notFound != null) writeFileSync(join(outDir, "404.html"), injectMarkup(shellHtml, notFound));
+    const result = await mod.renderRoute("/__otfw_404__");
+    if (result) {
+      const head = mod.renderHead({ robots: "noindex", ...result.metadata }, { baseUrl });
+      writeFileSync(join(outDir, "404.html"), injectMarkup(injectHead(shellHtml, head), result.html));
+    }
   } catch {
     /* no 404 page */
   }
 
-  rmSync(tmp, { recursive: true, force: true });
-  return { count: paths.length - failed.length, skipped, failed };
+  // Crawl infrastructure: sitemap.xml + robots.txt (honoring public/ overrides).
+  const publicDir = join(root, "public");
+  const hasSitemap = writeSitemap(outDir, publicDir, baseUrl, rendered);
+  writeRobots(outDir, publicDir, baseUrl, hasSitemap);
+
+  cleanup();
+  return { count: rendered.length, skipped, failed };
 }

package/src/reporter.js

@@ -0,0 +1,73 @@
+// Pretty, TTY-aware build reporter. Each phase is a "step" that shows an animated
+// spinner with live detail while it runs, then collapses to a green ✅ line with the
+// elapsed time when done. On a non-interactive stream (CI, piped logs) the spinner and
+// in-place updates are skipped — only the final ✅/✗ line per step is printed — so logs
+// stay clean and free of carriage returns.
+
+const TTY = !!process.stdout.isTTY;
+const FRAMES = ["⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"];
+
+const paint = (code, s) => (TTY ? `\x1b[${code}m${s}\x1b[0m` : s);
+const dim = (s) => paint(2, s);
+const cyan = (s) => paint(36, s);
+const red = (s) => paint(31, s);
+
+/** Humanize a millisecond duration: 940ms, 5.2s. */
+export function fmtMs(n) {
+  return n < 1000 ? `${Math.round(n)}ms` : `${(n / 1000).toFixed(1)}s`;
+}
+
+class Step {
+  constructor(label) {
+    this.label = label;
+    this.detail = "";
+    this.t0 = performance.now();
+    this.frame = 0;
+    this.timer = null;
+    if (TTY) {
+      this.render();
+      this.timer = setInterval(() => this.render(), 80);
+    } else {
+      process.stdout.write(`  ${dim("•")} ${label}…\n`);
+    }
+  }
+
+  render() {
+    const spinner = cyan(FRAMES[(this.frame = (this.frame + 1) % FRAMES.length)]);
+    const detail = this.detail ? "  " + dim(this.detail) : "";
+    process.stdout.write(`\r\x1b[K  ${spinner} ${this.label}${detail}`);
+  }
+
+  /**
+   * Update the live detail (e.g. the current file or an `N/total` count) and repaint
+   * immediately. The immediate repaint matters because compilation runs synchronous
+   * subprocesses that block the event loop — the `setInterval` tick can't fire during
+   * them, so each `update()` is what actually advances the line.
+   */
+  update(detail) {
+    this.detail = detail || "";
+    if (TTY) this.render();
+  }
+
+  _stop() {
+    if (this.timer) clearInterval(this.timer);
+    if (TTY) process.stdout.write("\r\x1b[K");
+  }
+
+  /** Finish the step: green ✅ with `msg` (defaults to the label) and elapsed time. */
+  done(msg) {
+    this._stop();
+    process.stdout.write(`  ✅ ${msg || this.label}  ${dim(fmtMs(performance.now() - this.t0))}\n`);
+  }
+
+  /** Mark the step failed: red ✗ with `msg`. */
+  fail(msg) {
+    this._stop();
+    process.stdout.write(`  ${red("✗")} ${msg || this.label}\n`);
+  }
+}
+
+/** Start a build phase. Returns a `Step` you drive with `.update()` then `.done()`. */
+export function step(label) {
+  return new Step(label);
+}