@jxsuite/compiler 4.0.3 → 5.1.0

package/src/site/site-build.ts

@@ -39,6 +39,7 @@ import {
   resolveSidecarEntry,
 } from "./bundler.ts";
 import { loadProjectConfig } from "./site-loader.ts";
+import type { ResolvedProjectConfig } from "./site-loader.ts";
 import {
   discoverPages,
   expandDynamicRoutes,
@@ -95,6 +96,13 @@ import {
   registrationScript,
   tombstoneServiceWorker,
 } from "./service-worker.ts";
+import {
+  htmlDeclaresNoindex,
+  isNotFoundRoute,
+  outputUrlPath,
+  servedRoute,
+  withAlternateLink,
+} from "./static-host.ts";
 import {
   buildManifest,
   buildSecurityTxt,
@@ -579,7 +587,7 @@ export async function buildSite(
     translationKey: declaredKeys.get(r.urlPattern),
     urlPattern: r.urlPattern,
   }));
-  const alternateMap = localeAlternates(keyedRoutes, i18n, siteUrl ?? "");
+  const alternateMap = localeAlternates(keyedRoutes, i18n, siteUrl ?? "", trailingSlash);
   /*
    * The same sets, site-absolute, for `$page.alternates`. Computed from the whole table for the
    * same reason the alternates are — and separately from them because a switcher must work in a
@@ -745,16 +753,16 @@ export async function buildSite(
       // Determine output path
       const outPath = routeToOutputPath(route.urlPattern, outDir, trailingSlash);
       mkdirSync(dirname(outPath), { recursive: true });
-      /* Last, after the scans above: they ask what this page REFERENCES, and the answer is a
-         build-output path, not a deployed URL. Re-rooting first would make every one of them miss. */
-      writeFileSync(outPath, rewriteHtmlBase(result.html, basePath), "utf8");
-      fileCount += 1;
-
       // Record a sitemap entry for this concrete page (skip unexpanded dynamic routes and
       // Pages that opted out via $sitemap: false). <loc> is built like the canonical URL so
       // The two always agree.
       const isConcrete = !route.urlPattern.includes(":") && !route.urlPattern.includes("*");
-      if (sitemapEnabled && !result.excludeFromSitemap && isConcrete) {
+      /*
+       * A page the sitemap would list and the page itself disowns sends two answers to one question,
+       * so a page that asks crawlers for `noindex` is left out, and so is the not-found page.
+       */
+      const indexable = !isNotFoundRoute(route.urlPattern) && !htmlDeclaresNoindex(result.html);
+      if (sitemapEnabled && !result.excludeFromSitemap && indexable && isConcrete) {
         const routeAlternates = alternateMap.get(route.urlPattern) ?? [];
         sitemapEntries.push({
           /*
@@ -768,12 +776,17 @@ export async function buildSite(
             typeof route.sourceMtime === "string" && route.sourceMtime !== ""
               ? route.sourceMtime
               : toRfc3339(statSync(route.sourcePath).mtime),
-          loc: siteAbsoluteUrl(route.urlPattern, siteUrl),
+          loc: siteAbsoluteUrl(servedRoute(route.urlPattern, trailingSlash), siteUrl),
           ...(routeAlternates.length > 0 && { alternates: routeAlternates }),
         });
       }
 
-      // Write serialized export sidecars alongside HTML (formats with exportTarget: true)
+      /*
+       * Serialize the export sidecars (formats with exportTarget: true, e.g. Markdown) BEFORE the page
+       * is written, so its `<head>` advertises exactly the twins that exist: a link to a file the
+       * build did not write is worse than no link.
+       */
+      const sidecars: { content: string; path: string; type: string | null }[] = [];
       for (const fmt of formatRegistry.withCapability("serialize")) {
         if (!fmt.exportTarget) {
           continue;
@@ -793,14 +806,34 @@ export async function buildSite(
           })) as string;
           if (content) {
             const sidecarPath = outPath.replace(/\.html$/, fmt.extensions[0]!);
-            writeFileSync(sidecarPath, content, "utf8");
-            fileCount += 1;
+            sidecars.push({ content, path: sidecarPath, type: fmt.mediaTypeEssence });
           }
         } catch (error) {
           const err = error as Error;
           errors.push(`Error exporting ${fmt.name} for ${route.urlPattern}: ${err.message}`);
         }
       }
+      /* The not-found page is served for whatever URL was mistyped, so what it would advertise is
+         not a twin of anything. */
+      if (!isNotFoundRoute(route.urlPattern)) {
+        for (const sidecar of sidecars) {
+          if (sidecar.type !== null) {
+            result.html = withAlternateLink(result.html, {
+              href: outputUrlPath(outDir, sidecar.path),
+              type: sidecar.type,
+            });
+          }
+        }
+      }
+
+      /* Last, after the scans above: they ask what this page REFERENCES, and the answer is a
+         build-output path, not a deployed URL. Re-rooting first would make every one of them miss. */
+      writeFileSync(outPath, rewriteHtmlBase(result.html, basePath), "utf8");
+      fileCount += 1;
+      for (const sidecar of sidecars) {
+        writeFileSync(sidecar.path, sidecar.content, "utf8");
+        fileCount += 1;
+      }
 
       // Write any additional files (island modules, etc.)
       for (const file of result.files) {
@@ -1239,7 +1272,7 @@ export async function buildSite(
  */
 async function compilePage(
   route: SiteRoute,
-  projectConfig: ProjectConfig,
+  projectConfig: ResolvedProjectConfig,
   projectRoot: string,
   sections: Record<string, unknown>,
   imageCache: CacheManifest | null,
@@ -1427,7 +1460,14 @@ async function compilePage(
     ...(projectConfig.name != null && { siteName: projectConfig.name }),
     ...(projectConfig.url != null && { siteUrl: projectConfig.url }),
     ...(locale.alternates.length > 0 && { alternates: locale.alternates }),
-    pageUrl: route.urlPattern,
+    /*
+     * The canonical link and og:url name the URL the host answers with the page: the route as
+     * `build.trailingSlash` serves it. The not-found page gets neither, since the URL it answers at
+     * is whatever the visitor mistyped.
+     */
+    ...(!isNotFoundRoute(route.urlPattern) && {
+      pageUrl: servedRoute(route.urlPattern, projectConfig.build.trailingSlash),
+    }),
   });
 
   // Merge project-level $media into the layout document so responsive queries are available
@@ -1775,6 +1815,33 @@ function isBareSpecifier(s: string) {
   );
 }
 
+/**
+ * The resolved value of a text template that still contains `${`, as escaped `innerHTML`.
+ *
+ * A template's result is data: an excerpt, a heading, a title. When that data itself holds `${` (a
+ * note that explains template syntax) it is text to show, and left as `textContent` the compile
+ * phase would read it as a template a second time, bind it to an expression that throws or renders
+ * nothing, and the words would be gone. The same entity spelling the `innerHTML` resolution below
+ * uses keeps it text.
+ *
+ * @param {string} resolved
+ * @returns {string}
+ */
+function inertTextHtml(resolved: string): string {
+  return escapeHtml(resolved).replaceAll("${", "&#36;{");
+}
+
+/**
+ * A resolved attribute value with any `${` split by a zero-width space. An entity cannot be written
+ * into an attribute, and a value that kept the sequence would be bound as a template again.
+ *
+ * @param {string} resolved
+ * @returns {string}
+ */
+function inertAttribute(resolved: string): string {
+  return resolved.replaceAll("${", "$\u200B{");
+}
+
 /**
  * Deep-clone a map template, resolving template strings and $ref values against the given scope.
  *
@@ -1808,7 +1875,11 @@ function expandMapTemplate(template: JxElement, scope: Record<string, unknown>):
       const attrs = { ...(v as Record<string, JxAttributeValue>) };
       for (const [ak, av] of Object.entries(attrs)) {
         if (isTemplateString(av)) {
-          attrs[ak] = (evaluateMapTemplate(av, scope) as JxAttributeValue | undefined) ?? av;
+          const resolved = evaluateMapTemplate(av, scope) as JxAttributeValue | undefined;
+          attrs[ak] =
+            typeof resolved === "string" && resolved.includes("${")
+              ? inertAttribute(resolved)
+              : (resolved ?? av);
         }
       }
       node.attributes = attrs;
@@ -1823,7 +1894,12 @@ function expandMapTemplate(template: JxElement, scope: Record<string, unknown>):
       }
       node.$props = props;
     } else if (typeof v === "string" && isTemplateString(v)) {
-      node[k] = evaluateMapTemplate(v, scope) ?? v;
+      const resolved = evaluateMapTemplate(v, scope);
+      if (k === "textContent" && typeof resolved === "string" && resolved.includes("${")) {
+        node.innerHTML = inertTextHtml(resolved);
+      } else {
+        node[k] = resolved ?? v;
+      }
     } else {
       node[k] = v;
     }
@@ -1886,9 +1962,16 @@ function resolveDocTemplates(node: JxElement | string, scope: Record<string, unk
     }
   }
   if (typeof node.textContent === "string" && isTemplateString(node.textContent)) {
-    node.textContent =
-      (evaluateStaticTemplate(node.textContent, scope) as string | null) ??
-      (node.textContent as string | null);
+    const resolved = evaluateStaticTemplate(node.textContent, scope);
+    if (typeof resolved === "string" && resolved.includes("${")) {
+      // The resolved value is data (an excerpt, a heading, a title) that happens to contain `${`, not
+      // A template. Left as `textContent` the compile phase would read it as one again and bind it
+      // To an expression that throws or renders nothing, so it stays text as escaped `innerHTML`.
+      node.innerHTML = inertTextHtml(resolved);
+      delete node.textContent;
+    } else {
+      node.textContent = (resolved as string | null) ?? (node.textContent as string | null);
+    }
   }
   if (node.style && typeof node.style === "object") {
     for (const [k, v] of Object.entries(node.style)) {
@@ -1902,7 +1985,13 @@ function resolveDocTemplates(node: JxElement | string, scope: Record<string, unk
   if (node.attributes && typeof node.attributes === "object") {
     for (const [k, v] of Object.entries(node.attributes)) {
       if (typeof v === "string" && isTemplateString(v)) {
-        node.attributes[k] = (evaluateStaticTemplate(v, scope) as JxAttributeValue | null) ?? v;
+        const resolved = evaluateStaticTemplate(v, scope) as JxAttributeValue | null;
+        // A resolved value is data: a `${` left in it must not become a template a second time. An
+        // Entity cannot be written into an attribute, so the sequence is split with a zero-width space.
+        node.attributes[k] =
+          typeof resolved === "string" && resolved.includes("${")
+            ? inertAttribute(resolved)
+            : (resolved ?? v);
       }
     }
   }
@@ -2351,7 +2440,7 @@ function injectHead(
  * Convert a URL pattern to an output file path.
  *
  * "/" → dist/index.html "/about" → dist/about/index.html (with trailingSlash: "always")
- * "/blog/hello" → dist/blog/hello/index.html
+ * "/blog/hello" → dist/blog/hello/index.html "/404" → dist/404.html
  *
  * @param {string} urlPattern
  * @param {string} outDir
@@ -2363,6 +2452,12 @@ function routeToOutputPath(urlPattern: string, outDir: string, trailingSlash: st
     return join(outDir, "index.html");
   }
 
+  // The not-found page is the one file a static host looks for by name (specs/site-architecture.md
+  // §8.4.2), whatever the trailingSlash setting.
+  if (isNotFoundRoute(urlPattern)) {
+    return join(outDir, "404.html");
+  }
+
   // Remove leading slash
   const segments = urlPattern.replace(/^\//, "");
 

package/src/site/static-host.ts

@@ -0,0 +1,153 @@
+/**
+ * What a static host and a crawler are told about each page, decided in one place.
+ *
+ * Four facts, each of which the build used to leave to the author or get wrong:
+ *
+ * - The path a host serves a route at, which `build.trailingSlash` decides, so the canonical link,
+ *   `og:url`, the sitemap `<loc>` and the `hreflang` alternates name the URL that answers with the
+ *   page and not one that redirects to it ({@link servedRoute});
+ * - Which route is the site's not-found page ({@link isNotFoundRoute}), which hosts expect at
+ *   `/404.html` and which is never a page to index;
+ * - Whether a built page asks crawlers not to index it ({@link htmlDeclaresNoindex}), because a
+ *   sitemap that lists a page the page itself disowns sends two answers to the same question;
+ * - Where a page's machine-readable twin lives, and the `<link rel="alternate">` that tells a reader
+ *   of the HTML about it ({@link withAlternateLink}), because a file nothing points to is a file
+ *   nobody finds.
+ *
+ * @docs framework/site/deployment
+ * @docs framework/agents/machine-readable
+ */
+
+import { relative, sep } from "node:path";
+import { escapeHtml } from "../shared.ts";
+
+/**
+ * The route of the site's not-found page, written to `404.html` (specs/site-architecture.md
+ * §8.4.2).
+ */
+export const NOT_FOUND_ROUTE = "/404";
+
+/** Whether `urlPattern` is the site's not-found page. */
+export function isNotFoundRoute(urlPattern: string): boolean {
+  return urlPattern === NOT_FOUND_ROUTE;
+}
+
+/**
+ * The path a host serves `urlPattern` at, given `build.trailingSlash`.
+ *
+ * `"always"` writes `/about` as `about/index.html`, which hosts serve at `/about/` and answer
+ * `/about` with a redirect to it; `"never"` writes `about.html`, served at `/about`. The root is
+ * `/` either way, and any other setting leaves the route as written.
+ *
+ * @param {string} urlPattern - A concrete route (`/about`, `/kb/a/b`)
+ * @param {string} [trailingSlash] - `build.trailingSlash`
+ * @returns {string} The route as the host serves it
+ */
+export function servedRoute(urlPattern: string, trailingSlash?: string): string {
+  if (urlPattern === "/") {
+    return "/";
+  }
+  if (trailingSlash === "always") {
+    return urlPattern.endsWith("/") ? urlPattern : `${urlPattern}/`;
+  }
+  if (trailingSlash === "never") {
+    return urlPattern.replace(/\/+$/, "") || "/";
+  }
+  return urlPattern;
+}
+
+/** The value of one attribute in a start tag, quoted either way or bare; `null` when absent. */
+function attributeOf(tag: string, name: string): string | null {
+  const match = new RegExp(`\\s${name}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s"'>]+))`, "i").exec(
+    tag,
+  );
+  return match === null ? null : (match[1] ?? match[2] ?? match[3] ?? "");
+}
+
+/**
+ * Whether the `<head>` of a built page carries `<meta name="robots">` with `noindex` (or `none`,
+ * which means `noindex, nofollow`). Reads the finished HTML, like the CSP scan: a page can set the
+ * tag from a layout, a state entry or `$head`, and only the output says what it ended up with.
+ *
+ * @param {string} html - A built page
+ * @returns {boolean} True when crawlers are asked to leave the page out of their index
+ */
+export function htmlDeclaresNoindex(html: string): boolean {
+  const end = html.search(/<\/head\s*>/i);
+  const head = end === -1 ? html : html.slice(0, end);
+  for (const [tag] of head.matchAll(/<meta\b[^>]*>/gi)) {
+    if (attributeOf(tag, "name")?.trim().toLowerCase() !== "robots") {
+      continue;
+    }
+    const directives = new Set(
+      (attributeOf(tag, "content") ?? "")
+        .toLowerCase()
+        .split(",")
+        .map((part) => part.trim()),
+    );
+    if (directives.has("noindex") || directives.has("none")) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/** The `<head>` of a built page, or the whole string when it has no closing tag. */
+function headOf(html: string): { head: string; end: number } {
+  const end = html.search(/<\/head\s*>/i);
+  return { end, head: end === -1 ? html : html.slice(0, end) };
+}
+
+/**
+ * Whether a page's `<head>` already has a `<link rel="alternate">` of this media type, one the
+ * author wrote or an earlier step added. `type` is compared by essence and without regard to case.
+ *
+ * @param {string} html - A built page
+ * @param {string} type - A media type essence, `text/markdown`
+ * @returns {boolean} True when the page already advertises an alternate of that type
+ */
+export function hasAlternateLink(html: string, type: string): boolean {
+  const wanted = type.trim().toLowerCase();
+  for (const [tag] of headOf(html).head.matchAll(/<link\b[^>]*>/gi)) {
+    const rel = (attributeOf(tag, "rel") ?? "").toLowerCase().split(/\s+/);
+    const linked = (attributeOf(tag, "type") ?? "").split(";")[0]?.trim().toLowerCase();
+    if (rel.includes("alternate") && linked === wanted) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/**
+ * Adds `<link rel="alternate" type="…" href="…">` to the end of a built page's `<head>`, unless the
+ * page already has an alternate of that type (an author's own wins, like every auto-injected entry,
+ * specs/site-architecture.md §8.4) or has no `<head>` to put it in.
+ *
+ * This is the discovery mechanism the llms.txt proposal recommends for a Markdown twin
+ * (`rel="alternate" type="text/markdown"`): the twin's URL is not one a reader can guess, and
+ * `/templates/` is not `/templates/index.md` to anyone who has not been told.
+ *
+ * @param {string} html - A built page
+ * @param {{ href: string; type: string }} link - Site-absolute `href`, and the media type essence
+ * @returns {string} The page, with the link when it was needed
+ */
+export function withAlternateLink(html: string, link: { href: string; type: string }): string {
+  const { end } = headOf(html);
+  if (end === -1 || hasAlternateLink(html, link.type)) {
+    return html;
+  }
+  const tag = `<link href="${escapeHtml(link.href)}" rel="alternate" type="${escapeHtml(link.type)}">`;
+  return `${html.slice(0, end)}${tag}\n${html.slice(end)}`;
+}
+
+/**
+ * The site-absolute URL path a file in the build output is served at, percent-encoded the way a
+ * `<link href>` and a canonical URL are (`/kb/a b/index.md` is `/kb/a%20b/index.md`).
+ *
+ * @param {string} outDir - The build output directory
+ * @param {string} file - A file inside it
+ * @returns {string} `/templates/index.md`
+ */
+export function outputUrlPath(outDir: string, file: string): string {
+  return new URL(`/${relative(outDir, file).split(sep).join("/")}`, "http://localhost").pathname;
+}