@iterant/site-runtime 3.11.2 → 3.13.0

package/src/lib/sitemap/sitemap-with-custom-pages-plugin.ts

@@ -2,6 +2,11 @@ import { fileURLToPath } from "node:url";
 import sitemap, { type SitemapOptions } from "@astrojs/sitemap";
 import type { AstroIntegration } from "astro";
 
+import {
+  HOST_SCOPE,
+  inCanonicalScope,
+  type CanonicalScope,
+} from "../canonical-scope";
 import { getSitemapPaths, type SitemapPathsOptions } from "./get-sitemap-paths";
 
 // The platform patches the `site:` literal at deploy time, so we emit URLs against a
@@ -14,51 +19,69 @@ const PLACEHOLDER = "https://starter.invalid";
 // names them here.
 const PLATFORM_DENY = new Set(["/under-construction"]);
 
-/** Whether a sitemap URL may be listed: neither a platform route nor a denied
- *  entry route (a draft or a noindex page), whichever way the integration
- *  found it. */
+/** Whether a sitemap URL may be listed: in the canonical scope, and neither a
+ *  platform route nor a denied entry route (a draft or a noindex page),
+ *  whichever way the integration found it. An out-of-scope path is not served
+ *  on the origin the `<loc>` would name, so listing it points a crawler at the
+ *  customer's own site. */
 export function sitemapPageAllowed(
   url: string,
   deny: ReadonlySet<string> = new Set(),
+  scope: CanonicalScope = HOST_SCOPE,
 ): boolean {
+  const pathname = sitemapPath(url);
+  if (pathname === undefined) return true;
+  return (
+    !PLATFORM_DENY.has(pathname) &&
+    !deny.has(pathname) &&
+    inCanonicalScope(pathname, scope)
+  );
+}
+
+/** A candidate's path in entry-route form, or undefined when it is not a URL
+ *  the integration can read. */
+function sitemapPath(url: string): string | undefined {
   try {
-    const pathname = new URL(url).pathname.replace(/\/+$/, "") || "/";
-    return !PLATFORM_DENY.has(pathname) && !deny.has(pathname);
+    return new URL(url).pathname.replace(/\/+$/, "") || "/";
   } catch {
-    return true;
+    return undefined;
   }
 }
 
-/** The serialize step's URL work, extracted so it can be tested: resolve the
- *  placeholder origin against the real site, then drop a URL this build has
- *  already emitted. The integration de-duplicates by exact string before
- *  serialize runs, so a route that is both a page entry and a prerendered
- *  shell arrives twice, under the crawl's real origin and under ours, and
- *  becomes one URL only here. Returning nothing drops the item. */
-export function resolveSitemapUrl(
-  url: string,
-  site: string,
-  emitted: Set<string>,
-): string | undefined {
-  const resolved =
-    site && url.startsWith(PLACEHOLDER)
-      ? site + url.slice(PLACEHOLDER.length)
-      : url;
-  if (emitted.has(resolved)) return undefined;
-  emitted.add(resolved);
-  return resolved;
+/**
+ * The integration's page filter: the rule above, plus first-one-wins on a
+ * repeated path.
+ *
+ * @astrojs/sitemap discovers the prerendered pages itself AND is handed the
+ * entry routes as customPages, so a bespoke page that also has a page entry
+ * arrives twice and is listed twice. One pass over the candidates, so the state
+ * is the filter's own and a new build gets a new one.
+ */
+export function createSitemapFilter(
+  deny: ReadonlySet<string> = new Set(),
+  scope: CanonicalScope = HOST_SCOPE,
+): (url: string) => boolean {
+  const listed = new Set<string>();
+  return (url) => {
+    if (!sitemapPageAllowed(url, deny, scope)) return false;
+    const pathname = sitemapPath(url) ?? url;
+    if (listed.has(pathname)) return false;
+    listed.add(pathname);
+    return true;
+  };
 }
 
 export type SitemapWithCustomPagesOptions = SitemapOptions &
-  SitemapPathsOptions;
+  SitemapPathsOptions & {
+    /** Which paths the site origin serves. Defaults to the whole host. */
+    scope?: CanonicalScope;
+  };
 
 export function sitemapWithCustomPages(
   options: SitemapWithCustomPagesOptions = {},
 ): AstroIntegration[] {
   let resolvedSite = "";
-  // Emptied per build in astro:config:done.
-  const emitted = new Set<string>();
-  const { pagesDir, root, ...sitemapOptions } = options;
+  const { pagesDir, root, scope = HOST_SCOPE, ...sitemapOptions } = options;
   const userSerialize = sitemapOptions.serialize;
   const userFilter = sitemapOptions.filter;
 
@@ -69,8 +92,10 @@ export function sitemapWithCustomPages(
   // populated before anything reads it.
   const customPages: string[] = [...(sitemapOptions.customPages ?? [])];
   // Filled beside customPages: the entry routes the integration must not list
-  // from its own crawl of prerendered pages.
+  // from its own crawl of prerendered pages. The filter reads it by reference,
+  // so it sees whatever astro:config:setup put there.
   const denied = new Set<string>();
+  const allowed = createSitemapFilter(denied, scope);
 
   return [
     {
@@ -109,7 +134,6 @@ export function sitemapWithCustomPages(
           }
         },
         "astro:config:done": ({ config }) => {
-          emitted.clear();
           if (config.site) {
             resolvedSite = String(config.site).replace(/\/$/, "");
           }
@@ -119,13 +143,11 @@ export function sitemapWithCustomPages(
     sitemap({
       ...sitemapOptions,
       customPages,
-      filter: (page) =>
-        sitemapPageAllowed(page, denied) &&
-        (userFilter ? userFilter(page) : true),
+      filter: (page) => (userFilter ? userFilter(page) : true) && allowed(page),
       serialize(item) {
-        const url = resolveSitemapUrl(item.url, resolvedSite, emitted);
-        if (!url) return undefined;
-        item.url = url;
+        if (resolvedSite && item.url.startsWith(PLACEHOLDER)) {
+          item.url = resolvedSite + item.url.slice(PLACEHOLDER.length);
+        }
         return userSerialize ? userSerialize(item) : item;
       },
     }),

package/src/routes/index.ts

@@ -1,8 +1,18 @@
 // Platform route handlers. Every one of these stays behind a VISIBLE file in the
-// repo's src/pages/ — existence and addressing are repo-owned, behavior rides a
-// package bump. No route is injected: a route that exists in no repo file breaks
-// the src/pages/ mental model and the debuggability contract.
-export { createLlmsTxtRoute, type LlmsTxtOptions } from "./llms-txt";
+// repo's src/pages/: existence and addressing are repo-owned, behavior rides a
+// package bump. A route that exists in no repo file breaks the src/pages/ mental
+// model and the debuggability contract.
+//
+// One route is addressed by the platform, llms.txt under a brand folder
+// (site-runtime 3.12.0): the folder is per-brand data, so no file name the
+// template ships can carry it, and the preset injects the same handler at
+// /<folder>/llms.txt (../config/site-module.ts).
+export {
+  createLlmsTxtRoute,
+  renderLlmsTxt,
+  type LlmsTxtOptions,
+  type LlmsTxtPage,
+} from "./llms-txt";
 export {
   AI_ANSWER_CRAWLERS,
   AI_POLICY_PRESETS,

package/src/routes/llms-txt-injected.ts

@@ -0,0 +1,11 @@
+import { SITE_CONFIG } from "virtual:iterant/site";
+
+import { createLlmsTxtRoute } from "./llms-txt";
+
+// The body of /<folder>/llms.txt, as the preset injects it (see
+// ../config/site-module.ts). The brand's name and description come from the
+// repo's site config through the virtual module, since an injected route has no
+// repo file to import them from; the folder the handler writes into its links
+// comes from the same module.
+export const prerender = true;
+export const GET = createLlmsTxtRoute({ siteConfig: SITE_CONFIG });

package/src/routes/llms-txt.ts

@@ -1,5 +1,8 @@
 import { getCollection } from "astro:content";
+import { CANONICAL_SCOPE, FOLDER, PLATFORM_SITE } from "virtual:iterant/site";
 import { isAdvertised } from "../lib/advertised";
+import { inCanonicalScope, type CanonicalScope } from "../lib/canonical-scope";
+import { folderPath } from "../lib/folder";
 import type { APIRoute } from "astro";
 
 // /llms.txt, generated at dev/build time with no crawler or AI step: a curated
@@ -11,59 +14,121 @@ import type { APIRoute } from "astro";
 //   ---
 //   export const prerender = true;
 //   export const GET = createLlmsTxtRoute({ siteConfig: SITE_CONFIG });
+//
+// Under a brand folder the same handler is addressed by the preset instead, at
+// /<folder>/llms.txt, because no file name the template ships can carry a
+// per-brand segment (../config/site-module.ts).
 
 export interface LlmsTxtOptions {
   /** The brand's SITE_CONFIG; name and description head the file when set. */
   siteConfig: { name: string; description: string };
 }
 
+/** One published page, as the file lists it. */
+export interface LlmsTxtPage {
+  route: string;
+  title: string;
+  description?: string;
+}
+
 const normalizeText = (value: string) => value.trim().replace(/\s+/g, " ");
 
 const escapeLinkText = (value: string) =>
   normalizeText(value).replaceAll("[", "\\[").replaceAll("]", "\\]");
 
-export function createLlmsTxtRoute({ siteConfig }: LlmsTxtOptions): APIRoute {
-  return async ({ site }) => {
-    const origin = site?.origin ?? "https://example.com";
-    const pages = (await getCollection("pages")).filter((entry) =>
-      isAdvertised(
-        { draft: entry.data.draft, noindex: entry.data.noindex },
-        { includeDrafts: !import.meta.env.PROD },
-      ),
-    );
+/**
+ * The file's body. EVERY link in it is a path this origin serves: the file is
+ * read on whichever host it was fetched from, so a link to a path the customer
+ * forwards to their own site sends a reader there, and an absolute link to our
+ * host advertises it on their domain. Under folder scope that leaves out the
+ * pages outside the folder, the brand home and the root robots policy; the
+ * generated sitemap index is inside the folder and stays. A section whose every
+ * line went is dropped with its heading.
+ */
+export function renderLlmsTxt(params: {
+  siteConfig: { name: string; description: string };
+  pages: LlmsTxtPage[];
+  origin: string;
+  folder: string;
+  scope: CanonicalScope;
+}): string {
+  const { siteConfig, pages, origin, folder, scope } = params;
+  const served = (path: string) => inCanonicalScope(path, scope);
+  const absolute = (path: string) => new URL(path, origin).href;
+
+  const contentLinks = pages
+    .filter((page) => served(page.route))
+    .toSorted((a, b) => a.title.localeCompare(b.title))
+    .map((page) => {
+      const title = escapeLinkText(page.title);
+      const description = page.description
+        ? `: ${normalizeText(page.description)}`
+        : "";
+
+      return `- [${title}](${absolute(page.route)})${description}`;
+    });
 
-    const contentLinks = pages
-      .toSorted((a, b) => a.data.meta.title.localeCompare(b.data.meta.title))
-      .map((entry) => {
-        const title = escapeLinkText(entry.data.meta.title);
-        const url = new URL(entry.data.route, origin).href;
-        const description = entry.data.meta.description
-          ? `: ${normalizeText(entry.data.meta.description)}`
-          : "";
+  const coreLinks = served("/")
+    ? [`- [Home](${absolute("/")}): Primary overview of the site.`]
+    : [];
+  const sitemapPath = folderPath(folder, "sitemap-index.xml");
+  const discoveryLinks = [
+    ...(served(sitemapPath)
+      ? [
+          `- [Sitemap](${absolute(sitemapPath)}): Complete search-engine sitemap, if available.`,
+        ]
+      : []),
+    ...(served("/robots.txt")
+      ? [
+          `- [Robots policy](${absolute("/robots.txt")}): Crawler permissions for automated agents.`,
+        ]
+      : []),
+  ];
 
-        return `- [${title}](${url})${description}`;
-      });
+  const section = (heading: string, links: string[]) =>
+    links.length > 0 ? [heading, "", ...links, ""] : [];
 
-    const lines = [
-      ...(siteConfig.name ? [`# ${siteConfig.name}`, ""] : []),
-      ...(siteConfig.description ? [`> ${siteConfig.description}`, ""] : []),
-      "This file is generated from the site's source content. It is a curated index for AI assistants, not a crawler permissions file. For crawler permissions, see `/robots.txt`.",
-      "",
-      "## Core Pages",
-      "",
-      `- [Home](${new URL("/", origin).href}): Primary overview of the site.`,
-      "",
-      ...(contentLinks.length > 0
-        ? ["## Public Content", "", ...contentLinks, ""]
-        : []),
-      "## Discovery",
-      "",
-      `- [Sitemap](${new URL("/sitemap-index.xml", origin).href}): Complete search-engine sitemap, if available.`,
-      `- [Robots policy](${new URL("/robots.txt", origin).href}): Crawler permissions for automated agents.`,
-      "",
-    ];
+  const lines = [
+    ...(siteConfig.name ? [`# ${siteConfig.name}`, ""] : []),
+    ...(siteConfig.description ? [`> ${siteConfig.description}`, ""] : []),
+    "This file is generated from the site's source content. It is a curated index for AI assistants, not a crawler permissions file. For crawler permissions, see `/robots.txt`.",
+    "",
+    ...section("## Core Pages", coreLinks),
+    ...section("## Public Content", contentLinks),
+    ...section("## Discovery", discoveryLinks),
+  ];
+
+  return lines.join("\n");
+}
+
+export function createLlmsTxtRoute({ siteConfig }: LlmsTxtOptions): APIRoute {
+  return async ({ site }) => {
+    const pages = (await getCollection("pages"))
+      .filter((entry) =>
+        isAdvertised(
+          { draft: entry.data.draft, noindex: entry.data.noindex },
+          { includeDrafts: !import.meta.env.PROD },
+        ),
+      )
+      .map((entry) => ({
+        route: entry.data.route,
+        title: entry.data.meta.title,
+        description: entry.data.meta.description,
+      }));
+
+    const body = renderLlmsTxt({
+      siteConfig,
+      pages,
+      origin: site?.origin ?? "https://example.com",
+      folder: FOLDER,
+      scope: {
+        name: CANONICAL_SCOPE,
+        folder: FOLDER,
+        platformSite: PLATFORM_SITE,
+      },
+    });
 
-    return new Response(lines.join("\n"), {
+    return new Response(body, {
       headers: {
         "content-type": "text/plain; charset=utf-8",
       },

package/src/routes/robots-txt.ts

@@ -1,5 +1,12 @@
 import type { APIRoute } from "astro";
-
+import { CANONICAL_SCOPE, FOLDER, PLATFORM_SITE } from "virtual:iterant/site";
+
+import {
+  HOST_SCOPE,
+  inCanonicalScope,
+  type CanonicalScope,
+} from "../lib/canonical-scope";
+import { folderPath } from "../lib/folder";
 import { normalizeSourceUrls } from "../lib/sitemap/shared";
 
 // /robots.txt: crawler permissions, the site's AI policy and the sitemap
@@ -108,15 +115,30 @@ function legacyDisallows(rules: string[] | undefined): string[] {
 export function renderRobotsTxt(
   config: RobotsSiteConfig,
   site: URL | undefined,
+  // The brand's folder (site-runtime 3.12.0). robots.txt itself stays at the
+  // root, where a crawler looks for it; its index line points into the folder,
+  // where the build writes the sitemap. Resolved against the ORIGIN, because the
+  // folder is what the customer forwards from their domain's root, while a site
+  // URL that carries a path keeps resolving relative to it without one.
+  folder: string = "",
+  // Which paths that origin serves (site-runtime 3.13.0). The generated index
+  // is inside the folder and is always advertised; the MIRRORED sitemap is at
+  // the root, which a customer forwarding the folder alone answers with their
+  // own site, so under folder scope it is not advertised at all.
+  scope: CanonicalScope = HOST_SCOPE,
 ): string {
   const policy = resolveAiPolicy(config.aiPolicy);
+  const indexPath = folder
+    ? folderPath(folder, "sitemap-index.xml")
+    : "sitemap-index.xml";
+  const mirrored =
+    normalizeSourceUrls(config.sourceSitemapUrl ?? "").length > 0 &&
+    inCanonicalScope("/sitemap.xml", scope);
   const sitemap = site
     ? [
-        `Sitemap: ${new URL("sitemap-index.xml", site).href}`,
+        `Sitemap: ${new URL(indexPath, site).href}`,
         // Mirrored sitemap, only when sourceSitemapUrl is configured.
-        ...(normalizeSourceUrls(config.sourceSitemapUrl ?? "").length > 0
-          ? [`Sitemap: ${new URL("sitemap.xml", site).href}`]
-          : []),
+        ...(mirrored ? [`Sitemap: ${new URL("sitemap.xml", site).href}`] : []),
       ].join("\n") + "\n"
     : "";
   const wildcard = [
@@ -141,7 +163,12 @@ export function createRobotsTxtRoute(options: RobotsTxtOptions): APIRoute {
   const { siteConfig, ...flat } = options;
   const config: RobotsSiteConfig = { ...flat, ...(siteConfig ?? {}) };
   return ({ site }) =>
-    new Response(renderRobotsTxt(config, site), {
-      headers: { "Content-Type": "text/plain" },
-    });
+    new Response(
+      renderRobotsTxt(config, site, FOLDER, {
+        name: CANONICAL_SCOPE,
+        folder: FOLDER,
+        platformSite: PLATFORM_SITE,
+      }),
+      { headers: { "Content-Type": "text/plain" } },
+    );
 }