@iterant/site-runtime 3.12.0 → 3.13.0

package/src/routes/llms-txt.ts

@@ -1,6 +1,7 @@
 import { getCollection } from "astro:content";
-import { FOLDER } from "virtual:iterant/site";
+import { CANONICAL_SCOPE, FOLDER, PLATFORM_SITE } from "virtual:iterant/site";
 import { isAdvertised } from "../lib/advertised";
+import { inCanonicalScope, type CanonicalScope } from "../lib/canonical-scope";
 import { folderPath } from "../lib/folder";
 import type { APIRoute } from "astro";
 
@@ -23,53 +24,111 @@ export interface LlmsTxtOptions {
   siteConfig: { name: string; description: string };
 }
 
+/** One published page, as the file lists it. */
+export interface LlmsTxtPage {
+  route: string;
+  title: string;
+  description?: string;
+}
+
 const normalizeText = (value: string) => value.trim().replace(/\s+/g, " ");
 
 const escapeLinkText = (value: string) =>
   normalizeText(value).replaceAll("[", "\\[").replaceAll("]", "\\]");
 
-export function createLlmsTxtRoute({ siteConfig }: LlmsTxtOptions): APIRoute {
-  return async ({ site }) => {
-    const origin = site?.origin ?? "https://example.com";
-    const pages = (await getCollection("pages")).filter((entry) =>
-      isAdvertised(
-        { draft: entry.data.draft, noindex: entry.data.noindex },
-        { includeDrafts: !import.meta.env.PROD },
-      ),
-    );
+/**
+ * The file's body. EVERY link in it is a path this origin serves: the file is
+ * read on whichever host it was fetched from, so a link to a path the customer
+ * forwards to their own site sends a reader there, and an absolute link to our
+ * host advertises it on their domain. Under folder scope that leaves out the
+ * pages outside the folder, the brand home and the root robots policy; the
+ * generated sitemap index is inside the folder and stays. A section whose every
+ * line went is dropped with its heading.
+ */
+export function renderLlmsTxt(params: {
+  siteConfig: { name: string; description: string };
+  pages: LlmsTxtPage[];
+  origin: string;
+  folder: string;
+  scope: CanonicalScope;
+}): string {
+  const { siteConfig, pages, origin, folder, scope } = params;
+  const served = (path: string) => inCanonicalScope(path, scope);
+  const absolute = (path: string) => new URL(path, origin).href;
+
+  const contentLinks = pages
+    .filter((page) => served(page.route))
+    .toSorted((a, b) => a.title.localeCompare(b.title))
+    .map((page) => {
+      const title = escapeLinkText(page.title);
+      const description = page.description
+        ? `: ${normalizeText(page.description)}`
+        : "";
+
+      return `- [${title}](${absolute(page.route)})${description}`;
+    });
 
-    const contentLinks = pages
-      .toSorted((a, b) => a.data.meta.title.localeCompare(b.data.meta.title))
-      .map((entry) => {
-        const title = escapeLinkText(entry.data.meta.title);
-        const url = new URL(entry.data.route, origin).href;
-        const description = entry.data.meta.description
-          ? `: ${normalizeText(entry.data.meta.description)}`
-          : "";
+  const coreLinks = served("/")
+    ? [`- [Home](${absolute("/")}): Primary overview of the site.`]
+    : [];
+  const sitemapPath = folderPath(folder, "sitemap-index.xml");
+  const discoveryLinks = [
+    ...(served(sitemapPath)
+      ? [
+          `- [Sitemap](${absolute(sitemapPath)}): Complete search-engine sitemap, if available.`,
+        ]
+      : []),
+    ...(served("/robots.txt")
+      ? [
+          `- [Robots policy](${absolute("/robots.txt")}): Crawler permissions for automated agents.`,
+        ]
+      : []),
+  ];
 
-        return `- [${title}](${url})${description}`;
-      });
+  const section = (heading: string, links: string[]) =>
+    links.length > 0 ? [heading, "", ...links, ""] : [];
 
-    const lines = [
-      ...(siteConfig.name ? [`# ${siteConfig.name}`, ""] : []),
-      ...(siteConfig.description ? [`> ${siteConfig.description}`, ""] : []),
-      "This file is generated from the site's source content. It is a curated index for AI assistants, not a crawler permissions file. For crawler permissions, see `/robots.txt`.",
-      "",
-      "## Core Pages",
-      "",
-      `- [Home](${new URL("/", origin).href}): Primary overview of the site.`,
-      "",
-      ...(contentLinks.length > 0
-        ? ["## Public Content", "", ...contentLinks, ""]
-        : []),
-      "## Discovery",
-      "",
-      `- [Sitemap](${new URL(folderPath(FOLDER, "sitemap-index.xml"), origin).href}): Complete search-engine sitemap, if available.`,
-      `- [Robots policy](${new URL("/robots.txt", origin).href}): Crawler permissions for automated agents.`,
-      "",
-    ];
+  const lines = [
+    ...(siteConfig.name ? [`# ${siteConfig.name}`, ""] : []),
+    ...(siteConfig.description ? [`> ${siteConfig.description}`, ""] : []),
+    "This file is generated from the site's source content. It is a curated index for AI assistants, not a crawler permissions file. For crawler permissions, see `/robots.txt`.",
+    "",
+    ...section("## Core Pages", coreLinks),
+    ...section("## Public Content", contentLinks),
+    ...section("## Discovery", discoveryLinks),
+  ];
+
+  return lines.join("\n");
+}
+
+export function createLlmsTxtRoute({ siteConfig }: LlmsTxtOptions): APIRoute {
+  return async ({ site }) => {
+    const pages = (await getCollection("pages"))
+      .filter((entry) =>
+        isAdvertised(
+          { draft: entry.data.draft, noindex: entry.data.noindex },
+          { includeDrafts: !import.meta.env.PROD },
+        ),
+      )
+      .map((entry) => ({
+        route: entry.data.route,
+        title: entry.data.meta.title,
+        description: entry.data.meta.description,
+      }));
+
+    const body = renderLlmsTxt({
+      siteConfig,
+      pages,
+      origin: site?.origin ?? "https://example.com",
+      folder: FOLDER,
+      scope: {
+        name: CANONICAL_SCOPE,
+        folder: FOLDER,
+        platformSite: PLATFORM_SITE,
+      },
+    });
 
-    return new Response(lines.join("\n"), {
+    return new Response(body, {
       headers: {
         "content-type": "text/plain; charset=utf-8",
       },

package/src/routes/robots-txt.ts

@@ -1,6 +1,11 @@
 import type { APIRoute } from "astro";
-import { FOLDER } from "virtual:iterant/site";
+import { CANONICAL_SCOPE, FOLDER, PLATFORM_SITE } from "virtual:iterant/site";
 
+import {
+  HOST_SCOPE,
+  inCanonicalScope,
+  type CanonicalScope,
+} from "../lib/canonical-scope";
 import { folderPath } from "../lib/folder";
 import { normalizeSourceUrls } from "../lib/sitemap/shared";
 
@@ -116,18 +121,24 @@ export function renderRobotsTxt(
   // folder is what the customer forwards from their domain's root, while a site
   // URL that carries a path keeps resolving relative to it without one.
   folder: string = "",
+  // Which paths that origin serves (site-runtime 3.13.0). The generated index
+  // is inside the folder and is always advertised; the MIRRORED sitemap is at
+  // the root, which a customer forwarding the folder alone answers with their
+  // own site, so under folder scope it is not advertised at all.
+  scope: CanonicalScope = HOST_SCOPE,
 ): string {
   const policy = resolveAiPolicy(config.aiPolicy);
   const indexPath = folder
     ? folderPath(folder, "sitemap-index.xml")
     : "sitemap-index.xml";
+  const mirrored =
+    normalizeSourceUrls(config.sourceSitemapUrl ?? "").length > 0 &&
+    inCanonicalScope("/sitemap.xml", scope);
   const sitemap = site
     ? [
         `Sitemap: ${new URL(indexPath, site).href}`,
         // Mirrored sitemap, only when sourceSitemapUrl is configured.
-        ...(normalizeSourceUrls(config.sourceSitemapUrl ?? "").length > 0
-          ? [`Sitemap: ${new URL("sitemap.xml", site).href}`]
-          : []),
+        ...(mirrored ? [`Sitemap: ${new URL("sitemap.xml", site).href}`] : []),
       ].join("\n") + "\n"
     : "";
   const wildcard = [
@@ -152,7 +163,12 @@ export function createRobotsTxtRoute(options: RobotsTxtOptions): APIRoute {
   const { siteConfig, ...flat } = options;
   const config: RobotsSiteConfig = { ...flat, ...(siteConfig ?? {}) };
   return ({ site }) =>
-    new Response(renderRobotsTxt(config, site, FOLDER), {
-      headers: { "Content-Type": "text/plain" },
-    });
+    new Response(
+      renderRobotsTxt(config, site, FOLDER, {
+        name: CANONICAL_SCOPE,
+        folder: FOLDER,
+        platformSite: PLATFORM_SITE,
+      }),
+      { headers: { "Content-Type": "text/plain" } },
+    );
 }