@iterant/site-runtime 3.0.2

package/src/lib/sitemap/shared.ts

@@ -0,0 +1,95 @@
+// Shared helpers for the sitemap-proxy routes (./sitemap.ts and ./proxy-sitemap.ts):
+// fetch the upstream, validate it's XML, parse <sitemapindex> entries, rewrite every
+// URL's host to this site's domain. Plus the response headers and empty-urlset body
+// both routes return.
+
+// Successful responses cache for 1h; the empty fallback caches for 5m so a
+// transient bad upstream self-heals on the next crawl.
+const CACHE_SECONDS = 3600;
+const EMPTY_CACHE_SECONDS = 300;
+
+export const SITEMAP_HEADERS = {
+  "Content-Type": "application/xml; charset=utf-8",
+  "Cache-Control": `public, max-age=${CACHE_SECONDS}`,
+};
+
+export const EMPTY_SITEMAP_HEADERS = {
+  "Content-Type": "application/xml; charset=utf-8",
+  "Cache-Control": `public, max-age=${EMPTY_CACHE_SECONDS}`,
+};
+
+export const EMPTY_URLSET =
+  '<?xml version="1.0" encoding="UTF-8"?>\n' +
+  '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"></urlset>\n';
+
+// Trim a string-or-array config to a clean, non-empty array.
+export function normalizeSourceUrls(raw: string | string[]): string[] {
+  const arr = Array.isArray(raw) ? raw : [raw];
+  return arr.map((s) => s.trim()).filter((s) => s.length > 0);
+}
+
+// Only rewrite hosts in <loc>, <image:loc>, and <xhtml:link href>.
+const LOC_RE = /(<(?:loc|image:loc)>)([^<]+)(<\/(?:loc|image:loc)>)/g;
+const XHTML_HREF_RE = /(<xhtml:link\b[^>]*?\bhref=")([^"]+)(")/g;
+
+export function rewriteSitemapDomain(
+  xml: string,
+  targetOrigin: string,
+): string {
+  const target = new URL(targetOrigin);
+
+  const rewriteUrl = (raw: string): string => {
+    try {
+      const url = new URL(raw.trim());
+      url.protocol = target.protocol;
+      url.host = target.host;
+      return url.toString();
+    } catch {
+      return raw;
+    }
+  };
+
+  let out = xml.replace(
+    LOC_RE,
+    (_m, open, url, close) => `${open}${rewriteUrl(url)}${close}`,
+  );
+  if (out.includes("xhtml:link")) {
+    out = out.replace(
+      XHTML_HREF_RE,
+      (_m, pre, url, post) => `${pre}${rewriteUrl(url)}${post}`,
+    );
+  }
+  return out;
+}
+
+// Returns the entries inside `xml` if it's a <sitemapindex>, else null.
+export function parseSitemapIndex(xml: string): string[] | null {
+  if (!/<sitemapindex[\s>]/.test(xml)) return null;
+  const urls: string[] = [];
+  const re = /<loc>([^<]+)<\/loc>/g;
+  let m: RegExpExecArray | null;
+  while ((m = re.exec(xml)) !== null) urls.push(m[1].trim());
+  return urls;
+}
+
+// Fetch a sitemap from `url`. Returns the body only if it looks like XML, so an
+// HTML error / parked page can't get served back as a sitemap. null on any
+// failure (logged).
+export async function fetchUpstream(url: string): Promise<string | null> {
+  try {
+    const res = await fetch(url, {
+      headers: { Accept: "application/xml, text/xml" },
+      cf: { cacheTtl: CACHE_SECONDS, cacheEverything: true },
+    } as RequestInit);
+    if (!res.ok) {
+      console.warn(`[sitemap-proxy] upstream ${url} returned ${res.status}`);
+      return null;
+    }
+    const text = await res.text();
+    if (/^\s*<\??(xml|urlset|sitemapindex)/i.test(text)) return text;
+    console.warn(`[sitemap-proxy] upstream ${url} is not XML; ignoring`);
+  } catch (err) {
+    console.warn(`[sitemap-proxy] failed to fetch ${url}`, err);
+  }
+  return null;
+}

package/src/lib/sitemap/sitemap-with-custom-pages-plugin.ts

@@ -0,0 +1,74 @@
+import { fileURLToPath } from "node:url";
+import sitemap, { type SitemapOptions } from "@astrojs/sitemap";
+import type { AstroIntegration } from "astro";
+
+import { getSitemapPaths, type SitemapPathsOptions } from "./get-sitemap-paths";
+
+// The platform patches the `site:` literal at deploy time, so we emit URLs against a
+// placeholder and swap it for the resolved `config.site` at sitemap emit time.
+const PLACEHOLDER = "https://starter.invalid";
+
+export type SitemapWithCustomPagesOptions = SitemapOptions &
+  SitemapPathsOptions;
+
+export function sitemapWithCustomPages(
+  options: SitemapWithCustomPagesOptions = {},
+): AstroIntegration[] {
+  let resolvedSite = "";
+  const { pagesDir, root, ...sitemapOptions } = options;
+  const userSerialize = sitemapOptions.serialize;
+
+  // Filled in at astro:config:setup and read by @astrojs/sitemap at
+  // astro:build:done, which is what lets the entry routes be discovered against
+  // the RESOLVED project root instead of whatever cwd the build was driven from.
+  // This integration is ordered ahead of @astrojs/sitemap, so the array is
+  // populated before anything reads it.
+  const customPages: string[] = [...(sitemapOptions.customPages ?? [])];
+
+  return [
+    {
+      name: "capture-site-for-sitemap",
+      hooks: {
+        "astro:config:setup": ({ config, logger }) => {
+          const { paths, pagesDir: searched, entryCount } = getSitemapPaths({
+            pagesDir,
+            root: root ?? fileURLToPath(config.root),
+          });
+          if (entryCount === 0) {
+            // SSR routes are invisible to @astrojs/sitemap, so an empty read
+            // means a sitemap with no content pages in it. Never silent.
+            logger.warn(
+              `no page entries found in ${searched}; SSR routes will be missing from the sitemap`,
+            );
+          } else if (paths.length === 0) {
+            // Entries exist but every one is a draft or the site root, which
+            // the sitemap already carries; expected on a fresh template.
+            logger.info(
+              `page entries in ${searched} are all drafts or the site root; the sitemap lists no SSR routes yet`,
+            );
+          }
+          for (const path of paths) {
+            customPages.push(
+              `${PLACEHOLDER}${path.startsWith("/") ? path : `/${path}`}`,
+            );
+          }
+        },
+        "astro:config:done": ({ config }) => {
+          if (config.site) {
+            resolvedSite = String(config.site).replace(/\/$/, "");
+          }
+        },
+      },
+    },
+    sitemap({
+      ...sitemapOptions,
+      customPages,
+      serialize(item) {
+        if (resolvedSite && item.url.startsWith(PLACEHOLDER)) {
+          item.url = resolvedSite + item.url.slice(PLACEHOLDER.length);
+        }
+        return userSerialize ? userSerialize(item) : item;
+      },
+    }),
+  ];
+}

package/src/routes/UnderConstruction.astro

@@ -0,0 +1,43 @@
+---
+import type { AstroComponentFactory } from "astro/runtime/server/index.js";
+
+// The fallback target for src/integrations/preview-error-shell.mjs: when any
+// route 5xxes in dev (typically a bespoke page mid-build), the preview serves
+// this body instead of the raw error overlay. Deliberately dependency-light —
+// it must render even when page components are broken.
+//
+// The route stays a visible repo file (src/pages/under-construction.astro), a
+// shim that hands its Layout in:
+//
+//   ---
+//   import Layout from "../layouts/Layout.astro";
+//   import UnderConstruction from "@iterant/site-runtime/under-construction";
+//   ---
+//   <UnderConstruction Layout={Layout} />
+//
+// Do not delete, rename, or link to that route from site navigation.
+
+interface Props {
+  /** The repo's src/layouts/Layout.astro. */
+  Layout: AstroComponentFactory;
+}
+
+const { Layout } = Astro.props;
+---
+
+<Layout
+  title="Page in progress"
+  description="This page is being built."
+  noindex={true}
+>
+  <section
+    class="flex min-h-[60vh] flex-col items-center justify-center gap-3 px-6 text-center"
+  >
+    <p class="text-it-text-primary text-2xl font-semibold">
+      This page is in progress
+    </p>
+    <p class="text-it-text-secondary max-w-md text-sm">
+      We're still putting it together — check back in a moment.
+    </p>
+  </section>
+</Layout>

package/src/routes/index.ts

@@ -0,0 +1,11 @@
+// Platform route handlers. Every one of these stays behind a VISIBLE file in the
+// repo's src/pages/ — existence and addressing are repo-owned, behavior rides a
+// package bump. No route is injected: a route that exists in no repo file breaks
+// the src/pages/ mental model and the debuggability contract.
+export { createLlmsTxtRoute, type LlmsTxtOptions } from "./llms-txt";
+export { createRobotsTxtRoute, type RobotsTxtOptions } from "./robots-txt";
+export {
+  createProxiedSitemapRoute,
+  createSitemapRoute,
+  type SitemapRouteOptions,
+} from "../lib/sitemap/routes";

package/src/routes/llms-txt.ts

@@ -0,0 +1,68 @@
+import { getCollection } from "astro:content";
+import type { APIRoute } from "astro";
+
+// /llms.txt, generated at dev/build time with no crawler or AI step: a curated
+// index for AI assistants built from the site's own published content. The repo
+// keeps the route file (so the URL is visible where every other route is) and
+// this supplies the body:
+//
+//   src/pages/llms.txt.ts
+//   ---
+//   export const prerender = true;
+//   export const GET = createLlmsTxtRoute({ siteConfig: SITE_CONFIG });
+
+export interface LlmsTxtOptions {
+  /** The brand's SITE_CONFIG; name and description head the file when set. */
+  siteConfig: { name: string; description: string };
+}
+
+const normalizeText = (value: string) => value.trim().replace(/\s+/g, " ");
+
+const escapeLinkText = (value: string) =>
+  normalizeText(value).replaceAll("[", "\\[").replaceAll("]", "\\]");
+
+export function createLlmsTxtRoute({ siteConfig }: LlmsTxtOptions): APIRoute {
+  return async ({ site }) => {
+    const origin = site?.origin ?? "https://example.com";
+    const pages = (await getCollection("pages")).filter(
+      (entry) => !entry.data.draft || !import.meta.env.PROD,
+    );
+
+    const contentLinks = pages
+      .toSorted((a, b) => a.data.meta.title.localeCompare(b.data.meta.title))
+      .map((entry) => {
+        const title = escapeLinkText(entry.data.meta.title);
+        const url = new URL(entry.data.route, origin).href;
+        const description = entry.data.meta.description
+          ? `: ${normalizeText(entry.data.meta.description)}`
+          : "";
+
+        return `- [${title}](${url})${description}`;
+      });
+
+    const lines = [
+      ...(siteConfig.name ? [`# ${siteConfig.name}`, ""] : []),
+      ...(siteConfig.description ? [`> ${siteConfig.description}`, ""] : []),
+      "This file is generated from the site's source content. It is a curated index for AI assistants, not a crawler permissions file. For crawler permissions, see `/robots.txt`.",
+      "",
+      "## Core Pages",
+      "",
+      `- [Home](${new URL("/", origin).href}): Primary overview of the site.`,
+      "",
+      ...(contentLinks.length > 0
+        ? ["## Public Content", "", ...contentLinks, ""]
+        : []),
+      "## Discovery",
+      "",
+      `- [Sitemap](${new URL("/sitemap-index.xml", origin).href}): Complete search-engine sitemap, if available.`,
+      `- [Robots policy](${new URL("/robots.txt", origin).href}): Crawler permissions for automated agents.`,
+      "",
+    ];
+
+    return new Response(lines.join("\n"), {
+      headers: {
+        "content-type": "text/plain; charset=utf-8",
+      },
+    });
+  };
+}

package/src/routes/robots-txt.ts

@@ -0,0 +1,56 @@
+import type { APIRoute } from "astro";
+
+import { normalizeSourceUrls } from "../lib/sitemap/shared";
+
+// /robots.txt: crawler permissions plus the sitemap pointers. The repo keeps the
+// route file and this supplies the body:
+//
+//   src/pages/robots.txt.ts
+//   ---
+//   export const prerender = true;
+//   export const GET = createRobotsTxtRoute({
+//     sourceSitemapUrl: SITE_CONFIG.sourceSitemapUrl,
+//   });
+
+export interface RobotsTxtOptions {
+  /** The brand's `SITE_CONFIG.sourceSitemapUrl`; a mirrored sitemap is only
+   *  advertised when one is configured. */
+  sourceSitemapUrl: string | string[];
+}
+
+export function createRobotsTxtRoute({
+  sourceSitemapUrl,
+}: RobotsTxtOptions): APIRoute {
+  return ({ site }) => {
+    const sitemap = site
+      ? [
+          `Sitemap: ${new URL("sitemap-index.xml", site).href}`,
+          // Mirrored sitemap, only when sourceSitemapUrl is configured.
+          ...(normalizeSourceUrls(sourceSitemapUrl).length > 0
+            ? [`Sitemap: ${new URL("sitemap.xml", site).href}`]
+            : []),
+        ].join("\n") + "\n"
+      : "";
+
+    const body = `User-agent: Googlebot
+Allow: /
+
+User-agent: Bingbot
+Allow: /
+
+User-agent: Twitterbot
+Allow: /
+
+User-agent: facebookexternalhit
+Allow: /
+
+User-agent: *
+Allow: /
+
+${sitemap}`;
+
+    return new Response(body, {
+      headers: { "Content-Type": "text/plain" },
+    });
+  };
+}

package/src/version.ts

@@ -0,0 +1,9 @@
+import packageJson from "../package.json";
+
+/**
+ * The runtime version every page announces (`<meta name="it-site-runtime">`).
+ * Read straight from the installed package, so the repo's pinned version IS
+ * the version the rendered site reports — no second place to edit, nothing to
+ * re-stamp on a bump.
+ */
+export const SITE_RUNTIME_VERSION: string = packageJson.version;