@iterant/site-runtime 3.10.0 → 3.11.0

package/docs/runtime-contract.md

@@ -50,7 +50,7 @@ runtime and says so.
 
 <!-- generated: available libraries -->
 
-_Generated from package.json by scripts/generate-kit-table.mjs. Runtime 3.10.0._
+_Generated from package.json by scripts/generate-kit-table.mjs. Runtime 3.11.0._
 
 **Toolchain** (this package owns the version; do NOT declare these):
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@iterant/site-runtime",
-  "version": "3.10.0",
+  "version": "3.11.0",
   "type": "module",
   "description": "The platform layer every Iterant brand site runs on: content grammar, collection schemas, SEO head and JSON-LD, layout core, Astro config preset, dev integrations and the verify gates.",
   "scripts": {

package/scripts/scan-bespoke-siblings.mjs

@@ -113,6 +113,7 @@ async function bespokeInventory() {
  * `/es` (and `/es/<base>`) to `src/pages/[locale]/index.astro` (respectively
  * `[locale]/<base>.astro`) before any rest-param catch-all, so when that file
  * exists the sibling never renders through the catch-all this gate asserts.
+ * @param {string} base
  */
 async function siblingRouteShadowed(base) {
   const file = base === "home" ? "index.astro" : `${base}.astro`;

package/src/content/schema.ts

@@ -135,6 +135,12 @@ export function createContentSchemas({
       // design ships its own nav/footer (a full-design import) so the layout
       // mounts no chrome. Additive — existing entries omit it and keep chrome.
       chrome: z.boolean().default(true),
+      // Kept out of search (site-runtime 3.11.0): the ONE owner of noindex.
+      // The layout renders noindex,nofollow from it, and the sitemap, llms.txt
+      // and a sibling's hreflang leave the page out. Each locale sibling
+      // carries its own value, as draft does. Additive: existing entries omit
+      // it and stay indexed.
+      noindex: z.boolean().default(false),
       // Which SHELL this page renders in (site-runtime 3.2.0). A replicated
       // site can carry more than one frame: a page whose chrome differed from
       // the site's kept its own under a scope suffix, and the shell registry
@@ -159,16 +165,23 @@ export function createContentSchemas({
         .object({
           title: z.string(),
           description: z.string().optional(),
-          // Social/share image. src must be an absolute URL (crawlers 403 on
-          // signed S3 URLs — publish CDN URLs only). assetId links back to the
-          // brand asset; alt is the accessible/og:image:alt text.
+          // Social/share image. src is a root-relative path, which the layout
+          // resolves against the site origin at render time, or an https URL
+          // (a published CDN URL; crawlers 403 on signed S3 URLs, and a
+          // scheme-relative or http src never reaches a share card). assetId
+          // links back to the brand asset; alt is the og:image:alt text.
           ogImage: z
             .object({
               // Every other image leaf in an entry is tagged type: "image",
               // and the model reaches for the same tag here; the tag is
               // accepted so a share image never refuses a whole page write.
               type: z.literal("image").optional(),
-              src: z.string().url(),
+              src: z
+                .string()
+                .regex(
+                  /^(?:\/(?!\/)\S*|https:\/\/\S+)$/,
+                  "ogImage.src must be a root-relative path or an https URL",
+                ),
               assetId: z.string().optional(),
               alt: z.string().optional(),
             })

package/src/layouts/LayoutCore.astro

@@ -86,7 +86,7 @@ const {
   canonical,
   image,
   imageAlt,
-  noindex = false,
+  noindex,
   type = "website",
   siteName = siteConfig.name,
   jsonLd,
@@ -144,6 +144,11 @@ const structuredData = resolveStructuredData(
 // prop, which is what makes the forward safe to ship ahead of any emit.
 const shellId = shell ?? pageEntry?.data.shell;
 
+// Kept out of search, from the same entry, so the page's own noindex reaches
+// <meta robots> through a catch-all that threads nothing. A shell that passes
+// the prop wins, like shell does; no entry and no prop means indexed.
+const noindexResolved = noindex ?? pageEntry?.data.noindex ?? false;
+
 // Machine-fed related links (ILV-6): the machine-owned src/content/links.json
 // manifest, resolved by this page's own route key. A locale sibling looks up
 // its full prefixed route; there is no fallback to the base route, because the
@@ -225,7 +230,7 @@ const fontPreload = fonts.named ? [{ style: "normal", subset: "latin" }] : false
       canonicalUrl={canonicalUrl}
       imageUrl={imageUrl}
       imageAlt={imageAlt}
-      noindex={noindex}
+      noindex={noindexResolved}
       type={type}
       pageType={structuredData.pageType}
       datePublished={structuredData.datePublished}

package/src/lib/advertised.ts

@@ -0,0 +1,17 @@
+// Whether a page entry is advertised to crawlers and agents: listed in the
+// sitemap and llms.txt, and named by its siblings' hreflang. One flag with one
+// meaning: `noindex` keeps a page out everywhere, and a draft is out except
+// where drafts are shown (the dev server). Absent flags read as live and
+// indexed, which is what every entry written before the flags carries.
+export interface AdvertisedFlags {
+  draft?: boolean;
+  noindex?: boolean;
+}
+
+export function isAdvertised(
+  entry: AdvertisedFlags,
+  options: { includeDrafts: boolean },
+): boolean {
+  if (entry.noindex) return false;
+  return !entry.draft || options.includeDrafts;
+}

package/src/lib/hreflang.ts

@@ -1,3 +1,4 @@
+import { isAdvertised } from "./advertised";
 import { DEFAULT_LOCALE, normalizeBcp47, parseEntryId } from "./locales";
 
 // hreflang alternates for locale sibling pages (starter 2.9.0). A page with
@@ -21,12 +22,14 @@ export interface HreflangEntry {
   /** The entry's `route` field (`/example`, `/es/example`). */
   route: string;
   draft: boolean;
+  /** Kept out of search; never advertised as an alternate. */
+  noindex?: boolean;
 }
 
 /** A `pages` collection entry, as `pageLocaleHead` consumes it. */
 export interface PagesCollectionEntry {
   id: string;
-  data: { route: string; draft: boolean };
+  data: { route: string; draft: boolean; noindex?: boolean };
 }
 
 /**
@@ -58,6 +61,7 @@ export function pageLocaleHead(params: {
         id: entry.id,
         route: entry.data.route,
         draft: entry.data.draft,
+        noindex: entry.data.noindex,
       })),
       baseName: parseEntryId(params.entryId).base,
       site: params.site,
@@ -74,8 +78,8 @@ function absolute(route: string, site: URL | string | undefined): string {
  * The reciprocal hreflang set for the group `baseName` belongs to, derived from
  * ALL page entries (the caller passes the whole collection). Every non-draft
  * entry in the group contributes one alternate; the base entry also seeds the
- * single `x-default`. Draft siblings are never advertised (unpublished
- * translations must not leak to crawlers). Deduped on the normalized locale key
+ * single `x-default`. Draft and noindex siblings are never advertised
+ * (unpublished translations and private pages must not leak to crawlers). Deduped on the normalized locale key
  * (first entry wins); hrefs absolute via `site` (`Astro.site`). Returns `[]`
  * when the group has no non-draft siblings — a lone page emits no hreflang.
  */
@@ -92,7 +96,8 @@ export function deriveHreflangAlternates(params: {
   >();
   let base: { route: string } | undefined;
   for (const entry of entries) {
-    if (entry.draft) continue; // never advertise a draft sibling/base
+    // Never advertise a draft or a noindex sibling or base.
+    if (!isAdvertised(entry, { includeDrafts: false })) continue;
     const { base: entryBase, locale } = parseEntryId(entry.id);
     if (entryBase !== baseName) continue;
     const isBase = !locale;

package/src/lib/sitemap/get-sitemap-paths.ts

@@ -6,8 +6,8 @@
 //
 // Locale siblings (`<page>.<locale>.json`) are ordinary entries here: each
 // contributes its own prefixed route (`/es/example`) so every published locale
-// is in the sitemap. Drafts (siblings inherit the base's `draft`) are skipped,
-// so an unpublished translation never appears.
+// is in the sitemap. Each file's own `draft` and `noindex` are read, so an
+// unpublished translation or a private page never appears.
 //
 // `root` is the CONSUMING repo's project root, which the caller reads off
 // Astro's resolved config rather than guessing from cwd: the package sits in
@@ -16,6 +16,7 @@
 import { readdirSync, readFileSync } from "node:fs";
 import { isAbsolute, join } from "node:path";
 
+import { isAdvertised } from "../advertised";
 import { DEFAULT_PAGES_DIR } from "../content-paths";
 
 export interface SitemapPathsOptions {
@@ -31,8 +32,8 @@ function resolvePagesDir({ pagesDir, root }: SitemapPathsOptions): string {
 }
 
 /**
- * Every non-draft page entry's route, home excluded (@astrojs/sitemap already
- * emits the site root). Returns `{ paths, pagesDir }` so a caller can report
+ * Every advertised page entry's route (neither draft nor noindex), home
+ * excluded (@astrojs/sitemap already emits the site root). Returns `{ paths, pagesDir }` so a caller can report
  * WHERE it looked when the answer is empty.
  */
 export function getSitemapPaths(options: SitemapPathsOptions = {}): {
@@ -54,7 +55,8 @@ export function getSitemapPaths(options: SitemapPathsOptions = {}): {
     .map((rel) => {
       try {
         const data = JSON.parse(readFileSync(join(pagesDir, rel), "utf8"));
-        if (data.draft || typeof data.route !== "string") return null;
+        if (typeof data.route !== "string") return null;
+        if (!isAdvertised(data, { includeDrafts: false })) return null;
         return data.route as string;
       } catch {
         return null; // malformed entry; the build will surface the schema error

package/src/lib/sitemap/sitemap-with-custom-pages-plugin.ts

@@ -8,6 +8,22 @@ import { getSitemapPaths, type SitemapPathsOptions } from "./get-sitemap-paths";
 // placeholder and swap it for the resolved `config.site` at sitemap emit time.
 const PLACEHOLDER = "https://starter.invalid";
 
+// Routes the platform serves that are never a page of the site: the holding
+// page a brand shows before its first publish. @astrojs/sitemap lists every
+// prerendered route, and these have no entry to carry noindex, so the filter
+// names them here.
+const PLATFORM_DENY = new Set(["/under-construction"]);
+
+/** Whether a sitemap URL may be listed: everything but the platform's own routes. */
+export function sitemapPageAllowed(url: string): boolean {
+  try {
+    const pathname = new URL(url).pathname.replace(/\/+$/, "") || "/";
+    return !PLATFORM_DENY.has(pathname);
+  } catch {
+    return true;
+  }
+}
+
 export type SitemapWithCustomPagesOptions = SitemapOptions &
   SitemapPathsOptions;
 
@@ -17,6 +33,7 @@ export function sitemapWithCustomPages(
   let resolvedSite = "";
   const { pagesDir, root, ...sitemapOptions } = options;
   const userSerialize = sitemapOptions.serialize;
+  const userFilter = sitemapOptions.filter;
 
   // Filled in at astro:config:setup and read by @astrojs/sitemap at
   // astro:build:done, which is what lets the entry routes be discovered against
@@ -67,6 +84,8 @@ export function sitemapWithCustomPages(
     sitemap({
       ...sitemapOptions,
       customPages,
+      filter: (page) =>
+        sitemapPageAllowed(page) && (userFilter ? userFilter(page) : true),
       serialize(item) {
         if (resolvedSite && item.url.startsWith(PLACEHOLDER)) {
           item.url = resolvedSite + item.url.slice(PLACEHOLDER.length);

package/src/routes/index.ts

@@ -3,7 +3,18 @@
 // package bump. No route is injected: a route that exists in no repo file breaks
 // the src/pages/ mental model and the debuggability contract.
 export { createLlmsTxtRoute, type LlmsTxtOptions } from "./llms-txt";
-export { createRobotsTxtRoute, type RobotsTxtOptions } from "./robots-txt";
+export {
+  AI_ANSWER_CRAWLERS,
+  AI_POLICY_PRESETS,
+  AI_TRAINING_CRAWLERS,
+  createRobotsTxtRoute,
+  renderRobotsTxt,
+  resolveAiPolicy,
+  type AiPolicy,
+  type AiPolicyPreset,
+  type RobotsSiteConfig,
+  type RobotsTxtOptions,
+} from "./robots-txt";
 export {
   createProxiedSitemapRoute,
   createSitemapRoute,

package/src/routes/llms-txt.ts

@@ -1,4 +1,5 @@
 import { getCollection } from "astro:content";
+import { isAdvertised } from "../lib/advertised";
 import type { APIRoute } from "astro";
 
 // /llms.txt, generated at dev/build time with no crawler or AI step: a curated
@@ -24,8 +25,11 @@ const escapeLinkText = (value: string) =>
 export function createLlmsTxtRoute({ siteConfig }: LlmsTxtOptions): APIRoute {
   return async ({ site }) => {
     const origin = site?.origin ?? "https://example.com";
-    const pages = (await getCollection("pages")).filter(
-      (entry) => !entry.data.draft || !import.meta.env.PROD,
+    const pages = (await getCollection("pages")).filter((entry) =>
+      isAdvertised(
+        { draft: entry.data.draft, noindex: entry.data.noindex },
+        { includeDrafts: !import.meta.env.PROD },
+      ),
     );
 
     const contentLinks = pages

package/src/routes/robots-txt.ts

@@ -2,55 +2,146 @@ import type { APIRoute } from "astro";
 
 import { normalizeSourceUrls } from "../lib/sitemap/shared";
 
-// /robots.txt: crawler permissions plus the sitemap pointers. The repo keeps the
-// route file and this supplies the body:
+// /robots.txt: crawler permissions, the site's AI policy and the sitemap
+// pointers. The repo keeps the route file and this supplies the body:
 //
 //   src/pages/robots.txt.ts
 //   ---
 //   export const prerender = true;
-//   export const GET = createRobotsTxtRoute({
-//     sourceSitemapUrl: SITE_CONFIG.sourceSitemapUrl,
-//   });
+//   export const GET = createRobotsTxtRoute({ siteConfig: SITE_CONFIG });
+//
+// A shim written before 3.11.0 passes `{ sourceSitemapUrl }` alone and keeps
+// working: the policy then renders its recommended default.
 
-export interface RobotsTxtOptions {
-  /** The brand's `SITE_CONFIG.sourceSitemapUrl`; a mirrored sitemap is only
-   *  advertised when one is configured. */
-  sourceSitemapUrl: string | string[];
+/** What the site lets AI systems do with its content. The Content-Signal
+ *  line states it and the per-crawler blocks enforce it. */
+export interface AiPolicy {
+  /** Search engines may index and show the pages. */
+  search: boolean;
+  /** AI answers and assistants may read the pages to answer a person. */
+  aiInput: boolean;
+  /** AI models may train on the pages. */
+  aiTrain: boolean;
 }
 
-export function createRobotsTxtRoute({
-  sourceSitemapUrl,
-}: RobotsTxtOptions): APIRoute {
-  return ({ site }) => {
-    const sitemap = site
-      ? [
-          `Sitemap: ${new URL("sitemap-index.xml", site).href}`,
-          // Mirrored sitemap, only when sourceSitemapUrl is configured.
-          ...(normalizeSourceUrls(sourceSitemapUrl).length > 0
-            ? [`Sitemap: ${new URL("sitemap.xml", site).href}`]
-            : []),
-        ].join("\n") + "\n"
-      : "";
+export type AiPolicyPreset = "recommended" | "open" | "closed";
+
+export const AI_POLICY_PRESETS: Record<AiPolicyPreset, AiPolicy> = {
+  // Findable, quotable, not trained on: what the readiness playbook recommends.
+  recommended: { search: true, aiInput: true, aiTrain: false },
+  open: { search: true, aiInput: true, aiTrain: true },
+  closed: { search: true, aiInput: false, aiTrain: false },
+};
+
+// The AI crawlers the public readiness scanner names, by the purpose each one
+// declares. One list per language: the platform's robots parser carries the
+// same fifteen names, and the test here pins them.
+export const AI_TRAINING_CRAWLERS = [
+  "GPTBot",
+  "ClaudeBot",
+  "Claude-Web",
+  "anthropic-ai",
+  "CCBot",
+  "Bytespider",
+  "cohere-ai",
+  "Google-Extended",
+  "GoogleOther",
+  "Applebot-Extended",
+  "FacebookBot",
+] as const;
+
+export const AI_ANSWER_CRAWLERS = [
+  "OAI-SearchBot",
+  "ChatGPT-User",
+  "PerplexityBot",
+  "Amazonbot",
+] as const;
+
+const SEARCH_CRAWLERS = [
+  "Googlebot",
+  "Bingbot",
+  "Twitterbot",
+  "facebookexternalhit",
+] as const;
+
+/** The fields of `SITE_CONFIG` this route reads. */
+export interface RobotsSiteConfig {
+  /** A mirrored legacy sitemap, advertised only when configured. */
+  sourceSitemapUrl?: string | string[];
+  /** The AI policy, explicit or by preset. Absent: `recommended`. */
+  aiPolicy?: AiPolicy | AiPolicyPreset;
+  /** Disallow paths carried over from the site's previous robots.txt, rendered
+   *  under the wildcard agent so what the old site kept out stays out. */
+  legacyRules?: string[];
+}
 
-    const body = `User-agent: Googlebot
-Allow: /
+export interface RobotsTxtOptions extends RobotsSiteConfig {
+  /** The brand's `SITE_CONFIG`, whole, so a field added later needs no shim
+   *  change. Its fields win over the flat ones. */
+  siteConfig?: RobotsSiteConfig;
+}
 
-User-agent: Bingbot
-Allow: /
+export function resolveAiPolicy(
+  policy: AiPolicy | AiPolicyPreset | undefined,
+): AiPolicy {
+  if (policy === undefined) return AI_POLICY_PRESETS.recommended;
+  if (typeof policy === "string") {
+    return AI_POLICY_PRESETS[policy] ?? AI_POLICY_PRESETS.recommended;
+  }
+  return policy;
+}
 
-User-agent: Twitterbot
-Allow: /
+const grant = (allowed: boolean) => (allowed ? "Allow: /" : "Disallow: /");
+const yesNo = (value: boolean) => (value ? "yes" : "no");
 
-User-agent: facebookexternalhit
-Allow: /
+function agentBlocks(agents: readonly string[], allowed: boolean): string[] {
+  return agents.map((agent) => `User-agent: ${agent}\n${grant(allowed)}`);
+}
 
-User-agent: *
-Allow: /
+function legacyDisallows(rules: string[] | undefined): string[] {
+  return (rules ?? [])
+    .map((rule) => rule.replace(/^\s*disallow\s*:/i, "").trim())
+    .filter((path) => path.length > 0)
+    .map((path) => `Disallow: ${path}`);
+}
 
-${sitemap}`;
+export function renderRobotsTxt(
+  config: RobotsSiteConfig,
+  site: URL | undefined,
+): string {
+  const policy = resolveAiPolicy(config.aiPolicy);
+  const sitemap = site
+    ? [
+        `Sitemap: ${new URL("sitemap-index.xml", site).href}`,
+        // Mirrored sitemap, only when sourceSitemapUrl is configured.
+        ...(normalizeSourceUrls(config.sourceSitemapUrl ?? "").length > 0
+          ? [`Sitemap: ${new URL("sitemap.xml", site).href}`]
+          : []),
+      ].join("\n") + "\n"
+    : "";
+  const wildcard = [
+    "User-agent: *",
+    "Allow: /",
+    ...legacyDisallows(config.legacyRules),
+    `Content-Signal: search=${yesNo(policy.search)}, ai-input=${yesNo(policy.aiInput)}, ai-train=${yesNo(policy.aiTrain)}`,
+  ].join("\n");
+  return (
+    [
+      ...agentBlocks(SEARCH_CRAWLERS, policy.search),
+      ...agentBlocks(AI_ANSWER_CRAWLERS, policy.aiInput),
+      ...agentBlocks(AI_TRAINING_CRAWLERS, policy.aiTrain),
+      wildcard,
+    ].join("\n\n") +
+    "\n\n" +
+    sitemap
+  );
+}
 
-    return new Response(body, {
+export function createRobotsTxtRoute(options: RobotsTxtOptions): APIRoute {
+  const { siteConfig, ...flat } = options;
+  const config: RobotsSiteConfig = { ...flat, ...(siteConfig ?? {}) };
+  return ({ site }) =>
+    new Response(renderRobotsTxt(config, site), {
       headers: { "Content-Type": "text/plain" },
     });
-  };
 }