@sandboxsonny/ace-kit 0.10.0 → 0.12.0

package/CHANGELOG.md

@@ -1,5 +1,17 @@
 # @sandboxsonny/ace-kit
 
+## 0.12.0
+
+### Minor Changes
+
+- 4d64734: Add a sitemap and `robots.txt`. `getSitemapResponse` and `getRobotsResponse` (Astro), and `sitemapLoader` and `robotsLoader` (React Router), each make the file in one line. The sitemap lists the site's live pages from the CMS, leaving out content types whose entries aren't pages of their own, with when each last changed. Also adds `client.getSitemap()`, and the pure `sitemapXml`, `robotsTxt` and `publicOrigin` helpers.
+
+## 0.11.0
+
+### Minor Changes
+
+- f0a08c3: Follow the redirects set up in the CMS. When a request is for an address the site has no page for, `getPage` (and the React Router page loader) now asks the CMS whether the site's owners redirected it, and answers with that redirect (301 or 302, to a path on the site or another address) instead of "not found". Your page already handles redirects, so there is nothing to change. A page shown in the CMS's preview frame is never redirected. Also adds `client.getRedirect(address)`.
+
 ## 0.10.0
 
 ### Minor Changes

package/dist/astro.d.ts

@@ -53,6 +53,24 @@ type AceSite = {
   favicon: AceFavicon | null;
 };
 
+/** One page of the site as the CMS lists it for the sitemap. */
+type AceSitemapEntry = {
+  /** The address visitors use: "/" for the home entry, otherwise "/{content type}/{entry}". */
+  path: string;
+  /** When the entry was last changed, as an ISO date and time. */
+  updated_at: string;
+};
+
+/** A redirect the CMS has for an address, as its public API returns it. */
+type AceRedirect = {
+  /** The old address the redirect is for. */
+  from: string;
+  /** Where it goes: a path on the site, or a full web address. */
+  to: string;
+  /** 301 is permanent; 302 is temporary. */
+  status: 301 | 302;
+};
+
 /** One link of a menu as the public API returns it, with its address already worked out. */
 type AceMenuItem = {
   label: string;
@@ -112,6 +130,13 @@ type AceValuesOf<Handle extends string> = Handle extends keyof AceContentTypes
   ? AceContentTypes[Handle]
   : Record<string, unknown>;
 
+type RobotsOptions = {
+    /** Asks every crawler to stay away, for a site that isn't live yet. */
+    disallowAll?: boolean;
+    /** The site's address, e.g. `https://example.com`. Left out, the address the request came to is used. */
+    origin?: string;
+};
+
 /** What to do with a request for a page: show an entry, redirect, or explain. */
 type AcePage<Values = Record<string, unknown>> =
   | { kind: "entry"; status: 200; entry: AceEntry<Values>; headers: Record<string, string> }
@@ -129,6 +154,8 @@ declare const ace: {
         limit?: number;
     }): Promise<AceEntry<AceValuesOf<Handle>>[]>;
     getMenu(handle: string): Promise<AceMenu | null>;
+    getRedirect(address: string): Promise<AceRedirect | null>;
+    getSitemap(): Promise<AceSitemapEntry[]>;
     getSite(): Promise<AceSite | null>;
     getContentTypes(): Promise<AceContentTypeSchema[]>;
 };
@@ -177,5 +204,31 @@ declare function getPage<Values = Record<string, unknown>>(astro: {
         headers: Headers;
     };
 }): Promise<AcePage<Values>>;
+/**
+ * The response for `/sitemap.xml`: the site's live pages (of the content types that have pages), with when each last
+ * changed. Make `src/pages/sitemap.xml.ts` with
+ *
+ *     import { getSitemapResponse } from "@sandboxsonny/ace-kit/astro";
+ *     export const GET = ({ url }) => getSitemapResponse({ url });
+ *
+ * The addresses use https unless the site is on localhost; pass `{ origin: "https://example.com" }` to choose.
+ */
+declare const getSitemapResponse: (astro: {
+    url: URL;
+}, options?: {
+    origin?: string;
+}) => Promise<Response>;
+/**
+ * The response for `/robots.txt`, which lets crawlers in and points them at the sitemap. Make `src/pages/robots.txt.ts`
+ * with
+ *
+ *     import { getRobotsResponse } from "@sandboxsonny/ace-kit/astro";
+ *     export const GET = ({ url }) => getRobotsResponse({ url });
+ *
+ * Pass `{ disallowAll: true }` for a site that isn't live yet.
+ */
+declare const getRobotsResponse: (astro: {
+    url: URL;
+}, options?: RobotsOptions) => Response;
 
-export { ace, applyPage, getEntries, getEntry, getMenu, getMenuLinks, getPage, getSite };
+export { ace, applyPage, getEntries, getEntry, getMenu, getMenuLinks, getPage, getRobotsResponse, getSite, getSitemapResponse };

package/dist/astro.js

@@ -1,4 +1,4 @@
-import { createAceClient, loadMenuLinks, loadPage } from './chunk-EKZRLWNO.js';
+import { createAceClient, loadMenuLinks, loadPage, sitemapResponse, robotsResponse } from './chunk-AJLQFATM.js';
 import './chunk-M5WR5TCR.js';
 import './chunk-YLO6IAXF.js';
 import { ACE_SITE_DOMAIN, ACE_API_URL } from 'astro:env/server';
@@ -21,5 +21,7 @@ async function getPage(astro) {
   applyPage(astro.response, page);
   return page;
 }
+var getSitemapResponse = (astro, options = {}) => sitemapResponse(ace, astro.url, options);
+var getRobotsResponse = (astro, options = {}) => robotsResponse(astro.url, options);
 
-export { ace, applyPage, getEntries, getEntry, getMenu, getMenuLinks, getPage, getSite };
+export { ace, applyPage, getEntries, getEntry, getMenu, getMenuLinks, getPage, getRobotsResponse, getSite, getSitemapResponse };

package/dist/{chunk-EKZRLWNO.js → chunk-AJLQFATM.js}

@@ -89,6 +89,19 @@ function createAceClient({
     getMenu(handle) {
       return get(`/public/menus/${encodeURIComponent(handle)}`);
     },
+    /**
+     * Where an address the site has no page for should redirect to, if the site's owners set that up; null if not.
+     * `address` is the path with its query string, e.g. "/old-page?id=7".
+     */
+    async getRedirect(address) {
+      const redirect = await get(`/public/redirects?from=${encodeURIComponent(address)}`);
+      const goesSomewhereSafe = redirect !== null && typeof redirect.to === "string" && (redirect.to.startsWith("/") && !redirect.to.startsWith("//") && !redirect.to.includes("\\") || /^https?:\/\/[^\s]+$/i.test(redirect.to)) && (redirect.status === 301 || redirect.status === 302);
+      return goesSomewhereSafe ? redirect : null;
+    },
+    /** The site's live pages for its sitemap, by path. Pages of content types that have no pages of their own are left out. */
+    async getSitemap() {
+      return await get("/public/sitemap") ?? [];
+    },
     /** The site itself: its name, and the social image its owners chose (null if none). */
     getSite() {
       return get("/public/site");
@@ -132,12 +145,12 @@ async function loadPage(client, { slug, url }) {
   if (slug && address && isHomeAddress(address)) {
     return { kind: "redirect", status: url.search ? 302 : 301, location: `/${url.search}`, headers };
   }
-  if (!address) return { kind: "not-found", status: 404, headers };
+  if (!address) return notFoundOrRedirect(client, url, previewToken, headers);
   try {
     const entry = await client.getEntry(address.contentTypeHandle, address.entryHandle, {
       previewToken
     });
-    return entry ? { kind: "entry", status: 200, entry, headers } : { kind: "not-found", status: 404, headers };
+    return entry ? { kind: "entry", status: 200, entry, headers } : notFoundOrRedirect(client, url, previewToken, headers);
   } catch (error) {
     if (error instanceof AcePreviewExpiredError) return { kind: "preview-expired", status: 403, headers };
     if (error instanceof AceUnavailableError) {
@@ -147,5 +160,66 @@ async function loadPage(client, { slug, url }) {
     throw error;
   }
 }
+async function notFoundOrRedirect(client, url, previewToken, headers) {
+  if (previewToken) return { kind: "not-found", status: 404, headers };
+  try {
+    const redirect = await client.getRedirect(`${url.pathname}${url.search}`);
+    if (redirect) return { kind: "redirect", status: redirect.status, location: redirect.to, headers };
+  } catch (error) {
+    if (!(error instanceof AceUnavailableError)) throw error;
+    console.error(error.message);
+    return { kind: "unavailable", status: 503, headers };
+  }
+  return { kind: "not-found", status: 404, headers };
+}
+
+// src/helpers/sitemap.ts
+var escapeXml = (text) => text.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&apos;");
+function publicOrigin(url) {
+  const local = url.hostname === "localhost" || url.hostname.endsWith(".localhost") || /^[\d.]+$|:/.test(url.hostname);
+  return local ? url.origin : `https://${url.host}`;
+}
+function sitemapXml(entries, origin) {
+  const base = origin.replace(/\/+$/, "");
+  const urls = entries.map((entry) => {
+    const lastChanged = Number.isNaN(Date.parse(entry.updated_at)) ? "" : new Date(entry.updated_at).toISOString();
+    return `  <url>
+    <loc>${escapeXml(`${base}${entry.path}`)}</loc>${lastChanged ? `
+    <lastmod>${lastChanged}</lastmod>` : ""}
+  </url>`;
+  }).join("\n");
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${urls}${urls ? "\n" : ""}</urlset>
+`;
+}
+function robotsTxt({ origin, disallowAll = false }) {
+  if (disallowAll) return "User-agent: *\nDisallow: /\n";
+  return `User-agent: *
+Allow: /
+
+Sitemap: ${origin.replace(/\/+$/, "")}/sitemap.xml
+`;
+}
+async function sitemapResponse(client, url, options = {}) {
+  try {
+    const entries = await client.getSitemap();
+    return new Response(sitemapXml(entries, options.origin ?? publicOrigin(url)), {
+      headers: { "Content-Type": "application/xml; charset=utf-8", "Cache-Control": "public, max-age=300" }
+    });
+  } catch (error) {
+    if (!(error instanceof AceUnavailableError)) throw error;
+    console.error(error.message);
+    return new Response("The sitemap is unavailable right now.", {
+      status: 503,
+      headers: { "Content-Type": "text/plain; charset=utf-8", "Retry-After": "300" }
+    });
+  }
+}
+function robotsResponse(url, options = {}) {
+  return new Response(robotsTxt({ origin: options.origin ?? publicOrigin(url), disallowAll: options.disallowAll }), {
+    headers: { "Content-Type": "text/plain; charset=utf-8", "Cache-Control": "public, max-age=3600" }
+  });
+}
 
-export { AcePreviewExpiredError, AceUnavailableError, createAceClient, isCurrentPage, linksFromMenu, loadMenuLinks, loadPage };
+export { AcePreviewExpiredError, AceUnavailableError, createAceClient, isCurrentPage, linksFromMenu, loadMenuLinks, loadPage, publicOrigin, robotsResponse, robotsTxt, sitemapResponse, sitemapXml };

package/dist/cli.js

@@ -9,8 +9,8 @@ if (major < 22 || major === 22 && minor < 12) {
   process.exit(1);
 }
 try {
-  const { createRealEnvironment } = await import("./environment-M3B4UZ5T.js");
-  const { runCli } = await import("./main-6O2IIVHO.js");
+  const { createRealEnvironment } = await import("./environment-4FBNWIHO.js");
+  const { runCli } = await import("./main-4QMQF3L3.js");
   const environment = await createRealEnvironment();
   process.exit(await runCli(process.argv.slice(2), environment));
 } catch (error) {

package/dist/{environment-M3B4UZ5T.js → environment-4FBNWIHO.js}

@@ -145,7 +145,7 @@ function createLastApiUrl(path) {
 }
 
 // src/cli/version.ts
-var aceKitVersion = true ? "0.10.0" : "development";
+var aceKitVersion = true ? "0.12.0" : "development";
 
 // src/cli/environment.ts
 var runCommand = (command, commandArguments, input) => new Promise((resolve, reject) => {

package/dist/index.d.ts

@@ -84,6 +84,16 @@ type AceNavigationLink = {
   children?: AceNavigationLink[];
 };
 
+/** A redirect the CMS has for an address, as its public API returns it. */
+type AceRedirect = {
+  /** The old address the redirect is for. */
+  from: string;
+  /** Where it goes: a path on the site, or a full web address. */
+  to: string;
+  /** 301 is permanent; 302 is temporary. */
+  status: 301 | 302;
+};
+
 /** An image as the CMS Ace public API returns it. Use `src` and `srcset` in an `<img>`. */
 type AceImageData = {
   id: number;
@@ -123,6 +133,14 @@ type AceSite = {
   favicon: AceFavicon | null;
 };
 
+/** One page of the site as the CMS lists it for the sitemap. */
+type AceSitemapEntry = {
+  /** The address visitors use: "/" for the home entry, otherwise "/{content type}/{entry}". */
+  path: string;
+  /** When the entry was last changed, as an ISO date and time. */
+  updated_at: string;
+};
+
 /**
  * A client for the CMS Ace public API. It runs on the server only, so the visitor's browser never talks to the
  * API. The API finds the site from X-Ace-Site-Domain, which this client sets to the site's domain. X-Forwarded-Host
@@ -143,6 +161,13 @@ declare function createAceClient({ apiUrl, siteDomain, cacheSeconds, fetchImplem
     }): Promise<AceEntry<AceValuesOf<Handle>>[]>;
     /** One of the site's navigation menus by its handle, with every link worked out; null if the site has none. */
     getMenu(handle: string): Promise<AceMenu | null>;
+    /**
+     * Where an address the site has no page for should redirect to, if the site's owners set that up; null if not.
+     * `address` is the path with its query string, e.g. "/old-page?id=7".
+     */
+    getRedirect(address: string): Promise<AceRedirect | null>;
+    /** The site's live pages for its sitemap, by path. Pages of content types that have no pages of their own are left out. */
+    getSitemap(): Promise<AceSitemapEntry[]>;
     /** The site itself: its name, and the social image its owners chose (null if none). */
     getSite(): Promise<AceSite | null>;
     /** What the site's content types and fields are, which is what type generation reads. */
@@ -415,7 +440,8 @@ type AcePage<Values = Record<string, unknown>> =
  * - `slug` is the part of the path after the site's root, e.g. "pages/about" ("" or undefined for the home page).
  * - `url` is the full request address; it carries the preview token the CMS puts in its preview frame.
  *
- * Addresses are `/{content type}/{entry}` as on Shopify, and `/` is the home entry. A page shown with a preview
+ * Addresses are `/{content type}/{entry}` as on Shopify, and `/` is the home entry. An address with no page is
+ * redirected if the site's owners set up a redirect for it (in the CMS), and otherwise not found. A page shown with a preview
  * token is never cached or indexed (the answer's `headers` say so).
  */
 declare function loadPage<Values = Record<string, unknown>>(client: AceClient, { slug, url }: {
@@ -535,6 +561,35 @@ type AceSeoProperties = {
  */
 declare function seoProperties(options: AceSeoOptions): AceSeoProperties;
 
+/**
+ * The address a visitor reached the site at, for the full addresses in a sitemap. A site on a real domain is served
+ * over https, but behind a proxy the server often sees plain http, so anything that isn't `localhost` or an IP address
+ * is given https. Pass `origin` to the response helpers to choose it yourself.
+ */
+declare function publicOrigin(url: URL): string;
+/** The sitemap file for a site: one `<url>` per page, with when it last changed. `origin` is e.g. `https://example.com`. */
+declare function sitemapXml(entries: AceSitemapEntry[], origin: string): string;
+type RobotsOptions = {
+    /** Asks every crawler to stay away, for a site that isn't live yet. */
+    disallowAll?: boolean;
+    /** The site's address, e.g. `https://example.com`. Left out, the address the request came to is used. */
+    origin?: string;
+};
+/** The `robots.txt` for a site: everything may be crawled, and the sitemap is pointed at. */
+declare function robotsTxt({ origin, disallowAll }: {
+    origin: string;
+    disallowAll?: boolean;
+}): string;
+/**
+ * The response for `/sitemap.xml`: the site's live pages from the CMS. If the CMS can't be reached it answers 503, so a
+ * crawler tries again later instead of reading an empty sitemap.
+ */
+declare function sitemapResponse(client: AceClient, url: URL, options?: {
+    origin?: string;
+}): Promise<Response>;
+/** The response for `/robots.txt`. */
+declare function robotsResponse(url: URL, options?: RobotsOptions): Response;
+
 /**
  * A text area field's text as HTML. With `multiline` (the default) blank lines start new paragraphs (`<p>`) and a
  * single new line is a line break; without it the text is one line, with no element around it. Every piece of text
@@ -576,4 +631,4 @@ type AceSection = {
   blocks: AceSectionBlock[];
 };
 
-export { type AceAddress, type AceClient, type AceClientOptions, type AceContentTypeHandle, type AceContentTypeSchema, type AceContentTypes, type AceDateFormat, type AceEntry, type AceFavicon, type AceFieldSchema, type AceImageAttributes, type AceImageData, type AceImageOptions, type AceLinkAttributes, type AceLinkData, type AceMenu, type AceMenuItem, type AceNavigationLink, type AcePage, AcePreviewExpiredError, type AceRichTextDocument, type AceRichTextMark, type AceRichTextNode, type AceSection, type AceSectionBlock, type AceSeoOptions, type AceSeoProperties, type AceSite, AceUnavailableError, type AceValuesOf, type BlockDefinition, type SectionDefinition, type SectionFieldDefinition, type SectionFieldType, type SectionOf, type SectionTypesPayload, type ValuesOfFields, addFramingPolicy, addressFromPath, checkSectionDefinition, createAceClient, defineSection, formatAceDate, framingPolicy, headingLevel, homeAddress, imageAttributes, isCurrentPage, isHomeAddress, isInternalPath, isRefreshRequest, linkAddress, linkAttributes, linksFromMenu, listStart, listenForPreviewRefresh, loadMenuLinks, loadPage, originOf, pageNotices, pathOfAddress, previewHeaders, previewParameter, previewTokenFrom, refreshMessageType, refreshPageContent, renderRichText, renderTextArea, richTextImageSizes, sectionTypesPayload, seoDescriptionHandle, seoProperties, seoTitleHandle, swapPageContent, visitablePath };
+export { type AceAddress, type AceClient, type AceClientOptions, type AceContentTypeHandle, type AceContentTypeSchema, type AceContentTypes, type AceDateFormat, type AceEntry, type AceFavicon, type AceFieldSchema, type AceImageAttributes, type AceImageData, type AceImageOptions, type AceLinkAttributes, type AceLinkData, type AceMenu, type AceMenuItem, type AceNavigationLink, type AcePage, AcePreviewExpiredError, type AceRedirect, type AceRichTextDocument, type AceRichTextMark, type AceRichTextNode, type AceSection, type AceSectionBlock, type AceSeoOptions, type AceSeoProperties, type AceSite, type AceSitemapEntry, AceUnavailableError, type AceValuesOf, type BlockDefinition, type RobotsOptions, type SectionDefinition, type SectionFieldDefinition, type SectionFieldType, type SectionOf, type SectionTypesPayload, type ValuesOfFields, addFramingPolicy, addressFromPath, checkSectionDefinition, createAceClient, defineSection, formatAceDate, framingPolicy, headingLevel, homeAddress, imageAttributes, isCurrentPage, isHomeAddress, isInternalPath, isRefreshRequest, linkAddress, linkAttributes, linksFromMenu, listStart, listenForPreviewRefresh, loadMenuLinks, loadPage, originOf, pageNotices, pathOfAddress, previewHeaders, previewParameter, previewTokenFrom, publicOrigin, refreshMessageType, refreshPageContent, renderRichText, renderTextArea, richTextImageSizes, robotsResponse, robotsTxt, sectionTypesPayload, seoDescriptionHandle, seoProperties, seoTitleHandle, sitemapResponse, sitemapXml, swapPageContent, visitablePath };

package/dist/index.js

@@ -1,7 +1,7 @@
 export { pageNotices } from './chunk-XPZY5R5A.js';
 export { headingLevel, imageAttributes, isInternalPath, linkAddress, linkAttributes, listStart, renderRichText, renderTextArea, richTextImageSizes } from './chunk-FVR55GIU.js';
 export { listenForPreviewRefresh, refreshPageContent } from './chunk-YN7LJWGI.js';
-export { AcePreviewExpiredError, AceUnavailableError, createAceClient, isCurrentPage, linksFromMenu, loadMenuLinks, loadPage } from './chunk-EKZRLWNO.js';
+export { AcePreviewExpiredError, AceUnavailableError, createAceClient, isCurrentPage, linksFromMenu, loadMenuLinks, loadPage, publicOrigin, robotsResponse, robotsTxt, sitemapResponse, sitemapXml } from './chunk-AJLQFATM.js';
 export { addressFromPath, homeAddress, isHomeAddress, pathOfAddress, visitablePath } from './chunk-M5WR5TCR.js';
 export { addFramingPolicy, framingPolicy, isRefreshRequest, originOf, previewHeaders, previewParameter, previewTokenFrom, refreshMessageType, swapPageContent } from './chunk-YLO6IAXF.js';
 

package/dist/{main-6O2IIVHO.js → main-4QMQF3L3.js}

@@ -1367,6 +1367,19 @@ function createAceClient({
     getMenu(handle) {
       return get(`/public/menus/${encodeURIComponent(handle)}`);
     },
+    /**
+     * Where an address the site has no page for should redirect to, if the site's owners set that up; null if not.
+     * `address` is the path with its query string, e.g. "/old-page?id=7".
+     */
+    async getRedirect(address) {
+      const redirect = await get(`/public/redirects?from=${encodeURIComponent(address)}`);
+      const goesSomewhereSafe = redirect !== null && typeof redirect.to === "string" && (redirect.to.startsWith("/") && !redirect.to.startsWith("//") && !redirect.to.includes("\\") || /^https?:\/\/[^\s]+$/i.test(redirect.to)) && (redirect.status === 301 || redirect.status === 302);
+      return goesSomewhereSafe ? redirect : null;
+    },
+    /** The site's live pages for its sitemap, by path. Pages of content types that have no pages of their own are left out. */
+    async getSitemap() {
+      return await get("/public/sitemap") ?? [];
+    },
     /** The site itself: its name, and the social image its owners chose (null if none). */
     getSite() {
       return get("/public/site");

package/dist/react-router-server.d.ts

@@ -126,6 +126,24 @@ type AceContentTypeSchema = {
   fields: AceFieldSchema[];
 };
 
+/** A redirect the CMS has for an address, as its public API returns it. */
+type AceRedirect = {
+  /** The old address the redirect is for. */
+  from: string;
+  /** Where it goes: a path on the site, or a full web address. */
+  to: string;
+  /** 301 is permanent; 302 is temporary. */
+  status: 301 | 302;
+};
+
+/** One page of the site as the CMS lists it for the sitemap. */
+type AceSitemapEntry = {
+  /** The address visitors use: "/" for the home entry, otherwise "/{content type}/{entry}". */
+  path: string;
+  /** When the entry was last changed, as an ISO date and time. */
+  updated_at: string;
+};
+
 /**
  * A client for the CMS Ace public API. It runs on the server only, so the visitor's browser never talks to the
  * API. The API finds the site from X-Ace-Site-Domain, which this client sets to the site's domain. X-Forwarded-Host
@@ -146,6 +164,13 @@ declare function createAceClient({ apiUrl, siteDomain, cacheSeconds, fetchImplem
     }): Promise<AceEntry<AceValuesOf<Handle>>[]>;
     /** One of the site's navigation menus by its handle, with every link worked out; null if the site has none. */
     getMenu(handle: string): Promise<AceMenu | null>;
+    /**
+     * Where an address the site has no page for should redirect to, if the site's owners set that up; null if not.
+     * `address` is the path with its query string, e.g. "/old-page?id=7".
+     */
+    getRedirect(address: string): Promise<AceRedirect | null>;
+    /** The site's live pages for its sitemap, by path. Pages of content types that have no pages of their own are left out. */
+    getSitemap(): Promise<AceSitemapEntry[]>;
     /** The site itself: its name, and the social image its owners chose (null if none). */
     getSite(): Promise<AceSite | null>;
     /** What the site's content types and fields are, which is what type generation reads. */
@@ -154,6 +179,13 @@ declare function createAceClient({ apiUrl, siteDomain, cacheSeconds, fetchImplem
 /** The client `createAceClient` returns. */
 type AceClient = ReturnType<typeof createAceClient>;
 
+type RobotsOptions = {
+    /** Asks every crawler to stay away, for a site that isn't live yet. */
+    disallowAll?: boolean;
+    /** The site's address, e.g. `https://example.com`. Left out, the address the request came to is used. */
+    origin?: string;
+};
+
 /** What the root loader (`createRootLoader`) puts in the root route's data, for `useAceMenu` and `useAcePreview`. */
 type AceRootData = {
   ace: {
@@ -214,6 +246,31 @@ declare function createPageLoader<Values = Record<string, unknown>>(): ({ reques
     entry: AceEntry<Values>;
     headers: Record<string, string>;
 }>>;
+/**
+ * The loader for `/sitemap.xml`: the site's live pages (of the content types that have pages), with when each last
+ * changed. Add a route `route("sitemap.xml", "routes/sitemap.tsx")` (before the catch-all) whose file is
+ *
+ *     export { sitemapLoader as loader } from "@sandboxsonny/ace-kit/react-router/server";
+ *
+ * It uses https unless the site is on localhost; for another address use `createSitemapLoader({ origin })`.
+ */
+declare const createSitemapLoader: (options?: {
+    origin?: string;
+}) => ({ request }: {
+    request: Request;
+}) => Promise<Response>;
+/** `createSitemapLoader()` with the defaults. */
+declare const sitemapLoader: ({ request }: {
+    request: Request;
+}) => Promise<Response>;
+/** The loader for `/robots.txt`, which lets crawlers in and points them at the sitemap. See `createSitemapLoader`. */
+declare const createRobotsLoader: (options?: RobotsOptions) => ({ request }: {
+    request: Request;
+}) => Response;
+/** `createRobotsLoader()` with the defaults. */
+declare const robotsLoader: ({ request }: {
+    request: Request;
+}) => Response;
 /** `createPageLoader()` for a site that has not generated types for its content. */
 declare const loader: ({ request }: {
     request: Request;
@@ -266,4 +323,4 @@ declare const streamTimeout = 5000;
  */
 declare function handleRequest(request: Request, responseStatusCode: number, responseHeaders: Headers, routerContext: EntryContext, _loadContext: AppLoadContext): Promise<Response>;
 
-export { createPageLoader, createRootLoader, getAceClient, getEntries, getEntry, getMenu, getMenuLinks, getSite, handleRequest, headers, loader, streamTimeout };
+export { createPageLoader, createRobotsLoader, createRootLoader, createSitemapLoader, getAceClient, getEntries, getEntry, getMenu, getMenuLinks, getSite, handleRequest, headers, loader, robotsLoader, sitemapLoader, streamTimeout };

package/dist/react-router-server.js

@@ -1,4 +1,4 @@
-import { createAceClient, loadMenuLinks, loadPage } from './chunk-EKZRLWNO.js';
+import { createAceClient, loadMenuLinks, loadPage, sitemapResponse, robotsResponse } from './chunk-AJLQFATM.js';
 import './chunk-M5WR5TCR.js';
 import { hostedAdminOrigin, hostedApiUrl } from './chunk-N5DIMG3X.js';
 import { previewTokenFrom, addFramingPolicy } from './chunk-YLO6IAXF.js';
@@ -43,6 +43,14 @@ function createPageLoader() {
     return data(page, { status: page.status, headers: page.headers });
   };
 }
+var createSitemapLoader = (options = {}) => function loader2({ request }) {
+  return sitemapResponse(getAceClient(), new URL(request.url), options);
+};
+var sitemapLoader = createSitemapLoader();
+var createRobotsLoader = (options = {}) => function loader2({ request }) {
+  return robotsResponse(new URL(request.url), options);
+};
+var robotsLoader = createRobotsLoader();
 var loader = createPageLoader();
 var headers = ({ loaderHeaders }) => loaderHeaders;
 function createRootLoader({ menu, fallback = [] }) {
@@ -88,4 +96,4 @@ function handleRequest(request, responseStatusCode, responseHeaders, routerConte
   });
 }
 
-export { createPageLoader, createRootLoader, getAceClient, getEntries, getEntry, getMenu, getMenuLinks, getSite, handleRequest, headers, loader, streamTimeout };
+export { createPageLoader, createRobotsLoader, createRootLoader, createSitemapLoader, getAceClient, getEntries, getEntry, getMenu, getMenuLinks, getSite, handleRequest, headers, loader, robotsLoader, sitemapLoader, streamTimeout };

package/docs/api-reference.md

@@ -86,13 +86,14 @@ Types: `AceSection`, `AceSectionBlock`, `SectionDefinition`, `SectionFieldDefini
 
 `@sandboxsonny/ace-kit/astro` (uses the integration's settings, so there is no client to create):
 
-|                                                                                           |                                                                                                                                                           |
-| ----------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `getPage(Astro)`                                                                          | What to show for the page being requested; also sets the response's status and headers. A `redirect` is returned for you to act on with `Astro.redirect`. |
-| `getEntry(contentType, handle)`, `getEntries(contentType, { limit? })`, `getMenu(handle)` | As on the client.                                                                                                                                         |
-| `getMenuLinks(handle, { fallback? })`                                                     | A menu's links, or the fallback if the menu is missing or the CMS is down.                                                                                |
-| `applyPage(response, page)`                                                               | Puts a page's status and headers on a response.                                                                                                           |
-| `ace`                                                                                     | The client these use.                                                                                                                                     |
+|                                                                                                     |                                                                                                                                                           |
+| --------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `getPage(Astro)`                                                                                    | What to show for the page being requested; also sets the response's status and headers. A `redirect` is returned for you to act on with `Astro.redirect`. |
+| `getEntry(contentType, handle)`, `getEntries(contentType, { limit? })`, `getMenu(handle)`           | As on the client.                                                                                                                                         |
+| `getSitemapResponse({ url }, { origin? })`, `getRobotsResponse({ url }, { origin?, disallowAll? })` | The responses for `/sitemap.xml` and `/robots.txt` (see [Astro](./astro.md)).                                                                             |
+| `getMenuLinks(handle, { fallback? })`                                                               | A menu's links, or the fallback if the menu is missing or the CMS is down.                                                                                |
+| `applyPage(response, page)`                                                                         | Puts a page's status and headers on a response.                                                                                                           |
+| `ace`                                                                                               | The client these use.                                                                                                                                     |
 
 Components, by file: `AceImage.astro`, `AceLink.astro`, `AceSections.astro`, `AceRichText.astro`, `AceTextArea.astro`, `AceMenu.astro`, `AceNotice.astro`, `AcePreviewRefresh.astro`.
 
@@ -110,13 +111,14 @@ Components, by file: `AceImage.astro`, `AceLink.astro`, `AceSections.astro`, `Ac
 
 `@sandboxsonny/ace-kit/react-router/server` (server only):
 
-|                                                                     |                                                                  |
-| ------------------------------------------------------------------- | ---------------------------------------------------------------- |
-| `createPageLoader<Values>()`, `loader`                              | The loader for the catch-all route that draws every page.        |
-| `headers`                                                           | The route's `headers` export.                                    |
-| `createRootLoader({ menu, fallback? })`                             | The root loader: the menu's links and whether this is a preview. |
-| `handleRequest`, `streamTimeout`                                    | The server entry.                                                |
-| `getEntry`, `getEntries`, `getMenu`, `getMenuLinks`, `getAceClient` | Read content from your own loaders.                              |
+|                                                                                                |                                                                                           |
+| ---------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------- |
+| `createPageLoader<Values>()`, `loader`                                                         | The loader for the catch-all route that draws every page.                                 |
+| `headers`                                                                                      | The route's `headers` export.                                                             |
+| `createRootLoader({ menu, fallback? })`                                                        | The root loader: the menu's links and whether this is a preview.                          |
+| `handleRequest`, `streamTimeout`                                                               | The server entry.                                                                         |
+| `sitemapLoader`, `robotsLoader`, `createSitemapLoader(options)`, `createRobotsLoader(options)` | The loaders for `/sitemap.xml` and `/robots.txt` (see [React Router](./react-router.md)). |
+| `getEntry`, `getEntries`, `getMenu`, `getMenuLinks`, `getAceClient`                            | Read content from your own loaders.                                                       |
 
 ## React (`@sandboxsonny/ace-kit/react`)
 

package/docs/astro.md

@@ -46,7 +46,25 @@ if (page.kind === "redirect") return Astro.redirect(page.location, page.status);
 {page.kind === "entry" ? <h1>{page.entry.title}</h1> : <AceNotice page={page} />}
 ```
 
-Addresses are `/{content type}/{entry}` as on Shopify, `/` is the `home` entry of `pages`, and `/pages/home` redirects to `/`. `page.kind` is `entry`, `redirect`, `not-found`, `unavailable` or `preview-expired`. `AceNotice` draws a heading and a message for the last three; draw your own if you prefer.
+Addresses are `/{content type}/{entry}` as on Shopify, `/` is the `home` entry of `pages`, and `/pages/home` redirects to `/`. An address with no page is redirected too if the site's owners set up a redirect for it in the CMS (Redirects), so old addresses from a site you moved keep working. `page.kind` is `entry`, `redirect`, `not-found`, `unavailable` or `preview-expired`. `AceNotice` draws a heading and a message for the last three; draw your own if you prefer.
+
+## Sitemap and robots.txt
+
+Search engines find a site's pages through `/sitemap.xml` and are pointed at it by `/robots.txt`. Both are one small file each; the sitemap lists the live pages from the CMS (only the content types that have "Entries have their own pages" ticked), with when each last changed.
+
+```ts
+// src/pages/sitemap.xml.ts
+import { getSitemapResponse } from "@sandboxsonny/ace-kit/astro";
+export const GET = ({ url }) => getSitemapResponse({ url });
+```
+
+```ts
+// src/pages/robots.txt.ts
+import { getRobotsResponse } from "@sandboxsonny/ace-kit/astro";
+export const GET = ({ url }) => getRobotsResponse({ url });
+```
+
+Addresses in the sitemap use `https` unless the site is on `localhost`; pass `{ origin: "https://www.example.com" }` to either to choose. `getRobotsResponse({ url }, { disallowAll: true })` asks crawlers to stay away, for a site that isn't live yet. Delete any `robots.txt` in `public/`, which would take its place.
 
 ## Reading content
 

package/docs/react-router.md

@@ -69,7 +69,7 @@ export default function Page({ loaderData: page }: Route.ComponentProps) {
 }
 ```
 
-The loader reads the address (`/{content type}/{entry}`, and `/` is the `home` entry of `pages`), loads the entry and answers with the right status and headers: a missing page is a real 404, an unreachable CMS a 503, an expired preview a 403, `/pages/home` redirects to `/`, and a preview is never cached or indexed. `page.kind` is `entry`, `not-found`, `unavailable` or `preview-expired`. `createMeta` titles the page from the entry (followed by the site's name), adds its description, and asks search engines to ignore a preview. With [typed content](./typed-content.md), `createPageLoader<PageValues>()` gives `page.entry.values` its real shape.
+The loader reads the address (`/{content type}/{entry}`, and `/` is the `home` entry of `pages`), loads the entry and answers with the right status and headers: a missing page is a real 404, an unreachable CMS a 503, an expired preview a 403, `/pages/home` redirects to `/`, an address with no page follows the redirect the site's owners set up in the CMS (if any), and a preview is never cached or indexed. `page.kind` is `entry`, `not-found`, `unavailable` or `preview-expired`. `createMeta` titles the page from the entry (followed by the site's name), adds its description, and asks search engines to ignore a preview. With [typed content](./typed-content.md), `createPageLoader<PageValues>()` gives `page.entry.values` its real shape.
 
 **4. The root route** loads the header menu and keeps previews fresh:
 
@@ -102,6 +102,22 @@ From `@sandboxsonny/ace-kit/react-router` (these are safe in the browser):
 - **AceImage** is an `<img>` with `src`, `srcset`, `sizes`, size and alt text. Set `sizes` to how wide it is shown, and `lazy={false}` for the image at the top of the page.
 - **AceNotice** is the heading and message for a page that isn't an entry. Give it a `className`.
 
+## Sitemap and robots.txt
+
+Search engines find a site's pages through `/sitemap.xml` and are pointed at it by `/robots.txt`. Add two routes before the catch-all in `app/routes.ts`: `route("sitemap.xml", "routes/sitemap.tsx")` and `route("robots.txt", "routes/robots.tsx")`, whose files are one line each:
+
+```ts
+// app/routes/sitemap.tsx
+export { sitemapLoader as loader } from "@sandboxsonny/ace-kit/react-router/server";
+```
+
+```ts
+// app/routes/robots.tsx
+export { robotsLoader as loader } from "@sandboxsonny/ace-kit/react-router/server";
+```
+
+The sitemap lists the live pages from the CMS (only the content types that have "Entries have their own pages" ticked), with when each last changed. Addresses use `https` unless the site is on `localhost`. For another address, or to keep crawlers away from a site that isn't live yet, use `createSitemapLoader({ origin })` and `createRobotsLoader({ origin, disallowAll })`.
+
 ## Reading content in your own loaders
 
 From `@sandboxsonny/ace-kit/react-router/server`, with no client to create:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sandboxsonny/ace-kit",
-  "version": "0.10.0",
+  "version": "0.12.0",
   "description": "Everything a website needs to use CMS Ace: a typed client, addressing, images, menus, rich text, live preview, and Astro and React adapters.",
   "keywords": [
     "cms",