@junejs/server 1.0.0-dev.28 → 1.0.0-dev.36

package/src/pipeline.ts

@@ -24,20 +24,43 @@ import {
   type RenderTarget,
   type RouteContext,
 } from "@junejs/core/route";
-import { Document, type DocumentConfig } from "@junejs/core/document";
 import {
+  Document,
+  documentTitle,
+  pageCanonical,
+  PRERENDER_ORIGIN,
+  type Breadcrumb,
+  type DocumentConfig,
+} from "@junejs/core/document";
+import {
+  AGENT_SKILLS_INDEX_PATH,
+  AI_CATALOG_PATH,
+  API_CATALOG_CONTENT_TYPE,
+  ARD_PATH,
+  agentSkillsIndex,
+  aiCatalog,
   apiCatalog,
   buildLinkHeader,
   llmsTxt,
+  MCP_SERVER_CARD_TYPE,
   mcpServerCard,
   robotsTxt,
+  siteSkill,
   sitemapXml,
 } from "@junejs/core/discovery";
-import { mcpHandler, mcpTools } from "@junejs/core/mcp";
+import { mcpHandler, mcpServerIdentity, mcpTools } from "@junejs/core/mcp";
+import {
+  apiActionId,
+  apiHandler,
+  apiNamespaceResponse,
+  isApiNamespace,
+  OPENAPI_MEDIA_TYPE,
+  openApiDocument,
+} from "@junejs/core/api";
 import type { Principal, Session } from "@junejs/core/context";
 
 import { ensureScope, runInScope, setRequestLocale } from "@junejs/db";
-import type { AgentConfig } from "@junejs/core/config";
+import { siteShortName, type AgentConfig } from "@junejs/core/config";
 import {
   LOCALE_COOKIE,
   localeAlternates,
@@ -50,6 +73,7 @@ import {
 import type { Resources } from "@junejs/core/resources";
 
 import { iconLetter } from "./icon-letter";
+import { collectLlmsLinks, collectSitemapPages } from "./llms-links";
 import { webManifest } from "./web-manifest";
 import { negotiate, TITLE_HEADER, SEGMENT_HEADER, encodeTitle } from "./negotiate";
 
@@ -121,6 +145,11 @@ export type PipelineConfig = {
   // re-scan the filesystem; the worker returns a frozen array.
   routeList: () => Promise<string[]> | string[];
   resolve: RouteResolver;
+  // True only while the static() build prerenders: /sitemap.xml may then list a
+  // dynamic route's `staticPaths`, which the route contract confines to that
+  // build (they may do build-only or expensive work). At runtime a crawler's
+  // /sitemap.xml enumerates dynamic pages from their `llms` entries alone.
+  staticBuild?: boolean;
   // Opened data resources (db/kv/blob) injected onto ctx before load(). A
   // provider so opening is lazy/memoized; absent → no resources on ctx.
   resources?: () => Promise<Resources> | Resources;
@@ -170,6 +199,9 @@ const DefaultNotFound: React.ComponentType<{ pathname: string }> = ({ pathname }
     React.createElement("p", null, pathname),
   );
 
+// Catalogs crawlers and browser-based agents fetch cross-origin (ARD requires it).
+const CORS = { "access-control-allow-origin": "*" };
+
 function text(body: string, contentType: string, init?: ResponseInit) {
   const headers = new Headers(init?.headers);
   headers.set("content-type", contentType);
@@ -217,19 +249,91 @@ function letterFavicon(siteName: string | undefined): Response {
   );
 }
 
-type PageProps = { pageUrl: string; isHome: boolean; onLocaleDomain: boolean };
+// A page URL's markdown twin — the same paths `june build` prerenders: /index.md
+// for the home page (and a locale home, /zh-cn/index.md), <path>.md otherwise.
+// A trailing slash is dropped first: /users/ is the same page as /users, whose
+// twin is the flat /users.md (never /users/index.md). A home reached through its
+// /index alias (/index, /de/index) is the same home: /de/index → /de/index.md.
+// Only a HOME folds /index — /docs/index is its own route, twin /docs/index.md.
+export function markdownPath(pathname: string, isHome: boolean): string {
+  const path = pathname.replace(/\/+$/, "");
+  return isHome ? `${path.replace(/\/index$/, "")}/index.md` : `${path}.md`;
+}
+
+// The inverse: the path a page's HTML form lives at, for the markdown
+// projection's canonical — /docs/x.md → /docs/x; for a home, its /index alias
+// folds away (/index.md → /, /de/index.md and /de/index → /de). A non-home
+// /docs/index(.md) stays /docs/index.
+export function htmlPath(pathname: string, isHome: boolean): string {
+  const path = pathname.endsWith(".md") ? pathname.slice(0, -3) : pathname;
+  if (!isHome) return path;
+  return path.replace(/\/+$/, "").replace(/\/index$/, "") || "/";
+}
+
+// The MCP Server Card's discovery contract (experimental-ext-server-card
+// docs/discovery.md): its own media type (MCP_SERVER_CARD_TYPE, shared with the
+// api-catalog and Link header), and CORS so browser clients can read it.
+const SERVER_CARD_PATH = "/.well-known/mcp/server-card.json";
+const SERVER_CARD_CORS = {
+  "access-control-allow-origin": "*",
+  // HEAD too: the discovery gate answers it on every surface (same headers, no body).
+  "access-control-allow-methods": "GET, HEAD",
+  "access-control-allow-headers": "Content-Type, If-None-Match",
+  "access-control-expose-headers": "ETag",
+};
+
+type PageProps = {
+  pageUrl: string;
+  isHome: boolean;
+  onLocaleDomain: boolean;
+  markdownHref?: string;
+  breadcrumbs?: Breadcrumb[];
+};
+
+// A URL segment as a crumb label, for a page whose title isn't static: decoded,
+// "-"/"_" as spaces, first letter capitalized (scripts without case are unchanged).
+function humanizeSegment(segment: string): string {
+  let s = segment;
+  try {
+    s = decodeURIComponent(segment);
+  } catch {
+    // an undecodable segment never reaches here (resolveRoute refuses the path)
+  }
+  s = s.replace(/[-_]+/g, " ").trim();
+  return s.charAt(0).toUpperCase() + s.slice(1);
+}
 
 // A request's matched route pathname (locale stripped, /index → /), keyed by its
 // ctx — RouteContext is public API, so the pipeline tracks it on the side.
 const routePathOf = new WeakMap<RouteContext, string>();
 
+// Whether every "/"-separated segment of a path percent-decodes (a malformed
+// escape like "%ZZ" or a truncated UTF-8 sequence doesn't). The only thing
+// decodeURIComponent can throw is URIError, so any failure means "no".
+export function isDecodablePath(pathname: string): boolean {
+  try {
+    for (const segment of pathname.split("/")) decodeURIComponent(segment);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
 export function createPipeline(cfg: PipelineConfig): Pipeline {
   const { docConfig, agent } = cfg;
   const NotFound = cfg.notFoundComponent ?? DefaultNotFound;
+  // The origin-independent half of catalogOrigin() below — all a response header
+  // can know (a static build publishes no headers, so the prerender-origin half
+  // never applies to one).
+  const publishesCatalogs = agent.discovery && !docConfig.basePath;
 
   function htmlHeaders(): Headers {
-    const headers = new Headers({ "content-type": "text/html; charset=utf-8" });
-    const links = [buildLinkHeader(agent), ...(cfg.earlyHints ?? [])].filter(Boolean) as string[];
+    // Every pipeline document is one variant of an Accept-negotiated URL (its .md
+    // and .json answer at the same path) — including streamed pages and the 404.
+    const headers = new Headers({ "content-type": "text/html; charset=utf-8", vary: "accept" });
+    const links = [buildLinkHeader(agent, { catalogs: publishesCatalogs }), ...(cfg.earlyHints ?? [])].filter(
+      Boolean,
+    ) as string[];
     if (links.length) headers.set("link", links.join(", "));
     if (cfg.htmlCacheControl) headers.set("cache-control", cfg.htmlCacheControl);
     return headers;
@@ -261,15 +365,60 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
   // The document's page-identity props: the URL for og:url/canonical, whether the
   // MATCHED route is home (so /de and /index count, per negotiate's normalized
   // pathname), and whether the host is a locale's own domain (its public origin).
-  function pageProps(ctx: RouteContext): PageProps {
-    const host = ctx.url.hostname.toLowerCase();
+  function pageProps(ctx: RouteContext, def: BrandedRoute): PageProps {
+    const isHome = routePathOf.get(ctx) === "/";
     return {
       pageUrl: ctx.url.href,
-      isHome: routePathOf.get(ctx) === "/",
-      onLocaleDomain: cfg.i18n
-        ? Object.values(cfg.i18n.locales).some((l) => l.domain?.toLowerCase() === host)
-        : false,
+      isHome,
+      onLocaleDomain: onLocaleDomain(ctx),
+      // The page's markdown twin, advertised as a discovery signal (so gated with
+      // the rest of discovery). A disabled md projection gets no link to a 404.
+      markdownHref: agent.discovery && def.md !== false ? markdownPath(ctx.url.pathname, isHome) : undefined,
+    };
+  }
+
+  // The page's breadcrumb trail (Metadata.breadcrumb): home first, this page last.
+  // Derived from the URL unless the route declares its own (or false). An ancestor
+  // path gets a crumb only if it RESOLVES to a page matched with no params — a
+  // static route, or an optional segment's bare match: a page that exists by
+  // declaration. A dynamic match (/docs/[slug] for /docs/x) exists only if its data
+  // does, which would take a load to know, so it is skipped, never guessed. Each
+  // ancestor costs one route resolve (no load); its label is the route's static
+  // metadata.title, else its humanized URL segment — the URL is verified, only the
+  // label is derived, and a gap in the trail would misstate the hierarchy more.
+  // Paths are request paths: a locale prefix (/de/…) carries through, so the
+  // home crumb is the locale's home.
+  async function breadcrumbsFor(ctx: RouteContext, meta: Metadata | undefined): Promise<Breadcrumb[] | undefined> {
+    const routePath = routePathOf.get(ctx);
+    if (!routePath || routePath === "/" || meta?.breadcrumb === false) return undefined;
+    const urlPath = ctx.url.pathname.replace(/\/+$/, "") || "/";
+    const route = routePath.replace(/\/+$/, "");
+    // The request path minus the route path = the locale prefix ("" on the default
+    // locale or a locale's own domain).
+    const prefix = urlPath.endsWith(route) ? urlPath.slice(0, urlPath.length - route.length) : "";
+    const home: Breadcrumb = {
+      name: siteShortName(docConfig.site) ?? documentTitle(undefined, docConfig.site),
+      path: prefix || "/",
     };
+    if (Array.isArray(meta?.breadcrumb)) return [home, ...meta.breadcrumb];
+    const segments = route.split("/").filter(Boolean);
+    const trail: Breadcrumb[] = [home];
+    for (let i = 1; i < segments.length; i++) {
+      const ancestor = "/" + segments.slice(0, i).join("/");
+      const r = await resolveRoute(ancestor);
+      if (!r || "handler" in r || Object.keys(r.params).length > 0) continue;
+      const title = typeof r.def.metadata === "object" ? r.def.metadata?.title : undefined;
+      trail.push({ name: title || humanizeSegment(segments[i - 1]!), path: prefix + ancestor });
+    }
+    trail.push({ name: meta?.title || humanizeSegment(segments[segments.length - 1]!), path: urlPath });
+    return trail;
+  }
+
+  function onLocaleDomain(ctx: RouteContext): boolean {
+    const host = ctx.url.hostname.toLowerCase();
+    return cfg.i18n
+      ? Object.values(cfg.i18n.locales).some((l) => l.domain?.toLowerCase() === host)
+      : false;
   }
 
   async function renderDocument(
@@ -289,7 +438,7 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
     );
     const stream = await renderToReadableStream(
       React.createElement(Document, {
-        config: docConfigForRender(),
+        config: docConfigForRender(page?.pageUrl),
         metadata,
         children: wrapped,
         shellKey: boundaryKey, // stamps data-june-shell on [data-june-root]
@@ -348,9 +497,17 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
   // WebMCP: register the app's actions as browser tools. Computed per render
   // from the live registry (stable after warmup), gated on agent.webmcp + mcp
   // (execute proxies to /mcp). No actions → no script → page stays zero-JS.
-  function docConfigForRender(): DocumentConfig {
+  function docConfigForRender(pageUrl?: string): DocumentConfig {
     const webmcpTools = agent.webmcp && agent.mcp ? mcpTools() : null;
-    return webmcpTools?.length ? { ...docConfig, webmcpTools } : docConfig;
+    // <link rel="ai-catalog"> — the AI Catalog spec's in-page pointer (ARD) —
+    // only where this pipeline actually serves the catalog for the page's origin
+    // (never under a basePath, never on a static build without a public origin).
+    const servesCatalog = !!pageUrl && catalogOrigin(new URL(pageUrl)) !== null;
+    return {
+      ...docConfig,
+      ...(webmcpTools?.length ? { webmcpTools } : {}),
+      ...(servesCatalog ? { aiCatalog: AI_CATALOG_PATH } : {}),
+    };
   }
 
   // Streaming Suspense: the shell (layout chain + the loading.tsx fallback)
@@ -362,6 +519,7 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
     resolved: Resolved,
     loadPromise: Promise<unknown>,
     ctx: RouteContext,
+    breadcrumbs: Breadcrumb[] | undefined,
   ): Promise<Response> {
     const { def, chain, loading: Loading, boundaryKey } = resolved;
     const leaf = React.createElement(StreamedView, { loadPromise, def, ctx });
@@ -375,14 +533,17 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
       boundary,
     );
     const metadata = typeof def.metadata === "object" ? def.metadata : undefined;
+    // No await between here and React taking loadPromise (see the caller); the
+    // breadcrumb trail is resolved before load() starts.
     const stream = await renderToReadableStream(
       React.createElement(Document, {
-        config: docConfigForRender(),
+        config: docConfigForRender(ctx.url.href),
         metadata,
         children: wrapped,
         shellKey: boundaryKey, // stamps data-june-shell on [data-june-root]
         alternates: alternatesFor(ctx),
-        ...pageProps(ctx),
+        ...pageProps(ctx, def),
+        breadcrumbs,
         ...langDir(ctx.locale),
       }),
       { onError: (e: unknown) => console.error("[june] streaming render error:", e) },
@@ -392,21 +553,94 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
     return new Response(stream, { status: 200, headers: htmlHeaders() });
   }
 
+  // Route resolution for a request path. Every resolver (the dev tree matcher,
+  // the built worker's dynamic + resource tables) percent-decodes URL segments,
+  // and would throw URIError on a malformed escape (/docs/%ZZ). Such a path can
+  // match no route by definition, so it is refused BEFORE resolving — a miss
+  // (the normal negotiated 404, or the API's JSON 404 under /api) — rather than
+  // catching around cfg.resolve(), which also imports route modules: a genuine
+  // error there, URIError included, must still surface. Checking each segment
+  // suffices for every resolver: their captures are whole segments or runs of
+  // them joined by "/", which decode iff each segment does.
+  async function resolveRoute(pathname: string): ReturnType<RouteResolver> {
+    return isDecodablePath(pathname) ? cfg.resolve(pathname) : null;
+  }
+
   function notFoundResponse(
     target: RenderTarget,
     pathname: string,
     locale?: string,
+    // Passed only when ROUTING found nothing (not when a matched route 404s): an
+    // unmatched path in June's REST namespace (/api) is an API miss, answered
+    // with the API's own JSON error/index whatever Accept negotiated — app
+    // routes under /api were already given their chance.
+    unmatched?: Request,
   ): Promise<Response> | Response {
-    // Data clients get a JSON 404; humans get the rendered NotFound document.
-    return target !== "view"
-      ? Response.json({ error: "Not Found", path: pathname }, { status: 404 })
-      : renderDocument(
-          React.createElement(NotFound, { pathname }),
-          { title: "Not found", robots: "noindex" },
-          404,
-          [],
-          locale,
-        );
+    if (unmatched && agent.api) {
+      const url = new URL(unmatched.url);
+      if (isApiNamespace(url.pathname)) return apiNamespaceResponse(unmatched, url.origin);
+    }
+    // Agents get a 404 they can act on: Markdown for an Accept: text/markdown (or
+    // .md) request, structured JSON for other data clients — both pointing at the
+    // discovery surfaces. Humans get the rendered NotFound document. The body
+    // depends on Accept, so every variant says so to caches.
+    if (target === "md") {
+      return discoveryLinks().then((links) =>
+        text(notFoundMarkdown(pathname, links), "text/markdown; charset=utf-8", {
+          status: 404,
+          headers: { vary: "accept" },
+        }),
+      );
+    }
+    if (target !== "view") {
+      return discoveryLinks().then((links) =>
+        Response.json(
+          { error: "Not Found", code: "not_found", path: pathname, hint: notFoundHint(links) },
+          { status: 404, headers: { vary: "accept" } },
+        ),
+      );
+    }
+    return renderDocument(
+      React.createElement(NotFound, { pathname }),
+      { title: "Not found", robots: "noindex" },
+      404,
+      [],
+      locale,
+    );
+  }
+
+  // Where an agent that hit a dead end should look next — only surfaces this app
+  // actually serves: /index.md only when a root page exists with its md live.
+  async function discoveryLinks(): Promise<Array<[label: string, href: string]>> {
+    const links: Array<[string, string]> = [];
+    if (agent.discovery) {
+      links.push(["llms.txt — the site index for agents", "/llms.txt"], ["sitemap.xml — every page", "/sitemap.xml"]);
+    }
+    if (agent.mcp) links.push(["MCP server — this site's tools", "/mcp"]);
+    if (agent.api) links.push(["OpenAPI — the same tools over HTTP", "/openapi.json"]);
+    const home = await cfg.resolve("/");
+    if (home && "def" in home && home.def.md !== false) links.push(["Home page (Markdown)", "/index.md"]);
+    return links;
+  }
+
+  function notFoundHint(links: Array<[string, string]>): string {
+    return links.length
+      ? `No page at this path. Start from ${links.map(([, h]) => h).join(", ")}.`
+      : "No page at this path.";
+  }
+
+  function notFoundMarkdown(pathname: string, links: Array<[string, string]>): string {
+    const site = docConfig.site.name ? ` on ${docConfig.site.name}` : "";
+    return [
+      "# 404 — Not found",
+      "",
+      links.length
+        ? `There is no page at \`${pathname}\`${site}. Try one of these instead:`
+        : `There is no page at \`${pathname}\`${site}.`,
+      "",
+      ...links.map(([label, href]) => `- [${label}](${href})`),
+      "",
+    ].join("\n");
   }
 
   function resolveMeta(def: BrandedRoute, data: unknown, ctx: RouteContext): Metadata | undefined {
@@ -420,16 +654,38 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
     // json is also absent). md/json === false is handled as 404 by the caller.
     const jsonData =
       typeof def.json === "function" ? await def.json(data, ctx) : def.json === false ? null : data;
-    const body =
+    const content =
       typeof def.md === "function"
         ? await def.md(data, ctx)
         : "```json\n" + JSON.stringify(jsonData, null, 2) + "\n```\n";
+    const body = withFrontmatter(content, resolveMeta(def, data, ctx), ctx);
     // x-markdown-tokens: a rough estimate (~4 chars/token) agents use to budget.
     return text(body, "text/markdown; charset=utf-8", {
       headers: { "x-markdown-tokens": String(Math.ceil(body.length / 4)) },
     });
   }
 
+  // Open served markdown with a frontmatter block (title, description, canonical)
+  // so an agent gets the page's metadata without scraping the HTML. A body that
+  // already opens with its own frontmatter (e.g. a content file served verbatim)
+  // is left alone. Values are JSON strings — valid YAML double-quoted scalars.
+  function withFrontmatter(body: string, meta: Metadata | undefined, ctx: RouteContext): string {
+    if (/^---\r?\n/.test(body)) return body;
+    // The page's OWN title, not the templated <title> ("Users", not "Users · Site"):
+    // the template is browser-tab branding, and authored frontmatter (served
+    // verbatim) carries the bare title too — so every .md reads the same way. The
+    // site name is only the fallback for a page with no title — an empty title
+    // counts as none, exactly as documentTitle() treats it.
+    const title = meta?.title || documentTitle(meta, docConfig.site);
+    const description = meta?.description ?? docConfig.site.description;
+    const canonical = pageCanonical(docConfig, meta, ctx.url.href, htmlPath(ctx.url.pathname, routePathOf.get(ctx) === "/"), onLocaleDomain(ctx));
+    const lines = ["---", `title: ${JSON.stringify(title)}`];
+    if (description) lines.push(`description: ${JSON.stringify(description)}`);
+    if (canonical) lines.push(`canonical: ${JSON.stringify(canonical)}`);
+    lines.push("---", "");
+    return lines.join("\n") + body;
+  }
+
   async function renderProjection(
     resolved: Resolved,
     target: RenderTarget,
@@ -475,27 +731,124 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
     // renders the whole chain (a hard load of the URL is never segment-scoped) —
     // but it stamps the shell key so the client knows which shell is mounted.
     if (target === "fragment") return renderFragment(node, meta, chain, boundaryIndex, boundaryKey);
-    return renderDocument(node, meta, 200, chain, ctx.locale, boundaryKey, alternatesFor(ctx), pageProps(ctx));
+    const page = { ...pageProps(ctx, def), breadcrumbs: await breadcrumbsFor(ctx, meta) };
+    return renderDocument(node, meta, 200, chain, ctx.locale, boundaryKey, alternatesFor(ctx), page);
+  }
+
+  // The origin the agent catalogs name, or null when this site publishes none —
+  // the single rule for serving them, prerendering them, and advertising them
+  // (head link, Link header, robots Agentmap):
+  //   • agent.discovery off → no discovery surface at all;
+  //   • a basePath site doesn't own the domain root, where /.well-known lives;
+  //   • a static build renders against the placeholder prerender host, which must
+  //     never reach a published file: use the configured public origin (site.url,
+  //     else deploy.domain) there, and publish nothing when neither is set.
+  function catalogOrigin(url: URL): string | null {
+    return publishesCatalogs ? publicOrigin(url) : null;
+  }
+
+  // The public origin that published discovery files (llms.txt, sitemap.xml,
+  // robots.txt, the agent catalogs) name. Live, it is the request's own origin.
+  // A prerender runs against the placeholder host, which must never reach a
+  // published file, so it uses the configured public origin: site.url's origin,
+  // else the deploy domain. null → unknown: publish no absolute URL at all.
+  function publicOrigin(url: URL): string | null {
+    if (url.origin !== PRERENDER_ORIGIN) return url.origin;
+    return docConfig.site.url ? new URL(docConfig.site.url).origin : (docConfig.deployOrigin ?? null);
+  }
+
+  // Where the site's pages live: the public origin plus the deploy basePath, so
+  // every absolute URL a discovery file lists resolves (a GitHub Pages project
+  // site is https://user.github.io/repo/…). null when the origin is unknown.
+  function publicBase(url: URL): string | null {
+    const origin = publicOrigin(url);
+    return origin === null ? null : origin + (docConfig.basePath ?? "");
+  }
+
+  // The agent catalogs: Agent Skills (index + the generated SKILL.md) and the
+  // ARD / AI Catalog. CORS-open — crawlers and browser agents fetch them cross-origin.
+  async function agentCatalog(url: URL): Promise<Response | null> {
+    const { pathname } = url;
+    if (!pathname.startsWith("/.well-known/")) return null;
+    const origin = catalogOrigin(url);
+    if (!origin) return null;
+    if (pathname === AGENT_SKILLS_INDEX_PATH) {
+      return Response.json(await agentSkillsIndex(origin, agent, docConfig.site), { headers: CORS });
+    }
+    if (pathname === AI_CATALOG_PATH || pathname === ARD_PATH) {
+      return Response.json(aiCatalog(origin, agent, docConfig.site), { headers: CORS });
+    }
+    // The skill's path carries its name; only the generated skill's own path answers.
+    const skill = siteSkill(origin, agent, docConfig.site);
+    if (pathname === new URL(skill.url).pathname) {
+      return text(skill.markdown, "text/markdown; charset=utf-8", { headers: CORS });
+    }
+    // The skills directory is ours while we serve it: any other path under it is
+    // a skill that doesn't exist, a 404 the RFC requires — never handed to app
+    // routing, where a catch-all would answer 200.
+    if (pathname.startsWith("/.well-known/agent-skills/")) {
+      return Response.json({ error: "Not Found", path: pathname }, { status: 404, headers: CORS });
+    }
+    return null;
   }
 
   async function discovery(url: URL): Promise<Response | null> {
+    const catalog = await agentCatalog(url);
+    if (catalog) return catalog;
     switch (url.pathname) {
-      case "/llms.txt":
-        return text(
-          llmsTxt(url.origin, await cfg.routeList(), agent, docConfig.site),
-          "text/markdown; charset=utf-8",
-        );
-      case "/robots.txt":
-        return text(robotsTxt(url.origin), "text/plain; charset=utf-8");
-      case "/sitemap.xml":
+      case "/llms.txt": {
+        const routes = await cfg.routeList();
+        // With no known public origin (a static build without site.url or a deploy
+        // domain) the links are root-relative — llmstxt.org allows it — never the
+        // prerender placeholder.
+        const base = publicBase(url) ?? docConfig.basePath ?? "";
+        // sections, descriptions, and Optional come from each route's `llms` export
+        const links = await collectLlmsLinks(base, routes, cfg.resolve);
+        return text(llmsTxt(base, routes, agent, docConfig.site, links), "text/markdown; charset=utf-8");
+      }
+      case "/robots.txt": {
+        const base = publicBase(url);
+        if (base === null) return text("Not Found\n", "text/plain; charset=utf-8", { status: 404 });
+        return text(robotsTxt(base, { catalogs: catalogOrigin(url) !== null }), "text/plain; charset=utf-8");
+      }
+      case "/sitemap.xml": {
+        // The sitemap protocol requires absolute <loc>s: with no known public origin
+        // there is no correct sitemap, so none (the static build warns).
+        const base = publicBase(url);
+        if (base === null) return text("Not Found\n", "text/plain; charset=utf-8", { status: 404 });
         return text(
-          sitemapXml(url.origin, await cfg.routeList(), cfg.i18n),
+          sitemapXml(
+            base,
+            await collectSitemapPages(await cfg.routeList(), cfg.resolve, {
+              i18n: !!cfg.i18n,
+              staticPaths: cfg.staticBuild === true,
+            }),
+            cfg.i18n,
+          ),
           "application/xml; charset=utf-8",
         );
+      }
       case "/.well-known/api-catalog":
-        return text(JSON.stringify(apiCatalog(url.origin, agent)), "application/linkset+json");
-      case "/.well-known/mcp/server-card.json":
-        return agent.mcp ? Response.json(mcpServerCard(url.origin)) : null;
+        // RFC 9727 §2: a HEAD here SHALL carry a Link with rel="api-catalog". The
+        // gate answers HEAD with this GET's headers, so both carry it.
+        return text(JSON.stringify(apiCatalog(url.origin, agent)), API_CATALOG_CONTENT_TYPE, {
+          headers: { link: `</.well-known/api-catalog>; rel="api-catalog"` },
+        });
+      case SERVER_CARD_PATH:
+        return agent.mcp
+          ? text(
+              JSON.stringify(
+                mcpServerCard(url.origin, {
+                  site: docConfig.site,
+                  agent,
+                  icons: docConfig.icons,
+                  basePath: docConfig.basePath,
+                }),
+              ),
+              MCP_SERVER_CARD_TYPE,
+              { headers: { ...SERVER_CARD_CORS, "cache-control": "public, max-age=3600" } },
+            )
+          : null;
       default:
         return null;
     }
@@ -512,11 +865,42 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
         // the same principal: requiresPrincipal actions and per-call connection
         // auth are live on this surface. ctx carries identity only — not resources.
         const identity = cfg.identity ? await cfg.identity(request) : undefined;
-        return mcpHandler(request, { request, ...identity });
+        return mcpHandler(
+          request,
+          { request, ...identity },
+          mcpServerIdentity(url.origin, { site: docConfig.site, agent }),
+        );
+      }
+      // The REST projection of the same actions: POST /api/<id> dispatches through
+      // the same invokeAction with the same principal as /mcp. Only a registered
+      // action's canonical path is claimed — any other /api/* falls through to the app.
+      // /openapi.json is the API's own description, so it is gated on agent.api (not
+      // agent.discovery) and answered here; the discovery gate below never sees it.
+      if (agent.api) {
+        if (url.pathname === "/openapi.json" && (request.method === "GET" || request.method === "HEAD")) {
+          const doc = Response.json(openApiDocument(url.origin, docConfig.site), {
+            // the same type the Link header + api-catalog advertise
+            headers: { "content-type": OPENAPI_MEDIA_TYPE, "access-control-allow-origin": "*" },
+          });
+          // HEAD: the same status + headers, no body.
+          return request.method === "HEAD" ? new Response(null, { status: doc.status, headers: doc.headers }) : doc;
+        }
+        const actionId = apiActionId(url.pathname);
+        if (actionId) {
+          const identity = cfg.identity ? await cfg.identity(request) : undefined;
+          return apiHandler(request, actionId, { request, ...identity });
+        }
+      }
+      // The server card is read cross-origin by browser-based MCP clients; its
+      // If-None-Match revalidation is a non-safelisted header → a CORS preflight.
+      if (request.method === "OPTIONS" && agent.discovery && agent.mcp && url.pathname === SERVER_CARD_PATH) {
+        return new Response(null, { status: 204, headers: SERVER_CARD_CORS });
       }
-      if (request.method === "GET" && agent.discovery) {
+      // GET and HEAD (Agent Skills Discovery RFC v0.2.0 requires both): a HEAD
+      // gets the same status and headers, no body.
+      if ((request.method === "GET" || request.method === "HEAD") && agent.discovery) {
         const d = await discovery(url);
-        if (d) return d;
+        if (d) return request.method === "HEAD" ? new Response(null, { status: d.status, headers: d.headers }) : d;
       }
 
       // --- durable agent surface (chat + channels) -------------------------
@@ -581,8 +965,8 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
 
       // --- routes ----------------------------------------------------------
       const { target, pathname, speculative } = negotiate(url, request, routeBase);
-      const resolved = await cfg.resolve(pathname);
-      if (!resolved) return notFoundResponse(target, pathname, locale);
+      const resolved = await resolveRoute(pathname);
+      if (!resolved) return notFoundResponse(target, pathname, locale, request);
 
       // ctx is identity/request only; db/kv/blob are ambient (read from the
       // request scope this whole handler runs inside — see runInScope below).
@@ -603,10 +987,16 @@ export function createPipeline(cfg: PipelineConfig): Pipeline {
       // static metadata flushes the shell + fallback before load() resolves.
       // (data-derived metadata can't stream — the <head> needs the title.)
       if (target === "view" && resolved.loading && typeof resolved.def.metadata !== "function") {
+        // Everything the shell awaits happens BEFORE load() starts: once the load
+        // promise exists, it must reach React (use()) with no await in between, or
+        // a load that rejects meanwhile is an unhandled rejection (fatal on a Node
+        // host with --unhandled-rejections=throw). Metadata is static on this path,
+        // so the breadcrumb trail doesn't depend on the load.
+        const breadcrumbs = await breadcrumbsFor(ctx, resolved.def.metadata);
         const loadPromise = Promise.resolve(
           resolved.def.load ? resolved.def.load(ctx) : undefined,
         );
-        return renderStreamingDocument(resolved, loadPromise, ctx);
+        return renderStreamingDocument(resolved, loadPromise, ctx, breadcrumbs);
       }
 
       let data: unknown;