@supersoniks/concorde 5.1.1 → 5.1.2

package/scripts/generate-crawl-docs.mjs

@@ -1,13 +1,23 @@
+/**
+ * Post-build agent/SEO artifacts for the docs site.
+ *
+ * Canonical page URLs are the SPA pathnames (`/docs/…`, `/core/…`):
+ * - Browsers → index.html (pushState SPA)
+ * - Accept: text/markdown → sibling `*.md` (same pathname + `.md`)
+ * - Known crawler UAs → sibling `*.html` static pages (same path + `.html`)
+ *
+ * No redirects for old URL shapes (`/crawl/`, `/docs/…`, `…/name/name`).
+ */
 import fs from "fs";
 import path from "path";
 import { fileURLToPath } from "url";
 import { marked } from "marked";
 import { gfmHeadingId } from "marked-gfm-heading-id";
+import { toPublicPath } from "./docs-public-url.mjs";
 
 const SITE_ORIGIN = "https://concorde.supersoniks.org";
 const SRC_DIR = path.resolve("src");
 const DOCS_OUT = path.resolve("docs");
-const CRAWL_DIR = path.join(DOCS_OUT, "crawl");
 
 const SONIC_CODE_BLOCK_RE = /<sonic-code\b[\s\S]*?<\/sonic-code>/gi;
 const DOC_HASH_LINK_RE = /#([^)\s"']+)\.md\/([^)\s"'#]+)/g;
@@ -48,28 +58,44 @@ function mdRelativePath(absPath) {
   return path.relative(SRC_DIR, absPath).replace(/\\/g, "/");
 }
 
-function crawlHtmlPath(mdRel) {
-  return path.join("crawl", mdRel.replace(/\.mdx?$/, ".html")).replace(/\\/g, "/");
-}
-
 function docId(mdRel) {
   return mdRel.replace(/\.mdx?$/, "");
 }
 
-function relativeCrawlLink(fromMdRel, toMdPath, anchor) {
-  const fromHtml = crawlHtmlPath(fromMdRel);
-  const toHtml = crawlHtmlPath(`${toMdPath}.md`);
-  const rel = path
-    .relative(path.dirname(fromHtml), toHtml)
-    .replace(/\\/g, "/");
-  const href = rel.startsWith(".") ? rel : `./${rel}`;
-  return `${href}#${anchor}`;
+/** Public pathname (SPA + agents): /docs/getting-started/start */
+export function pagePath(mdRel) {
+  return toPublicPath(mdRel);
+}
+
+/** On-disk static HTML next to the SPA build: docs/getting-started/start.html */
+function staticHtmlRel(mdRel) {
+  return `${docId(mdRel)}.html`;
 }
 
-function rewriteDocLinks(text, fromMdRel) {
-  return text.replace(DOC_HASH_LINK_RE, (match, mdPath, anchor) => {
-    return relativeCrawlLink(fromMdRel, mdPath, anchor);
+function pageUrl(mdRel) {
+  return `${SITE_ORIGIN}${pagePath(mdRel)}`;
+}
+
+function docLink(toMdPath, anchor) {
+  const base = pagePath(`${toMdPath}.md`);
+  const slug = toMdPath.split("/").pop() || "page";
+  if (!anchor || anchor === slug) return base;
+  return `${base}#${anchor}`;
+}
+
+function rewriteDocLinks(text) {
+  let out = text.replace(DOC_HASH_LINK_RE, (match, mdPath, anchor) => {
+    return docLink(mdPath, anchor);
   });
+  out = out.replace(
+    /\]\((\/(?:docs|core)\/[^)\s"'#]+)(#[^)\s"']*)?\)/g,
+    (match, pathname, fragment) => {
+      const mdPath = pathname.replace(/^\//, "");
+      const anchor = fragment ? fragment.slice(1) : mdPath.split("/").pop() || "page";
+      return `](${docLink(mdPath, anchor)})`;
+    },
+  );
+  return out;
 }
 
 function extractTitle(markdown) {
@@ -133,11 +159,10 @@ function extractDescription(markdown, title) {
 }
 
 function docKind(mdRel) {
-  if (mdRel.includes("/_decorators/")) return "decorator";
-  if (mdRel.includes("/_directives/")) return "directive";
-  if (mdRel.includes("/_getting-started/")) return "guide";
-  if (mdRel.includes("/_core-concept/")) return "concept";
-  if (mdRel.includes("/_misc/")) return "reference";
+  if (mdRel.includes("/decorators/")) return "decorator";
+  if (mdRel.includes("/directives/")) return "directive";
+  if (mdRel.includes("/getting-started/")) return "guide";
+  if (mdRel.includes("/misc/")) return "reference";
   if (mdRel.includes("/components/functional/")) return "functional component";
   if (mdRel.includes("/components/ui/")) return "UI component";
   return "documentation";
@@ -167,10 +192,10 @@ function extractKeywords(mdRel, title, raw) {
     keywords.add(`sonic-${baseName}`);
     keywords.add("web component", "Lit");
   }
-  if (mdRel.includes("/_decorators/")) {
+  if (mdRel.includes("/decorators/")) {
     keywords.add("decorator", "DataProvider", "ApiResult", "Endpoint");
   }
-  if (mdRel.includes("/_core-concept/")) {
+  if (mdRel.endsWith("/dataFlow.md")) {
     keywords.add("DataProvider", "data flow");
   }
 
@@ -184,16 +209,6 @@ function extractKeywords(mdRel, title, raw) {
   return [...keywords].join(", ");
 }
 
-function pageHash(mdRel) {
-  const base = mdRel.replace(/\.mdx?$/, "");
-  const anchor = base.split("/").pop() || "page";
-  return `#${base}.md/${anchor}`;
-}
-
-function crawlPageUrl(crawlRel) {
-  return `${SITE_ORIGIN}/${crawlRel}`;
-}
-
 function buildLeadParagraph({ title, mdRel, description, canonical, keywords }) {
   const id = docId(mdRel);
   const kind = docKind(mdRel);
@@ -201,11 +216,7 @@ function buildLeadParagraph({ title, mdRel, description, canonical, keywords })
 }
 
 function renderPage({ mdRel, title, description, keywords, bodyHtml }) {
-  const crawlRel = crawlHtmlPath(mdRel);
-  const canonical = crawlPageUrl(crawlRel);
-  const interactiveDoc = `${SITE_ORIGIN}/${pageHash(mdRel)}`;
-  const depth = path.dirname(crawlRel).split("/").length - 1;
-  const indexHref = `${"../".repeat(depth)}index.html`;
+  const canonical = pageUrl(mdRel);
   const id = docId(mdRel);
   const leadHtml = buildLeadParagraph({
     title,
@@ -216,19 +227,20 @@ function renderPage({ mdRel, title, description, keywords, bodyHtml }) {
   });
 
   return `<!DOCTYPE html>
-<html lang="fr" data-crawl-url="${escapeHtml(canonical)}" data-doc-id="${escapeHtml(id)}">
+<html lang="en" data-doc-url="${escapeHtml(canonical)}" data-doc-id="${escapeHtml(id)}">
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>${escapeHtml(title)} — Concorde</title>
+  <title>${escapeHtml(title)} — Concorde by Supersoniks</title>
   <meta name="description" content="${escapeHtml(description)}">
   <meta name="keywords" content="${escapeHtml(keywords)}">
   <meta name="concorde-doc-id" content="${escapeHtml(id)}">
   <link rel="canonical" href="${escapeHtml(canonical)}">
+  <link rel="alternate" type="text/markdown" href="${SITE_ORIGIN}${pagePath(mdRel)}.md">
 </head>
 <body>
   <header>
-    <p><a href="${indexHref}">Concorde documentation (crawl)</a> · <a href="${escapeHtml(interactiveDoc)}" rel="nofollow">interactive version</a></p>
+    <p><a href="/docs/index.html">Concorde documentation</a> · <a href="${escapeHtml(canonical)}">same URL (interactive)</a> · <a href="/llms.txt">llms.txt</a></p>
   </header>
   <main>
     <article>
@@ -245,29 +257,29 @@ function renderIndex(pages) {
   const items = pages
     .map(
       (p) =>
-        `    <li><a href="${escapeHtml(p.crawlHref)}">${escapeHtml(p.title)}</a> <small>(${escapeHtml(p.docId)})</small></li>`,
+        `    <li><a href="${escapeHtml(p.pagePath)}">${escapeHtml(p.title)}</a> <small>(${escapeHtml(p.docId)})</small></li>`,
     )
     .join("\n");
 
   return `<!DOCTYPE html>
-<html lang="fr" data-doc-id="crawl-index">
+<html lang="en" data-doc-id="docs-index">
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>Concorde documentation index</title>
-  <meta name="description" content="Static crawl index for Concorde framework documentation. All pages use .html under /crawl/.">
-  <meta name="keywords" content="Concorde, documentation index, web components, Lit, DataProvider">
-  <link rel="canonical" href="${SITE_ORIGIN}/crawl/index.html">
+  <title>Concorde documentation index — Supersoniks</title>
+  <meta name="description" content="Documentation index for Concorde by Supersoniks. Same pathnames as the interactive docs; static HTML for crawlers, Markdown via Accept: text/markdown.">
+  <meta name="keywords" content="Concorde, Supersoniks, documentation index, web components, Lit, DataProvider">
+  <link rel="canonical" href="${SITE_ORIGIN}/docs/index.html">
 </head>
 <body>
   <header>
     <h1>Concorde documentation</h1>
-    <p>Static pages for crawlers and AI indexing. Interactive docs: <a href="${SITE_ORIGIN}/">${SITE_ORIGIN}/</a></p>
-    <p>Full text mirror: <a href="${SITE_ORIGIN}/llms-full.txt">llms-full.txt</a></p>
+    <p>Canonical URLs match the interactive docs (<code>/docs/…</code>, <code>/core/…</code>). Crawlers receive static HTML; agents can request <code>Accept: text/markdown</code>.</p>
+    <p>Interactive site: <a href="${SITE_ORIGIN}/">${SITE_ORIGIN}/</a> · Full text: <a href="${SITE_ORIGIN}/llms-full.txt">llms-full.txt</a></p>
     <ul>
-      <li><a href="hubs/decorators.html">Decorators hub (@get, @post, …)</a></li>
-      <li><a href="hubs/remote-data.html">Remote data hub (@get, List, Queue)</a></li>
-      <li><a href="hubs/form-components.html">Form components hub</a></li>
+      <li><a href="/hubs/decorators.html">Decorators hub (@get, @post, …)</a></li>
+      <li><a href="/hubs/remote-data.html">Remote data hub (@get, List, Queue)</a></li>
+      <li><a href="/hubs/form-components.html">Form components hub</a></li>
     </ul>
   </header>
   <main>
@@ -287,15 +299,27 @@ function writeFileEnsuringDir(filePath, content) {
 
 function generateLlmsTxt(pages, hubs = []) {
   const lines = [
-    "# Concorde",
+    "# Concorde by Supersoniks",
     "",
-    "> Web components framework (Lit, DataProvider, decorators).",
+    "> Concorde is the Supersoniks Lit web-components library (UI, DataProvider, decorators, forms, SDUI).",
+    "> npm: `@supersoniks/concorde` — https://www.npmjs.com/package/@supersoniks/concorde",
+    `> Site: ${SITE_ORIGIN}/`,
     "",
-    "Documentation for AI indexers lives under `/crawl/` as **HTML** (`.html`), not Markdown.",
-    "Example: https://concorde.supersoniks.org/crawl/docs/_decorators/post.html",
-    "Do not fetch `/crawl/**/*.md` (does not exist) nor `/src/docs/**/*.md` (SPA source only).",
+    "## Developer resources",
     "",
-    `Full text: ${SITE_ORIGIN}/llms-full.txt`,
+    `- [Homepage (HTML)](${SITE_ORIGIN}/) — interactive docs SPA`,
+    `- [Homepage (Markdown)](${SITE_ORIGIN}/index.md) — also via \`Accept: text/markdown\` on ${SITE_ORIGIN}/`,
+    `- [llms-full.txt](${SITE_ORIGIN}/llms-full.txt) — full plain-text mirror`,
+    `- [Docs index](${SITE_ORIGIN}/docs/index.html)`,
+    `- [Sitemap](${SITE_ORIGIN}/sitemap.xml)`,
+    `- [npm package @supersoniks/concorde](https://www.npmjs.com/package/@supersoniks/concorde)`,
+    `- [Getting started](${SITE_ORIGIN}/docs/getting-started/start)`,
+    `- [AI agents (skills)](${SITE_ORIGIN}/docs/getting-started/ai-agents)`,
+    `- [Components index](${SITE_ORIGIN}/docs/getting-started/components)`,
+    "",
+    "Doc pages use the same pathnames as the interactive site (`/docs/…`, `/core/…`).",
+    "Markdown: same URL with `.md` (e.g. `/docs/getting-started/start.md`), or `Accept: text/markdown` on the pathname.",
+    "HTML crawlers receive a static HTML variant of the same URL.",
     "",
     "## Topic hubs (start here for RAG)",
     "",
@@ -303,7 +327,7 @@ function generateLlmsTxt(pages, hubs = []) {
 
   for (const h of hubs) {
     lines.push(
-      `- [${h.title}](${SITE_ORIGIN}/${h.crawlPath}) (${h.docId}, ${h.sectionCount} pages)`,
+      `- [${h.title}](${SITE_ORIGIN}/${h.hubPath}) (${h.docId}, ${h.sectionCount} pages)`,
     );
   }
 
@@ -311,27 +335,29 @@ function generateLlmsTxt(pages, hubs = []) {
 
   for (const p of pages) {
     lines.push(
-      `- [${p.title}](${SITE_ORIGIN}/${p.crawlPath}) (${p.docId}): ${p.description}`,
+      `- [${p.title}](${SITE_ORIGIN}${p.pagePath}) (${p.docId}): ${p.description}`,
     );
   }
 
-  lines.push("", "## Optional", "", `- [Full index](${SITE_ORIGIN}/crawl/index.html)`);
+  lines.push("", "## Optional", "", `- [Full index](${SITE_ORIGIN}/docs/index.html)`);
   return lines.join("\n");
 }
 
 function generateLlmsFullTxt(pages, hubs = []) {
   const lines = [
-    "# Concorde — full documentation (plain text)",
+    "# Concorde by Supersoniks — full documentation (plain text)",
     "",
     "> Mirror for AI indexers. Each page is a section below.",
-    "> HTML URLs use /crawl/**/*.html — never .md under /crawl/.",
+    "> Canonical URLs: /docs/** and /core/** (Markdown via Accept: text/markdown).",
+    `> Homepage Markdown: ${SITE_ORIGIN}/index.md`,
+    `> npm: https://www.npmjs.com/package/@supersoniks/concorde`,
     "",
     "## Topic hubs",
     "",
   ];
 
   for (const h of hubs) {
-    lines.push(`- ${h.title}: ${SITE_ORIGIN}/${h.crawlPath} (${h.docId})`);
+    lines.push(`- ${h.title}: ${SITE_ORIGIN}/${h.hubPath} (${h.docId})`);
   }
 
   lines.push("");
@@ -342,7 +368,7 @@ function generateLlmsFullTxt(pages, hubs = []) {
       "",
       `## ${p.title}`,
       "",
-      `- URL: ${SITE_ORIGIN}/${p.crawlPath}`,
+      `- URL: ${SITE_ORIGIN}${p.pagePath}`,
       `- Doc ID: ${p.docId}`,
       `- Keywords: ${p.keywords}`,
       "",
@@ -363,7 +389,7 @@ const TOPIC_HUBS = [
       "Hub for all Concorde Lit decorators: @get @post @put @patch @subscribe @publish @bind @handle and related. Use this page when asking about any decorator.",
     keywords:
       "decorator, @get, @post, @put, @patch, @subscribe, @publish, @bind, @handle, ApiResult, Endpoint, DataProviderKey",
-    filter: (p) => p.docId.startsWith("docs/_decorators/"),
+    filter: (p) => p.docId.startsWith("docs/decorators/"),
   },
   {
     slug: "remote-data",
@@ -375,14 +401,14 @@ const TOPIC_HUBS = [
       "@get, fetch, remote data, API, Endpoint, ApiResult, serviceURL, sonic-list, sonic-queue, HTTP GET",
     filter: (p) =>
       [
-        "docs/_decorators/get",
-        "docs/_decorators/post",
-        "docs/_decorators/put",
-        "docs/_decorators/patch",
-        "docs/_misc/endpoint",
-        "docs/_misc/api-configuration",
-        "docs/_misc/dynamic-path",
-        "docs/_core-concept/dataFlow",
+        "docs/decorators/get",
+        "docs/decorators/post",
+        "docs/decorators/put",
+        "docs/decorators/patch",
+        "docs/misc/endpoint",
+        "docs/misc/api-configuration",
+        "docs/misc/dynamic-path",
+        "docs/getting-started/dataFlow",
         "core/components/functional/fetch/fetch",
         "core/components/functional/list/list",
         "core/components/functional/queue/queue",
@@ -400,26 +426,19 @@ const TOPIC_HUBS = [
   },
 ];
 
-function hubCrawlPath(slug) {
-  return `crawl/hubs/${slug}.html`;
-}
-
-function relativeHubLink(fromHubSlug, targetCrawlPath) {
-  const from = hubCrawlPath(fromHubSlug);
-  const rel = path.relative(path.dirname(from), targetCrawlPath).replace(/\\/g, "/");
-  return rel.startsWith(".") ? rel : `./${rel}`;
+function hubPath(slug) {
+  return `hubs/${slug}.html`;
 }
 
 function renderHubPage(hub, sections) {
-  const crawlRel = hubCrawlPath(hub.slug);
-  const canonical = crawlPageUrl(crawlRel);
+  const hubRel = hubPath(hub.slug);
+  const canonical = `${SITE_ORIGIN}/${hubRel}`;
   const sectionHtml = sections
     .map((p) => {
       const excerpt = p.plainText.slice(0, 1200);
-      const href = relativeHubLink(hub.slug, p.crawlPath);
       return `<section id="${escapeHtml(p.docId.replace(/\//g, "-"))}" data-doc-id="${escapeHtml(p.docId)}">
-  <h2><a href="${escapeHtml(href)}">${escapeHtml(p.title)}</a></h2>
-  <p><strong>Doc ID:</strong> <code>${escapeHtml(p.docId)}</code> · <a href="${escapeHtml(href)}">full page</a></p>
+  <h2><a href="${escapeHtml(p.pagePath)}">${escapeHtml(p.title)}</a></h2>
+  <p><strong>Doc ID:</strong> <code>${escapeHtml(p.docId)}</code> · <a href="${escapeHtml(p.pagePath)}">full page</a></p>
   <p>${escapeHtml(p.description)}</p>
   <pre class="doc-excerpt">${escapeHtml(excerpt)}</pre>
 </section>`;
@@ -435,11 +454,11 @@ function renderHubPage(hub, sections) {
   });
 
   return `<!DOCTYPE html>
-<html lang="fr" data-crawl-url="${escapeHtml(canonical)}" data-doc-id="${escapeHtml(hub.docId)}">
+<html lang="en" data-doc-url="${escapeHtml(canonical)}" data-doc-id="${escapeHtml(hub.docId)}">
 <head>
   <meta charset="UTF-8">
   <meta name="viewport" content="width=device-width, initial-scale=1.0">
-  <title>${escapeHtml(hub.title)} — Concorde</title>
+  <title>${escapeHtml(hub.title)} — Concorde by Supersoniks</title>
   <meta name="description" content="${escapeHtml(hub.intro)}">
   <meta name="keywords" content="${escapeHtml(hub.keywords)}">
   <meta name="concorde-doc-id" content="${escapeHtml(hub.docId)}">
@@ -447,7 +466,7 @@ function renderHubPage(hub, sections) {
 </head>
 <body>
   <header>
-    <p><a href="../index.html">Concorde documentation (crawl)</a></p>
+    <p><a href="/docs/index.html">Concorde documentation</a></p>
   </header>
   <main>
     <article>
@@ -465,14 +484,11 @@ function generateTopicHubs(pages) {
   for (const hub of TOPIC_HUBS) {
     const sections = pages.filter(hub.filter);
     if (sections.length === 0) continue;
-    const crawlRel = hubCrawlPath(hub.slug);
-    writeFileEnsuringDir(
-      path.join(DOCS_OUT, crawlRel),
-      renderHubPage(hub, sections),
-    );
+    const rel = hubPath(hub.slug);
+    writeFileEnsuringDir(path.join(DOCS_OUT, rel), renderHubPage(hub, sections));
     hubs.push({
       ...hub,
-      crawlPath: crawlRel,
+      hubPath: rel,
       sectionCount: sections.length,
     });
   }
@@ -480,16 +496,20 @@ function generateTopicHubs(pages) {
 }
 
 function generateSitemap(pages, hubs = []) {
-  const indexUrl = `  <url>\n    <loc>${SITE_ORIGIN}/crawl/index.html</loc>\n  </url>`;
+  const homeUrl = `  <url>\n    <loc>${SITE_ORIGIN}/</loc>\n  </url>`;
+  const homeMdUrl = `  <url>\n    <loc>${SITE_ORIGIN}/index.md</loc>\n  </url>`;
+  const indexUrl = `  <url>\n    <loc>${SITE_ORIGIN}/docs/index.html</loc>\n  </url>`;
   const hubUrls = hubs
-    .map((h) => `  <url>\n    <loc>${SITE_ORIGIN}/${h.crawlPath}</loc>\n  </url>`)
+    .map((h) => `  <url>\n    <loc>${SITE_ORIGIN}/${h.hubPath}</loc>\n  </url>`)
     .join("\n");
   const urls = pages
-    .map((p) => `  <url>\n    <loc>${SITE_ORIGIN}/${p.crawlPath}</loc>\n  </url>`)
+    .map((p) => `  <url>\n    <loc>${SITE_ORIGIN}${p.pagePath}</loc>\n  </url>`)
     .join("\n");
 
   return `<?xml version="1.0" encoding="UTF-8"?>
 <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${homeUrl}
+${homeMdUrl}
 ${indexUrl}
 ${hubUrls}
 ${urls}
@@ -497,15 +517,91 @@ ${urls}
 `;
 }
 
+/** Homepage Markdown for Accept: text/markdown negotiation (acceptmarkdown.com). */
+export function generateHomepageMarkdown() {
+  return `# Concorde by Supersoniks
+
+Concorde is an open-source Lit web-components library from Supersoniks. It provides UI primitives, data binding, forms, routing helpers and theming that run the same way in React, Vue, PHP templates or a static HTML page.
+
+Install the npm package [\`@supersoniks/concorde\`](https://www.npmjs.com/package/@supersoniks/concorde).
+
+## Developer resources
+
+- Interactive docs: ${SITE_ORIGIN}/
+- Agent index: ${SITE_ORIGIN}/llms.txt
+- Full text mirror: ${SITE_ORIGIN}/llms-full.txt
+- Docs index: ${SITE_ORIGIN}/docs/index.html
+- Sitemap: ${SITE_ORIGIN}/sitemap.xml
+- Getting started: ${SITE_ORIGIN}/docs/getting-started/start
+- AI agents (skills): ${SITE_ORIGIN}/docs/getting-started/ai-agents
+- Components: ${SITE_ORIGIN}/docs/getting-started/components
+
+Doc pages share pathnames with the interactive site (\`/docs/…\`, \`/core/…\`). Markdown is at the same path with a \`.md\` suffix (e.g. \`/docs/getting-started/start.md\`), or via \`Accept: text/markdown\` on the pathname (\`Content-Type: text/markdown\`, \`Vary: Accept\`).
+
+## What Concorde includes
+
+Custom elements (\`sonic-*\`) for buttons, forms, lists, queues, SDUI trees and more. Decorators such as \`@get\`, \`@post\`, \`@subscribe\` and \`@bind\` wire components to a shared DataProvider without framework lock-in. Concorde is MIT-licensed and maintained by Supersoniks.
+`;
+}
+
+/** Markdown 404 body for agents (HTTP 404 + text/markdown). */
+export function generateNotFoundMarkdown() {
+  return `# Page not found
+
+The requested URL does not exist on the Concorde documentation site (Concorde by Supersoniks).
+
+Try the [homepage](${SITE_ORIGIN}/), the [documentation index](${SITE_ORIGIN}/docs/index.html), [llms.txt](${SITE_ORIGIN}/llms.txt), or the [sitemap](${SITE_ORIGIN}/sitemap.xml).
+`;
+}
+
+export function generateNotFoundHtml() {
+  return `<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+  <title>Page not found — Concorde by Supersoniks</title>
+  <meta name="robots" content="noindex">
+  <link rel="canonical" href="${SITE_ORIGIN}/">
+</head>
+<body>
+  <h1>Page not found</h1>
+  <p>The requested URL does not exist on the Concorde documentation site (Concorde by Supersoniks).</p>
+  <p>Try the <a href="${SITE_ORIGIN}/">homepage</a>, the <a href="${SITE_ORIGIN}/docs/index.html">documentation index</a>, <a href="${SITE_ORIGIN}/llms.txt">llms.txt</a>, or the <a href="${SITE_ORIGIN}/sitemap.xml">sitemap</a>.</p>
+</body>
+</html>
+`;
+}
+
+/**
+ * Apache rules live in scripts/docs.htaccess (pretty URLs + negotiation).
+ */
+export function generateHtaccess() {
+  const htaccessPath = fileURLToPath(new URL("./docs.htaccess", import.meta.url));
+  return fs.readFileSync(htaccessPath, "utf8");
+}
+
+export function writeAgentSiteArtifacts(docsOut = DOCS_OUT) {
+  fs.writeFileSync(path.join(docsOut, "index.md"), generateHomepageMarkdown());
+  fs.writeFileSync(path.join(docsOut, "404.md"), generateNotFoundMarkdown());
+  fs.writeFileSync(path.join(docsOut, "404.html"), generateNotFoundHtml());
+  fs.writeFileSync(path.join(docsOut, ".htaccess"), generateHtaccess());
+}
+
+function removeLegacyCrawlDir() {
+  const crawlDir = path.join(DOCS_OUT, "crawl");
+  if (fs.existsSync(crawlDir)) {
+    fs.rmSync(crawlDir, { recursive: true, force: true });
+  }
+}
+
 export function generateCrawlDocs() {
   if (!fs.existsSync(DOCS_OUT)) {
     console.warn("generate-crawl-docs: docs/ not found, skipping");
     return;
   }
 
-  if (fs.existsSync(CRAWL_DIR)) {
-    fs.rmSync(CRAWL_DIR, { recursive: true, force: true });
-  }
+  removeLegacyCrawlDir();
 
   const mdFiles = listMdFiles(SRC_DIR).sort();
   const pages = [];
@@ -517,14 +613,13 @@ export function generateCrawlDocs() {
     const description = extractDescription(raw, title);
     const keywords = extractKeywords(mdRel, title, raw);
     const plainText = markdownToPlainText(raw);
-    const markdown = rewriteDocLinks(preprocessSonicCode(raw), mdRel);
+    const markdown = rewriteDocLinks(preprocessSonicCode(raw));
     const bodyHtml = marked.parse(markdown);
-    const crawlRel = crawlHtmlPath(mdRel);
-    const outPath = path.join(DOCS_OUT, crawlRel);
+    const htmlRel = staticHtmlRel(mdRel);
     const id = docId(mdRel);
 
     writeFileEnsuringDir(
-      outPath,
+      path.join(DOCS_OUT, htmlRel),
       renderPage({ mdRel, title, description, keywords, bodyHtml }),
     );
 
@@ -535,19 +630,14 @@ export function generateCrawlDocs() {
       description,
       keywords,
       plainText,
-      crawlPath: crawlRel,
-      crawlHref: crawlRel.replace(/^crawl\//, ""),
+      pagePath: pagePath(mdRel),
+      htmlRel,
     });
   }
 
   writeFileEnsuringDir(
-    path.join(CRAWL_DIR, "index.html"),
-    renderIndex(
-      pages.map((p) => ({
-        ...p,
-        crawlHref: p.mdRel.replace(/\.mdx?$/, ".html"),
-      })),
-    ),
+    path.join(DOCS_OUT, "docs/index.html"),
+    renderIndex(pages),
   );
 
   const hubs = generateTopicHubs(pages);
@@ -558,9 +648,10 @@ export function generateCrawlDocs() {
     path.join(DOCS_OUT, "llms-full.txt"),
     generateLlmsFullTxt(pages, hubs),
   );
+  writeAgentSiteArtifacts(DOCS_OUT);
 
   console.info(
-    `generate-crawl-docs: ${pages.length} pages, ${hubs.length} hubs → docs/crawl/, sitemap.xml, llms.txt, llms-full.txt`,
+    `generate-crawl-docs: ${pages.length} pages, ${hubs.length} hubs → pathnames /docs|/core + hubs/, sitemap, llms*, index.md (no /crawl/)`,
   );
 }