blume 0.7.0 → 0.8.0

package/src/ai/llms.ts

@@ -2,54 +2,163 @@ import { normalizeBasePath, withBasePath } from "../core/base-path.ts";
 import matter from "../core/frontmatter.ts";
 import type { BlumeProject } from "../core/project-graph.ts";
 import { readEntryText } from "../core/sources/read.ts";
-import type { PageRecord } from "../core/types.ts";
+import type { NavNode, Navigation, PageRecord } from "../core/types.ts";
+import { downlevelComponents } from "./component-markdown.ts";
+import { applyAgentVisibility } from "./visibility.ts";
 
-// Routes carry `basePath`; a `deployment.base` subdirectory is layered on top so
-// the emitted URL matches where the page is served.
+// Routes carry `basePath`; a `deployment.base` subdirectory is layered on top —
+// with or without a `site` (the mcp.json convention) — so the emitted URL
+// matches where the page is served. Encoded like the sitemap: a route with
+// spaces or non-ASCII must still yield a valid Markdown link.
 const pageUrl = (route: string, site?: string, base = ""): string => {
-  if (!site) {
-    return route;
-  }
-  return `${site.replace(/\/$/u, "")}${withBasePath(base, route)}`;
+  const path = withBasePath(base, route);
+  return encodeURI(site ? `${site.replace(/\/$/u, "")}${path}` : path);
 };
 
-const orderedPages = (project: BlumeProject): PageRecord[] =>
-  [...project.graph.pages]
-    .filter((page) => !page.meta.draft)
-    .sort((a, b) => a.route.localeCompare(b.route));
+// Drafts, hidden, and `noindex` pages are excluded, matching the sitemap.
+// Generated API reference pages are excluded when `ai.llmsTxt.openapi` is off
+// (they arrive through the internal staged "openapi" source).
+const eligiblePages = (project: BlumeProject): PageRecord[] =>
+  project.graph.pages.filter(
+    (page) =>
+      !(page.meta.draft || page.meta.sidebar.hidden || page.meta.seo.noindex) &&
+      (project.config.ai.llmsTxt.openapi || page.source.name !== "openapi")
+  );
 
-/** Build the compact `llms.txt` index: title, summary, and links per page. */
+/**
+ * The navigation trees the index mirrors: the site tree, or one per locale
+ * under i18n (each labeled with the locale except the default, so sections
+ * don't repeat ambiguously).
+ */
+const indexedNavigations = (
+  project: BlumeProject
+): { label?: string; nav: Navigation }[] => {
+  const { i18n } = project.config;
+  if (i18n) {
+    return i18n.locales.flatMap(({ code, label }) => {
+      const nav = project.graph.navigationByLocale[code];
+      if (!nav) {
+        return [];
+      }
+      return [{ label: code === i18n.defaultLocale ? undefined : label, nav }];
+    });
+  }
+  return [{ nav: project.graph.navigation }];
+};
+
+/**
+ * Build the compact `llms.txt` index: title and summary, then the sidebar tree
+ * rendered as sections — group labels become headings, pages become link lists —
+ * so the file mirrors how the docs are organized rather than one flat blob.
+ */
 const buildIndex = (project: BlumeProject): string => {
   const { config } = project;
   const { site } = config.deployment;
-  const lines = [`# ${config.title}`];
-  if (config.description) {
-    lines.push("", `> ${config.description}`);
+  const base = normalizeBasePath(config.deployment.base);
+  const eligible = eligiblePages(project);
+  const byRoute = new Map(eligible.map((page) => [page.route, page]));
+  const seen = new Set<string>();
+
+  const line = (page: PageRecord): string => {
+    seen.add(page.route);
+    const summary = page.description ? `: ${page.description}` : "";
+    return `- [${page.title}](${pageUrl(page.route, site, base)})${summary}`;
+  };
+
+  // One nav level -> Markdown blocks: the level's loose pages as a link list,
+  // then each group as a heading (depth-capped at h6) followed by its own
+  // blocks. Nav entries whose route has no eligible page — external links,
+  // padded i18n fallbacks, `noindex` pages, excluded API references — are
+  // skipped, and a group left with nothing emits no heading at all.
+  const renderLevel = (nodes: NavNode[], depth: number): string[] => {
+    const list: string[] = [];
+    const groupBlocks: string[] = [];
+    for (const node of nodes) {
+      if (node.kind === "page") {
+        const page = byRoute.get(node.route);
+        if (page && !seen.has(page.route)) {
+          list.push(line(page));
+        }
+        continue;
+      }
+      // An explicit-config group may link its index page on the group itself
+      // (`root`) rather than as a child; keep it at the top of the section.
+      const rootPage = node.route ? byRoute.get(node.route) : undefined;
+      const blocks = renderLevel(node.children, depth + 1);
+      if (rootPage && !seen.has(rootPage.route)) {
+        blocks.unshift(line(rootPage));
+      }
+      if (blocks.length > 0) {
+        groupBlocks.push(
+          `${"#".repeat(Math.min(depth, 6))} ${node.label}`,
+          ...blocks
+        );
+      }
+    }
+    return list.length > 0 ? [list.join("\n"), ...groupBlocks] : groupBlocks;
+  };
+
+  // Loose pages at a tree's root get a "Docs" section of their own, so every
+  // link sits under an h2 as llms.txt consumers expect.
+  const renderNav = (nav: Navigation, depth: number): string[] => {
+    const loose = nav.sidebar.filter((node) => node.kind === "page");
+    const groups = nav.sidebar.filter((node) => node.kind === "group");
+    const looseBlocks = renderLevel(loose, depth + 1);
+    return [
+      ...(looseBlocks.length > 0
+        ? [`${"#".repeat(depth)} Docs`, ...looseBlocks]
+        : []),
+      ...renderLevel(groups, depth),
+    ];
+  };
+
+  const blocks: string[] = [];
+  for (const { label, nav } of indexedNavigations(project)) {
+    if (label) {
+      const localized = renderNav(nav, 3);
+      if (localized.length > 0) {
+        blocks.push(`## ${label}`, ...localized);
+      }
+      continue;
+    }
+    blocks.push(...renderNav(nav, 2));
   }
-  lines.push("", "## Docs", "");
 
-  for (const page of orderedPages(project)) {
-    const url = pageUrl(
-      page.route,
-      site,
-      normalizeBasePath(config.deployment.base)
+  // Pages the navigation doesn't reach (an explicit sidebar that omits them,
+  // or a hand-rolled tree) still belong in the index.
+  const leftover = eligible
+    .filter((page) => !seen.has(page.route))
+    .toSorted((a, b) => a.route.localeCompare(b.route));
+  if (leftover.length > 0) {
+    blocks.push(
+      blocks.length > 0 ? "## Other" : "## Docs",
+      leftover.map(line).join("\n")
     );
-    const summary = page.description ? `: ${page.description}` : "";
-    lines.push(`- [${page.title}](${url})${summary}`);
   }
 
-  return `${lines.join("\n")}\n`;
+  const header = config.description
+    ? `# ${config.title}\n\n> ${config.description}`
+    : `# ${config.title}`;
+  return `${[header, ...blocks].join("\n\n")}\n`;
 };
 
 /** Build `llms-full.txt`: the full Markdown body of every page. */
 const buildFull = async (project: BlumeProject): Promise<string> => {
   const { config } = project;
-  const pages = orderedPages(project);
+  const pages = eligiblePages(project).toSorted((a, b) =>
+    a.route.localeCompare(b.route)
+  );
 
   const sections = await Promise.all(
     pages.map(async (page) => {
       const raw = await readEntryText(project, page);
-      const body = matter(raw).content.trim();
+      // Resolve `<Visibility>` audiences (web-only content omitted from the
+      // agent-facing output, agents-only unwrapped), then downlevel supported
+      // components to plain Markdown.
+      const body = downlevelComponents(
+        applyAgentVisibility(matter(raw).content),
+        config.ai.markdownComponents
+      ).trim();
       const url = pageUrl(
         page.route,
         config.deployment.site,

package/src/ai/markdown.ts

@@ -3,15 +3,37 @@ import { readFile } from "node:fs/promises";
 import type { BlumeProject } from "../core/project-graph.ts";
 import { readEntryText } from "../core/sources/read.ts";
 import type { RouteManifestEntry } from "../core/types.ts";
+import { downlevelComponents } from "./component-markdown.ts";
+import { applyAgentVisibility } from "./visibility.ts";
+
+/** One route's raw-Markdown variants. */
+export interface RawMarkdownEntry {
+  /**
+   * The agent-facing Markdown served at `/<route>.md`: supported components
+   * downleveled to plain Markdown (`<TypeTable>` → table, `<Callout>` →
+   * blockquote, …). Present only when downleveling changed something, so
+   * component-free pages aren't stored twice.
+   */
+  md?: string;
+  /** The original source, served verbatim at `/<route>.mdx`. */
+  mdx: string;
+}
+
+/** The Markdown an agent should read for a route. */
+export const agentMarkdown = (entry: RawMarkdownEntry): string =>
+  entry.md ?? entry.mdx;
 
 /**
  * Map every route to its raw source Markdown. Powers the `<route>.md` and
- * `<route>.mdx` endpoints, which serve the original source so AI tools — and
- * readers — can fetch any page as plain Markdown.
+ * `<route>.mdx` endpoints: `.mdx` serves the original source so tools can see
+ * exactly what the author wrote, while `.md` downlevels supported components
+ * to plain Markdown for consumers that can't interpret JSX. `<Visibility>`
+ * audiences are resolved for agents in both variants: web-only content is
+ * removed, agents-only unwrapped.
  */
 export const buildRawMarkdown = async (
   project: BlumeProject
-): Promise<Record<string, string>> => {
+): Promise<Record<string, RawMarkdownEntry>> => {
   const pageById = new Map(project.graph.pages.map((page) => [page.id, page]));
 
   const readRoute = async (route: RouteManifestEntry): Promise<string> => {
@@ -23,9 +45,16 @@ export const buildRawMarkdown = async (
   };
 
   const entries = await Promise.all(
-    project.manifest.routes.map(
-      async (route) => [route.path, await readRoute(route)] as const
-    )
+    project.manifest.routes.map(async (route) => {
+      const source = applyAgentVisibility(await readRoute(route));
+      const md = downlevelComponents(
+        source,
+        project.config.ai.markdownComponents
+      );
+      const entry: RawMarkdownEntry =
+        md === source ? { mdx: source } : { md, mdx: source };
+      return [route.path, entry] as const;
+    })
   );
   return Object.fromEntries(entries);
 };

package/src/ai/mcp/data.ts

@@ -1,8 +1,9 @@
+import { normalizeBasePath } from "../../core/base-path.ts";
 import type { BlumeProject } from "../../core/project-graph.ts";
 import type { Navigation } from "../../core/types.ts";
 import { buildSearchDocuments } from "../../search/documents.ts";
 import type { OramaDoc } from "../../search/orama-index.ts";
-import { buildRawMarkdown } from "../markdown.ts";
+import { agentMarkdown, buildRawMarkdown } from "../markdown.ts";
 
 /** A page entry surfaced by the `list_pages` MCP tool. */
 export interface McpRoute {
@@ -21,6 +22,12 @@ export interface McpRoute {
  * access at request time. Serialized to `generated/mcp-data.json`.
  */
 export interface McpData {
+  /**
+   * Normalized `deployment.base` (`""` or `/seg`), layered onto routes when
+   * emitting URLs — the site is base-less and routes are base-less manifest
+   * paths, matching the sitemap/llms.txt convention.
+   */
+  base: string;
   documents: OramaDoc[];
   instructions?: string;
   name: string;
@@ -34,13 +41,26 @@ export interface McpData {
 /** Build the MCP data snapshot from a resolved project. */
 export const buildMcpData = async (project: BlumeProject): Promise<McpData> => {
   const { config, graph, manifest } = project;
-  const [documents, pages] = await Promise.all([
+  const [documents, rawMarkdown] = await Promise.all([
     // The MCP server is independent of on-page search, so index docs even when
-    // the search provider is `none`.
-    buildSearchDocuments(project, { includeWhenDisabled: true }),
+    // the search provider is `none`. Documents are agent-facing, so
+    // `<Visibility>` resolves like `get_page`/llms-full.txt (web-only content
+    // removed, agents-only kept).
+    buildSearchDocuments(project, {
+      audience: "agents",
+      includeWhenDisabled: true,
+    }),
     buildRawMarkdown(project),
   ]);
 
+  // `get_page` serves the agent variant: components downleveled to Markdown.
+  const pages = Object.fromEntries(
+    Object.entries(rawMarkdown).map(([route, entry]) => [
+      route,
+      agentMarkdown(entry),
+    ])
+  );
+
   const descriptionById = new Map(
     graph.pages.map((page) => [page.id, page.description])
   );
@@ -61,6 +81,7 @@ export const buildMcpData = async (project: BlumeProject): Promise<McpData> => {
   }
 
   return {
+    base: normalizeBasePath(config.deployment.base),
     documents: documents.map((doc) => ({
       content: doc.content,
       description: doc.description,

package/src/ai/mcp/discovery.ts

@@ -1,7 +1,10 @@
+import { withBasePath } from "../../core/base-path.ts";
 import { MCP_TOOLS } from "./tools.ts";
 
 /** Inputs needed to describe the MCP server in discovery documents. */
 export interface McpDiscoveryInput {
+  /** Normalized `deployment.base` (`""` or `/seg`); the route is base-less. */
+  base: string;
   name: string;
   route: string;
   site: string | null;
@@ -9,10 +12,14 @@ export interface McpDiscoveryInput {
 }
 
 /** The MCP server's address — absolute when a site is configured. */
-const serverUrl = (input: McpDiscoveryInput): string =>
-  // Concatenate rather than `new URL(route, site)` — a root-absolute route
+const serverUrl = (input: McpDiscoveryInput): string => {
+  // The endpoint is a generated Astro page, so it's served under
+  // `deployment.base` like every other route (the sitemap/llms.txt convention).
+  const path = withBasePath(input.base, input.route);
+  // Concatenate rather than `new URL(path, site)` — a root-absolute path
   // would drop the base path of a subpath deployment (`acme.com/docs`).
-  input.site ? `${input.site.replace(/\/+$/u, "")}${input.route}` : input.route;
+  return input.site ? `${input.site.replace(/\/+$/u, "")}${path}` : path;
+};
 
 /**
  * The `/.well-known/mcp.json` discovery document: the minimal pointer agents use

package/src/ai/mcp/server.ts

@@ -5,6 +5,7 @@ import {
   ListToolsRequestSchema,
 } from "@modelcontextprotocol/sdk/types.js";
 
+import { withBasePath } from "../../core/base-path.ts";
 import { buildOramaIndex, queryOramaIndex } from "../../search/orama-index.ts";
 import type { OramaDoc } from "../../search/orama-index.ts";
 import type { McpData } from "./data.ts";
@@ -89,10 +90,24 @@ const normalizeRoute = (input: string): string => {
 };
 
 /** Build the absolute (or root-relative) URL for a route. */
-const urlFor = (route: string, site: string | null): string =>
-  // Concatenate rather than `new URL(route, site)` — a root-absolute route
+const urlFor = (route: string, data: McpData): string => {
+  // Routes are base-less manifest paths; layer `deployment.base` on top so the
+  // URL matches where the page is served (the sitemap/llms.txt convention).
+  const path = withBasePath(data.base, route);
+  // Concatenate rather than `new URL(path, site)` — a root-absolute path
   // would drop the base path of a subpath deployment (`acme.com/docs`).
-  site ? `${site.replace(/\/+$/u, "")}${route}` : route;
+  return data.site ? `${data.site.replace(/\/+$/u, "")}${path}` : path;
+};
+
+/** A hit's excerpt: its description, else the head of its content with an
+ * ellipsis only when something was actually cut off. */
+const excerptFor = (doc: OramaDoc): string => {
+  if (doc.description) {
+    return doc.description;
+  }
+  const head = doc.content.slice(0, EXCERPT_LENGTH).trim();
+  return doc.content.length > EXCERPT_LENGTH ? `${head}…` : head;
+};
 
 const text = (value: string, isError = false) => ({
   content: [{ text: value, type: "text" as const }],
@@ -127,10 +142,9 @@ const buildServer = (
         asLimit(args.limit)
       );
       const results = hits.map((doc: OramaDoc) => ({
-        excerpt:
-          doc.description || `${doc.content.slice(0, EXCERPT_LENGTH)}…`.trim(),
+        excerpt: excerptFor(doc),
         title: doc.title,
-        url: urlFor(doc.route, data.site),
+        url: urlFor(doc.route, data),
       }));
       return text(JSON.stringify(results, null, 2));
     }
@@ -156,7 +170,7 @@ const buildServer = (
             lastModified: route.lastModified,
             route: route.route,
             title: route.title,
-            url: urlFor(route.route, data.site),
+            url: urlFor(route.route, data),
           })),
           null,
           2

package/src/ai/visibility.ts

@@ -0,0 +1,74 @@
+// Mirrors the fence masking in `core/sources/assets.ts`: a fenced code sample
+// that *shows* `<Visibility>` markup must keep showing what the author wrote.
+const CODE_FENCE_BLOCK =
+  /^(?<fence>`{3,}|~{3,})[^\n]*\n[\s\S]*?^\k<fence>[^\n]*(?=\n|$)/gmu;
+// NUL delimiters cannot appear in authored markdown, so tokens never collide.
+// oxlint-disable-next-line no-control-regex -- the NUL is the collision guard.
+const FENCE_TOKEN = /\u0000blume-fence-(?<index>\d+)\u0000/gu;
+
+/** The audience an output surface serves — the component's two `for` values. */
+export type VisibilityAudience = "agents" | "web";
+
+// `<Visibility for="…">…</Visibility>` in either quote style, tolerant of
+// whitespace around the attribute and inside the tags. Non-greedy bodies stop
+// at the first close tag, so nesting is not supported: a nested block closes
+// the outer match early and any remainder passes through verbatim.
+const visibilityBlock = (audience: VisibilityAudience): RegExp =>
+  new RegExp(
+    `<Visibility\\s+for\\s*=\\s*(?:"${audience}"|'${audience}')\\s*>(?<inner>[\\s\\S]*?)</Visibility\\s*>`,
+    "gu"
+  );
+
+const BLOCKS: Record<VisibilityAudience, RegExp> = {
+  agents: visibilityBlock("agents"),
+  web: visibilityBlock("web"),
+};
+
+/**
+ * Resolve `<Visibility>` blocks for one audience: blocks addressed to the
+ * other audience are removed entirely and blocks addressed to `audience` are
+ * unwrapped (tags dropped, body kept), matching what the Astro component
+ * renders on the web. Other `for` values (the component's default) are left
+ * untouched, and markdown with no matching blocks is returned byte-identical,
+ * so raw sources stay raw.
+ */
+export const applyAudienceVisibility = (
+  markdown: string,
+  audience: VisibilityAudience
+): string => {
+  // Mask fenced code blocks so documentation *about* Visibility survives.
+  const fences: string[] = [];
+  const masked = markdown.replace(CODE_FENCE_BLOCK, (block) => {
+    fences.push(block);
+    return `\u0000blume-fence-${fences.length - 1}\u0000`;
+  });
+
+  let touched = false;
+  const filtered = masked
+    .replaceAll(BLOCKS[audience === "agents" ? "web" : "agents"], () => {
+      touched = true;
+      return "";
+    })
+    .replaceAll(BLOCKS[audience], (_match, inner: string) => {
+      touched = true;
+      return inner;
+    });
+
+  // Removing/unwrapping block-level tags leaves runs of blank lines behind;
+  // collapse them only when something matched so untouched files round-trip
+  // exactly. Fences are masked as single-line tokens, so they are unaffected.
+  const tidied = touched ? filtered.replaceAll(/\n{3,}/gu, "\n\n") : filtered;
+
+  return tidied.replaceAll(
+    FENCE_TOKEN,
+    (token, index) => fences[Number(index)] ?? token
+  );
+};
+
+/**
+ * Resolve `<Visibility>` blocks for agent-facing Markdown (llms-full.txt, the
+ * `.md`/`.mdx` mirrors, MCP tools, Ask AI grounding): `for="web"` content is
+ * removed and `for="agents"` content is unwrapped.
+ */
+export const applyAgentVisibility = (markdown: string): string =>
+  applyAudienceVisibility(markdown, "agents");

package/src/astro/component-slots.ts

@@ -91,8 +91,18 @@ ${clause}
 `;
 };
 
+/**
+ * A filesystem-safe, injective token for an override key. Distinct keys must
+ * never share a wrapper file ("Foo.Bar" vs "Foo_Bar" used to collide, racing
+ * the same temp file and silently rendering the wrong component), so every
+ * non-alphanumeric character is hex-escaped rather than collapsed — the same
+ * hardening as `exampleSlug` in templates.ts.
+ */
 const sanitize = (value: string): string =>
-  value.replaceAll(/[^A-Za-z0-9]/gu, "_");
+  value.replaceAll(
+    /[^A-Za-z0-9]/gu,
+    (char) => `_${(char.codePointAt(0) ?? 0).toString(16)}_`
+  );
 
 export const planComponentSlots = (
   componentsFile: string | null,