blume 1.3.0 → 1.4.0

package/src/ai/llms.ts

@@ -60,8 +60,10 @@ const indexedNavigations = (
  * Build the compact `llms.txt` index: title and summary, then the sidebar tree
  * rendered as sections — group labels become headings, pages become link lists —
  * so the file mirrors how the docs are organized rather than one flat blob.
+ * Also serves as the homepage's synthesized Markdown mirror when the home
+ * route is a landing page (see `buildRawMarkdown`).
  */
-const buildIndex = (project: BlumeProject): string => {
+export const buildLlmsIndex = (project: BlumeProject): string => {
   const { config } = project;
   const { site } = config.deployment;
   const base = normalizeBasePath(config.deployment.base);
@@ -222,5 +224,5 @@ export const buildLlmsFiles = async (
   project: BlumeProject
 ): Promise<{ index: string; full: string }> => ({
   full: await buildFull(project),
-  index: buildIndex(project),
+  index: buildLlmsIndex(project),
 });

package/src/ai/markdown.ts

@@ -9,6 +9,7 @@ import {
   downlevelComponents,
   exampleComponentSerializers,
 } from "./component-markdown.ts";
+import { buildLlmsIndex } from "./llms.ts";
 import { applyAgentVisibility } from "./visibility.ts";
 
 /** One route's raw-Markdown variants. */
@@ -28,6 +29,16 @@ export interface RawMarkdownEntry {
 export const agentMarkdown = (entry: RawMarkdownEntry): string =>
   entry.md ?? entry.mdx;
 
+/**
+ * Estimated token count of a Markdown document, for the `x-markdown-tokens`
+ * response header (the convention Cloudflare's Markdown for Agents ships; the
+ * tokenizer is unspecified there too, so this is the common ~4-characters-
+ * per-token estimate). Kept in sync with the inline computation in
+ * `rawMarkdownEndpointTemplate`, which can't import build-time code.
+ */
+export const markdownTokenCount = (text: string): number =>
+  Math.ceil(text.length / 4);
+
 /**
  * Map every route to its raw source Markdown. Powers the `<route>.md` and
  * `<route>.mdx` endpoints: `.mdx` serves the original source so tools can see
@@ -79,5 +90,27 @@ export const buildRawMarkdown = async (
       return [route.path, entry] as const;
     })
   );
-  return Object.fromEntries(entries);
+  const map = Object.fromEntries(entries);
+  // A landing-page homepage (user `.astro` page, or no home route at all) has
+  // no Markdown source, but agents negotiating `Accept: text/markdown` on `/`
+  // still expect a Markdown answer. The llms.txt index — the machine-readable
+  // representation of the site a landing page fronts — becomes its mirror, so
+  // `/index.md` always exists (see `markdownRoutePaths`).
+  if (!map["/"]) {
+    map["/"] = { mdx: buildLlmsIndex(project) };
+  }
+  return map;
+};
+
+/**
+ * Every route path with a raw-Markdown mirror: the manifest routes, plus the
+ * homepage when its mirror is the synthesized llms.txt fallback (see
+ * `buildRawMarkdown`). This is the route list the negotiation surfaces (dev
+ * middleware, Vercel routing config) and the homepage `Link` header build
+ * from, so `Accept: text/markdown` on `/` resolves even when the homepage is
+ * a landing page.
+ */
+export const markdownRoutePaths = (project: BlumeProject): string[] => {
+  const paths = project.manifest.routes.map((route) => route.path);
+  return paths.includes("/") ? paths : [...paths, "/"];
 };

package/src/ai/mcp/data.ts

@@ -2,6 +2,7 @@ import { normalizeBasePath } from "../../core/base-path.ts";
 import type { BlumeProject } from "../../core/project-graph.ts";
 import type { Navigation } from "../../core/types.ts";
 import { buildSearchDocuments } from "../../search/documents.ts";
+import { pageFacets } from "../../search/facets.ts";
 import type { OramaDoc } from "../../search/orama-index.ts";
 import { agentMarkdown, buildRawMarkdown } from "../markdown.ts";
 
@@ -9,6 +10,8 @@ import { agentMarkdown, buildRawMarkdown } from "../markdown.ts";
 export interface McpRoute {
   contentType: string;
   description?: string;
+  /** Declared facet values (`content.types.<type>.facets`), key → value. */
+  facets?: Record<string, string>;
   indexable: boolean;
   lastModified: string | null;
   route: string;
@@ -67,18 +70,19 @@ export const buildMcpData = async (project: BlumeProject): Promise<McpData> => {
     ])
   );
 
-  const descriptionById = new Map(
-    graph.pages.map((page) => [page.id, page.description])
-  );
+  const pageById = new Map(graph.pages.map((page) => [page.id, page]));
 
   const routes: McpRoute[] = [];
   for (const route of manifest.routes) {
     if (route.hidden) {
       continue;
     }
+    const page = pageById.get(route.id);
+    const facets = page ? pageFacets(page, config) : undefined;
     routes.push({
       contentType: route.contentType,
-      description: descriptionById.get(route.id),
+      description: page?.description,
+      ...(facets ? { facets } : {}),
       indexable: route.indexable,
       lastModified: route.lastModified ?? null,
       route: route.path,
@@ -91,7 +95,9 @@ export const buildMcpData = async (project: BlumeProject): Promise<McpData> => {
     defaultLocale: config.i18n?.defaultLocale,
     documents: documents.map((doc) => ({
       content: doc.content,
+      contentType: doc.contentType,
       description: doc.description,
+      ...(doc.facets ? { facets: doc.facets } : {}),
       route: doc.route,
       title: doc.title,
     })),

package/src/ai/mcp/server.ts

@@ -33,6 +33,22 @@ const CORS_HEADERS: Record<string, string> = {
   "Access-Control-Expose-Headers": "Mcp-Session-Id",
 };
 
+/** The optional content-type filter `search_docs` and `list_pages` share. */
+const CONTENT_TYPES_SCHEMA = {
+  description:
+    'Only include pages of these content types (frontmatter `type`, e.g. `["doc", "rfc"]`). `list_pages` shows each page\'s type. Omit to include every type.',
+  items: { type: "string" },
+  type: "array",
+} as const;
+
+/** The optional facet filter `search_docs` and `list_pages` share. */
+const FILTERS_SCHEMA = {
+  additionalProperties: { type: "string" },
+  description:
+    'Only include pages matching every facet, key → required value (e.g. `{"status": "enforced"}`). Facets are metadata the site declares per content type; `list_pages` shows each page\'s facet values. Omit for no facet filtering.',
+  type: "object",
+} as const;
+
 /** JSON Schema for each tool's input, keyed by tool name. */
 const INPUT_SCHEMAS: Record<string, Record<string, unknown>> = {
   get_navigation: { properties: {}, type: "object" },
@@ -46,9 +62,17 @@ const INPUT_SCHEMAS: Record<string, Record<string, unknown>> = {
     required: ["route"],
     type: "object",
   },
-  list_pages: { properties: {}, type: "object" },
+  list_pages: {
+    properties: {
+      contentTypes: CONTENT_TYPES_SCHEMA,
+      filters: FILTERS_SCHEMA,
+    },
+    type: "object",
+  },
   search_docs: {
     properties: {
+      contentTypes: CONTENT_TYPES_SCHEMA,
+      filters: FILTERS_SCHEMA,
       limit: {
         description: `Maximum hits to return (default ${DEFAULT_SEARCH_LIMIT}).`,
         maximum: MAX_SEARCH_LIMIT,
@@ -74,6 +98,39 @@ const TOOL_DEFINITIONS = MCP_TOOLS.map((tool) => ({
 const asString = (value: unknown): string =>
   typeof value === "string" ? value : "";
 
+/**
+ * The `contentTypes` filter as a string array, or `undefined` when absent or
+ * empty — an agent sending `[]` means "no filter", not "match nothing". A bare
+ * string is accepted as a one-element list.
+ */
+const asContentTypes = (value: unknown): string[] | undefined => {
+  const list = Array.isArray(value)
+    ? value.filter((entry): entry is string => typeof entry === "string")
+    : [value].filter((entry): entry is string => typeof entry === "string");
+  return list.length > 0 ? list : undefined;
+};
+
+/**
+ * The `filters` facet map with only its string-valued entries, or `undefined`
+ * when nothing usable remains — an empty `{}` means "no filter".
+ */
+const asFacetFilters = (value: unknown): Record<string, string> | undefined => {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    return;
+  }
+  const entries = Object.entries(value).filter(
+    (entry): entry is [string, string] => typeof entry[1] === "string"
+  );
+  return entries.length > 0 ? Object.fromEntries(entries) : undefined;
+};
+
+/** Whether a page's facet values satisfy every requested filter entry. */
+const matchesFacets = (
+  facets: Record<string, string> | undefined,
+  filters: Record<string, string>
+): boolean =>
+  Object.entries(filters).every(([key, value]) => facets?.[key] === value);
+
 const asLimit = (value: unknown): number => {
   const num = typeof value === "number" ? value : Number(value);
   if (!Number.isFinite(num)) {
@@ -181,12 +238,18 @@ export const buildServer = (
       const hits = await queryOramaIndex(
         db,
         asString(args.query),
-        asLimit(args.limit)
+        asLimit(args.limit),
+        {
+          contentTypes: asContentTypes(args.contentTypes),
+          facets: asFacetFilters(args.filters),
+        }
       );
       // `route` is the key `get_page` takes (the tool descriptions promise
       // it); `url` is where the page is served.
       const results = hits.map((doc: OramaDoc) => ({
+        contentType: doc.contentType,
         excerpt: excerptFor(doc),
+        facets: doc.facets,
         route: doc.route,
         title: doc.title,
         url: urlFor(doc.route, data),
@@ -207,11 +270,19 @@ export const buildServer = (
     }
 
     if (name === "list_pages") {
+      const contentTypes = asContentTypes(args.contentTypes);
+      const filters = asFacetFilters(args.filters);
+      const routes = data.routes.filter(
+        (route) =>
+          (!contentTypes || contentTypes.includes(route.contentType)) &&
+          (!filters || matchesFacets(route.facets, filters))
+      );
       return text(
         JSON.stringify(
-          data.routes.map((route) => ({
+          routes.map((route) => ({
             contentType: route.contentType,
             description: route.description,
+            facets: route.facets,
             lastModified: route.lastModified,
             route: route.route,
             title: route.title,

package/src/ai/mcp/tools.ts

@@ -19,7 +19,7 @@ export const MCP_TOOLS: McpToolMeta[] = [
   {
     annotations: READ_ONLY,
     description:
-      "Full-text search across the documentation. Returns matching pages with their title, route, and a short excerpt. Use this first to discover relevant pages, then `get_page` to read one in full.",
+      'Full-text search across the documentation. Returns matching pages with their title, route, content type, and a short excerpt; pass `contentTypes` to search only pages of certain types (e.g. `rfc`, `changelog`), and `filters` to require facet values the site declares per type (e.g. `{"status": "enforced"}`). Use this first to discover relevant pages, then `get_page` to read one in full.',
     name: "search_docs",
     title: "Search documentation",
   },
@@ -33,7 +33,7 @@ export const MCP_TOOLS: McpToolMeta[] = [
   {
     annotations: READ_ONLY,
     description:
-      "List every documentation page with its route, title, description, and content type. Useful for enumerating the docs or finding a page when search is too narrow.",
+      "List every documentation page with its route, title, description, content type, and any declared facet values; pass `contentTypes` and/or `filters` to narrow the list. Useful for enumerating the docs, discovering the types and facets in use, or finding a page when search is too narrow.",
     name: "list_pages",
     title: "List pages",
   },

package/src/astro/generate.ts

@@ -19,7 +19,7 @@ import { glob } from "tinyglobby";
 
 import { buildAskData } from "../ai/ask-data.ts";
 import { resolveAskBackend } from "../ai/ask.ts";
-import { buildRawMarkdown } from "../ai/markdown.ts";
+import { buildRawMarkdown, markdownRoutePaths } from "../ai/markdown.ts";
 import { buildMcpData } from "../ai/mcp/data.ts";
 import { buildMcpDiscovery, buildMcpServerCard } from "../ai/mcp/discovery.ts";
 import { normalizeBasePath } from "../core/base-path.ts";
@@ -1608,7 +1608,7 @@ export const generateRuntime = async (
           aliases: resolveTsconfigAliases(context.root),
           askPath,
           config,
-          contentRoutes: project.manifest.routes.map((route) => route.path),
+          contentRoutes: markdownRoutePaths(project),
           contentWatchesRuntimeDir: contentWatchesRuntimeDir(
             hasFilesystemSource,
             docsCollection.base,

package/src/astro/integration.ts

@@ -114,9 +114,13 @@ const isHomeUrl = (rawUrl: string | undefined, base?: string): boolean => {
  * production the content pages are prerendered and served from the platform's
  * static layer, which this middleware never fronts. Vercel server builds get
  * the same negotiation from routing rules spliced into the Build Output config
- * (see `deploy/vercel-negotiation.ts`); every other build exposes the same
+ * (see `deploy/vercel-negotiation.ts`), Cloudflare server builds from a
+ * wrapper Worker routed to by `assets.run_worker_first` (see
+ * `deploy/cloudflare-negotiation.ts`); every other build exposes the same
  * content at the `.md` URL. Only routes with a Markdown variant are rewritten,
- * so landing pages and user `.astro` pages keep serving HTML. The same
+ * so user `.astro` pages keep serving HTML — except the homepage, whose
+ * variant falls back to the synthesized llms.txt mirror when it's a landing
+ * page (see `markdownRoutePaths`). The same
  * middleware also stamps the homepage agent-discovery `Link` header, mirroring
  * what the deployed site sends via `_headers` / the Vercel routing config.
  */

package/src/astro/markdown-negotiation.ts

@@ -3,6 +3,11 @@
  * endpoints already serve a page's source verbatim; these helpers let the dev
  * server honor `Accept: text/markdown` by transparently rewriting a page
  * request to its `.md` variant.
+ *
+ * The Cloudflare negotiation Worker embeds a plain-JavaScript copy of these
+ * helpers (`deploy/cloudflare-negotiation.ts` — its deploy bundle is uploaded
+ * unbundled, so it cannot import this module); when editing here, mirror the
+ * change there. Parity is enforced by `test/cloudflare-negotiation.test.ts`.
  */
 
 interface AcceptEntry {

package/src/astro/templates.ts

@@ -370,6 +370,38 @@ const renderImageOption = (config: ResolvedConfig): string =>
     ? `\n  image: ${JSON.stringify(config.image)},`
     : "";
 
+/**
+ * Startup-scan entry points and forced includes for the dev dep optimizer:
+ * the Vite root is the generated runtime, so user pages, convention islands,
+ * and alias-reachable components all live outside it and are otherwise only
+ * crawled when first requested. The compiler runtime rides the include list
+ * because it is Babel-injected and no source scan can see it. See the
+ * optimizeDeps comment in the generated config for the failure this prevents.
+ */
+const resolveOptimizeDeps = (options: {
+  aliases: Record<string, string> | undefined;
+  context: ProjectContext;
+  needsReact: boolean;
+  reactCompilerPath: string | null | undefined;
+}): { optimizeDepsEntries: string[]; optimizeDepsInclude: string[] } => {
+  const { context } = options;
+  const optimizeDepsEntries = [
+    ...(context.pagesRoot ? [`${context.pagesRoot}/**/*.astro`] : []),
+    `${context.root}/islands/**/*.{jsx,svelte,tsx,vue}`,
+    ...[...new Set(Object.values(options.aliases ?? {}))]
+      .toSorted()
+      .map((dir) => `${dir}/**/*.{astro,jsx,svelte,tsx,vue}`),
+  ];
+  const optimizeDepsInclude = [
+    "blume > mermaid",
+    "blume > epub-gen-memory/bundle",
+    ...(options.needsReact && options.reactCompilerPath
+      ? ["react/compiler-runtime"]
+      : []),
+  ];
+  return { optimizeDepsEntries, optimizeDepsInclude };
+};
+
 export const astroConfigTemplate = (options: {
   context: ProjectContext;
   config: ResolvedConfig;
@@ -424,6 +456,13 @@ export const astroConfigTemplate = (options: {
   // KaTeX fonts under a monorepo's root node_modules) stay servable in dev.
   const fsAllow = [...new Set([findWorkspaceRoot(context.root), context.root])];
 
+  const { optimizeDepsEntries, optimizeDepsInclude } = resolveOptimizeDeps({
+    aliases: options.aliases,
+    context,
+    needsReact,
+    reactCompilerPath: options.reactCompilerPath,
+  });
+
   const adapterImport =
     server && deployment.adapter
       ? `import adapter from "${ADAPTER_IMPORTS[deployment.adapter]}";\n`
@@ -642,24 +681,32 @@ ${userConfigSetup}export default defineConfig({
   devToolbar: { enabled: false },
   vite: {
     plugins: [tailwindcss(), prerenderDepsPlugin(), serverAppResolvePlugin()],
-    // The lazy client-side imports both land on CJS/UMD files: mermaid (for
-    // diagrams) statically imports dayjs as CJS (\`dayjs/dayjs.min.js\`), and
-    // epub-gen-memory's browser bundle is a browserified UMD. In dev, an
-    // un-pre-bundled dependency is served as raw ESM, where such a file
-    // exposes no \`default\` export — mermaid throws on load and diagrams
-    // render blank, and the EPUB export throws \`epub is not a function\`
-    // (the UMD finds no \`exports\`/\`define\` and strands its callable on
-    // \`window.epubGen\` instead). Forcing them through the dep optimizer
-    // restores the CJS interop. In a standalone install these dynamic imports
-    // live inside \`node_modules/blume\`, which Vite's optimizer scan doesn't
-    // crawl, so neither is discovered on its own — hence the explicit
-    // includes. They resolve through the \`blume\` package (they aren't direct
-    // deps of the generated project), so the nested \`blume > x\` form is
-    // required, and epub-gen-memory must name the \`/bundle\` subpath that is
-    // actually imported: optimizing the package root leaves that entry out.
-    // Production (Rollup) already handles the interop, so this only affects dev.
+    // Everything hydration can reach must be part of the dev dep optimizer's
+    // FIRST run. The Vite root is the generated runtime, so user pages,
+    // islands, and aliased components live outside it and are only crawled
+    // when first requested — and \`react/compiler-runtime\` is Babel-injected,
+    // so no source scan can ever see it. A dependency discovered after
+    // hydration begins triggers a mid-session re-optimization whose new
+    // generation imports React through new \`?v=\` URLs; the browser then
+    // evaluates a second React copy and every island tears down with
+    // "Invalid hook call" (#157). \`entries\` points the startup scanner at
+    // the user's files (the scanner follows their imports, so their deps land
+    // in the initial optimization); the compiler runtime rides the include
+    // list because only the transform pipeline knows it exists.
+    //
+    // The mermaid/epub includes fix CJS interop instead: both lazy client-side
+    // imports land on CJS/UMD files (mermaid statically imports dayjs as CJS,
+    // epub-gen-memory's browser bundle is a browserified UMD) that break when
+    // served as raw ESM — mermaid throws on load and the EPUB export throws
+    // \`epub is not a function\`. They resolve through the \`blume\` package
+    // (they aren't direct deps of the generated project), so the nested
+    // \`blume > x\` form is required, and epub-gen-memory must name the
+    // \`/bundle\` subpath that is actually imported: optimizing the package
+    // root leaves that entry out. Production (Rollup) already handles the
+    // interop, so all of this only affects dev.
     optimizeDeps: {
-      include: ["blume > mermaid", "blume > epub-gen-memory/bundle"],
+      entries: ${JSON.stringify(optimizeDepsEntries)},
+      include: ${JSON.stringify(optimizeDepsInclude)},
     },
     // Blume's render-time deps are forced external on both build environments so
     // native bindings resolve at runtime and isolated linkers don't bundle
@@ -1181,10 +1228,15 @@ export function getStaticPaths() {
 export function GET({ props }: { props: { route: string } }) {
   const entries = raw as Record<string, { md?: string; mdx?: string }>;
   const entry = entries[props.route];
-  return new Response(entry ? ${
+  const body = entry ? ${
     kind === "md" ? '(entry.md ?? entry.mdx ?? "")' : '(entry.mdx ?? "")'
-  } : "", {
-    headers: { "Content-Type": "text/markdown; charset=utf-8" },
+  } : "";
+  return new Response(body, {
+    headers: {
+      "Content-Type": "text/markdown; charset=utf-8",
+      // ~4 characters per token; keep in sync with markdownTokenCount.
+      "x-markdown-tokens": String(Math.ceil(body.length / 4)),
+    },
   });
 }
 `;
@@ -1304,11 +1356,12 @@ export const mcpEndpointTemplate = (route: string): string => {
   return `// Generated by Blume. Do not edit.
 import type { APIRoute } from "astro";
 import { createMcpFetchHandler } from "blume/ai/mcp/server.ts";
+import type { McpData } from "blume/ai/mcp/data.ts";
 import data from "${up}generated/mcp-data.json";
 
 export const prerender = false;
 
-const handler = createMcpFetchHandler(data);
+const handler = createMcpFetchHandler(data as McpData);
 
 export const ALL: APIRoute = ({ request }) => handler(request);
 `;

package/src/audit/url.ts

@@ -1,4 +1,8 @@
-import { stripBasePath } from "../core/base-path.ts";
+import { normalizePath, stripBasePath } from "../core/base-path.ts";
+
+// Re-exported from its home next to the other path helpers; the audit checks
+// (and their tests) import it from here.
+export { normalizePath } from "../core/base-path.ts";
 
 /** What an `href` in built HTML turned out to point at. */
 export type ResolvedHref =
@@ -13,15 +17,6 @@ export type ResolvedHref =
 
 const NON_HTTP_SCHEME = /^(?!https?:)[a-z][a-z0-9+.-]*:/iu;
 
-/**
- * Normalize a site path for comparison: drop the trailing slash (Astro serves
- * `/docs` and `/docs/` as the same page) and collapse an empty path to `/`.
- */
-export const normalizePath = (path: string): string => {
-  const trimmed = path.replace(/\/+$/u, "");
-  return trimmed === "" ? "/" : trimmed;
-};
-
 /**
  * Percent-decode a pathname for comparison against the built file tree. Page
  * URLs and file-index keys come from raw on-disk names, while `URL#pathname`