blume 1.4.2 → 1.5.0

package/src/openapi/scalar.ts

@@ -4,6 +4,7 @@ import { isAbsolute, join } from "pathe";
 
 import { scalarReferenceTemplate } from "../astro/templates.ts";
 import type { ResolvedConfig } from "../core/schema.ts";
+import { trimChar } from "../core/trim.ts";
 import { resolveAccent, resolveRadius } from "../theme/palette.ts";
 import { resolveReferences } from "./references.ts";
 import type { ReferenceSource } from "./references.ts";
@@ -24,17 +25,14 @@ export interface ReferenceFile {
 }
 
 const URL_SPEC = /^https?:\/\//u;
-const ROUTE_EDGES = /^\/+|\/+$/gu;
 
 /** The `src/pages`-relative file path for a reference route. */
 const referencePagePath = (route: string): string => {
-  const segments = route.replace(ROUTE_EDGES, "");
+  const segments = trimChar(route, "/");
   return `${segments === "" ? "index" : segments}.astro`;
 };
 
-const darkModeConfig = (
-  mode: ResolvedConfig["theme"]["mode"]
-): Record<string, boolean> => {
+const darkModeConfig = (mode: ResolvedConfig["theme"]["mode"]) => {
   if (mode === "dark") {
     return { darkMode: true };
   }
@@ -51,10 +49,7 @@ const darkModeConfig = (
  * `customCss`. Scalar re-injects `customCss` after its bundled theme, so these
  * variables reliably override the defaults. Best-effort, not pixel-exact.
  */
-const themeConfiguration = (
-  config: ResolvedConfig,
-  override?: string
-): Record<string, unknown> => {
+const themeConfiguration = (config: ResolvedConfig, override?: string) => {
   if (override) {
     return { theme: override };
   }
@@ -70,7 +65,10 @@ const themeConfiguration = (
 const specConfiguration = async (
   spec: string,
   root: string
-): Promise<{ config: Record<string, unknown>; warning?: string }> => {
+): Promise<{
+  config: { content: string } | { url: string };
+  warning?: string;
+}> => {
   if (URL_SPEC.test(spec)) {
     return { config: { url: spec } };
   }

package/src/openapi/source.ts

@@ -1,5 +1,6 @@
 import { withBasePath } from "../core/base-path.ts";
 import matter from "../core/frontmatter.ts";
+import type { FolderMeta } from "../core/schema.ts";
 import { hashText } from "../core/sources/cache.ts";
 import type {
   ContentSource,
@@ -8,19 +9,30 @@ import type {
   SourceLoadResult,
 } from "../core/sources/types.ts";
 import type { Diagnostic } from "../core/types.ts";
+import { extractAsyncApiOperations } from "./asyncapi.ts";
+import type { AsyncApiDocument } from "./asyncapi.ts";
 import { extractOperations } from "./model.ts";
-import type { ApiOperationRef, ApiSpecData, OpenApiData } from "./model.ts";
-import { InvalidSpecError, parseSpec } from "./parse.ts";
+import type {
+  ApiDocument,
+  ApiOperationRef,
+  ApiSpecData,
+  ApiTagRef,
+  OpenApiData,
+} from "./model.ts";
+import { InvalidSpecError, parseAsyncApiSpec, parseSpec } from "./parse.ts";
 import type { ReferenceSource } from "./references.ts";
 import { operationMdx, overviewMdx } from "./render-mdx.ts";
 import type { RenderedPage } from "./render-mdx.ts";
 
 /**
- * The staged content source behind Blume's own OpenAPI renderer. Each configured
- * spec is parsed once here, then lowered into one MDX page per operation plus an
- * overview page — so operations become first-class Blume pages (real routes,
- * sidebar, search, i18n, OG) and the parsed documents are handed to the
- * generated `blume:openapi` module for the UI components to render.
+ * The staged content source behind Blume's own API reference renderer (OpenAPI
+ * and AsyncAPI alike). Each configured spec is parsed once here, then lowered
+ * into one MDX page per operation plus an overview page — so operations become
+ * first-class Blume pages (real routes, sidebar, search, i18n, OG) and the
+ * parsed documents are handed to the generated `blume:openapi` module for the
+ * UI components to render. The source keeps its historical `openapi` name for
+ * both kinds — downstream consumers (`ai.llmsTxt.openapi`, the llms noindex
+ * exemption) key on it as "the generated API reference source".
  */
 
 /** A content source that also exposes the specs it parsed during `load()`. */
@@ -34,7 +46,7 @@ export interface OpenApiContentSource extends ContentSource {
 export const isOpenApiSource = (
   source: ContentSource
 ): source is OpenApiContentSource =>
-  (source as Partial<OpenApiContentSource>).kind === "openapi-source";
+  "kind" in source && source.kind === "openapi-source";
 
 /** Route (`/reference/pet/add-pet`) to a staged content ref, without extension. */
 const routeToRef = (route: string): string => route.replace(/^\/+/u, "");
@@ -43,7 +55,9 @@ const toEntry = (rendered: RenderedPage, ref: string): SourceEntry => {
   const raw = matter.stringify(`${rendered.body}\n`, rendered.data);
   return {
     body: { format: "mdx", text: rendered.body },
-    data: rendered.data,
+    // Spread so the named frontmatter shape satisfies the open metadata
+    // dictionary every source entry carries.
+    data: { ...rendered.data },
     hash: hashText(raw),
     raw,
     ref,
@@ -75,14 +89,81 @@ const specEntries = (
   return entries;
 };
 
+/**
+ * Label each tag's sidebar group with the spec's own tag name. The group label
+ * is otherwise re-humanized from the tag's route slug (split on hyphens,
+ * title-cased), which mangles authored casing and symbols — `OAuth2` →
+ * "Oauth2", `Größe` → "Größe" only by luck of the slug. Keys are the tag
+ * directories under the reference route, the same group paths `meta.ts` files
+ * use, so user-authored meta still overrides these.
+ */
+const tagFolderMeta = (
+  spec: ApiSpecData,
+  tags: { slug: string; name: string }[]
+): Record<string, FolderMeta> => {
+  const base = routeToRef(spec.route);
+  return Object.fromEntries(
+    tags.map((tag) => [
+      base ? `${base}/${tag.slug}` : tag.slug,
+      { title: tag.name },
+    ])
+  );
+};
+
 interface LoadedSpec {
   slug: string;
   spec: ApiSpecData;
   entries: SourceEntry[];
+  /** Sidebar-group labels for the spec's tag directories. */
+  folderMeta: Record<string, FolderMeta>;
   /** Non-fatal notes from the load (e.g. an offline cache fallback). */
   diagnostics: Diagnostic[];
 }
 
+/** One parsed spec, whichever front-end read it — the kind dispatch seam. */
+interface ParsedReference {
+  document: ApiDocument | AsyncApiDocument;
+  warnings: string[];
+  operations: ApiOperationRef[];
+  tags: ApiTagRef[];
+  extractWarnings: string[];
+}
+
+const parseReference = async (
+  reference: ReferenceSource,
+  ctx: SourceContext
+): Promise<ParsedReference> => {
+  const options = { cacheDir: ctx.cacheDir, refresh: ctx.refresh };
+  if (reference.kind === "asyncapi") {
+    const { document, warnings } = await parseAsyncApiSpec(
+      reference.spec,
+      ctx.projectRoot,
+      options
+    );
+    const extracted = extractAsyncApiOperations(document, reference.route);
+    return {
+      document,
+      extractWarnings: extracted.warnings,
+      operations: extracted.operations,
+      tags: extracted.tags,
+      warnings,
+    };
+  }
+  const { document, warnings } = await parseSpec(
+    reference.spec,
+    ctx.projectRoot,
+    options
+  );
+  const extracted = extractOperations(document, reference.route);
+  return {
+    document,
+    extractWarnings: extracted.warnings,
+    operations: extracted.operations,
+    tags: extracted.tags,
+    warnings,
+  };
+};
+
 export const openApiSource = (
   references: ReferenceSource[],
   ctx: SourceContext
@@ -92,23 +173,21 @@ export const openApiSource = (
   const loadReference = async (
     reference: ReferenceSource
   ): Promise<LoadedSpec | Diagnostic> => {
+    // Human label and diagnostic-code prefix for the spec's kind, so an
+    // AsyncAPI failure never reads as an OpenAPI one.
+    const kindLabel = reference.kind === "asyncapi" ? "AsyncAPI" : "OpenAPI";
+    const codePrefix =
+      reference.kind === "asyncapi" ? "BLUME_ASYNCAPI" : "BLUME_OPENAPI";
     try {
-      const { document, warnings } = await parseSpec(
-        reference.spec,
-        ctx.projectRoot,
-        { cacheDir: ctx.cacheDir, refresh: ctx.refresh }
-      );
-      const {
-        operations,
-        tags,
-        warnings: extractWarnings,
-      } = extractOperations(document, reference.route);
+      const { document, warnings, operations, tags, extractWarnings } =
+        await parseReference(reference, ctx);
       const info = document.info ?? { title: reference.label, version: "" };
       const spec: ApiSpecData = {
         codeSamples: reference.display.codeSamples,
         description: info.description ?? "",
         document,
         expandSchemas: reference.display.expandSchemas,
+        kind: reference.kind,
         label: reference.label,
         // Operation pages flow through the content pipeline, which mounts them
         // under the site-wide `basePath` (staged entry refs below stay
@@ -132,13 +211,24 @@ export const openApiSource = (
       return {
         diagnostics: [
           ...warnings.map((message) => ({
-            code: "BLUME_OPENAPI_STALE",
+            // Parse-level notes: an offline cache fallback for either kind,
+            // plus lossy 2.x→3.0 conversion notes for AsyncAPI — hence the
+            // broader code on that side (OpenAPI keeps its historical one).
+            code:
+              reference.kind === "asyncapi"
+                ? "BLUME_ASYNCAPI_SPEC_WARNING"
+                : "BLUME_OPENAPI_STALE",
             message,
             severity: "warning" as const,
           })),
           ...extractWarnings.map((message) => ({
-            code: "BLUME_OPENAPI_REF_PATH_ITEM",
-            message: `In OpenAPI spec "${reference.spec}": ${message}`,
+            // OpenAPI keeps its historical code (the only extract warning it
+            // emits is the unresolved $ref path item).
+            code:
+              reference.kind === "asyncapi"
+                ? "BLUME_ASYNCAPI_SKIPPED_OPERATION"
+                : "BLUME_OPENAPI_REF_PATH_ITEM",
+            message: `In ${kindLabel} spec "${reference.spec}": ${message}`,
             severity: "warning" as const,
           })),
           // A document with no operations (say, a config file that happens to
@@ -147,23 +237,28 @@ export const openApiSource = (
           ...(operations.length === 0
             ? [
                 {
-                  code: "BLUME_OPENAPI_EMPTY",
-                  message: `OpenAPI spec "${reference.spec}" for ${reference.route} declares no operations; its API reference is empty.`,
+                  code: `${codePrefix}_EMPTY`,
+                  message: `${kindLabel} spec "${reference.spec}" for ${reference.route} declares no operations; its API reference is empty.`,
                   severity: "warning" as const,
                   suggestion:
-                    "Check the spec points at an OpenAPI document with operations under `paths`.",
+                    reference.kind === "asyncapi"
+                      ? "Check the spec points at an AsyncAPI document with `channels` and `operations`."
+                      : "Check the spec points at an OpenAPI document with operations under `paths`.",
                 },
               ]
             : []),
         ],
         entries: specEntries(spec, operations, reference),
+        folderMeta: tagFolderMeta(spec, tags),
         slug: reference.slug,
         spec,
       };
     } catch (error) {
       return {
-        code: "BLUME_OPENAPI_UNAVAILABLE",
-        message: `Could not load OpenAPI spec "${reference.spec}" for ${reference.route} (${(error as Error).message}); its reference pages were skipped.`,
+        code: `${codePrefix}_UNAVAILABLE`,
+        // SAFETY: spec loading fails with Error instances (fetch, read, and
+        // parse errors alike); only the message is read for the diagnostic.
+        message: `Could not load ${kindLabel} spec "${reference.spec}" for ${reference.route} (${(error as Error).message}); its reference pages were skipped.`,
         // A configured-but-unloadable spec ships a dead nav tab (a 404 route),
         // so fail loudly in build (blocks under --strict) while staying a warning
         // in dev so offline work still runs.
@@ -172,7 +267,7 @@ export const openApiSource = (
         // only point at reachability for actual fetch/read failures.
         suggestion:
           error instanceof InvalidSpecError
-            ? "Point the spec at an OpenAPI document (a YAML or JSON file with an object at the top level)."
+            ? `Point the spec at ${reference.kind === "asyncapi" ? "an AsyncAPI" : "an OpenAPI"} document (a YAML or JSON file with an object at the top level).`
             : "Check the spec URL/path is reachable from the build environment; behind a proxy, set HTTP(S)_PROXY.",
       };
     }
@@ -192,6 +287,7 @@ export const openApiSource = (
       }))
     );
     const data: OpenApiData = {};
+    const folderMeta: Record<string, FolderMeta> = {};
     for (const result of results) {
       if ("severity" in result) {
         diagnostics.push(result);
@@ -199,10 +295,11 @@ export const openApiSource = (
       }
       data[result.slug] = result.spec;
       entries.push(...result.entries);
+      Object.assign(folderMeta, result.folderMeta);
       diagnostics.push(...result.diagnostics);
     }
     parsed = data;
-    return { diagnostics, entries };
+    return { diagnostics, entries, folderMeta };
   };
 
   return {

package/src/registry/eject.ts

@@ -49,6 +49,7 @@ import { scanProject } from "../core/project-graph.ts";
 import type { BlumeProject } from "../core/project-graph.ts";
 import type { ProjectContext } from "../core/types.ts";
 import { buildRssFeeds, renderRssFeed } from "../deploy/rss.ts";
+import type { OpenApiData } from "../openapi/model.ts";
 import { hasScalarReferences } from "../openapi/references.ts";
 import { buildReferenceFiles } from "../openapi/scalar.ts";
 import { isOpenApiSource } from "../openapi/source.ts";
@@ -97,7 +98,7 @@ export const blumeSourceGlob = (
 };
 
 /** The `blume:openapi` payload for the ejected app (`{}` when none). */
-const ejectOpenApiData = (project: BlumeProject): unknown => {
+const ejectOpenApiData = (project: BlumeProject): OpenApiData => {
   const source = project.sources.find(isOpenApiSource);
   return source ? source.openApiData() : {};
 };
@@ -126,7 +127,11 @@ const askFiles = async (
   const grounded = ask.provider !== "inkeep";
   const files = [
     {
-      content: askEndpointTemplate(resolveAskBackend(ask), grounded),
+      content: askEndpointTemplate(
+        resolveAskBackend(ask),
+        grounded,
+        ask.instructions
+      ),
       path: join(srcDir, "pages", "api", "ask.ts"),
     },
   ];

package/src/search/documents.ts

@@ -1,3 +1,8 @@
+import type { Nodes } from "mdast";
+import { fromMarkdown } from "mdast-util-from-markdown";
+import { gfmFromMarkdown } from "mdast-util-gfm";
+import { gfm } from "micromark-extension-gfm";
+
 import { applyAudienceVisibility } from "../ai/visibility.ts";
 import type { VisibilityAudience } from "../ai/visibility.ts";
 import matter from "../core/frontmatter.ts";
@@ -21,6 +26,8 @@ export interface SearchDocument {
   locale: string;
   /** Resolved page `type` (`doc`, `blog`, a custom `rfc`…), for type filters. */
   contentType: string;
+  /** Docs version (`""` for the current docs), so results scope to the viewed version. */
+  version: string;
   /** Frontmatter `search.tags`, surfaced for hosted-provider faceting. */
   tags?: string[];
   /**
@@ -43,51 +50,96 @@ export interface SearchRecord {
   content: string;
   /** Locale code, carried as a facet for per-language filtering. */
   locale: string;
+  /**
+   * Docs version, carried as a facet for per-version filtering. The current
+   * docs upload as `"current"` — hosted backends treat an empty facet value
+   * unreliably, so the sentinel stands in for the empty version id.
+   */
+  version: string;
   /** Single faceting tag (the first frontmatter tag, when present). */
   tag?: string;
 }
 
-const CODE_FENCE = /```[\s\S]*?```/gu;
-const INLINE_CODE = /`(?<code>[^`]+)`/gu;
 // Tag-shaped only: a name (or closing slash/fragment) right after `<`, and no
-// newline inside. A bare `<` in prose ("costs < 5 credits") must not swallow
-// everything up to some later `>` — potentially whole paragraphs.
+// newline inside. Applied *within* html/JSX nodes so their inner prose is
+// kept; the surrounding Markdown is walked as a tree, so a bare `<` in prose
+// ("costs < 5 credits") is ordinary text and never at risk.
 const HTML_OR_JSX = /<\/?[a-zA-Z][^\n<>]*>|<\/?>/gu;
-const IMAGE = /!\[[^\]]*\]\([^)]*\)/gu;
-const LINK = /\[(?<text>[^\]]*)\]\([^)]*\)/gu;
-const HEADING_MARK = /^#{1,6}\s+/gmu;
-const MARKDOWN_PUNCT = /[*_~>]+/gu;
 const WHITESPACE = /\s+/gu;
 
-/** Reduce Markdown/MDX to plain, searchable text. */
-const toPlainText = (markdown: string): string => {
-  const withoutBlocks = markdown
-    .replaceAll(CODE_FENCE, " ")
-    .replaceAll(IMAGE, " ")
-    .replaceAll(LINK, "$<text>");
+// Parents whose children are inline: no separator is inserted after them, or
+// `re*ally*` would index as `re ally`. Every other parent is block-shaped and
+// ends with a space so adjacent paragraphs/headings/cells don't fuse.
+const INLINE_PARENTS = new Set([
+  "delete",
+  "emphasis",
+  "footnoteReference",
+  "link",
+  "linkReference",
+  "strong",
+]);
 
-  // Strip HTML/JSX from the prose, but keep the contents of inline code — an
-  // angle-bracket span like `<T>` inside `Array<T>` is a type parameter, not a
-  // tag, and stripping it would drop those tokens from the search index. Split
-  // on inline-code spans and only run the HTML strip on the text between them.
-  const pieces: string[] = [];
-  let cursor = 0;
-  for (const match of withoutBlocks.matchAll(INLINE_CODE)) {
-    const start = match.index ?? 0;
-    pieces.push(
-      withoutBlocks.slice(cursor, start).replaceAll(HTML_OR_JSX, " "),
-      match.groups?.code ?? ""
-    );
-    cursor = start + match[0].length;
+/** Fold one mdast node into the plain-text accumulator. */
+const collectText = (node: Nodes, out: string[]): void => {
+  switch (node.type) {
+    // Fenced code is excluded from the plain index (ranking noise) — the
+    // "markdown" extraction keeps it for Ask AI grounding — and image alt
+    // text was never indexed.
+    case "code":
+    case "image":
+    case "imageReference": {
+      return;
+    }
+    // Inline code is kept verbatim — `Array<T>` is a type parameter, not a
+    // tag, and its tokens must stay searchable.
+    case "inlineCode": {
+      out.push(node.value);
+      return;
+    }
+    // A raw-HTML/JSX run. CommonMark parses a block-level `<Callout>` with no
+    // blank lines as ONE html node holding all its inner prose, so the node
+    // can't just be dropped — strip the tag-shaped runs and keep the text.
+    case "html": {
+      out.push(node.value.replaceAll(HTML_OR_JSX, " "));
+      return;
+    }
+    case "break": {
+      out.push(" ");
+      return;
+    }
+    default: {
+      break;
+    }
+  }
+  if ("value" in node) {
+    out.push(node.value);
+    return;
   }
-  pieces.push(withoutBlocks.slice(cursor).replaceAll(HTML_OR_JSX, " "));
+  if ("children" in node) {
+    for (const child of node.children) {
+      collectText(child, out);
+    }
+    if (!INLINE_PARENTS.has(node.type)) {
+      out.push(" ");
+    }
+  }
+};
 
-  return pieces
-    .join("")
-    .replaceAll(HEADING_MARK, "")
-    .replaceAll(MARKDOWN_PUNCT, " ")
-    .replaceAll(WHITESPACE, " ")
-    .trim();
+/**
+ * Reduce Markdown/MDX to plain, searchable text: parse (GFM included) and walk
+ * the tree instead of regex-stripping the source, so reference-style links,
+ * autolinks, setext headings, tables, and literal `*`/`~`/`>` in prose all
+ * reduce correctly. This feeds the client index *and* every hosted-provider
+ * record, so anything lost here is a permanent search-quality loss.
+ */
+const toPlainText = (markdown: string): string => {
+  const tree = fromMarkdown(markdown, {
+    extensions: [gfm()],
+    mdastExtensions: [gfmFromMarkdown()],
+  });
+  const out: string[] = [];
+  collectText(tree, out);
+  return out.join("").replaceAll(WHITESPACE, " ").trim();
 };
 
 interface Crumbs {
@@ -169,10 +221,17 @@ export const buildSearchDocuments = async (
   // locale-prefixed routes), so localized pages get the right section/breadcrumb.
   // Falls back to the single default-locale nav when i18n is off.
   const byLocale = Object.values(project.graph.navigationByLocale ?? {});
-  const sidebars =
-    byLocale.length > 0
+  // Archived versions' trees contribute too, so snapshot pages get their own
+  // section/breadcrumb instead of falling through to the "Docs" default.
+  const byVersion = Object.values(project.graph.navigationByVersion ?? {})
+    .flatMap((locales) => Object.values(locales))
+    .map((nav) => nav.sidebar);
+  const sidebars = [
+    ...(byLocale.length > 0
       ? byLocale.map((nav) => nav.sidebar)
-      : [project.graph.navigation?.sidebar ?? []];
+      : [project.graph.navigation?.sidebar ?? []]),
+    ...byVersion,
+  ];
   const crumbs = new Map<string, Crumbs>();
   for (const sidebar of sidebars) {
     for (const [route, crumb] of buildCrumbIndex(sidebar)) {
@@ -202,18 +261,22 @@ export const buildSearchDocuments = async (
       const tags = page?.meta?.search?.tags;
       const crumb = crumbs.get(route.path);
       const facets = page ? pageFacets(page, project.config) : undefined;
-      return {
+      const document: SearchDocument = {
         breadcrumb: crumb?.breadcrumb ?? [],
         content: body,
         contentType: route.contentType,
         description: page?.description ?? "",
-        ...(facets ? { facets } : {}),
         locale: route.locale,
         route: route.path,
         section: crumb?.section || "Docs",
         tags: tags && tags.length > 0 ? tags : undefined,
         title: route.title,
+        version: route.version,
       };
+      if (facets) {
+        document.facets = facets;
+      }
+      return document;
     })
   );
 };
@@ -232,4 +295,5 @@ export const toSearchRecords = (documents: SearchDocument[]): SearchRecord[] =>
     tag: doc.tags?.[0],
     title: doc.title,
     url: doc.route,
+    version: doc.version || "current",
   }));

package/src/search/facets.ts

@@ -10,6 +10,12 @@ import type { PageRecord } from "../core/types.ts";
  * when nothing facets, so the field stays absent from serialized documents
  * rather than shipping as `{}` on every page.
  */
+/** Whether a custom frontmatter value stringifies into a usable facet value. */
+const isFacetValue = <T>(value: T): value is T & (string | number | boolean) =>
+  typeof value === "string" ||
+  typeof value === "number" ||
+  typeof value === "boolean";
+
 export const pageFacets = (
   page: Pick<PageRecord, "contentType" | "custom">,
   config: ResolvedConfig
@@ -21,11 +27,7 @@ export const pageFacets = (
   const facets: Record<string, string> = {};
   for (const key of declared) {
     const value = page.custom[key];
-    if (
-      typeof value === "string" ||
-      typeof value === "number" ||
-      typeof value === "boolean"
-    ) {
+    if (isFacetValue(value)) {
       facets[key] = String(value);
     }
   }