blume 2.1.3 → 2.2.1

package/src/deploy/generated-files.ts

@@ -0,0 +1,128 @@
+import { join } from "pathe";
+
+import { buildAgentReadability } from "../ai/agent-readability.ts";
+import {
+  AI_CATALOG_PATH,
+  ARD_MANIFEST_PATH,
+  buildAiCatalog,
+} from "../ai/ai-catalog.ts";
+import { API_CATALOG_PATH, buildApiCatalog } from "../ai/api-catalog.ts";
+import {
+  API_NAVIGATION_PATH,
+  API_PAGES_PATH,
+  OPENAPI_PATH,
+} from "../ai/api/paths.ts";
+import { SKILL_MD_PATH } from "../ai/site-skill.ts";
+import { AGENT_SKILLS_INDEX_PATH, servesSkillsIndex } from "../ai/skills.ts";
+import type { SkillArtifact } from "../ai/skills.ts";
+import {
+  buildSignaturesDirectory,
+  SIGNATURES_DIRECTORY_PATH,
+} from "../ai/web-bot-auth.ts";
+import type { BlumeProject } from "../core/project-graph.ts";
+import type { ResolvedConfig } from "../core/schema.ts";
+import { staticFileResolver } from "../core/static-files.ts";
+import { buildRobots } from "./robots.ts";
+import { buildRssFeeds } from "./rss.ts";
+import { buildSitemapFiles } from "./sitemap.ts";
+
+/** A generated `.well-known` discovery document: `null` content when it's off. */
+export interface WellKnownFile {
+  content: string | null;
+  label: string;
+  path: string;
+}
+
+/**
+ * The generated `.well-known` discovery documents — the Web Bot Auth signature
+ * directory, the RFC 9727 API catalog, and the AI catalog under both the
+ * ai-catalog spec's well-known URI and the ARD v0.91 one (see
+ * `ai/ai-catalog.ts`) — each with `null` content when its feature is off.
+ */
+export const wellKnownFiles = (
+  config: ResolvedConfig,
+  skills: readonly SkillArtifact[]
+): WellKnownFile[] => {
+  const aiCatalog = buildAiCatalog(config, skills);
+  return [
+    {
+      content: buildSignaturesDirectory(config),
+      label: "Web Bot Auth",
+      path: SIGNATURES_DIRECTORY_PATH,
+    },
+    {
+      content: buildApiCatalog(config),
+      label: "RFC 9727",
+      path: API_CATALOG_PATH,
+    },
+    { content: aiCatalog, label: "AI Catalog", path: AI_CATALOG_PATH },
+    { content: aiCatalog, label: "ARD manifest", path: ARD_MANIFEST_PATH },
+  ];
+};
+
+/**
+ * The files a build serves beside its pages that don't come from `public/`,
+ * as base-less root-relative paths: what `publishBuildArtifacts` writes once
+ * Astro is done (llms.txt, the sitemap, robots.txt, `agent-readability.json`,
+ * the `.well-known` discovery documents, the Agent Skills index, `skill.md`)
+ * and the generated endpoints at fixed paths (the RSS feeds, the JSON docs
+ * API and its OpenAPI description, the MCP discovery documents). Each is
+ * listed under the gate its writer reads, so a feature the config switches
+ * off drops its file here too: `blume validate` accepts a link to
+ * `/llms.txt` exactly when the build writes one.
+ *
+ * Read from the config alone, before a build: the skill archives a build
+ * publishes beside the index, and the platform files (`_redirects`,
+ * `_headers`, `vercel.json`), aren't listed. A page's Markdown copy
+ * (`/guide.md`) is its route's, and resolves through the page.
+ */
+export const generatedFilePaths = (project: BlumeProject): string[] => {
+  const { config } = project;
+  const paths: string[] = [];
+  if (config.agents.llmsTxt.enabled) {
+    paths.push("/llms.txt", "/llms-full.txt");
+  }
+  paths.push(
+    ...(buildSitemapFiles(project) ?? []).map((file) => `/${file.name}`)
+  );
+  if (buildRobots(project)) {
+    paths.push("/robots.txt");
+  }
+  if (buildAgentReadability(project)) {
+    paths.push("/agent-readability.json");
+  }
+  paths.push(
+    ...wellKnownFiles(config, []).flatMap((file) =>
+      file.content ? [file.path] : []
+    )
+  );
+  if (servesSkillsIndex(config)) {
+    paths.push(AGENT_SKILLS_INDEX_PATH);
+    if (config.agents.skillMd) {
+      paths.push(SKILL_MD_PATH);
+    }
+  }
+  paths.push(...buildRssFeeds(project).map((feed) => feed.path));
+  if (config.agents.api) {
+    paths.push(OPENAPI_PATH, API_PAGES_PATH, API_NAVIGATION_PATH);
+  }
+  if (config.agents.mcp.enabled) {
+    paths.push("/.well-known/mcp.json", "/.well-known/mcp/server-card.json");
+  }
+  return paths;
+};
+
+/**
+ * Whether a base-less path is a file the project's site serves beside its
+ * pages: one in `public/` (`/spec.pdf`, a folder's `index.html`) or one the
+ * build generates (`/llms.txt`). What `generateRuntime` and `blume doctor`
+ * check navigation entries against, beside the routes, as `blume validate`
+ * checks links.
+ */
+export const servedFiles = (
+  project: BlumeProject
+): ((path: string) => boolean) =>
+  staticFileResolver(
+    join(project.context.root, "public"),
+    generatedFilePaths(project)
+  );

package/src/deploy/platforms/vercel.ts

@@ -1,5 +1,5 @@
 import { existsSync } from "node:fs";
-import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
 
 import { dirname, join } from "pathe";
 
@@ -99,6 +99,28 @@ export const checkVercelFunctionBundles = async (
   return !fatal;
 };
 
+/**
+ * Remove the routing config of a build that failed after the adapter wrote
+ * it. The adapter's config is complete as far as the adapter goes, but the
+ * redirects, negotiation, and base routes Blume adds were never spliced in,
+ * so `vercel deploy --prebuilt` would ship a site missing them. Without a
+ * `config.json`, the tree is no Build Output a deploy accepts; the function
+ * bundles stay for inspection.
+ */
+const withdrawBuildOutput = async (
+  context: ProjectContext,
+  log: BuildLog
+): Promise<void> => {
+  const configPath = join(buildOutputDir(context), "config.json");
+  if (!existsSync(configPath)) {
+    return;
+  }
+  await rm(configPath, { force: true });
+  log.info(
+    "Removed .vercel/output/config.json, so this failed build can't be deployed."
+  );
+};
+
 /**
  * Splice `Accept: text/markdown` negotiation routes into the adapter's Build
  * Output config, so a content-page request that prefers Markdown gets the
@@ -299,6 +321,7 @@ export const vercelPlatform: DeployPlatform = {
       log
     );
     if (!ok) {
+      await withdrawBuildOutput(context, log);
       return false;
     }
     // A verify build moves its static files too, where the budget gate reads
@@ -310,6 +333,7 @@ export const vercelPlatform: DeployPlatform = {
       return true;
     }
     if (!(await rebaseVercelRoutes(project, log))) {
+      await withdrawBuildOutput(context, log);
       return false;
     }
     await emitVercelNegotiation(project, log);

package/src/deploy/redirects.ts

@@ -8,6 +8,11 @@ import {
 } from "../core/base-path.ts";
 import { routeSetFor, servesRoute } from "../core/locale-links.ts";
 import type { RouteSet } from "../core/locale-links.ts";
+import {
+  MIRROR_EXTENSIONS,
+  mirrorName,
+  mirrorOwner,
+} from "../core/markdown-mirrors.ts";
 import type { BlumeProject } from "../core/project-graph.ts";
 import {
   compileRedirects,
@@ -113,29 +118,49 @@ export const applyBaseToPlatformRedirects = (
 };
 
 /**
- * The raw-Markdown copies Blume serves beside every page (`/guide.md`,
- * `/guide.mdx`; see `rawMarkdownEndpointTemplate`).
+ * A path naming an `.html` file (`/guide.html`), the URL another generator
+ * served a page at. No Blume page was ever served there, so it had no
+ * Markdown copy (`/guide.html.md`) for a redirect to move.
  */
-const MIRROR_EXTENSIONS = [".md", ".mdx"] as const;
+const HTML_FILE_PATH = /\.html$/u;
+
+/** A path naming a folder's `index.html` (`/guide/index.html`, `/index.html`). */
+const INDEX_FILE_PATH = /^(?<folder>.*)\/index\.html$/u;
 
 /**
- * Where a page's Markdown copy is served, from the page's served path and the
- * deployment base that path carries: the endpoints name the root `index`.
+ * Whether `from` names a page's own `index.html` (`/guide/index.html` for the
+ * page `/guide`, `/index.html` for the root), a URL a migrated site may have
+ * linked. A static build writes the page there, and Astro writes a redirect
+ * page to `<from>/index.html`, which would need that file to be a folder:
+ * the build fails with `EISDIR`. Hosts serve the page at that URL anyway,
+ * unless a redirect rule answers it first. `pages` are the served page
+ * routes, which carry `basePath`, as `from` does.
  */
-const mirrorPath = (path: string, base: string, extension: string): string => {
-  const route = stripBasePath(base, path);
-  return `${base}${route === "/" ? "/index" : route}${extension}`;
+export const isPageIndexFile = (from: string, pages: RouteSet): boolean => {
+  const folder = INDEX_FILE_PATH.exec(normalizePath(from))?.groups?.folder;
+  return folder !== undefined && servesRoute(pages, folder || "/");
 };
 
+/**
+ * Where a page's Markdown copy is served, from the page's served path and the
+ * deployment base that path carries.
+ */
+const mirrorPath = (path: string, base: string, extension: string): string =>
+  `${base}${mirrorName(stripBasePath(base, path))}${extension}`;
+
 /**
  * Based `redirects`, each exact one that moves a page followed by the same
- * redirect for the page's Markdown copies: from a path no page is served at
- * to one a page is, `/old.md` goes to `/new.md` (and `.mdx` alike), where it
- * would otherwise 404 for an agent still reading the old URL. A pattern gets
- * none; one that carries its capture over (`/beta/:slug*` → `/v2/:slug*`)
- * already moves `/beta/guide.md` along. `pages` are the served page routes
- * (carrying `basePath`, not `deployment.base`), and `bases` the deployment
- * base each end carries: Astro's config leaves it off `from` (see
+ * redirect for the page's Markdown copies: from a path whose copies no page
+ * serves to one a page is served at, `/old.md` goes to `/new.md` (and `.mdx`
+ * alike), where it would otherwise 404 for an agent still reading the old
+ * URL. A served copy never gives way: `/index` names the root's copies, so
+ * `/index` → `/` moving them would turn the home page's `/index.md` into a
+ * redirect to itself. A pattern gets none; one that carries its capture over
+ * (`/beta/:slug*` → `/v2/:slug*`) already moves `/beta/guide.md` along. Nor
+ * does a redirect from an `.html` file (`/guide.html`, from a migrated site),
+ * which had no copy to move. `pages` are the served page routes (carrying
+ * `basePath`, not `deployment.base`), and `bases` the deployment base each
+ * end carries: Astro's config leaves it off `from` (see
  * {@link applyBaseToAstroRedirects}).
  */
 export const withMirrorRedirects = (
@@ -148,8 +173,10 @@ export const withMirrorRedirects = (
     const to = normalizePath(redirect.to);
     const movesPage =
       !isPatternPath(redirect.from) &&
+      !HTML_FILE_PATH.test(from) &&
       servesRoute(pages, stripBasePath(bases.to, to)) &&
-      !servesRoute(pages, stripBasePath(bases.from, from));
+      mirrorOwner(pages, mirrorName(stripBasePath(bases.from, from))) ===
+        undefined;
     return movesPage
       ? [
           redirect,

package/src/deploy/sitemap.ts

@@ -11,6 +11,7 @@ import {
   normalizeBasePath,
   normalizePath,
 } from "../core/base-path.ts";
+import { isHiddenPage } from "../core/hidden-pages.ts";
 import type { BlumeProject } from "../core/project-graph.ts";
 import { siteRoot } from "../core/site-url.ts";
 
@@ -163,7 +164,7 @@ export const buildSitemapFiles = (
     const served = mountBasePath(deployBase, page.route);
     if (
       page.meta.draft ||
-      page.meta.sidebar.hidden ||
+      isHiddenPage(page, project.graph) ||
       page.meta.seo.noindex ||
       ERROR_ROUTES.has(page.route) ||
       archivedExcluded(page) ||

package/src/markdown/base-links.ts

@@ -4,22 +4,51 @@ import {
   withBasePath,
 } from "../core/base-path.ts";
 import { servesRoute } from "../core/locale-links.ts";
-import type { MdastNode } from "./mdast.ts";
+import { rewriteElementUrls } from "../core/sources/normalize.ts";
+import type { MdastNode, MdastValue } from "./mdast.ts";
 import { routeSnapshotReader } from "./route-snapshot.ts";
 
 interface UrlNode extends MdastNode {
   url?: string | null;
 }
 
+/** Raw HTML in a `.md` page. */
+interface HtmlNode extends MdastNode {
+  value: string;
+}
+
+/** An MDX JSX attribute, as the plugin reads it. */
+interface JsxAttribute {
+  [key: string]: MdastValue;
+  name?: string;
+  type?: string;
+  value?: MdastValue;
+}
+
+/** An MDX JSX element (`<a href="/guide">`), as the plugin reads it. */
+interface JsxNode extends MdastNode {
+  attributes?: JsxAttribute[];
+  children?: MdastValue[];
+  name?: string | null;
+}
+
 /**
  * The slice of Satteri's MDAST visitor context this plugin needs. Nodes are
  * read-only (the tree compiles to an op-stream), so a URL edit is recorded via
  * `setProperty`, not by mutating the node object.
  */
 interface MdastUrlContext {
-  setProperty: (node: MdastNode, key: "url", value: string) => void;
+  replaceNode: (node: MdastNode, replacement: MdastNode) => void;
+  setProperty: (node: MdastNode, key: "url" | "value", value: string) => void;
 }
 
+/** An HTML element's name, as opposed to a component's (`Card`). */
+const HTML_ELEMENT = /^[a-z]/u;
+
+/** A plain string attribute value; an expression value isn't a URL. */
+const isStringValue = (value: MdastValue): value is string =>
+  typeof value === "string";
+
 /** Only a string URL can be rebased; MDAST allows null or absent urls. */
 const isUrl = (url: string | null | undefined): url is string =>
   typeof url === "string";
@@ -53,10 +82,14 @@ export interface BaseLinksPluginOptions {
  * gains `deployment.base` layered over the site-wide `basePath` (`[x](/guide)`
  * -> `/base/docs/guide`); a public asset — an image, or a link to a file like
  * `/spec.pdf` (see `withAuthoredBasePath`) — gains `deployment.base` alone,
- * since Astro serves `public/` under it but never under `basePath`.
- * Idempotent per layer (a hand-written `/docs/x` isn't double-prefixed) and
- * inert for external URLs, fragments, and relative paths. Only constructed
- * when a base is set (see `markdown/index.ts`).
+ * since Astro serves `public/` under it but never under `basePath`. A raw
+ * HTML element's `href` or `src` (`<a href="/guide">`, `<img src="/x.png">`,
+ * in a `.md` page or as an `.mdx` element) gains `deployment.base` alone: the
+ * whole site moves under it, but a raw `href` keeps no `basePath`, as
+ * `blume validate` reads it. Idempotent per layer (a hand-written `/docs/x`
+ * isn't double-prefixed) and inert for external URLs, fragments, and
+ * relative paths. Only constructed when a base is set (see
+ * `markdown/index.ts`).
  */
 export const baseLinksPlugin = (
   deployBase: string,
@@ -95,12 +128,60 @@ export const baseLinksPlugin = (
   // endpoint), served under the deployment base but not `basePath`.
   const rebaseImage = (node: UrlNode, ctx: MdastUrlContext): void =>
     rebase(node, ctx, (url) => withBasePath(deployBase, url));
+
+  /** A raw element's root-relative URL under the deployment base, if it moves. */
+  const based = (url: string): string | null => {
+    const next = withBasePath(deployBase, url);
+    return next === url ? null : next;
+  };
+
+  const rebaseHtml = (node: HtmlNode, ctx: MdastUrlContext): void => {
+    const next = rewriteElementUrls(node.value, ({ value }) => based(value));
+    if (next !== node.value) {
+      ctx.setProperty(node, "value", next);
+    }
+  };
+
+  const rebaseElement = (node: JsxNode, ctx: MdastUrlContext): void => {
+    if (!(node.name && HTML_ELEMENT.test(node.name))) {
+      return;
+    }
+    let changed = false;
+    const attributes = (node.attributes ?? []).map((attribute) => {
+      const { name, value } = attribute;
+      const next =
+        attribute.type === "mdxJsxAttribute" &&
+        (name === "href" || name === "src") &&
+        isStringValue(value)
+          ? based(value)
+          : null;
+      if (next === null) {
+        return attribute;
+      }
+      changed = true;
+      return { name, type: attribute.type, value: next };
+    });
+    if (changed) {
+      // Sätteri can't set a JSX element's attributes in place, so the element
+      // is swapped for a copy; its children carry over and are still visited.
+      ctx.replaceNode(node, {
+        attributes,
+        children: node.children ?? [],
+        name: node.name,
+        type: node.type,
+      });
+    }
+  };
+
   // `link` covers inline links; `definition` covers reference-style
   // definitions (`[x]: /guide`), which a link or an image may cite.
   return {
     definition: rebaseLink,
+    html: rebaseHtml,
     image: rebaseImage,
     link: rebaseLink,
+    mdxJsxFlowElement: rebaseElement,
+    mdxJsxTextElement: rebaseElement,
     name: "blume-base-links",
   };
 };

package/src/markdown/code-title.ts

@@ -2,9 +2,9 @@
  * Code-fence meta. A Shiki transformer reads the tokens after the language and
  * promotes them to attributes on the rendered `<pre>`:
  *
- * - a title — the first bare token (```ts blume.config.ts) or `title="..."` —
- *   becomes `data-title`; the theme's code header shows it, falling back to the
- *   language label.
+ * - a title — the bare words (```ts blume.config.ts, ```js Install the
+ *   client) or `title="..."` — becomes `data-title`; the theme's code header
+ *   shows it, falling back to the language label.
  * - the `lineNumbers` keyword (```ts file.ts lineNumbers) becomes
  *   `data-line-numbers`; the theme renders a counter-driven line-number gutter.
  * - the `wrap` keyword becomes `data-wrap`: this block's long lines wrap
@@ -16,6 +16,7 @@
  */
 
 import {
+  isForeignKeyword,
   isLineRange,
   metaTokens,
   QUOTED_ATTR,
@@ -33,7 +34,7 @@ export const EXPANDABLE_MIN_LINES = 16;
 
 /** The slice of Shiki's transformer `this` context Blume reads. */
 interface CodeMetaContext {
-  options: { meta?: { __raw?: string } };
+  options: { lang?: string; meta?: { __raw?: string } };
   /** The block's code, as highlighted. */
   source?: string;
 }
@@ -51,17 +52,29 @@ export interface CodeTitleTransformer {
 
 const TITLE_ATTR = /(?:^|\s)title=(?:"(?<dq>[^"]*)"|'(?<sq>[^']*)')/u;
 
-// The first bare token is the title (```ts blume.config.ts): a non-empty token
-// that isn't a Shiki line range (`{1,3-5}`), a `key=value` attr, or a reserved
-// keyword.
-const isTitleToken = (token: string): boolean =>
+// The bare words are the title (```ts blume.config.ts, ```js Install the
+// client): every token that isn't a Shiki line range (`{1,3-5}`), a
+// `key=value` attr, a reserved keyword, or another tool's keyword for a
+// fence in `lang` (```rust ignore).
+const isTitleToken = (token: string, lang: string | undefined): boolean =>
   token.length > 0 &&
   !isLineRange(token) &&
   !token.includes("=") &&
-  !RESERVED_META_KEYWORDS.has(token);
+  !RESERVED_META_KEYWORDS.has(token) &&
+  !isForeignKeyword(token, lang);
 
-/** The title a fence's meta string promotes to `data-title`, if any. */
-export const parseCodeTitle = (raw: string | undefined): string | undefined => {
+// A title wrapped in one pair of brackets (```ts [file.ts], the Docus and
+// VitePress code-group spelling) shows without them.
+const BRACKETED_TITLE = /^\[(?<inner>[^\]]+)\]$/u;
+
+/**
+ * The title a fence's meta string promotes to `data-title`, if any. `lang` is
+ * the fence's language, which decides the other tools' keywords it drops.
+ */
+export const parseCodeTitle = (
+  raw: string | undefined,
+  lang?: string
+): string | undefined => {
   if (!raw) {
     return undefined;
   }
@@ -79,7 +92,10 @@ export const parseCodeTitle = (raw: string | undefined): string | undefined => {
   // The shared tokenizer keeps a quoted attr (rejected below by its `=`) and
   // a spaced line range (`{1, 3-5}`) whole, so neither can shed a fragment
   // that reads as a bare title.
-  return metaTokens(raw).find(isTitleToken);
+  const title = metaTokens(raw)
+    .filter((token) => isTitleToken(token, lang))
+    .join(" ");
+  return BRACKETED_TITLE.exec(title)?.groups?.inner ?? (title || undefined);
 };
 
 // The shared tokenizer keeps a quoted attr whole, so a keyword inside a
@@ -96,7 +112,7 @@ export const codeTitleTransformer = (): CodeTitleTransformer => ({
   name: "blume:code-meta",
   pre(node) {
     const raw = this.options.meta?.__raw;
-    const title = parseCodeTitle(raw);
+    const title = parseCodeTitle(raw, this.options.lang);
     if (title) {
       node.properties.dataTitle = title;
     }