blume 1.6.6 → 1.7.1

package/src/core/sources/assets.ts

@@ -1,4 +1,12 @@
-import { mkdir, writeFile } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import {
+  access,
+  mkdir,
+  readdir,
+  rename,
+  rm,
+  writeFile,
+} from "node:fs/promises";
 
 import { extname, join } from "pathe";
 
@@ -6,6 +14,12 @@ import type { Diagnostic } from "../types.ts";
 import { hashText } from "./cache.ts";
 
 const MD_IMAGE = /!\[(?<alt>[^\]]*)\]\((?<url>[^)\s]+)\)/gu;
+// A `<video>` tag's `src`, split so the rewrite can swap the URL and keep the
+// surrounding attributes untouched. Notion's uploaded videos arrive as signed,
+// expiring URLs exactly like its images, so they rot the same way. `[^>]`
+// bounds the attribute run to a single tag.
+const HTML_VIDEO_SRC =
+  /(?<open><video\b[^>]*?\ssrc=")(?<url>[^"]+)(?<close>")/gu;
 const REMOTE = /^https?:\/\//u;
 const SAFE_EXT = /^\.[a-z0-9]+$/iu;
 const CODE_FENCE_BLOCK =
@@ -14,32 +28,107 @@ const CODE_FENCE_BLOCK =
 // oxlint-disable-next-line no-control-regex -- the NUL is the collision guard.
 const FENCE_TOKEN = /\u0000blume-fence-(?<index>\d+)\u0000/gu;
 
+// The extension for a URL whose path carries none, keyed by the media type the
+// server reports. A static host serves by extension, so a `.png` holding JPEG
+// bytes is mislabeled and a `.png` holding a video is refused by strict
+// players — the response is the source of truth, not the reference kind.
+const EXT_BY_MIME = new Map([
+  ["image/avif", ".avif"],
+  ["image/gif", ".gif"],
+  ["image/jpeg", ".jpg"],
+  ["image/png", ".png"],
+  ["image/svg+xml", ".svg"],
+  ["image/webp", ".webp"],
+  ["video/mp4", ".mp4"],
+  ["video/ogg", ".ogv"],
+  ["video/quicktime", ".mov"],
+  ["video/webm", ".webm"],
+]);
+const UNKNOWN_EXT = ".bin";
+// Response types that are never a media file. A pasted Vimeo/Loom/Wistia link
+// is a video block in Notion whose URL is a watch page (200 text/html), and an
+// API endpoint answers JSON or XML; `image/svg+xml` is media and stays.
+const NON_MEDIA_TYPE =
+  /^(?:text\/|application\/(?:[\w.-]+\+)?(?:json|xml|javascript))/u;
+const PART_SUFFIX = ".part";
+// Generous enough for a multi-hundred-megabyte recording on an ordinary
+// connection; its job is to fail a stalled download rather than hang the build.
+const DEFAULT_TIMEOUT_MS = 120_000;
+
 /** Where to write downloaded assets and how to reference them publicly. */
 export interface AssetContext {
   assetsDir: string;
   assetsBaseUrl: string;
   /** Injected for tests; defaults to the global `fetch`. */
   fetchImpl?: typeof fetch;
+  /**
+   * Gate for concurrent downloads. A source shares one gate across every page
+   * it materializes, so a database of video-heavy pages doesn't open every
+   * download at once. Defaults to no gate.
+   */
+  limit?: <T>(task: () => Promise<T>) => Promise<T>;
+  /** Abort a download that hasn't completed within this many milliseconds. */
+  timeoutMs?: number;
 }
 
-/** Pick a file extension from a URL, defaulting to `.png`. */
-const extFor = (url: string): string => {
+/** The extension in a URL's path, or null when it carries none we'd trust. */
+const extFromUrl = (url: string): string | null => {
   const clean = url.split("?")[0] ?? url;
   const ext = extname(clean);
-  return SAFE_EXT.test(ext) ? ext.toLowerCase() : ".png";
+  return SAFE_EXT.test(ext) ? ext.toLowerCase() : null;
+};
+
+/** The response's media type, lowercased and stripped of parameters. */
+const mediaType = (res: Response): string =>
+  res.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? "";
+
+const exists = async (path: string): Promise<boolean> => {
+  try {
+    await access(path);
+    return true;
+  } catch {
+    return false;
+  }
+};
+
+/**
+ * The completed file for a stem whose extension came from an earlier response
+ * rather than the URL, so an extension-less asset is not fetched on every
+ * poll. Only a lone candidate counts: query-less URLs can collide, and picking
+ * between two files would be a guess.
+ */
+const completedFor = async (
+  dir: string,
+  stem: string
+): Promise<string | null> => {
+  let names: string[];
+  try {
+    names = await readdir(dir);
+  } catch {
+    return null;
+  }
+  const candidates = names.filter(
+    (name) => name.startsWith(`${stem}.`) && !name.endsWith(PART_SUFFIX)
+  );
+  return candidates.length === 1 ? (candidates[0] ?? null) : null;
 };
 
 /**
- * Download remote images referenced in a Markdown body into the asset dir and
- * rewrite their `src` to the local public path. Remote CMS URLs (notably
- * Notion's signed, expiring links) would otherwise rot a static build. Assets
- * are content-addressed by URL hash, so repeated builds are stable and deduped.
+ * Download remote media referenced in a Markdown body into the asset dir and
+ * rewrite the reference to the local public path. Markdown images and the
+ * `src` of a `<video>` tag are both covered. Remote CMS URLs (notably Notion's
+ * signed, expiring links) would otherwise rot a static build. Assets are
+ * content-addressed by URL hash, so repeated builds are stable and deduped —
+ * and a file already on disk is not fetched again, which keeps a dev poll
+ * from re-downloading every video on every tick.
  */
 export const materializeAssets = async (
   markdown: string,
   ctx: AssetContext
 ): Promise<{ markdown: string; diagnostics: Diagnostic[] }> => {
   const doFetch = ctx.fetchImpl ?? globalThis.fetch;
+  const limit = ctx.limit ?? ((task) => task());
+  const timeoutMs = ctx.timeoutMs ?? DEFAULT_TIMEOUT_MS;
   const diagnostics: Diagnostic[] = [];
 
   // Mask fenced code blocks so an image URL inside a code sample is neither
@@ -52,34 +141,77 @@ export const materializeAssets = async (
   });
 
   const urls = new Set<string>();
-  for (const match of masked.matchAll(MD_IMAGE)) {
-    const url = match.groups?.url;
-    if (url && REMOTE.test(url)) {
-      urls.add(url);
+  for (const pattern of [MD_IMAGE, HTML_VIDEO_SRC]) {
+    for (const match of masked.matchAll(pattern)) {
+      const url = match.groups?.url;
+      if (url && REMOTE.test(url)) {
+        urls.add(url);
+      }
     }
   }
 
+  /** Fetch one asset into the asset dir and return its file name. */
+  const download = async (url: string): Promise<string> => {
+    // Hash the query-less URL: CMS asset URLs are pre-signed, so the query
+    // changes on every fetch of the same file — hashing it would mint a new
+    // file each refresh and re-dirty the content digest. Two real assets
+    // sharing scheme+host+path and differing only in query are rare enough to
+    // accept colliding.
+    const stem = hashText(url.split("?")[0] ?? url);
+    const urlExt = extFromUrl(url);
+    // The name is known up front whenever the path has an extension (every
+    // Notion upload does), so a file from an earlier run is reused as is;
+    // otherwise the extension came from the last response, so look for it.
+    if (urlExt && (await exists(join(ctx.assetsDir, `${stem}${urlExt}`)))) {
+      return `${stem}${urlExt}`;
+    }
+    const completed = urlExt ? null : await completedFor(ctx.assetsDir, stem);
+    if (completed) {
+      return completed;
+    }
+    const res = await doFetch(url, { signal: AbortSignal.timeout(timeoutMs) });
+    if (!res.ok) {
+      throw new Error(`${res.status}`);
+    }
+    // Writing a watch page or an API response out as `.mp4` gives a player
+    // that can't play and a green build, so refuse what is never media.
+    const type = mediaType(res);
+    if (NON_MEDIA_TYPE.test(type)) {
+      throw new Error(`responded with ${type}, not a media file`);
+    }
+    if (!res.body) {
+      throw new Error("empty response body");
+    }
+    const file = `${stem}${urlExt ?? EXT_BY_MIME.get(type) ?? UNKNOWN_EXT}`;
+    const target = join(ctx.assetsDir, file);
+    await mkdir(ctx.assetsDir, { recursive: true });
+    // Stream the body to disk rather than buffering it: a video is hundreds of
+    // megabytes where an image was a hundred kilobytes. Write to a temporary
+    // name unique to this attempt and rename on completion, so a download that
+    // dies midway never leaves a truncated file the next run would trust as
+    // complete, and two pages fetching the same asset at once (the gate bounds
+    // concurrency, it doesn't dedupe) never write into each other's file —
+    // both publish identical bytes and the last rename wins.
+    const part = `${target}.${randomUUID()}${PART_SUFFIX}`;
+    try {
+      await writeFile(part, res.body);
+    } catch (error) {
+      await rm(part, { force: true });
+      throw error;
+    }
+    await rename(part, target);
+    return file;
+  };
+
   const rewrites = new Map<string, string>();
   await Promise.all(
     [...urls].map(async (url) => {
       try {
-        const res = await doFetch(url);
-        if (!res.ok) {
-          throw new Error(`${res.status}`);
-        }
-        const bytes = new Uint8Array(await res.arrayBuffer());
-        // Hash the query-less URL (as `extFor` does): CMS asset URLs are
-        // pre-signed, so the query changes on every fetch of the same image —
-        // hashing it would mint a new file each refresh and re-dirty the
-        // content digest. Two real assets sharing scheme+host+path and
-        // differing only in query are rare enough to accept colliding.
-        const file = `${hashText(url.split("?")[0] ?? url)}${extFor(url)}`;
-        await mkdir(ctx.assetsDir, { recursive: true });
-        await writeFile(join(ctx.assetsDir, file), bytes);
+        const file = await limit(() => download(url));
         rewrites.set(url, `${ctx.assetsBaseUrl}/${file}`);
       } catch (error) {
         // SAFETY: everything thrown in this block is an Error — the manual
-        // `!res.ok` throw above, and fetch/fs failures.
+        // throws in `download`, and fetch/fs failures.
         diagnostics.push({
           code: "BLUME_ASSET_FETCH_FAILED",
           message: `Failed to download asset ${url}: ${(error as Error).message}`,
@@ -94,6 +226,10 @@ export const materializeAssets = async (
       const local = rewrites.get(url);
       return local ? `![${alt}](${local})` : match;
     })
+    .replaceAll(HTML_VIDEO_SRC, (match, open, url, close) => {
+      const local = rewrites.get(url);
+      return local ? `${open}${local}${close}` : match;
+    })
     .replaceAll(FENCE_TOKEN, (token, index) => fences[Number(index)] ?? token);
 
   return { diagnostics, markdown: rewritten };

package/src/core/sources/notion.ts

@@ -3,6 +3,7 @@ import { setTimeout as sleep } from "node:timers/promises";
 import pLimit from "p-limit";
 import { join } from "pathe";
 
+import { parseYouTubeId } from "../../components/content/youtube.ts";
 import { BlumeError } from "../diagnostics.ts";
 import matter from "../frontmatter.ts";
 import type { Diagnostic } from "../types.ts";
@@ -189,6 +190,7 @@ const MAX_RETRIES = 4;
 const BASE_DELAY_MS = 500;
 const SECOND_MS = 1000;
 const DEFAULT_CONCURRENCY = 3;
+const ASSET_DOWNLOAD_CONCURRENCY = 4;
 
 /**
  * Retry a Notion API call on a `429 rate_limited`, honoring the `Retry-After`
@@ -250,6 +252,55 @@ const LIST_BLOCKS = new Set([
 const isListItem = (block: NotionBlock | undefined): boolean =>
   block !== undefined && LIST_BLOCKS.has(block.type);
 
+const richToPlain = (rich: NotionRichText[] = []): string =>
+  rich.map((node) => node.plain_text).join("");
+
+/**
+ * A string prop in JSX expression form. The quoted form (`title="…"`) keeps
+ * `\"` and `\n` as literal characters — MDX decodes no escapes there, so a
+ * caption holding a double quote fails to compile the whole page — where the
+ * expression form (`title={"…"}`) is a JSON string literal and decodes them.
+ */
+const jsxString = (value: string): string => `{${JSON.stringify(value)}}`;
+
+const YOUTUBE_HOST = /(?:^|\.)(?:youtube(?:-nocookie)?\.com|youtu\.be)$/u;
+
+// `parseYouTubeId` is host-agnostic on purpose (the component accepts bare
+// ids), so gate on the hostname first: `https://cdn.example.com/live/promo.mp4`
+// matches its `/live/<11 chars>` shape but is a media file, not an embed.
+const isYouTubeUrl = (url: string): boolean => {
+  const host = URL.parse(url)?.hostname ?? "";
+  return YOUTUBE_HOST.test(host) && parseYouTubeId(url) !== null;
+};
+
+/**
+ * Render a Notion `video` block. Kept out of `renderLeaf`'s switch so that
+ * function stays under the complexity limit.
+ */
+const renderVideo = (data: NotionBlockPayload): string => {
+  const url = data.external?.url ?? data.file?.url;
+  if (!url) {
+    return "";
+  }
+  const caption = richToMarkdown(data.caption);
+  // A YouTube link pasted into Notion becomes a `video` block holding an
+  // external URL. That URL is a watch page, not a media file, so it has to
+  // become the embed component — a `<video src>` pointing at it plays nothing,
+  // and `materializeAssets` would download the HTML page. The iframe's title
+  // is its accessible name, so it gets the caption's plain text; the Markdown
+  // rendering goes to the Frame's caption, as it does for an upload.
+  const title = caption ? ` title=${jsxString(richToPlain(data.caption))}` : "";
+  // Everything else (a Notion upload, or a direct link to a media file) is a
+  // real video file: `materializeAssets` rewrites the `src`, which matters most
+  // for uploads, whose Notion URLs are signed and expire.
+  const media = isYouTubeUrl(url)
+    ? `<YouTube${title} url=${JSON.stringify(url)} />`
+    : `<video controls src=${JSON.stringify(url)} />`;
+  return caption
+    ? `<Frame caption=${jsxString(caption)}>\n${media}\n</Frame>`
+    : media;
+};
+
 /** Render a leaf (non-container) block to Markdown, or null for containers. */
 const renderLeaf = (block: NotionBlock): string | null => {
   const data = payloadOf(block) ?? {};
@@ -289,6 +340,9 @@ const renderLeaf = (block: NotionBlock): string | null => {
       const url = data.external?.url ?? data.file?.url;
       return url ? `![${richToMarkdown(data.caption)}](${url})` : "";
     }
+    case "video": {
+      return renderVideo(data);
+    }
     default: {
       return null;
     }
@@ -311,6 +365,10 @@ export const notionSource = (
   // container — an unbounded burst guarantees 429s that even the retry loop
   // can't recover from, so every API call funnels through this limiter.
   const limit = pLimit(Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY));
+  // Asset downloads get their own gate: they go to Notion's S3, not its rate-
+  // limited API, and a slow video must not hold API slots. One gate per source
+  // bounds the fan-out across every page, not just within one.
+  const downloads = pLimit(ASSET_DOWNLOAD_CONCURRENCY);
   // Every Notion API call goes through the limiter, inside the retry — so a
   // call sleeping through a backoff doesn't hold a slot while it waits.
   const notionCall = <T>(call: () => Promise<T>): Promise<T> =>
@@ -380,7 +438,7 @@ export const notionSource = (
       return `<Callout>\n${body}\n</Callout>`;
     }
     if (block.type === "toggle") {
-      const title = JSON.stringify(richToMarkdown(blockField(block)));
+      const title = jsxString(richToMarkdown(blockField(block)));
       return `<Accordion>\n<AccordionItem title=${title}>\n${await children(block)}\n</AccordionItem>\n</Accordion>`;
     }
     if (block.type === "column_list") {
@@ -510,6 +568,7 @@ export const notionSource = (
       assetsBaseUrl,
       assetsDir,
       fetchImpl: options.fetchImpl,
+      limit: downloads,
     });
     const raw = matter.stringify(assets.markdown, data);
     return {

package/src/markdown/language-icon.ts

@@ -49,9 +49,10 @@ import {
   siYaml,
 } from "simple-icons";
 
-/** The slice of a `simple-icons` icon Blume reads (the SVG path data). */
+/** The slice of a `simple-icons` icon Blume reads: its slug and path data. */
 interface SimpleIcon {
   path: string;
+  slug: string;
 }
 
 /** Fence language (and common aliases) → icon. Unmapped languages get none. */
@@ -146,23 +147,6 @@ export interface LanguageIconTransformer {
   pre: (this: IconContext, node: IconPreNode) => void;
 }
 
-/** Build an inline SVG hast node from a simple-icons path. */
-const iconNode = (path: string): HastNode => ({
-  children: [
-    { children: [], properties: { d: path }, tagName: "path", type: "element" },
-  ],
-  properties: {
-    ariaHidden: "true",
-    className: ["blume-lang-icon"],
-    fill: "currentColor",
-    height: 14,
-    viewBox: "0 0 24 24",
-    width: 14,
-  },
-  tagName: "svg",
-  type: "element",
-});
-
 /** Build the transformer. Runs after Shiki's built-in `data-language` hook. */
 export const languageIconTransformer = (): LanguageIconTransformer => ({
   name: "blume:language-icon",
@@ -171,7 +155,67 @@ export const languageIconTransformer = (): LanguageIconTransformer => ({
     if (!icon) {
       return;
     }
-    node.children.unshift(iconNode(icon.path));
-    node.properties.dataIcon = "";
+    // The icon itself is CSS: the theme paints `pre[data-icon="<slug>"]::after`
+    // with the brand path as a mask (see `languageIconCss`), so a block
+    // carries a short attribute instead of ~1 kB of SVG — on a reference page
+    // with twenty TypeScript blocks, the difference is most of the page.
+    node.properties.dataIcon = icon.slug;
   },
 });
+
+/** The icon slug for a fence language, or null for an unmapped language. */
+export const languageIconSlug = (language: string): string | null =>
+  LANGUAGE_ICONS[language.toLowerCase()]?.slug ?? null;
+
+// Fence openers (```ts, ~~~tsx) and the `lang`/`language` props of code
+// components (<CodeBlock lang="ts">), which highlight through the same
+// transformer. Word characters plus the few punctuation marks languages use.
+const FENCE_LANGUAGE = /^[ \t]*(?:`{3,}|~{3,})[ \t]*(?<lang>[\w+#.-]+)/gmu;
+const PROP_LANGUAGE = /\blang(?:uage)?=["'](?<lang>[\w+#.-]+)["']/gu;
+
+/**
+ * The icon slugs a site's Markdown uses, sorted and deduped, so the theme
+ * carries a mask rule for each of them and none for the other thirty.
+ */
+export const languageIconSlugsIn = (markdown: string): string[] => {
+  const slugs = new Set<string>();
+  for (const pattern of [FENCE_LANGUAGE, PROP_LANGUAGE]) {
+    for (const match of markdown.matchAll(pattern)) {
+      const slug = languageIconSlug(match.groups?.lang ?? "");
+      if (slug) {
+        slugs.add(slug);
+      }
+    }
+  }
+  return [...slugs].toSorted();
+};
+
+const iconBySlug = (slug: string): SimpleIcon | undefined =>
+  Object.values(LANGUAGE_ICONS).find((icon) => icon.slug === slug);
+
+/** A simple-icons path as a `mask-image` data URI (24×24 viewBox). */
+const maskUri = (path: string): string =>
+  `url("data:image/svg+xml,${encodeURIComponent(`<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="${path}"/></svg>`)}")`;
+
+/**
+ * The per-language rules that paint a code block's icon: each gives the
+ * block's `::after` (positioned by the theme) the brand path as a mask over
+ * the muted foreground. Only the listed slugs get a rule, so an unmapped or
+ * unused language paints nothing rather than a blank square.
+ */
+export const languageIconCss = (slugs: string[]): string =>
+  slugs
+    .map((slug) => {
+      const icon = iconBySlug(slug);
+      if (!icon) {
+        return "";
+      }
+      const mask = maskUri(icon.path);
+      return `.prose > :where(pre[data-language][data-icon="${slug}"])::after {
+  background-color: var(--blume-muted-foreground);
+  -webkit-mask-image: ${mask};
+  mask-image: ${mask};
+}`;
+    })
+    .filter((rule) => rule !== "")
+    .join("\n");

package/src/markdown/mermaid.ts

@@ -13,6 +13,17 @@ interface CodeNode extends MdastNode {
  * rendered on the client (Mermaid needs a DOM), so the source rides on a string
  * attribute rather than as child text (which MDX would try to parse).
  */
+/**
+ * A ```mermaid (or ~~~mermaid) fence opener at the start of a line. Used to
+ * decide, at generation time, whether the site needs the Mermaid client
+ * library at all — see `featuresTemplate`.
+ */
+const MERMAID_FENCE = /^[ \t]*(?:`{3,}|~{3,})[ \t]*mermaid\b/mu;
+
+/** Whether a page's Markdown/MDX source contains a mermaid fence. */
+export const hasMermaidFence = (text: string): boolean =>
+  MERMAID_FENCE.test(text);
+
 export const mermaidPlugin = () => ({
   code(node: CodeNode, ctx: MdastVisitorContext) {
     if (node.lang !== "mermaid") {