blume 1.4.2 → 1.5.0

package/src/core/sources/github-releases.ts

@@ -2,8 +2,10 @@ import { fromMarkdown } from "mdast-util-from-markdown";
 import { gfmFromMarkdown } from "mdast-util-gfm";
 import { toString as mdastToString } from "mdast-util-to-string";
 import { gfm } from "micromark-extension-gfm";
+import stringWidth from "string-width";
 
 import matter from "../frontmatter.ts";
+import { columnsPrefix } from "../text-width.ts";
 import {
   hashText,
   loadWithCache,
@@ -61,9 +63,10 @@ const LEADING_V = /^v/iu;
 const NON_SLUG = /[^a-z0-9]+/gu;
 const EDGE_DASHES = /^-+|-+$/gu;
 
-// `blume audit` grades meta descriptions against the 110–160 character search
-// snippet range (audit/types.ts thresholds), so the derived summary aims for
-// the longest word-boundary cut under the cap.
+// `blume audit` grades meta descriptions against the 110–160 display-column
+// search snippet range (audit/types.ts thresholds), so the derived summary
+// budgets in the same columns and aims for the longest word-boundary cut
+// under the cap.
 const DESCRIPTION_MAX = 160;
 const DESCRIPTION_MIN = 110;
 
@@ -103,15 +106,17 @@ const releaseDescription = (body: string): string | undefined => {
   if (!text) {
     return undefined;
   }
-  if (text.length <= DESCRIPTION_MAX) {
+  if (stringWidth(text) <= DESCRIPTION_MAX) {
     return text;
   }
   // Cut before the cap at a word boundary (kept only when it doesn't drop the
   // summary under the minimum), shed any dangling punctuation, and mark the cut.
-  const slice = text.slice(0, DESCRIPTION_MAX - 1);
+  const slice = columnsPrefix(text, DESCRIPTION_MAX - 1);
   const boundary = slice.lastIndexOf(" ");
   const head = (
-    boundary >= DESCRIPTION_MIN ? slice.slice(0, boundary) : slice
+    boundary !== -1 && stringWidth(slice.slice(0, boundary)) >= DESCRIPTION_MIN
+      ? slice.slice(0, boundary)
+      : slice
   ).replace(TRAILING_FRAGMENT, "");
   return `${head}…`;
 };
@@ -130,6 +135,19 @@ const githubHeaders = (): Headers => {
   return headers;
 };
 
+/**
+ * The changelog frontmatter one release lowers to. `title`/`type` are always
+ * assigned (optional only so assignment order can keep the emitted YAML key
+ * order — and so each entry's content hash — stable).
+ */
+interface ChangelogFrontmatter {
+  changelog: { category: string; version: string };
+  date: string;
+  seo?: { description: string };
+  title?: string;
+  type?: string;
+}
+
 /**
  * Lower one release to a staged Markdown entry: the notes become the body,
  * `type: changelog` frontmatter (title/date/version/category) drives the
@@ -146,19 +164,25 @@ const releaseToEntry = (release: GithubRelease): SourceEntry => {
   // description (instead of the site-wide fallback) without also rendering the
   // visible lede paragraph a top-level `description` would add.
   const description = releaseDescription(body);
-  const data = {
+  // Assignment order matters: js-yaml serializes keys in insertion order, so
+  // `seo` lands between `date` and `title` exactly as it always has.
+  const data: ChangelogFrontmatter = {
     changelog: { category, version },
     date,
-    ...(description ? { seo: { description } } : {}),
-    title,
-    type: "changelog",
   };
+  if (description) {
+    data.seo = { description };
+  }
+  data.title = title;
+  data.type = "changelog";
   const raw = matter.stringify(`${body}\n`, data);
   const fallbackRef = `release-${release.id}`;
   const ref = `${slugifyTag(release.tag_name) || fallbackRef}.md`;
   return {
     body: { format: "md", text: body },
-    data,
+    // Spread: `SourceEntry.data` is an open dictionary, which the interface
+    // (no index signature) only satisfies as a fresh object literal.
+    data: { ...data },
     editUrl: release.html_url,
     hash: hashText(raw),
     lastModified: date,
@@ -194,6 +218,8 @@ export const githubReleasesSource = (
     if (!res.ok) {
       throw new Error(`${url} -> ${res.status}`);
     }
+    // SAFETY: GitHub's releases endpoint returns a JSON array of release
+    // objects; `GithubRelease` models only the fields the adapter reads.
     return (await res.json()) as GithubRelease[];
   };
 
@@ -233,6 +259,8 @@ export const githubReleasesSource = (
       // repo), degrade to an empty changelog with a warning rather than failing
       // the whole build.
       snapshot = new Map();
+      // SAFETY: everything thrown on this path is an Error — fetch rejects
+      // with a TypeError, fetchPage and loadWithCache throw Error instances.
       return {
         diagnostics: [
           {

package/src/core/sources/mdx-remote.ts

@@ -82,6 +82,8 @@ const enumerateGithub = async (
   if (!res.ok) {
     throw new Error(`${treeUrl} -> ${res.status}`);
   }
+  // SAFETY: GitHub's git/trees endpoint returns this envelope; a missing or
+  // differently-typed field falls through the `?? []` and blob filters below.
   const body = (await res.json()) as {
     tree?: GithubTreeEntry[];
     truncated?: boolean;
@@ -198,6 +200,8 @@ export const mdxRemoteSource = (
             } catch (error) {
               skipped.push({
                 code: "BLUME_SOURCE_FETCH_FAILED",
+                // SAFETY: fetch and decode failures throw Error instances;
+                // only the message is read for the skip diagnostic.
                 message: `Source "${options.name}" skipped "${ref.ref}" (${(error as Error).message}); the rest were imported.`,
                 severity: "warning",
               });

package/src/core/sources/normalize.ts

@@ -9,6 +9,7 @@ import { localePlacement, localizeRoute } from "../i18n.ts";
 import { pageMetaSchema } from "../schema.ts";
 import type { FrontmatterExtend, PageMeta } from "../schema.ts";
 import type { Diagnostic, Heading, PageLink, PageRecord } from "../types.ts";
+import { detectVersionRef, versionizeRoute } from "../versions.ts";
 import type { NormalizeContext, SourceEntry } from "./types.ts";
 
 const NUMERIC_PREFIX = /^\d+[-_.]/u;
@@ -28,12 +29,19 @@ const groupLabel = (segment: string): string | null =>
  * anchor ids are *not* slugged here — they use a `github-slugger` in
  * {@link extractHeadings}, matching the renderer (see `markdown/heading-anchors`)
  * so `blume validate` checks anchors against the exact rendered heading ids.
+ *
+ * The keep-class is Unicode letters/marks/numbers, not `\w`: ASCII slugs are
+ * unchanged, but a CJK/Cyrillic/accented slug keeps its characters instead of
+ * collapsing to `""` (which forced Sanity/Notion routes onto their opaque
+ * document-id fallbacks) or dropping accents (`café` → `caf`). NFC first so a
+ * macOS-NFD `é` (e + combining mark) slugs identically to the composed form.
  */
 export const slugify = (text: string): string =>
   text
+    .normalize("NFC")
     .toLowerCase()
     .trim()
-    .replaceAll(/[^\w\s-]/gu, "")
+    .replaceAll(/[^\p{L}\p{M}\p{N}\s_-]/gu, "")
     .replaceAll(/[\s_]+/gu, "-")
     .replaceAll(/-+/gu, "-")
     .replaceAll(/^-|-$/gu, "");
@@ -55,6 +63,18 @@ const titleCase = (value: string): string =>
     .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
     .join(" ");
 
+/**
+ * Strip characters that cannot survive the route → URL → output-file round
+ * trip. A `:` ahead of the first `/` makes `new URL()` read the segment as a
+ * scheme (`Guide: Architecture.md` → `guide:`), which crashes Astro's
+ * prerender write with "The URL must be of scheme file"; control characters
+ * (an embedded newline in a filename) are silently dropped by the URL parser,
+ * desyncing the route from its output path. Both are legal in macOS/Linux
+ * filenames, so they are removed here rather than rejected.
+ */
+const sanitizeSegment = (segment: string): string =>
+  segment.replaceAll(/[:\p{Cc}]/gu, "");
+
 /** Fold one raw path part into the accumulating route segments/groups. */
 const addRouteSegment = (
   part: string,
@@ -75,13 +95,23 @@ const addRouteSegment = (
   if (clean === "index") {
     return;
   }
-  segments.push(clean);
+  const safe = sanitizeSegment(clean);
+  // A part that was nothing but stripped characters cannot name a segment.
+  if (safe === "") {
+    return;
+  }
+  segments.push(safe);
 };
 
+/** URL + nav metadata mapped from one content-root-relative path. */
+interface MappedRoute {
+  segments: string[];
+  groups: string[];
+  route: string;
+}
+
 /** Convert a content-root-relative path into URL + nav metadata. */
-const mapRoute = (
-  relativePath: string
-): { segments: string[]; groups: string[]; route: string } => {
+const mapRoute = (relativePath: string): MappedRoute => {
   const withoutExt = relativePath.slice(
     0,
     relativePath.length - extname(relativePath).length
@@ -114,10 +144,10 @@ export type FenceState = "```" | "~~~" | null;
  */
 export const nextFenceState = (line: string, fence: FenceState): FenceState => {
   const trimmed = line.trimStart();
-  const delimiter = trimmed.match(CODE_FENCE)?.groups?.delimiter as
-    | Exclude<FenceState, null>
-    | undefined;
-  if (delimiter === undefined) {
+  const delimiter = trimmed.match(CODE_FENCE)?.groups?.delimiter;
+  // The `delimiter` group matches exactly ``` or ~~~; comparing against both
+  // narrows it without a cast.
+  if (delimiter !== "```" && delimiter !== "~~~") {
     return fence;
   }
   if (fence === null) {
@@ -495,6 +525,10 @@ const deriveTitle = (
 const trimSlashes = (value: string): string =>
   value.replaceAll(/^\/+|\/+$/gu, "");
 
+/** Whether a raw frontmatter value is a string (e.g. the `type` override). */
+const isStringValue = (value: SourceEntry["data"][string]): value is string =>
+  typeof value === "string";
+
 const withPrefix = (prefix: string | undefined, path: string): string => {
   const clean = prefix ? trimSlashes(prefix) : "";
   return clean ? `${clean}/${path}` : path;
@@ -506,13 +540,27 @@ interface CustomKeyIssue {
   path: (string | number)[];
 }
 
+/** The validated custom keys (if any survived) plus every failure found. */
+interface CustomKeyValidation {
+  custom?: PageRecord["custom"];
+  issues: CustomKeyIssue[];
+}
+
+/** Whether a Standard Schema path segment is the wrapped `{ key }` form. */
+const isKeyCarrier = (
+  segment: PropertyKey | { readonly key: PropertyKey }
+): segment is { readonly key: PropertyKey } =>
+  typeof segment === "object" && segment !== null;
+
+const isSymbolKey = (key: PropertyKey): key is symbol =>
+  typeof key === "symbol";
+
 /** Lower a Standard Schema path segment (`key` or `{ key }`) for joining. */
 const segmentKey = (
   segment: PropertyKey | { readonly key: PropertyKey }
 ): string | number => {
-  const key =
-    typeof segment === "object" && segment !== null ? segment.key : segment;
-  return typeof key === "symbol" ? String(key) : key;
+  const key = isKeyCarrier(segment) ? segment.key : segment;
+  return isSymbolKey(key) ? String(key) : key;
 };
 
 /**
@@ -525,10 +573,10 @@ const segmentKey = (
  * synchronous, and frontmatter validation has no business awaiting I/O.
  */
 const validateCustomKeys = (
-  data: Record<string, unknown>,
+  data: SourceEntry["data"],
   extend: FrontmatterExtend
-): { custom?: Record<string, unknown>; issues: CustomKeyIssue[] } => {
-  const custom: Record<string, unknown> = {};
+): CustomKeyValidation => {
+  const custom: NonNullable<PageRecord["custom"]> = {};
   const issues: CustomKeyIssue[] = [];
   for (const [key, schema] of Object.entries(extend)) {
     const outcome = schema["~standard"].validate(data[key]);
@@ -573,13 +621,14 @@ const parseEntryMeta = (
   entry: SourceEntry,
   ctx: NormalizeContext
 ):
-  | { meta: PageMeta; custom?: Record<string, unknown>; diagnostics?: never }
+  | { meta: PageMeta; custom?: PageRecord["custom"]; diagnostics?: never }
   | { meta?: never; diagnostics: Diagnostic[] } => {
   // Resolved the same way `contentType` is after parsing (`meta.type` falling
   // back to `defaultType`); a non-string `type` fails the strict parse below,
   // so which per-type map was merged for that entry never matters.
-  const entryType =
-    typeof entry.data.type === "string" ? entry.data.type : ctx.defaultType;
+  const entryType = isStringValue(entry.data.type)
+    ? entry.data.type
+    : ctx.defaultType;
   const typeExtend = ctx.typeFrontmatter?.[entryType];
   // Config validation rejects a key declared both site-wide and per-type, so
   // this merge never has to pick a winner.
@@ -625,6 +674,12 @@ const parseEntryMeta = (
   };
 };
 
+/** The per-locale page records and diagnostics from one source entry. */
+export interface NormalizedEntry {
+  pages: PageRecord[];
+  diagnostics: Diagnostic[];
+}
+
 /**
  * Normalize one source entry into per-locale `PageRecord`s. This is the single
  * funnel every adapter's entries pass through, so route mapping, heading/link
@@ -633,7 +688,7 @@ const parseEntryMeta = (
 export const normalizeEntry = (
   entry: SourceEntry,
   ctx: NormalizeContext
-): { pages: PageRecord[]; diagnostics: Diagnostic[] } => {
+): NormalizedEntry => {
   const { format } = entry.body;
   const ext = format === "mdx" ? ".mdx" : ".md";
 
@@ -654,14 +709,22 @@ export const normalizeEntry = (
     meta.seo.noindex = true;
   }
 
+  // The version is detected first: a snapshot directory is outermost on disk
+  // (`v1.0/fr/page.mdx`), so the locale parser and route mapping must see a
+  // version-stripped ref. The current version is `""` and lives at the root.
+  const { versions } = ctx;
+  const { version, rest: versionlessRef } = versions
+    ? detectVersionRef(entry.ref, versions)
+    : { rest: entry.ref, version: "" };
+
   // Locale and the locale-stripped nav path come from the entry's ref (a leading
   // dir, or a filename suffix under the `dot` parser), not the slug — the slug is
   // the logical, locale-agnostic path within a locale. A shared `$` file maps to
   // every locale. Remote/CMS sources without i18n placement map to one locale.
   const { i18n } = ctx;
   const { navPath: rawNavPath, locales } = i18n
-    ? localePlacement(entry.ref, ext, i18n)
-    : { locales: [""], navPath: entry.ref };
+    ? localePlacement(versionlessRef, ext, i18n)
+    : { locales: [""], navPath: versionlessRef };
 
   const navPath = withPrefix(ctx.source.prefix, rawNavPath);
   // Frontmatter `slug` wins, then the adapter-supplied `entry.slug` (the typed
@@ -675,7 +738,13 @@ export const normalizeEntry = (
     slug ? `${slug}${ext}` : rawNavPath
   );
 
-  const { segments, groups, route: logicalRoute } = mapRoute(routeInput);
+  // The version prefixes the mapped route *after* `mapRoute` runs: the mapped
+  // route is the version-agnostic key, the config id is prepended verbatim
+  // (never numeric-prefix-stripped), a frontmatter `slug` gets versionized so
+  // snapshots can't collide with the live page, and `translationKey` becomes
+  // version-specific for free.
+  const { segments, groups, route: versionKey } = mapRoute(routeInput);
+  const logicalRoute = versionizeRoute(versionKey, version);
   const headings = extractHeadings(entry.body.text);
   const { staged } = ctx.source;
 
@@ -705,6 +774,8 @@ export const normalizeEntry = (
     sourcePath: entry.sourcePath,
     title: deriveTitle(meta, headings, navPath),
     translationKey: logicalRoute,
+    version,
+    versionKey,
   } satisfies Omit<PageRecord, "locale" | "route">;
 
   // One record per locale this entry maps to (one normally; every locale for a

package/src/core/sources/notion.ts

@@ -1,5 +1,6 @@
 import { setTimeout as sleep } from "node:timers/promises";
 
+import pLimit from "p-limit";
 import { join } from "pathe";
 
 import { BlumeError } from "../diagnostics.ts";
@@ -46,11 +47,21 @@ interface NotionPage {
   last_edited_time?: string;
 }
 
+/** The per-type payload a block carries under the key matching its `type`. */
+interface NotionBlockPayload {
+  caption?: NotionRichText[];
+  checked?: boolean;
+  external?: { url: string };
+  file?: { url: string };
+  language?: string;
+  rich_text?: NotionRichText[];
+}
+
 interface NotionBlock {
   id: string;
   type: string;
   has_children?: boolean;
-  [key: string]: unknown;
+  [key: string]: NotionBlockPayload | boolean | string | undefined;
 }
 
 interface NotionList<T> {
@@ -96,6 +107,12 @@ export interface NotionSourceOptions {
   prefix?: string;
   /** The Notion database id. */
   database: string;
+  /**
+   * Maximum concurrent Notion API requests. Notion allows an average of 3
+   * requests per second per integration, so a large database must pace its
+   * block-tree fan-out or every request 429s. Default 3.
+   */
+  concurrency?: number;
   /** Integration token; defaults to `NOTION_TOKEN`. */
   token?: string;
   properties?: NotionPropertyMap;
@@ -113,6 +130,17 @@ export interface NotionSourceOptions {
   fetchImpl?: typeof fetch;
 }
 
+/** The frontmatter Blume derives from a Notion page's properties. */
+interface NotionFrontmatter {
+  description?: string;
+  draft?: boolean;
+  sidebar?: { order: number };
+  title?: string;
+  // Frontmatter stays an open bag downstream (`SourceEntry.data`), so admit
+  // the value shapes this source writes under any future key.
+  [key: string]: boolean | string | { order: number } | undefined;
+}
+
 const richToMarkdown = (rich: NotionRichText[] = []): string =>
   rich
     .map((node) => {
@@ -133,20 +161,32 @@ const richToMarkdown = (rich: NotionRichText[] = []): string =>
     })
     .join("");
 
+const isBlockPayload = (
+  value: NotionBlockPayload | boolean | string | undefined
+): value is NotionBlockPayload => typeof value === "object";
+
+/** The payload object stored under a block's own `type` key, if present. */
+const payloadOf = (block: NotionBlock): NotionBlockPayload | undefined => {
+  const value = block[block.type];
+  return isBlockPayload(value) ? value : undefined;
+};
+
 const blockField = (block: NotionBlock): NotionRichText[] =>
-  ((block[block.type] as { rich_text?: NotionRichText[] })?.rich_text ??
-    []) as NotionRichText[];
+  payloadOf(block)?.rich_text ?? [];
 
 const RATE_LIMITED = 429;
 const MAX_RETRIES = 4;
 const BASE_DELAY_MS = 500;
 const SECOND_MS = 1000;
+const DEFAULT_CONCURRENCY = 3;
 
 /**
  * Retry a Notion API call on a `429 rate_limited`, honoring the `Retry-After`
  * header and otherwise backing off exponentially. A large workspace fans out
  * many concurrent block-children requests, so without this a single 429 would
- * reject the batch and abort the whole import.
+ * reject the batch and abort the whole import. The exponential wait is
+ * jittered so calls rate-limited together don't retry in lockstep and trip
+ * the limit again as a herd.
  */
 const withNotionRetry = async <T>(
   call: () => Promise<T>,
@@ -155,15 +195,23 @@ const withNotionRetry = async <T>(
   try {
     return await call();
   } catch (error) {
+    // SAFETY: Notion SDK failures are APIResponseError-shaped, carrying the
+    // failed request's HTTP status; anything else reads `undefined` and is
+    // rethrown below.
     const { status } = error as { status?: number };
     if (status !== RATE_LIMITED || attempt === MAX_RETRIES) {
       throw error;
     }
+    // SAFETY: same APIResponseError shape — `headers` maps lower-cased HTTP
+    // header names to their values; a missing header yields `NaN` and falls
+    // back to exponential backoff.
     const retryAfter = Number(
       (error as { headers?: Record<string, string> }).headers?.["retry-after"]
     );
     const wait =
-      retryAfter > 0 ? retryAfter * SECOND_MS : BASE_DELAY_MS * 2 ** attempt;
+      retryAfter > 0
+        ? retryAfter * SECOND_MS
+        : BASE_DELAY_MS * 2 ** attempt * (1 + Math.random());
     await sleep(wait);
     return withNotionRetry(call, attempt + 1);
   }
@@ -194,7 +242,7 @@ const isListItem = (block: NotionBlock | undefined): boolean =>
 
 /** Render a leaf (non-container) block to Markdown, or null for containers. */
 const renderLeaf = (block: NotionBlock): string | null => {
-  const data = (block[block.type] ?? {}) as Record<string, unknown>;
+  const data = payloadOf(block) ?? {};
   const text = richToMarkdown(blockField(block));
   switch (block.type) {
     case "paragraph": {
@@ -225,16 +273,11 @@ const renderLeaf = (block: NotionBlock): string | null => {
       return "---";
     }
     case "code": {
-      return `\`\`\`${(data.language as string) ?? ""}\n${text}\n\`\`\``;
+      return `\`\`\`${data.language ?? ""}\n${text}\n\`\`\``;
     }
     case "image": {
-      const media = data as {
-        external?: { url: string };
-        file?: { url: string };
-        caption?: NotionRichText[];
-      };
-      const url = media.external?.url ?? media.file?.url;
-      return url ? `![${richToMarkdown(media.caption)}](${url})` : "";
+      const url = data.external?.url ?? data.file?.url;
+      return url ? `![${richToMarkdown(data.caption)}](${url})` : "";
     }
     default: {
       return null;
@@ -252,6 +295,16 @@ export const notionSource = (
   ctx?: SourceContext
 ): ContentSource => {
   const props = options.properties ?? {};
+  // A FIFO semaphore: at most N calls run at once, the rest queue. Notion's
+  // rate limit is per-integration (an average of 3 req/s), and a large
+  // database fans out one block-children request per page plus one per nested
+  // container — an unbounded burst guarantees 429s that even the retry loop
+  // can't recover from, so every API call funnels through this limiter.
+  const limit = pLimit(Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY));
+  // Every Notion API call goes through the limiter, inside the retry — so a
+  // call sleeping through a backoff doesn't hold a slot while it waits.
+  const notionCall = <T>(call: () => Promise<T>): Promise<T> =>
+    withNotionRetry(() => limit(call));
   const cache = snapshotCache(
     ctx?.cacheDir ?? join(".blume", "cache", options.name)
   );
@@ -266,6 +319,9 @@ export const notionSource = (
     }
     let Client: new (config: { auth?: string }) => NotionClientLike;
     try {
+      // SAFETY: `@notionhq/client` exports a `Client` class constructable with
+      // an `auth` token whose instances cover the NotionClientLike slice; the
+      // local type keeps the SDK mockable without importing its types.
       ({ Client } = (await import("@notionhq/client")) as {
         Client: new (config: { auth?: string }) => NotionClientLike;
       });
@@ -284,7 +340,7 @@ export const notionSource = (
     blockId: string
   ): Promise<NotionBlock[]> =>
     collectAll((cursor) =>
-      withNotionRetry(() =>
+      notionCall(() =>
         client.blocks.children.list({ block_id: blockId, start_cursor: cursor })
       )
     );
@@ -399,13 +455,11 @@ export const notionSource = (
 
   const orderOf = (page: NotionPage): number | undefined => {
     const order = page.properties[props.order ?? "Order"]?.number;
-    return typeof order === "number" ? order : undefined;
+    return order === null ? undefined : order;
   };
 
-  const frontmatter = (
-    page: NotionPage
-  ): { data: Record<string, unknown>; slug: string } => {
-    const data: Record<string, unknown> = {};
+  const frontmatter = (page: NotionPage) => {
+    const data: NotionFrontmatter = {};
     const title = richToMarkdown(titleProperty(page)?.title);
     if (title) {
       data.title = title;
@@ -458,12 +512,12 @@ export const notionSource = (
   };
 
   // Hoisted out of `load` so the retry closure doesn't nest past the linter's
-  // 4-level limit (source factory → queryDatabase → withNotionRetry callback).
+  // 4-level limit (source factory → queryDatabase → notionCall callback).
   const queryDatabase = (
     client: NotionClientLike,
     cursor?: string
   ): Promise<NotionList<NotionPage>> =>
-    withNotionRetry(() =>
+    notionCall(() =>
       client.databases.query({
         database_id: options.database,
         start_cursor: cursor,