blume 1.4.2 → 1.4.3

package/src/openapi/scalar.ts

@@ -4,6 +4,7 @@ import { isAbsolute, join } from "pathe";
 
 import { scalarReferenceTemplate } from "../astro/templates.ts";
 import type { ResolvedConfig } from "../core/schema.ts";
+import { trimChar } from "../core/trim.ts";
 import { resolveAccent, resolveRadius } from "../theme/palette.ts";
 import { resolveReferences } from "./references.ts";
 import type { ReferenceSource } from "./references.ts";
@@ -24,11 +25,10 @@ export interface ReferenceFile {
 }
 
 const URL_SPEC = /^https?:\/\//u;
-const ROUTE_EDGES = /^\/+|\/+$/gu;
 
 /** The `src/pages`-relative file path for a reference route. */
 const referencePagePath = (route: string): string => {
-  const segments = route.replace(ROUTE_EDGES, "");
+  const segments = trimChar(route, "/");
   return `${segments === "" ? "index" : segments}.astro`;
 };
 

package/src/openapi/source.ts

@@ -1,5 +1,6 @@
 import { withBasePath } from "../core/base-path.ts";
 import matter from "../core/frontmatter.ts";
+import type { FolderMeta } from "../core/schema.ts";
 import { hashText } from "../core/sources/cache.ts";
 import type {
   ContentSource,
@@ -75,10 +76,33 @@ const specEntries = (
   return entries;
 };
 
+/**
+ * Label each tag's sidebar group with the spec's own tag name. The group label
+ * is otherwise re-humanized from the tag's route slug (split on hyphens,
+ * title-cased), which mangles authored casing and symbols — `OAuth2` →
+ * "Oauth2", `Größe` → "Größe" only by luck of the slug. Keys are the tag
+ * directories under the reference route, the same group paths `meta.ts` files
+ * use, so user-authored meta still overrides these.
+ */
+const tagFolderMeta = (
+  spec: ApiSpecData,
+  tags: { slug: string; name: string }[]
+): Record<string, FolderMeta> => {
+  const base = routeToRef(spec.route);
+  return Object.fromEntries(
+    tags.map((tag) => [
+      base ? `${base}/${tag.slug}` : tag.slug,
+      { title: tag.name },
+    ])
+  );
+};
+
 interface LoadedSpec {
   slug: string;
   spec: ApiSpecData;
   entries: SourceEntry[];
+  /** Sidebar-group labels for the spec's tag directories. */
+  folderMeta: Record<string, FolderMeta>;
   /** Non-fatal notes from the load (e.g. an offline cache fallback). */
   diagnostics: Diagnostic[];
 }
@@ -157,6 +181,7 @@ export const openApiSource = (
             : []),
         ],
         entries: specEntries(spec, operations, reference),
+        folderMeta: tagFolderMeta(spec, tags),
         slug: reference.slug,
         spec,
       };
@@ -192,6 +217,7 @@ export const openApiSource = (
       }))
     );
     const data: OpenApiData = {};
+    const folderMeta: Record<string, FolderMeta> = {};
     for (const result of results) {
       if ("severity" in result) {
         diagnostics.push(result);
@@ -199,10 +225,11 @@ export const openApiSource = (
       }
       data[result.slug] = result.spec;
       entries.push(...result.entries);
+      Object.assign(folderMeta, result.folderMeta);
       diagnostics.push(...result.diagnostics);
     }
     parsed = data;
-    return { diagnostics, entries };
+    return { diagnostics, entries, folderMeta };
   };
 
   return {

package/src/search/documents.ts

@@ -1,3 +1,8 @@
+import type { Nodes } from "mdast";
+import { fromMarkdown } from "mdast-util-from-markdown";
+import { gfmFromMarkdown } from "mdast-util-gfm";
+import { gfm } from "micromark-extension-gfm";
+
 import { applyAudienceVisibility } from "../ai/visibility.ts";
 import type { VisibilityAudience } from "../ai/visibility.ts";
 import matter from "../core/frontmatter.ts";
@@ -47,47 +52,86 @@ export interface SearchRecord {
   tag?: string;
 }
 
-const CODE_FENCE = /```[\s\S]*?```/gu;
-const INLINE_CODE = /`(?<code>[^`]+)`/gu;
 // Tag-shaped only: a name (or closing slash/fragment) right after `<`, and no
-// newline inside. A bare `<` in prose ("costs < 5 credits") must not swallow
-// everything up to some later `>` — potentially whole paragraphs.
+// newline inside. Applied *within* html/JSX nodes so their inner prose is
+// kept; the surrounding Markdown is walked as a tree, so a bare `<` in prose
+// ("costs < 5 credits") is ordinary text and never at risk.
 const HTML_OR_JSX = /<\/?[a-zA-Z][^\n<>]*>|<\/?>/gu;
-const IMAGE = /!\[[^\]]*\]\([^)]*\)/gu;
-const LINK = /\[(?<text>[^\]]*)\]\([^)]*\)/gu;
-const HEADING_MARK = /^#{1,6}\s+/gmu;
-const MARKDOWN_PUNCT = /[*_~>]+/gu;
 const WHITESPACE = /\s+/gu;
 
-/** Reduce Markdown/MDX to plain, searchable text. */
-const toPlainText = (markdown: string): string => {
-  const withoutBlocks = markdown
-    .replaceAll(CODE_FENCE, " ")
-    .replaceAll(IMAGE, " ")
-    .replaceAll(LINK, "$<text>");
+// Parents whose children are inline: no separator is inserted after them, or
+// `re*ally*` would index as `re ally`. Every other parent is block-shaped and
+// ends with a space so adjacent paragraphs/headings/cells don't fuse.
+const INLINE_PARENTS = new Set([
+  "delete",
+  "emphasis",
+  "footnoteReference",
+  "link",
+  "linkReference",
+  "strong",
+]);
 
-  // Strip HTML/JSX from the prose, but keep the contents of inline code — an
-  // angle-bracket span like `<T>` inside `Array<T>` is a type parameter, not a
-  // tag, and stripping it would drop those tokens from the search index. Split
-  // on inline-code spans and only run the HTML strip on the text between them.
-  const pieces: string[] = [];
-  let cursor = 0;
-  for (const match of withoutBlocks.matchAll(INLINE_CODE)) {
-    const start = match.index ?? 0;
-    pieces.push(
-      withoutBlocks.slice(cursor, start).replaceAll(HTML_OR_JSX, " "),
-      match.groups?.code ?? ""
-    );
-    cursor = start + match[0].length;
+/** Fold one mdast node into the plain-text accumulator. */
+const collectText = (node: Nodes, out: string[]): void => {
+  switch (node.type) {
+    // Fenced code is excluded from the plain index (ranking noise) — the
+    // "markdown" extraction keeps it for Ask AI grounding — and image alt
+    // text was never indexed.
+    case "code":
+    case "image":
+    case "imageReference": {
+      return;
+    }
+    // Inline code is kept verbatim — `Array<T>` is a type parameter, not a
+    // tag, and its tokens must stay searchable.
+    case "inlineCode": {
+      out.push(node.value);
+      return;
+    }
+    // A raw-HTML/JSX run. CommonMark parses a block-level `<Callout>` with no
+    // blank lines as ONE html node holding all its inner prose, so the node
+    // can't just be dropped — strip the tag-shaped runs and keep the text.
+    case "html": {
+      out.push(node.value.replaceAll(HTML_OR_JSX, " "));
+      return;
+    }
+    case "break": {
+      out.push(" ");
+      return;
+    }
+    default: {
+      break;
+    }
+  }
+  if ("value" in node) {
+    out.push(node.value);
+    return;
   }
-  pieces.push(withoutBlocks.slice(cursor).replaceAll(HTML_OR_JSX, " "));
+  if ("children" in node) {
+    for (const child of node.children) {
+      collectText(child, out);
+    }
+    if (!INLINE_PARENTS.has(node.type)) {
+      out.push(" ");
+    }
+  }
+};
 
-  return pieces
-    .join("")
-    .replaceAll(HEADING_MARK, "")
-    .replaceAll(MARKDOWN_PUNCT, " ")
-    .replaceAll(WHITESPACE, " ")
-    .trim();
+/**
+ * Reduce Markdown/MDX to plain, searchable text: parse (GFM included) and walk
+ * the tree instead of regex-stripping the source, so reference-style links,
+ * autolinks, setext headings, tables, and literal `*`/`~`/`>` in prose all
+ * reduce correctly. This feeds the client index *and* every hosted-provider
+ * record, so anything lost here is a permanent search-quality loss.
+ */
+const toPlainText = (markdown: string): string => {
+  const tree = fromMarkdown(markdown, {
+    extensions: [gfm()],
+    mdastExtensions: [gfmFromMarkdown()],
+  });
+  const out: string[] = [];
+  collectText(tree, out);
+  return out.join("").replaceAll(WHITESPACE, " ").trim();
 };
 
 interface Crumbs {

package/src/search/orama-index.ts

@@ -79,6 +79,23 @@ const BIGRAM_LANGUAGES = new Set(["ja", "zh"]);
 const BIGRAM_SCRIPTS =
   /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}々〆〇ーﾞﾟ]+$/u;
 
+/**
+ * One index term inside a word-like segment. Being word-like does not make a
+ * segment all letters: UAX #29 keeps connector punctuation, format characters
+ * and mid-number punctuation *within* a word, so `Intl.Segmenter` reports
+ * スネーク_ケース and robots.txt as one segment each. A term is a run of
+ * letters, combining marks and digits. Marks are spelling, not punctuation —
+ * Thai writes vowels and tones as combining marks, so dropping them leaves
+ * consonant skeletons that collapse distinct words (เสื้อ, shirt, and เสือ,
+ * tiger, differ by one mark). Two separators stay where the surrounding text
+ * makes them part of the word: an apostrophe followed by a letter (don't),
+ * and a decimal point or thousands separator flanked by digits (1.0.3,
+ * 1,000) — split, either would leave one-letter and one-digit fragments that
+ * co-occur on unrelated pages.
+ */
+const TERM =
+  /[\p{L}\p{M}\p{N}]+(?:(?:['’](?=\p{L})|(?<=\p{N})[.,](?=\p{N}))[\p{L}\p{M}\p{N}]+)*/gu;
+
 /**
  * Emit every overlapping 2-character window of `run`, or the lone character.
  * Windows are cut by code point: an ideograph outside the basic plane is a
@@ -109,8 +126,10 @@ const addBigrams = (run: string, tokens: Set<string>): void => {
 /**
  * A word-segmenting tokenizer for languages the default splitter can't handle,
  * built on `Intl.Segmenter` (the same engine `@orama/tokenizers` wraps).
- * Input is lowercased before segmenting — unlike the upstream tokenizers —
- * so Latin terms ("GDPR", English pages on a mixed-locale site) still match
+ * Input is NFC-normalized and lowercased before segmenting — unlike the
+ * upstream tokenizers — so decomposed text (macOS filenames, some CMS
+ * pipelines) indexes the same terms a composed query produces, and Latin
+ * terms ("GDPR", English pages on a mixed-locale site) still match
  * case-insensitively. Returns `undefined` for languages the default tokenizer
  * already serves, and on runtimes without `Intl.Segmenter`, where the caller
  * falls back to Orama's default.
@@ -118,9 +137,9 @@ const addBigrams = (run: string, tokens: Set<string>): void => {
  * On a {@link BIGRAM_LANGUAGES} index, runs of adjacent
  * {@link BIGRAM_SCRIPTS} segments are joined and re-cut into character
  * bigrams; everything else (Latin, digits, and every segment on a Korean or
- * Thai index) is emitted as the segmenter produced it. Punctuation and spaces
- * are not word-like, so they end a run — 「クーリング・オフ」 bigrams either
- * side of the interpunct rather than across it.
+ * Thai index) is emitted one {@link TERM} at a time. Separators end a run
+ * either way — whether they stand between segments, as 「クーリング・オフ」
+ * does, or inside one.
  */
 const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
   const language = locale?.toLowerCase().split(/[-_]/u)[0] ?? "";
@@ -146,17 +165,35 @@ const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
           run = "";
         }
       };
-      for (const segment of segmenter.segment(raw.toLowerCase())) {
+      const take = (term: string): void => {
+        if (bigram && BIGRAM_SCRIPTS.test(term)) {
+          run += term;
+          return;
+        }
+        flush();
+        tokens.add(term);
+      };
+      for (const segment of segmenter.segment(
+        raw.normalize("NFC").toLowerCase()
+      )) {
         if (!segment.isWordLike) {
           flush();
           continue;
         }
-        if (bigram && BIGRAM_SCRIPTS.test(segment.segment)) {
-          run += segment.segment;
-          continue;
+        let end = 0;
+        for (const match of segment.segment.matchAll(TERM)) {
+          // A gap means punctuation stood there, which ends the run as surely
+          // as a non-word-like segment would: スネーク_ケース pairs either side
+          // of the connector, never across it.
+          if (match.index > end) {
+            flush();
+          }
+          take(match[0]);
+          end = match.index + match[0].length;
+        }
+        if (end < segment.segment.length) {
+          flush();
         }
-        flush();
-        tokens.add(segment.segment);
       }
       flush();
       return [...tokens];
@@ -171,7 +208,9 @@ const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
  * `i18n.defaultLocale` — swaps in a word-segmenting tokenizer for languages
  * written without spaces (Japanese, Chinese, Korean, Thai); the tokenizer
  * belongs to the database, so on a mixed-locale site it applies to every
- * document, which is safe because Latin words survive segmentation intact.
+ * document, which is safe because words in other scripts — Latin, and
+ * mark-bearing scripts like Hebrew or Devanagari — survive segmentation
+ * intact.
  */
 export const buildOramaIndex = async (
   documents: OramaDoc[],

package/src/theme/palette.ts

@@ -2,8 +2,12 @@ import type { ResolvedConfig } from "../core/schema.ts";
 
 const FALLBACK_ACCENT = "oklch(0.62 0.16 250)";
 
-/** Named accent presets mapped to OKLCH values. */
-const ACCENTS: Record<string, string> = {
+/**
+ * Named accent presets mapped to OKLCH values. The single source of truth for
+ * preset colors: the theme CSS and the OG card (og/card.ts) both resolve from
+ * this table, so a site and its social cards can't disagree about "blue".
+ */
+export const ACCENTS: Record<string, string> = {
   blue: FALLBACK_ACCENT,
   green: "oklch(0.6 0.16 150)",
   orange: "oklch(0.68 0.17 50)",

package/src/translate/ledger.ts

@@ -36,8 +36,10 @@ export const emptyLedger = (): TranslationLedger => ({
 
 /**
  * Hash the raw source text (frontmatter included), so any edit invalidates
- * every locale's stamp. sha256-16 like the audit snapshot's content hash —
- * never `hashText` (djb2), which is an ephemeral-cache-only hash.
+ * every locale's stamp. sha256-16 like the audit snapshot's content hash.
+ * Kept as its own function rather than sharing core's `hashText`: ledger
+ * stamps persist in user repos, so this hash must never change shape when an
+ * ephemeral cache hash does.
  */
 export const hashSource = (text: string): string =>
   createHash("sha256").update(text).digest("hex").slice(0, 16);

package/src/translate/report.ts

@@ -3,6 +3,7 @@ import type { ColorFunction } from "consola/utils";
 
 import { AGENTS } from "../audit/agent.ts";
 import type { AgentKind } from "../audit/agent.ts";
+import { duration, money, seconds } from "../cli/report-format.ts";
 import { countBySeverity } from "../core/diagnostics.ts";
 import type { Diagnostic } from "../core/types.ts";
 import type {
@@ -59,20 +60,6 @@ export const SPINNER_INTERVAL_MS = 80;
 /** The clear-to-start-of-line prefix every TTY rewrite uses. */
 const REWRITE = `\r${ESC}[K`;
 
-const seconds = (ms: number): string => `${(ms / 1000).toFixed(1)}s`;
-
-const money = (cost: number | undefined): string =>
-  cost === undefined ? "" : `$${cost.toFixed(2)}`;
-
-const duration = (ms: number): string => {
-  if (ms < 60_000) {
-    return seconds(ms);
-  }
-  const minutes = Math.floor(ms / 60_000);
-  const rest = Math.round((ms % 60_000) / 1000);
-  return `${minutes}m ${rest}s`;
-};
-
 /** `docs/guides/install.mdx → fr`, or the batched meta call's label. */
 export const itemLabel = (item: WorkItem): string =>
   item.kind === "page"

package/src/translate/run.ts

@@ -2,9 +2,11 @@ import { existsSync } from "node:fs";
 import { mkdtemp, readFile } from "node:fs/promises";
 import { tmpdir } from "node:os";
 
+import pLimit from "p-limit";
+import pMap from "p-map";
 import { join } from "pathe";
 
-import { AGENTS, WINDOWS_COMMAND_NOT_FOUND } from "../audit/agent.ts";
+import { AGENTS } from "../audit/agent.ts";
 import type { AgentKind } from "../audit/agent.ts";
 import { writeTextAtomic } from "../core/fs-atomic.ts";
 import type { BlumeProject } from "../core/project-graph.ts";
@@ -89,9 +91,9 @@ interface RunContext {
 const metaDirKey = (dir: string): string => (dir === "" ? "." : dir);
 
 /**
- * One headless agent call. A Windows shell launch reports a missing executable
- * through exit code 9009 instead of a spawn error, so that is normalized to
- * the ENOENT rejection the command layer already turns into an install hint.
+ * One headless agent call. A missing executable rejects with ENOENT on every
+ * platform (the runner spawns without a shell), which the command layer turns
+ * into an install hint.
  */
 const invokeAgent = async (
   context: RunContext,
@@ -104,13 +106,6 @@ const invokeAgent = async (
     translateAgentArgs(context.kind, messagePath),
     { cwd: context.dir, prompt, timeoutMs: context.timeoutMs }
   );
-  if (!result.timedOut && result.code === WINDOWS_COMMAND_NOT_FOUND) {
-    const missing = new Error(
-      `${context.bin} was not found on PATH`
-    ) as NodeJS.ErrnoException;
-    missing.code = "ENOENT";
-    throw missing;
-  }
   return await readAgentOutput(context.kind, result, messagePath);
 };
 
@@ -277,42 +272,31 @@ export const runTranslate = async (
   };
 
   const { items } = options.workList;
-  const results: TranslateItemResult[] = Array.from({ length: items.length });
   const concurrency = Math.max(
     1,
     Math.min(options.concurrency ?? 1, items.length || 1)
   );
 
-  // The persist chain: whichever lane finishes next appends its flush after
-  // the previous one, so two lanes never write the ledger file concurrently.
-  let persisting: Promise<unknown> = Promise.resolve();
-  const persist = (): Promise<unknown> => {
-    // The chain is the mutex: appending with .then() serializes flushes.
-    // oxlint-disable-next-line promise/prefer-await-to-then
-    persisting = persisting.then(() => options.persistLedger?.());
-    return persisting;
-  };
+  // The persist mutex: ledger flushes from concurrent lanes are serialized so
+  // two lanes never write the ledger file at the same time.
+  const persistLimit = pLimit(1);
+  const persist = (): Promise<unknown> =>
+    persistLimit(() => options.persistLedger?.());
 
-  let nextIndex = 0;
-  const worker = async (): Promise<void> => {
-    while (nextIndex < items.length) {
-      const index = nextIndex;
-      nextIndex += 1;
-      const item = items[index] as WorkItem;
+  const results = await pMap(
+    items,
+    async (item: WorkItem, index): Promise<TranslateItemResult> => {
       options.onProgress?.({
         index,
         item,
         kind: "item-start",
         total: items.length,
       });
-      // oxlint-disable-next-line no-await-in-loop -- each worker is a serial lane
       const result = await (item.kind === "page"
         ? runPageItem(item, index, context, options.ledger)
         : runMetaItem(item, index, context, options.ledger));
-      results[index] = result;
-      // Flush this item's stamps before claiming the next one, so a kill
-      // loses at most the in-flight items.
-      // oxlint-disable-next-line no-await-in-loop
+      // Flush this item's stamps before the slot frees for the next item, so
+      // a kill loses at most the in-flight items.
       await persist();
       options.onProgress?.({
         index,
@@ -320,9 +304,10 @@ export const runTranslate = async (
         result,
         total: items.length,
       });
-    }
-  };
-  await Promise.all(Array.from({ length: concurrency }, () => worker()));
+      return result;
+    },
+    { concurrency }
+  );
 
   const diagnostics: Diagnostic[] = [];
   for (const result of results) {

package/src/cli/coalesce.ts

@@ -1,43 +0,0 @@
-/**
- * Wrap an async task so it never runs concurrently with itself. Triggering the
- * returned function while a run is in flight coalesces into a single trailing
- * run after the current one settles.
- *
- * Dev regeneration (`scanProject` + `generateRuntime`) is expensive on a large
- * project — a full content re-scan that allocates big strings. A plain debounce
- * still lets a fast burst of watch events (or, before it was fixed, a `.blume/`
- * watch storm) start a new scan before the previous finished, piling up
- * overlapping scans until the heap is exhausted (observed as an OOM after
- * minutes of looping). Single-flighting bounds it to one scan at a time while
- * still guaranteeing a final run reflects the latest change.
- *
- * The task must not reject: a rejection would surface as an unhandled promise
- * rejection, so callers handle their own errors and always resolve.
- */
-export const coalescedRunner = (task: () => Promise<void>): (() => void) => {
-  let inFlight: Promise<void> | null = null;
-  let pending = false;
-
-  // Drain any run requested during the current run, then release the lock. The
-  // `inFlight` promise is assigned synchronously by the caller below, so a
-  // re-entrant trigger sees the lock immediately and only sets `pending`.
-  const cycle = async (): Promise<void> => {
-    try {
-      do {
-        pending = false;
-        // oxlint-disable-next-line no-await-in-loop -- serialized by design
-        await task();
-      } while (pending);
-    } finally {
-      inFlight = null;
-    }
-  };
-
-  return () => {
-    if (inFlight) {
-      pending = true;
-      return;
-    }
-    inFlight = cycle();
-  };
-};