blume 2.2.2 → 2.2.4

package/src/search/fold.ts

@@ -0,0 +1,175 @@
+/**
+ * Latin folding, shared by the search index and everything that finds a query
+ * in a page's text afterwards: the search dialog's highlights and excerpts,
+ * and the assistant's excerpts. The index matches Prüfung for `prufung`, so
+ * the rest must too, or a page found by folding shows its opening with
+ * nothing marked.
+ */
+
+/** A Latin letter and the combining marks on it, which NFD splits off. */
+const LATIN_ACCENTS = /(?<letter>\p{Script=Latin})\p{M}+/gu;
+
+/** A Latin character outside ASCII, once NFD has split off its accents. */
+const NON_ASCII_LATIN = /(?=\P{ASCII})\p{Script=Latin}/gu;
+
+/**
+ * Latin letters NFD leaves whole, spelled the way an unaccented query types
+ * them. Orama's own diacritics table folds the same letters, though its
+ * English splitter cuts words at them first.
+ */
+const LATIN_FOLDS = new Map([
+  ["ß", "ss"],
+  ["æ", "ae"],
+  ["œ", "oe"],
+  ["ø", "o"],
+  ["ł", "l"],
+  ["đ", "d"],
+  ["ð", "d"],
+  ["ħ", "h"],
+  ["ı", "i"],
+  ["þ", "th"],
+]);
+
+/**
+ * U+00AD, the invisible break hint `&shy;` writes inside a long word. It
+ * belongs to no word: split there, Prüfungs&shy;ordnung would index as two.
+ */
+const SOFT_HYPHEN = /\u00AD/gu;
+
+/** `text` without its soft hyphens. */
+export const stripSoftHyphens = (text: string): string =>
+  text.replace(SOFT_HYPHEN, "");
+
+/**
+ * Fold lowercased text's Latin letters to the plain letters an unaccented
+ * query types: accents drop (café → cafe), letters NFD leaves whole are
+ * spelled out (Łódź → lodz, ışık and IŞIK → isik, Straße → strasse), and
+ * compatibility forms unpack (the fi ligature → fi, fullwidth ＡＰＩ → api).
+ * Soft hyphens go too. Marks on any other script, and marks on no letter
+ * (the keycap in 1️⃣), stay.
+ */
+export const foldLatin = (lowered: string): string =>
+  stripSoftHyphens(lowered)
+    .normalize("NFD")
+    .replace(
+      NON_ASCII_LATIN,
+      (letter) =>
+        LATIN_FOLDS.get(letter) ?? letter.normalize("NFKD").toLowerCase()
+    )
+    .replace(LATIN_ACCENTS, "$<letter>");
+
+/**
+ * A run of Latin letters and the marks on them (lowercase İ is i plus a
+ * combining dot), inside a term of any script.
+ */
+const LATIN_RUN = /\p{Script=Latin}[\p{Script=Latin}\p{M}]*/gu;
+
+/**
+ * {@link foldLatin} for the Latin letters of a lowercased term, whatever
+ * script the rest of it is in: a segmenter keeps caféनमस्ते together as one
+ * term. Only the Latin runs fold. Marks are spelling elsewhere (Thai vowels
+ * and tones; the breve that separates Cyrillic й from и), and NFD would split
+ * a Hangul syllable into its jamo.
+ */
+export const foldLatinTerm = (term: string): string =>
+  term.replace(LATIN_RUN, (run) => foldLatin(run));
+
+/**
+ * The Unicode blocks holding Latin letters outside ASCII: Latin-1 through
+ * Latin Extended-B, the phonetic extensions and Latin Extended Additional,
+ * superscripts through number forms, Latin Extended-C, -D and -E, the
+ * ligatures, and the fullwidth forms.
+ */
+const LATIN_BLOCKS = [
+  [0x00_80, 0x02_4f],
+  [0x1d_00, 0x1e_ff],
+  [0x20_70, 0x21_8f],
+  [0x2c_60, 0x2c_7f],
+  [0xa7_20, 0xa7_ff],
+  [0xab_30, 0xab_6f],
+  [0xfb_00, 0xfb_06],
+  [0xff_21, 0xff_5a],
+] as const;
+
+const LATIN = /^\p{Script=Latin}$/u;
+const PLAIN = /^[a-z]+$/u;
+
+let spellings: Map<string, string> | undefined;
+
+/**
+ * Every Latin character outside ASCII that folds to plain letters, keyed by
+ * those letters: `a` → àáâ…, `ss` → ßẞ. Built on first use, not at import.
+ */
+const latinSpellings = (): Map<string, string> => {
+  if (spellings) {
+    return spellings;
+  }
+  spellings = new Map();
+  for (const [first, last] of LATIN_BLOCKS) {
+    for (let code = first; code <= last; code += 1) {
+      const char = String.fromCodePoint(code);
+      const plain = foldLatin(char.toLowerCase());
+      if (LATIN.test(char) && PLAIN.test(plain)) {
+        spellings.set(plain, `${spellings.get(plain) ?? ""}${char}`);
+      }
+    }
+  }
+  return spellings;
+};
+
+/** A Latin letter in a query, with the marks a decomposed query puts on it. */
+const QUERY_LETTER = /\p{Script=Latin}\p{M}*/gu;
+
+const REGEXP_SPECIAL = /[$()*+.?[\\\]^{|}]/gu;
+
+/** What may follow a letter inside a word: its accents, or a soft hyphen. */
+const LETTER_TAIL = String.raw`[\p{M}\u00AD]*`;
+
+/**
+ * The longest plain spelling a single character folds to (ⅷ → viii), so the
+ * pattern builder knows how far ahead to look.
+ */
+const longestSpelling = (folds: Map<string, string>): number =>
+  Math.max(...[...folds.keys()].map((plain) => plain.length));
+
+/**
+ * A regex source that finds `term` in unfolded text whatever accents, soft
+ * hyphens or spelled-out letters its Latin words carry there: `prufung`
+ * matches Prüfung, `strasse` matches Straße. Compile it with the `iu` flags.
+ * Characters outside Latin match as themselves.
+ */
+export const accentInsensitive = (term: string): string => {
+  const folds = latinSpellings();
+  const longest = longestSpelling(folds);
+  const letter = (plain: string): string =>
+    `[${plain}${folds.get(plain) ?? ""}]${LETTER_TAIL}`;
+  const chars = [
+    ...stripSoftHyphens(term.toLowerCase()).replace(QUERY_LETTER, (latin) =>
+      foldLatin(latin)
+    ),
+  ];
+  let source = "";
+  let index = 0;
+  while (index < chars.length) {
+    let length = Math.min(longest, chars.length - index);
+    // The longest run here that a single character spells out — `ss` for ß.
+    while (
+      length > 1 &&
+      !folds.has(chars.slice(index, index + length).join(""))
+    ) {
+      length -= 1;
+    }
+    const run = chars.slice(index, index + length);
+    const plain = run.join("");
+    if (length > 1) {
+      const letters = run.map((char) => letter(char)).join("");
+      source += `(?:${letters}|[${folds.get(plain) ?? ""}]${LETTER_TAIL})`;
+    } else if (PLAIN.test(plain)) {
+      source += letter(plain);
+    } else {
+      source += plain.replaceAll(REGEXP_SPECIAL, String.raw`\$&`);
+    }
+    index += length;
+  }
+  return source;
+};

package/src/search/orama-index.ts

@@ -1,10 +1,4 @@
-import {
-  components,
-  create,
-  insert,
-  insertMultiple,
-  search,
-} from "@orama/orama";
+import { create, insert, insertMultiple, search } from "@orama/orama";
 import type {
   AnyOrama,
   EnumArrComparisonOperator,
@@ -14,6 +8,8 @@ import type {
   Tokenizer,
 } from "@orama/orama";
 
+import { foldLatin, foldLatinTerm, stripSoftHyphens } from "./fold.ts";
+
 /**
  * The minimal document shape both the client-side search dialog and the
  * server-side MCP `search_docs` tool index. Mirrors the `blume-search.json`
@@ -82,10 +78,9 @@ const byBoostedScore = (a: RankedMatch, b: RankedMatch): number =>
   b[1] * (b[2].boost ?? 1) - a[1] * (a[2].boost ?? 1);
 
 /**
- * The script whose text Orama's default tokenizer keeps mostly intact. Its
- * delimiter class is `/[^A-Za-zàèéìòóù0-9_'-]+/`, so every character outside
- * that set counts as a separator: text in any other script collapses to zero
- * tokens and every query silently returns no hits. Unspaced scripts are the
+ * The script {@link latinTokenizer} indexes. {@link LATIN_SEPARATOR} splits on
+ * every character of any other script, so such text collapses to zero tokens
+ * and every query silently returns no hits. Unspaced scripts are the
  * best-known casualty, but the failure is not about spacing — Russian, Greek,
  * Hebrew and Hindi lose their tokens the same way Japanese does.
  */
@@ -189,22 +184,23 @@ const BIGRAM_SCRIPTS =
 const TERM =
   /[\p{L}\p{M}\p{N}]+(?:(?:['’](?=\p{L})|(?<=\p{N})[.,](?=\p{N}))[\p{L}\p{M}\p{N}]+)*/gu;
 
+const LATIN_SEPARATOR = /[^\p{Script=Latin}0-9_'-]+/u;
+
 /**
- * A term written entirely in Latin script (plus digits and the separators
- * {@link TERM} keeps within a word). Orama's default tokenizer folds the
- * accented vowels it recognizes (café → cafe), so a segmented index folds
- * Latin terms too — otherwise switching a Cyrillic- or Greek-default site to
- * the segmenting tokenizer would silently drop the unaccented-query matches
- * the default tokenizer provided. Only all-Latin terms fold: marks are
- * spelling elsewhere (Thai vowels and tones; the breve that separates
- * Cyrillic й from и), so a term carrying any other script keeps its marks.
+ * Modeled on Orama's default tokenizer, with the splitter widened from the
+ * accented vowels it keeps (`àèéìòóù`) to every Latin letter: Prüfung, når and
+ * İletişim index as whole words rather than splitting at ü, å and İ. Words
+ * fold with {@link foldLatin}, and ASCII text tokenizes exactly as in Orama's.
  */
-const LATIN_TERM = /^[\p{Script=Latin}\p{N}'’.,]+$/u;
-
-const MARKS = /\p{M}+/gu;
-
-const foldDiacritics = (term: string): string =>
-  LATIN_TERM.test(term) ? term.normalize("NFD").replace(MARKS, "") : term;
+const latinTokenizer = (): Tokenizer => ({
+  language: "english",
+  normalizationCache: new Map(),
+  tokenize: (raw: string): string[] => [
+    ...new Set(
+      foldLatin(raw.toLowerCase()).split(LATIN_SEPARATOR).filter(Boolean)
+    ),
+  ],
+});
 
 /**
  * Emit every overlapping 2-character window of `run`, or the lone character.
@@ -249,10 +245,11 @@ const hasSegmenter = (
  * pipelines) indexes the same terms a composed query produces, and Latin
  * terms ("GDPR", English pages on a mixed-locale site) still match
  * case-insensitively, with their diacritics folded by
- * {@link foldDiacritics}. Returns `undefined` for scripts the default
+ * {@link foldLatinTerm}. Soft hyphens go first, so a word they break still
+ * segments whole. Returns `undefined` for scripts the default
  * tokenizer already serves ({@link resolveLocale} decides, so `sr-Latn` keeps
  * the default while `az-Cyrl` is segmented), and on runtimes without
- * `Intl.Segmenter`, where the caller falls back to Orama's default.
+ * `Intl.Segmenter`, where the caller falls back to {@link latinTokenizer}.
  *
  * On a {@link BIGRAM_INDEX_SCRIPTS} index, runs of adjacent
  * {@link BIGRAM_SCRIPTS} segments are joined and re-cut into character
@@ -292,10 +289,10 @@ const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
           return;
         }
         flush();
-        tokens.add(foldDiacritics(term));
+        tokens.add(foldLatinTerm(term));
       };
       for (const segment of segmenter.segment(
-        raw.normalize("NFC").toLowerCase()
+        stripSoftHyphens(raw).normalize("NFC").toLowerCase()
       )) {
         if (!segment.isWordLike) {
           flush();
@@ -329,7 +326,7 @@ const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
  */
 const translationTokenizers = new WeakMap<AnyOrama, Map<string, Tokenizer>>();
 
-/** A segmenting tokenizer for each document locale Orama's can't serve. */
+/** A segmenting tokenizer for each document locale in a non-Latin script. */
 const localeTokenizers = (documents: OramaDoc[]): Map<string, Tokenizer> => {
   const tokenizers = new Map<string, Tokenizer>();
   for (const code of new Set(documents.map((doc) => doc.locale ?? ""))) {
@@ -345,25 +342,18 @@ const localeTokenizers = (documents: OramaDoc[]): Map<string, Tokenizer> => {
  * One tokenizer that hands each text to its locale's: Orama passes the
  * language given to `insert` and `search` through to `tokenize`, so a page
  * inserted under `ja` and a query searched under `ja` both reach the Japanese
- * tokenizer, and everything else reaches Orama's own, exactly as on an index
- * with no translations.
+ * tokenizer, and everything else reaches {@link latinTokenizer}, exactly as on
+ * an index with no translations.
  */
 const dispatchingTokenizer = (
   tokenizers: Map<string, Tokenizer>
 ): Tokenizer => {
-  const standard = components.tokenizer.createTokenizer();
+  const standard = latinTokenizer();
   return {
     language: standard.language,
     normalizationCache: standard.normalizationCache,
-    tokenize: (raw, language, prop, withCache) =>
-      // Orama's tokenizer rejects a language other than its own, so the
-      // locale that picked it is not passed on.
-      (tokenizers.get(language ?? "") ?? standard).tokenize(
-        raw,
-        undefined,
-        prop,
-        withCache
-      ),
+    tokenize: (raw, language) =>
+      (tokenizers.get(language ?? "") ?? standard).tokenize(raw),
   };
 };
 
@@ -372,10 +362,10 @@ const dispatchingTokenizer = (
  * Orama client loader (browser), the MCP server, and assistant grounding (Node),
  * so ranking is identical wherever docs are queried. `locale` — the site's
  * `i18n.defaultLocale` — swaps in a word-segmenting tokenizer for every
- * non-Latin script, all of which Orama's default tokenizer reduces to zero
+ * non-Latin script, all of which {@link latinTokenizer} reduces to zero
  * tokens. A non-Latin default's tokenizer serves every page: Latin words
  * survive segmentation intact, so English pages on a segmented index stay
- * searchable. A Latin default keeps Orama's tokenizer for its Latin-script
+ * searchable. A Latin default keeps {@link latinTokenizer} for its Latin-script
  * pages, and each page in a non-Latin locale (the Japanese and Hindi
  * translations of an English site) is indexed with its own locale's tokenizer.
  */
@@ -400,9 +390,10 @@ export const buildOramaIndex = async (
     }
     translationTokenizers.set(db, translations);
   } else {
-    db = tokenizer
-      ? create({ components: { tokenizer }, schema: SCHEMA })
-      : create({ schema: SCHEMA });
+    db = create({
+      components: { tokenizer: tokenizer ?? latinTokenizer() },
+      schema: SCHEMA,
+    });
     await insertMultiple(db, rows);
   }
   if (documents.some((doc) => (doc.boost ?? 1) !== 1)) {
@@ -492,8 +483,8 @@ const boostedScore = (hit: Hit): number =>
  * On an index whose translations have tokenizers of their own, a query scoped
  * to a locale is tokenized with that locale's tokenizer. An unscoped query
  * searches each locale's pages with its own tokenizer, and the Latin-script
- * pages with Orama's, then merges the matches by score: every search runs over
- * the one index, so their scores share its term statistics.
+ * pages with {@link latinTokenizer}, then merges the matches by score: every
+ * search runs over the one index, so their scores share its term statistics.
  */
 export const queryOramaIndex = async (
   db: AnyOrama,

package/src/seo/jsonld.ts

@@ -64,8 +64,36 @@ export interface StructuredDataIdentity {
   software?: SoftwareIdentity;
 }
 
+/**
+ * `seo.jsonLd`: entities the docs share with a site they live inside
+ * (`example.com/docs`), referenced by `@id` instead of defined again.
+ */
+export interface JsonLdEntities {
+  /** Credited on pages without their own `authors`. */
+  author?: { "@id": string };
+  /** Cited as every page's publisher in place of the Organization's id. */
+  publisher?: { "@id": string };
+  /** The Organization's `@id`, defined here or elsewhere. */
+  organizationId?: string;
+  /** Emit the WebSite node. Defaults to true unless `websiteId` is set. */
+  website?: boolean;
+  /** The WebSite's `@id`, which every page names as `isPartOf`. */
+  websiteId?: string;
+}
+
+/** A front matter `authors` entry: a name, or a name with a profile URL. */
+export type AuthorInput = string | { name: string; url?: string };
+
 /** Inputs for a page's JSON-LD, all known at render time in RootLayout. */
 export interface StructuredDataInput {
+  jsonLd?: JsonLdEntities;
+  /**
+   * The page's `og:image`, emitted as the page node's `image`. Absolute when
+   * `siteUrl` is set, otherwise root-relative like every other URL here.
+   */
+  image?: string | null;
+  /** The page's own byline, which wins over `jsonLd.author`. */
+  authors?: AuthorInput | AuthorInput[];
   siteName: string;
   /** Absolute site origin, or null when `deployment.site` is unset. */
   siteUrl: string | null;
@@ -162,15 +190,17 @@ const organizationNode = (
   context: {
     absolutize: (path: string) => string;
     id: string;
-    rootUrl: string;
-    siteName: string;
+    /** The site's name and root, or null when the id names an outside entity. */
+    defaults: { name: string; url: string } | null;
   }
 ): JsonLdNode => {
   const node: JsonLdNode = {
     "@id": context.id,
     "@type": "Organization",
-    name: organization.name ?? context.siteName,
-    url: organization.url ?? context.rootUrl,
+    ...definedStrings({
+      name: organization.name ?? context.defaults?.name,
+      url: organization.url ?? context.defaults?.url,
+    }),
   };
   if (organization.logo) {
     node.logo = context.absolutize(organization.logo);
@@ -261,6 +291,19 @@ const pageSchemaType = (input: StructuredDataInput, home: boolean): string => {
   return isArticleType(pageType) ? ARTICLE_TYPES[pageType] : "TechArticle";
 };
 
+/** An `authors` entry is a bare name or a name with a profile URL. */
+const isBareName = (author: AuthorInput): author is string =>
+  typeof author === "string";
+
+/** `Person` nodes for a page's front matter `authors`. */
+const authorNodes = (authors: AuthorInput | AuthorInput[]): JsonLdNode[] =>
+  [authors].flat().map((author): JsonLdNode => ({
+    "@type": "Person",
+    ...(isBareName(author)
+      ? { name: author }
+      : definedStrings({ name: author.name, url: author.url })),
+  }));
+
 /**
  * The page's own node: a homepage is a `WebPage` (it isn't an article, but it
  * still needs a machine-readable date, or search engines take whatever other
@@ -298,11 +341,25 @@ const pageNode = (
   if (modified) {
     node.dateModified = modified;
   }
-  if (context.base) {
-    node.isPartOf = { "@id": `${context.base}#website` };
+  // A shared id is already absolute, so it holds without a site URL too.
+  const websiteId =
+    input.jsonLd?.websiteId ??
+    (context.base ? `${context.base}#website` : null);
+  if (websiteId) {
+    node.isPartOf = { "@id": websiteId };
   }
-  if (context.organizationId) {
-    node.publisher = { "@id": context.organizationId };
+  const publisher = input.jsonLd?.publisher?.["@id"] ?? context.organizationId;
+  if (publisher) {
+    node.publisher = { "@id": publisher };
+  }
+  const authors = input.authors ? authorNodes(input.authors) : [];
+  if (authors.length > 0) {
+    node.author = authors;
+  } else if (input.jsonLd?.author) {
+    node.author = input.jsonLd.author;
+  }
+  if (input.image) {
+    node.image = input.image;
   }
   return node;
 };
@@ -335,6 +392,32 @@ const breadcrumbNode = (
   };
 };
 
+/**
+ * The site's `WebSite` node, which needs the site URL. A shared `websiteId`
+ * names one the parent site defines, so the docs leave it out unless
+ * `jsonLd.website` asks for it — their own name and root would contradict it.
+ */
+const addWebsiteNode = (
+  graph: JsonLdNode[],
+  input: StructuredDataInput,
+  base: string | null,
+  rootUrl: string,
+  organizationId: string | null
+): void => {
+  if (base && (input.jsonLd?.website ?? !input.jsonLd?.websiteId)) {
+    const website: JsonLdNode = {
+      "@id": input.jsonLd?.websiteId ?? `${base}#website`,
+      "@type": "WebSite",
+      name: input.siteName,
+      url: rootUrl,
+    };
+    if (organizationId) {
+      website.publisher = { "@id": organizationId };
+    }
+    graph.push(website);
+  }
+};
+
 /**
  * Build a schema.org JSON-LD `@graph` for a page: site identity (the WebSite,
  * plus the configured Organization everywhere and the SoftwareApplication on
@@ -355,20 +438,11 @@ export const buildStructuredData = (input: StructuredDataInput): JsonLdNode => {
   // the same rule as the WebSite node they attach to.
   const organization = base ? input.identity?.organization : undefined;
   const software = base ? input.identity?.software : undefined;
-  const organizationId = organization ? `${base}#organization` : null;
+  const organizationId =
+    input.jsonLd?.organizationId ??
+    (organization ? `${base}#organization` : null);
 
-  if (base) {
-    const website: JsonLdNode = {
-      "@id": `${base}#website`,
-      "@type": "WebSite",
-      name: input.siteName,
-      url: rootUrl,
-    };
-    if (organizationId) {
-      website.publisher = { "@id": organizationId };
-    }
-    graph.push(website);
-  }
+  addWebsiteNode(graph, input, base, rootUrl, organizationId);
   if (organization && organizationId) {
     graph.push(
       organizationNode(organization, {
@@ -378,9 +452,12 @@ export const buildStructuredData = (input: StructuredDataInput): JsonLdNode => {
           path.startsWith("/")
             ? absolute(base, withBasePath(deployBase, path))
             : path,
+        // A shared id names an organization defined elsewhere, which the
+        // docs' title and root would contradict; only configured fields go in.
+        defaults: input.jsonLd?.organizationId
+          ? null
+          : { name: input.siteName, url: rootUrl },
         id: organizationId,
-        rootUrl,
-        siteName: input.siteName,
       })
     );
   }

package/src/theme/chrome-icons.ts

@@ -10,7 +10,8 @@
  * time via `resolveIcon` and inline as zero-JS SVG.
  *
  * Values are Lucide inner-SVG markup; the client `svg()` helpers wrap them in an
- * `<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" …>`.
+ * `<svg viewBox="0 0 24 24" fill="none" stroke="currentColor" …>`. Lucide's
+ * license is in THIRD-PARTY-NOTICES.md.
  */
 /** Inner-SVG markup keyed by the icon name a client script requests. */
 interface ChromeIconSet {

package/src/theme/entry.ts

@@ -9,7 +9,10 @@ import { CODE_PADDING_BLOCK_REM } from "./code-block-padding.ts";
  */
 const CODE_SCROLL_INSET_REM = 0.375;
 
-/** Lucide's arrow-up-right (the featured-link icon) as a `mask` data URI. */
+/**
+ * Lucide's arrow-up-right (the featured-link icon) as a `mask` data URI.
+ * Lucide's license is in THIRD-PARTY-NOTICES.md.
+ */
 const EXTERNAL_LINK_MASK = `url("data:image/svg+xml,${encodeURIComponent('<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="#000" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"><path d="M7 7h10v10M7 17L17 7"/></svg>')}")`;
 
 interface TailwindEntryOptions {