blume 1.5.0 → 1.5.2

package/src/search/orama-index.ts

@@ -56,19 +56,59 @@ const toFacetTerms = (facets: Record<string, string>): string[] =>
 const BOOST = { description: 2, title: 3 };
 
 /**
- * Scripts written without spaces between words. Orama's default tokenizer
- * splits on a Latin-centric delimiter class, so text in these languages
- * collapses to zero tokens and every query silently returns no hits. Keyed by
- * the primary language subtag of `i18n.defaultLocale`.
+ * The script whose text Orama's default tokenizer keeps mostly intact. Its
+ * delimiter class is `/[^A-Za-zàèéìòóù0-9_'-]+/`, so every character outside
+ * that set counts as a separator: text in any other script collapses to zero
+ * tokens and every query silently returns no hits. Unspaced scripts are the
+ * best-known casualty, but the failure is not about spacing — Russian, Greek,
+ * Hebrew and Hindi lose their tokens the same way Japanese does.
  */
-const SEGMENTED_LANGUAGES = new Set(["ja", "ko", "th", "zh"]);
+const LATIN_SCRIPT = "Latn";
 
 /**
- * Languages indexed as character bigrams rather than whole segments, and
- * queried to match accordingly. Japanese and Chinese write compounds in
- * {@link BIGRAM_SCRIPTS} without delimiters; Korean separates words with
- * spaces and Thai has no comparable bigram convention, so both keep the plain
- * segmented tokens even where a page mixes in Han or kana.
+ * Parse a locale tag, tolerating the legacy forms that reach here from
+ * hand-written config: underscores (`ru_RU`), POSIX suffixes (`ja_JP.UTF-8`,
+ * `zh_TW@Big5`), and extlang tags ICU rejects (`zh-cmn-Hans`). `Intl.Locale`
+ * throws on all of these, so a failed parse retries with the primary language
+ * subtag alone — enough to resolve the script, which is all that gates the
+ * tokenizer. Returns `undefined` when even that subtag is unparseable.
+ */
+const parseLocale = (tag: string): Intl.Locale | undefined => {
+  const hyphenated = tag.replaceAll("_", "-");
+  try {
+    return new Intl.Locale(hyphenated);
+  } catch {
+    // Retry below with the primary subtag.
+  }
+  const [primary = ""] = hyphenated.split("-");
+  try {
+    return new Intl.Locale(primary);
+  } catch {
+    return undefined;
+  }
+};
+
+/**
+ * The maximized locale — script and language filled in from CLDR likely-subtags
+ * and alias data, so `ru` resolves to `Cyrl`, `cmn` to the canonical `zh`, and
+ * an explicit script subtag is honored: `sr-Latn` reports `Latn` while
+ * `az-Cyrl` reports `Cyrl`. `undefined` for tags {@link parseLocale} cannot
+ * salvage; a well-formed tag ICU knows nothing about keeps an `undefined`
+ * script instead.
+ */
+const resolveLocale = (tag: string): Intl.Locale | undefined =>
+  parseLocale(tag)?.maximize();
+
+/**
+ * Indexes whose language maximizes to one of these scripts are built as
+ * character bigrams rather than whole segments, and queried to match
+ * accordingly. Japanese and Chinese write compounds in {@link BIGRAM_SCRIPTS}
+ * without delimiters, and the writing system — not the language subtag — is
+ * what carries that convention: `yue` (Cantonese) maximizes to `Hant` and
+ * needs bigrams the same way `zh` does. Korean maximizes to `Kore` and Thai
+ * to `Thai`; Korean separates words with spaces and Thai has no comparable
+ * bigram convention, so both keep the plain segmented tokens even where a
+ * page mixes in Han or kana.
  *
  * Dictionary segmentation alone drops the adjacency that makes a compound term
  * distinctive: 資金決済法 becomes 資金 / 決済 / 法, and because Orama scores a
@@ -77,7 +117,19 @@ const SEGMENTED_LANGUAGES = new Set(["ja", "ko", "th", "zh"]);
  * terms, and dropping the whole-segment tokens keeps a fragment as common as
  * 法 from matching on its own.
  */
-const BIGRAM_LANGUAGES = new Set(["ja", "zh"]);
+const BIGRAM_INDEX_SCRIPTS = new Set(["Hans", "Hant", "Jpan"]);
+
+/**
+ * Whether the index keyed to this tokenizer language is built from bigrams.
+ * {@link segmentingTokenizer} stores the canonical maximized language, so the
+ * builder and the strict pass in {@link queryOramaIndex} share this predicate
+ * — an index is never built from bigrams that the query side then matches
+ * loosely, or vice versa.
+ */
+const isBigramLanguage = (language: string): boolean => {
+  const script = resolveLocale(language)?.script;
+  return script !== undefined && BIGRAM_INDEX_SCRIPTS.has(script);
+};
 
 /**
  * Segments written entirely in these scripts are the ones re-cut into bigrams,
@@ -107,6 +159,23 @@ const BIGRAM_SCRIPTS =
 const TERM =
   /[\p{L}\p{M}\p{N}]+(?:(?:['’](?=\p{L})|(?<=\p{N})[.,](?=\p{N}))[\p{L}\p{M}\p{N}]+)*/gu;
 
+/**
+ * A term written entirely in Latin script (plus digits and the separators
+ * {@link TERM} keeps within a word). Orama's default tokenizer folds the
+ * accented vowels it recognizes (café → cafe), so a segmented index folds
+ * Latin terms too — otherwise switching a Cyrillic- or Greek-default site to
+ * the segmenting tokenizer would silently drop the unaccented-query matches
+ * the default tokenizer provided. Only all-Latin terms fold: marks are
+ * spelling elsewhere (Thai vowels and tones; the breve that separates
+ * Cyrillic й from и), so a term carrying any other script keeps its marks.
+ */
+const LATIN_TERM = /^[\p{Script=Latin}\p{N}'’.,]+$/u;
+
+const MARKS = /\p{M}+/gu;
+
+const foldDiacritics = (term: string): string =>
+  LATIN_TERM.test(term) ? term.normalize("NFD").replace(MARKS, "") : term;
+
 /**
  * Emit every overlapping 2-character window of `run`, or the lone character.
  * Windows are cut by code point: an ideograph outside the basic plane is a
@@ -134,6 +203,14 @@ const addBigrams = (run: string, tokens: Set<string>): void => {
   }
 };
 
+/**
+ * `Intl.Segmenter` is missing on some runtimes even though the lib type
+ * declares it, so the constructor's presence is probed before use.
+ */
+const hasSegmenter = (
+  segmenter: typeof Intl.Segmenter | undefined
+): segmenter is typeof Intl.Segmenter => typeof segmenter === "function";
+
 /**
  * A word-segmenting tokenizer for languages the default splitter can't handle,
  * built on `Intl.Segmenter` (the same engine `@orama/tokenizers` wraps).
@@ -141,37 +218,32 @@ const addBigrams = (run: string, tokens: Set<string>): void => {
  * upstream tokenizers — so decomposed text (macOS filenames, some CMS
  * pipelines) indexes the same terms a composed query produces, and Latin
  * terms ("GDPR", English pages on a mixed-locale site) still match
- * case-insensitively. Returns `undefined` for languages the default tokenizer
- * already serves, and on runtimes without `Intl.Segmenter`, where the caller
- * falls back to Orama's default.
+ * case-insensitively, with their diacritics folded by
+ * {@link foldDiacritics}. Returns `undefined` for scripts the default
+ * tokenizer already serves ({@link resolveLocale} decides, so `sr-Latn` keeps
+ * the default while `az-Cyrl` is segmented), and on runtimes without
+ * `Intl.Segmenter`, where the caller falls back to Orama's default.
  *
- * On a {@link BIGRAM_LANGUAGES} index, runs of adjacent
+ * On a {@link BIGRAM_INDEX_SCRIPTS} index, runs of adjacent
  * {@link BIGRAM_SCRIPTS} segments are joined and re-cut into character
  * bigrams; everything else (Latin, digits, and every segment on a Korean or
  * Thai index) is emitted one {@link TERM} at a time. Separators end a run
  * either way — whether they stand between segments, as 「クーリング・オフ」
  * does, or inside one.
  */
-/**
- * `Intl.Segmenter` is missing on some runtimes even though the lib type
- * declares it, so the constructor's presence is probed before use.
- */
-const hasSegmenter = (
-  segmenter: typeof Intl.Segmenter | undefined
-): segmenter is typeof Intl.Segmenter => typeof segmenter === "function";
-
 const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
-  const language = locale?.toLowerCase().split(/[-_]/u)[0] ?? "";
-  if (!SEGMENTED_LANGUAGES.has(language)) {
+  const resolved = resolveLocale(locale ?? "");
+  if (!resolved?.script || resolved.script === LATIN_SCRIPT) {
     return;
   }
   if (!hasSegmenter(Intl.Segmenter)) {
     return;
   }
+  // The canonical maximized language (`cmn` → `zh`), so the bigram predicate
+  // here and the strict query pass read the same name.
+  const { language } = resolved;
   const segmenter = new Intl.Segmenter(language, { granularity: "word" });
-  // Keyed off the same set the strict query pass reads, so an index is never
-  // built from bigrams that the query side then matches loosely.
-  const bigram = BIGRAM_LANGUAGES.has(language);
+  const bigram = isBigramLanguage(language);
   return {
     language,
     normalizationCache: new Map(),
@@ -190,7 +262,7 @@ const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
           return;
         }
         flush();
-        tokens.add(term);
+        tokens.add(foldDiacritics(term));
       };
       for (const segment of segmenter.segment(
         raw.normalize("NFC").toLowerCase()
@@ -224,12 +296,13 @@ const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
  * Build an in-memory Orama full-text index from search documents. Shared by the
  * Orama client loader (browser), the MCP server, and Ask AI grounding (Node),
  * so ranking is identical wherever docs are queried. `locale` — the site's
- * `i18n.defaultLocale` — swaps in a word-segmenting tokenizer for languages
- * written without spaces (Japanese, Chinese, Korean, Thai); the tokenizer
- * belongs to the database, so on a mixed-locale site it applies to every
- * document, which is safe because words in other scripts — Latin, and
- * mark-bearing scripts like Hebrew or Devanagari — survive segmentation
- * intact.
+ * `i18n.defaultLocale` — swaps in a word-segmenting tokenizer for every
+ * non-Latin script, all of which Orama's default tokenizer reduces to zero
+ * tokens; the tokenizer belongs to the database, so on a mixed-locale site it
+ * applies to every document. That is safe in one direction only: Latin words
+ * survive segmentation intact, so English pages on a segmented index stay
+ * searchable, but non-Latin translations on a Latin-default index still
+ * collapse to zero tokens.
  */
 export const buildOramaIndex = async (
   documents: OramaDoc[],
@@ -322,7 +395,7 @@ export const queryOramaIndex = async (
   };
   const params =
     Object.keys(where).length > 0 ? { ...unfiltered, where } : unfiltered;
-  const bigrammed = BIGRAM_LANGUAGES.has(db.tokenizer?.language ?? "");
+  const bigrammed = isBigramLanguage(db.tokenizer?.language ?? "");
   // The result-document generic is OramaDoc because `buildOramaIndex` is the
   // only writer to this database and inserts OramaDoc records (plus the
   // derived `facetTerms`).

package/src/theme/entry.ts

@@ -179,7 +179,11 @@ ${THEME_MAPPING}
     scroll-padding-top: 4.5rem;
     text-rendering: optimizeLegibility;
   }
-  /* Headings use the display font (defaults to the body font when unset). */
+  /* Headings use the display font (defaults to the body font when unset).
+     The tightened tracking is part of the theme, not the font: display-tuned
+     families bake it into their metrics, but a text family promoted to
+     headings (including the Inter default) reads loose without it. -0.05em
+     was matched visually against Inter Tight, the previous display default. */
   h1,
   h2,
   h3,
@@ -187,6 +191,7 @@ ${THEME_MAPPING}
   h5,
   h6 {
     font-family: var(--font-display);
+    letter-spacing: -0.05em;
   }
   :focus-visible {
     outline: 2px solid var(--blume-accent);
@@ -207,6 +212,13 @@ ${THEME_MAPPING}
   }
 }
 
+/* Same-origin navigations ride Astro's client router (the layouts render
+   ClientRouter), which owns the page-transition styling — including its own
+   prefers-reduced-motion guard — so the theme declares nothing about it. The
+   cross-document view-transition opt-in that briefly lived here is
+   deliberately gone: mixing it with the client router double-animates the
+   full-load navigations the router hands back to the browser. */
+
 /* Code reads left-to-right regardless of page direction; only the surrounding
    chrome mirrors for RTL. Inline code is isolated so LTR identifiers don't
    disturb the bidi flow of right-to-left prose. */
@@ -242,9 +254,10 @@ ${THEME_MAPPING}
   line-height: 1.7;
 }
 
+/* No letter-spacing here: prose headings inherit the base h1-h6 rule's
+   display tracking, same as headings outside the prose column. */
 .prose :where(h1, h2, h3, h4) {
   font-weight: 500;
-  letter-spacing: 0;
 }
 
 /* A heading can carry one long unbreakable token — an OpenAPI operation's title

package/src/theme/fonts.ts

@@ -352,6 +352,78 @@ export const buildFontsCss = (fonts: FontsConfig): string => {
     : "";
 };
 
-/** The CSS variables to feed Astro's `<Font>` component in the document head. */
-export const configuredCssVars = (fonts: FontsConfig): string[] =>
-  buildFontEntries(fonts).map((entry) => entry.cssVariable);
+/**
+ * Weights worth preloading per role — the faces above-the-fold text actually
+ * renders in: body copy and UI chrome at 400/500, headings at 500/600, code at
+ * 400. Every other face still loads on demand through its `@font-face` rule
+ * (and `font-display: swap` never blocks text on it), so preloading the long
+ * tail only competes with the critical CSS for bandwidth and pushes LCP out.
+ */
+const PRELOAD_WEIGHTS = {
+  body: [400, 500],
+  display: [500, 600],
+  mono: [400],
+} satisfies Record<FontSlot, number[]>;
+
+/** One `<Font>` render in the head: its CSS variable + weights to preload. */
+export interface FontHead {
+  cssVariable: string;
+  preloadWeights: number[];
+}
+
+/** The weights an entry's faces declare (`undefined` = inferred from files). */
+const entryWeights = (entry: FontEntry): (number | string | undefined)[] =>
+  entry.kind === "remote"
+    ? entry.weights
+    : entry.variants.map((variant) => variant.weight);
+
+/**
+ * The role's preferred preload weights, narrowed to faces the family loads.
+ * Variable ranges (`"100..900"`) and weight-inferred local files can serve any
+ * weight, so they keep the preferred list; a family whose numeric weights miss
+ * the preferred ones entirely preloads all of its faces instead — those are
+ * what its text renders in.
+ */
+const preloadWeightsFor = (slot: FontSlot, entry: FontEntry): number[] => {
+  const preferred = PRELOAD_WEIGHTS[slot];
+  const weights = entryWeights(entry);
+  const numeric = weights.filter(
+    (weight): weight is number => typeof weight === "number"
+  );
+  const hits = preferred.filter((weight) => numeric.includes(weight));
+  if (hits.length > 0) {
+    return hits;
+  }
+  return numeric.length === weights.length ? numeric : preferred;
+};
+
+/**
+ * The fonts to feed Astro's `<Font>` component in the document head, deduped
+ * by CSS variable with preload weights unioned across the roles that share a
+ * family (so `display` and `body` both set to Inter preload 400/500/600 once).
+ */
+export const configuredFonts = (fonts: FontsConfig): FontHead[] => {
+  if (!fonts) {
+    return [];
+  }
+  const heads = new Map<string, Set<number>>();
+  for (const slot of SLOTS) {
+    const value = fonts[slot];
+    if (value === undefined) {
+      continue;
+    }
+    const entry = resolveFontValue(slot, value);
+    if (!entry) {
+      continue;
+    }
+    const weights = heads.get(entry.cssVariable) ?? new Set();
+    for (const weight of preloadWeightsFor(slot, entry)) {
+      weights.add(weight);
+    }
+    heads.set(entry.cssVariable, weights);
+  }
+  return [...heads].map(([cssVariable, weights]) => ({
+    cssVariable,
+    preloadWeights: [...weights].toSorted((a, b) => a - b),
+  }));
+};