blume 1.2.0 → 1.3.0

package/src/og/derive.ts

@@ -0,0 +1,200 @@
+/**
+ * Bridges `theme.fonts` into the OG card renderer. A site that explicitly
+ * picks its typefaces gets matching cards (and non-Latin coverage) without
+ * configuring `seo.og.fonts`; untouched defaults derive nothing, so plain
+ * sites keep Takumi's built-in font and gain no build-time font fetch.
+ */
+
+import { existsSync } from "node:fs";
+
+import { isAbsolute, join } from "pathe";
+
+import type {
+  FontsConfig,
+  FontValue,
+  LocalFontConfig,
+  RemoteFontConfig,
+} from "../theme/fonts.ts";
+import { GOOGLE_FONTS, isFontSlug } from "../theme/fonts.ts";
+import type { OgFont, OgFontFamilies, OgLocalFont } from "./card.ts";
+
+/** Fonts plus per-role families for the generated OG endpoint. */
+export interface DerivedOgFonts {
+  families?: OgFontFamilies;
+  fonts: OgFont[];
+}
+
+/** The weights the card actually renders at (title 600, everything else 400). */
+const CARD_WEIGHTS = [400, 600];
+
+/** Resolve a config path against the project root. */
+const absoluteSrc = (root: string, src: string): string =>
+  isAbsolute(src) ? src : join(root, src);
+
+/**
+ * The weight spec to fetch for a derived Google family: the declared weights
+ * the card uses, the declared numeric weights otherwise, a lone variable
+ * range as-is, or nothing (family default) as the last resort.
+ */
+const googleWeights = (
+  weights: (number | string)[]
+): number[] | string | undefined => {
+  const numbers = weights.filter(
+    (weight): weight is number => typeof weight === "number"
+  );
+  const used = numbers.filter((weight) => CARD_WEIGHTS.includes(weight));
+  if (used.length > 0) {
+    return used;
+  }
+  if (numbers.length > 0) {
+    return numbers;
+  }
+  const [first] = weights;
+  return weights.length === 1 && typeof first === "string" ? first : undefined;
+};
+
+const googleOgFont = (name: string, weights: (number | string)[]): OgFont => {
+  const weight = googleWeights(weights);
+  return weight === undefined ? { name } : { name, weight };
+};
+
+/** Per-variant local entries for the renderer (paths made absolute). */
+const localOgFonts = (font: LocalFontConfig, root: string): OgLocalFont[] =>
+  font.variants.map((variant) => ({
+    name: font.name,
+    src: absoluteSrc(root, variant.src),
+    ...(typeof variant.weight === "number" ? { weight: variant.weight } : {}),
+    // Takumi's per-face style is normal/italic; oblique falls back to the file.
+    ...(variant.style === "normal" || variant.style === "italic"
+      ? { style: variant.style }
+      : {}),
+  }));
+
+/**
+ * The card fonts for one theme role, or null when the role can't flow into
+ * the renderer (an unknown slug string, or a provider Takumi can't fetch —
+ * `googleFonts` only speaks Google's css2 endpoint).
+ */
+const roleFonts = (value: FontValue, root: string): OgFont[] | null => {
+  if (typeof value === "string") {
+    if (!isFontSlug(value)) {
+      return null;
+    }
+    const def = GOOGLE_FONTS[value];
+    return [googleOgFont(def.family, def.weights)];
+  }
+  if ("variants" in value) {
+    return localOgFonts(value, root);
+  }
+  const remote = value as RemoteFontConfig;
+  if ((remote.provider ?? "google") !== "google") {
+    return null;
+  }
+  return [googleOgFont(remote.name, remote.weights ?? CARD_WEIGHTS)];
+};
+
+/** The family name a theme role registers under. */
+const roleFamily = (value: FontValue): string | null => {
+  if (typeof value === "string") {
+    return isFontSlug(value) ? GOOGLE_FONTS[value].family : null;
+  }
+  return value.name;
+};
+
+/**
+ * Derive the OG card fonts from the theme's display and body roles (the two
+ * the card renders), deduped, plus the per-role family names so the title
+ * keeps the display face and the body text the body face.
+ */
+export const deriveOgFonts = (
+  fonts: FontsConfig,
+  root: string
+): DerivedOgFonts => {
+  const derived: OgFont[] = [];
+  const seen = new Set<string>();
+  const families: OgFontFamilies = {};
+
+  const roles = [
+    ["title", fonts?.display],
+    ["body", fonts?.body],
+  ] as const;
+  for (const [role, value] of roles) {
+    if (value === undefined) {
+      continue;
+    }
+    const roleEntries = roleFonts(value, root);
+    if (!roleEntries) {
+      continue;
+    }
+    families[role] = roleFamily(value) ?? undefined;
+    for (const entry of roleEntries) {
+      const key = JSON.stringify(entry);
+      if (!seen.has(key)) {
+        seen.add(key);
+        derived.push(entry);
+      }
+    }
+  }
+
+  return {
+    fonts: derived,
+    ...(families.title || families.body ? { families } : {}),
+  };
+};
+
+/** Explicit `seo.og.fonts` with local `src` paths resolved to absolute. */
+export const resolveOgFontSources = (fonts: OgFont[], root: string): OgFont[] =>
+  fonts.map((font) =>
+    typeof font !== "string" && "src" in font
+      ? { ...font, src: absoluteSrc(root, font.src) }
+      : font
+  );
+
+/**
+ * The fonts baked into the generated OG endpoint. An explicit `seo.og.fonts`
+ * always wins (including `[]` to opt out, keeping the card's role styling
+ * untouched); otherwise a site that explicitly set `theme.fonts` gets its
+ * display/body fonts derived so cards match the site without extra config.
+ */
+export const resolveOgFonts = (
+  options: {
+    /** Explicit `seo.og.fonts`, or undefined when unset. */
+    ogFonts: OgFont[] | undefined;
+    themeFonts: FontsConfig;
+    /** Whether the config file set `theme.fonts` itself (gates derivation). */
+    themeFontsConfigured: boolean;
+  },
+  root: string
+): DerivedOgFonts => {
+  if (options.ogFonts) {
+    return { fonts: resolveOgFontSources(options.ogFonts, root) };
+  }
+  return options.themeFontsConfigured
+    ? deriveOgFonts(options.themeFonts, root)
+    : { fonts: [] };
+};
+
+/**
+ * Every configured local font file (theme roles and `seo.og.fonts`) that is
+ * missing on disk, as absolute paths. Generation fails on these up front — the
+ * alternative is Astro or the OG renderer crashing later with a bare ENOENT.
+ */
+export const missingFontFiles = (
+  options: { ogFonts: OgFont[]; themeFonts: FontsConfig },
+  root: string
+): string[] => {
+  const sources: string[] = [];
+  for (const value of Object.values(options.themeFonts ?? {})) {
+    if (typeof value !== "string" && "variants" in value) {
+      sources.push(
+        ...value.variants.map((variant) => absoluteSrc(root, variant.src))
+      );
+    }
+  }
+  for (const font of options.ogFonts) {
+    if (typeof font !== "string" && "src" in font) {
+      sources.push(absoluteSrc(root, font.src));
+    }
+  }
+  return [...new Set(sources)].filter((path) => !existsSync(path));
+};

package/src/og/index.ts

@@ -1,2 +1,7 @@
 export { renderOgImage } from "./card.ts";
-export type { OgCardOptions, OgCardPalette, OgFont } from "./card.ts";
+export type {
+  OgCardOptions,
+  OgCardPalette,
+  OgFont,
+  OgFontFamilies,
+} from "./card.ts";

package/src/search/orama-index.ts

@@ -1,5 +1,5 @@
 import { create, insertMultiple, search } from "@orama/orama";
-import type { AnyOrama } from "@orama/orama";
+import type { AnyOrama, Tokenizer } from "@orama/orama";
 
 /**
  * The minimal document shape both the client-side search dialog and the
@@ -31,23 +31,161 @@ const SCHEMA = {
 /** Title and description outrank body text, matching the search dialog. */
 const BOOST = { description: 2, title: 3 };
 
+/**
+ * Scripts written without spaces between words. Orama's default tokenizer
+ * splits on a Latin-centric delimiter class, so text in these languages
+ * collapses to zero tokens and every query silently returns no hits. Keyed by
+ * the primary language subtag of `i18n.defaultLocale`.
+ */
+const SEGMENTED_LANGUAGES = new Set(["ja", "ko", "th", "zh"]);
+
+/**
+ * Languages indexed as character bigrams rather than whole segments, and
+ * queried to match accordingly. Japanese and Chinese write compounds in
+ * {@link BIGRAM_SCRIPTS} without delimiters; Korean separates words with
+ * spaces and Thai has no comparable bigram convention, so both keep the plain
+ * segmented tokens even where a page mixes in Han or kana.
+ *
+ * Dictionary segmentation alone drops the adjacency that makes a compound term
+ * distinctive: 資金決済法 becomes 資金 / 決済 / 法, and because Orama scores a
+ * bag of words, a page that merely mentions each fragment somewhere outranks
+ * the page about the law itself. Bigrams put that adjacency back as index
+ * terms, and dropping the whole-segment tokens keeps a fragment as common as
+ * 法 from matching on its own.
+ */
+const BIGRAM_LANGUAGES = new Set(["ja", "zh"]);
+
+/**
+ * Segments written entirely in these scripts are the ones re-cut into bigrams,
+ * matching the scripts Lucene's CJK analyzer bigrams. Property escapes rather
+ * than ranges, so ideographs outside the basic plane are covered as well —
+ * 𠮟, the 常用漢字表 form of しかる, is one. The literals that follow belong to
+ * no script of their own but appear only inside such words: the iteration
+ * marks, the prolonged sound mark, and the halfwidth voiced sound marks.
+ */
+const BIGRAM_SCRIPTS =
+  /^[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}々〆〇ーﾞﾟ]+$/u;
+
+/**
+ * Emit every overlapping 2-character window of `run`, or the lone character.
+ * Windows are cut by code point: an ideograph outside the basic plane is a
+ * surrogate pair, and slicing by code unit would split it into halves that
+ * match nothing.
+ *
+ * Dropping the whole-segment tokens means a single-character query reaches
+ * only pages where the character opens a bigram: a run-final 法 sits in 示法,
+ * which the query 法 does not prefix-match. Lucene's CJK analyzer shares this
+ * property; indexing lone characters alongside the bigrams would reinvite the
+ * fragment noise this file exists to remove.
+ */
+const addBigrams = (run: string, tokens: Set<string>): void => {
+  const characters = [...run];
+  if (characters.length === 1) {
+    tokens.add(run);
+    return;
+  }
+  let previous = "";
+  for (const character of characters) {
+    if (previous) {
+      tokens.add(previous + character);
+    }
+    previous = character;
+  }
+};
+
+/**
+ * A word-segmenting tokenizer for languages the default splitter can't handle,
+ * built on `Intl.Segmenter` (the same engine `@orama/tokenizers` wraps).
+ * Input is lowercased before segmenting — unlike the upstream tokenizers —
+ * so Latin terms ("GDPR", English pages on a mixed-locale site) still match
+ * case-insensitively. Returns `undefined` for languages the default tokenizer
+ * already serves, and on runtimes without `Intl.Segmenter`, where the caller
+ * falls back to Orama's default.
+ *
+ * On a {@link BIGRAM_LANGUAGES} index, runs of adjacent
+ * {@link BIGRAM_SCRIPTS} segments are joined and re-cut into character
+ * bigrams; everything else (Latin, digits, and every segment on a Korean or
+ * Thai index) is emitted as the segmenter produced it. Punctuation and spaces
+ * are not word-like, so they end a run — 「クーリング・オフ」 bigrams either
+ * side of the interpunct rather than across it.
+ */
+const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
+  const language = locale?.toLowerCase().split(/[-_]/u)[0] ?? "";
+  if (!SEGMENTED_LANGUAGES.has(language)) {
+    return;
+  }
+  if (typeof Intl.Segmenter !== "function") {
+    return;
+  }
+  const segmenter = new Intl.Segmenter(language, { granularity: "word" });
+  // Keyed off the same set the strict query pass reads, so an index is never
+  // built from bigrams that the query side then matches loosely.
+  const bigram = BIGRAM_LANGUAGES.has(language);
+  return {
+    language,
+    normalizationCache: new Map(),
+    tokenize: (raw: string): string[] => {
+      const tokens = new Set<string>();
+      let run = "";
+      const flush = (): void => {
+        if (run) {
+          addBigrams(run, tokens);
+          run = "";
+        }
+      };
+      for (const segment of segmenter.segment(raw.toLowerCase())) {
+        if (!segment.isWordLike) {
+          flush();
+          continue;
+        }
+        if (bigram && BIGRAM_SCRIPTS.test(segment.segment)) {
+          run += segment.segment;
+          continue;
+        }
+        flush();
+        tokens.add(segment.segment);
+      }
+      flush();
+      return [...tokens];
+    },
+  };
+};
+
 /**
  * Build an in-memory Orama full-text index from search documents. Shared by the
- * Orama client loader (browser) and the MCP server (Node), so ranking is
- * identical wherever docs are queried.
+ * Orama client loader (browser), the MCP server, and Ask AI grounding (Node),
+ * so ranking is identical wherever docs are queried. `locale` — the site's
+ * `i18n.defaultLocale` — swaps in a word-segmenting tokenizer for languages
+ * written without spaces (Japanese, Chinese, Korean, Thai); the tokenizer
+ * belongs to the database, so on a mixed-locale site it applies to every
+ * document, which is safe because Latin words survive segmentation intact.
  */
 export const buildOramaIndex = async (
-  documents: OramaDoc[]
+  documents: OramaDoc[],
+  locale?: string
 ): Promise<AnyOrama> => {
-  const db = create({ schema: SCHEMA });
+  const tokenizer = segmentingTokenizer(locale);
+  const db = create({
+    schema: SCHEMA,
+    ...(tokenizer ? { components: { tokenizer } } : {}),
+  });
   await insertMultiple(db, documents);
   return db;
 };
 
+/** Orama keeps only documents matching every token at a threshold of 0. */
+const ALL_TOKENS = 0;
+
 /**
  * Query the index, returning the matching documents (highest-ranked first).
  * When `locale` is given, results are filtered to that language via an exact
  * `where` match on the `locale` enum.
+ *
+ * On a bigrammed index the strict pass runs first: a term is only meant to
+ * match where its bigrams sit together, and scoring them independently lets a
+ * page sharing a couple of windows outrank the page the term is about. Terms
+ * spanning several words rarely appear in full on one page, so an empty strict
+ * result falls back to the default pass rather than reporting no matches.
  */
 export const queryOramaIndex = async (
   db: AnyOrama,
@@ -55,12 +193,18 @@ export const queryOramaIndex = async (
   limit: number,
   locale?: string
 ): Promise<OramaDoc[]> => {
-  const found = await search(db, {
+  const params = {
     boost: BOOST,
     limit,
     properties: ["title", "description", "content"],
     term,
     ...(locale ? { where: { locale: { eq: locale } } } : {}),
-  });
+  };
+  const bigrammed = BIGRAM_LANGUAGES.has(db.tokenizer?.language ?? "");
+  const strict = bigrammed
+    ? await search(db, { ...params, threshold: ALL_TOKENS })
+    : undefined;
+  const found =
+    strict && strict.hits.length > 0 ? strict : await search(db, params);
   return found.hits.map((hit) => hit.document as unknown as OramaDoc);
 };

package/src/theme/entry.ts

@@ -452,19 +452,29 @@ blume-diff {
    even though its chrome wrapper is not-prose. */
 .prose :where(pre:not(.twoslash, .twoslash pre, blume-panel-tabs *) > code) {
   display: block;
-  overflow-x: auto;
-  padding: 0 1.25rem;
-  /* The scroller is only as tall as the code, so an overlay scrollbar would
-     draw on top of the last line; hide it (wheel/trackpad/keyboard scrolling
-     still works). */
-  scrollbar-width: none;
-}
-
-.prose
-  :where(
-    pre:not(.twoslash, .twoslash pre, blume-panel-tabs *) > code
-  )::-webkit-scrollbar {
-  display: none;
+  /* Tall blocks scroll vertically in place instead of taking the page. Both
+     axes live on the code element for the same reason: the pre stays static
+     so the header bar and copy button never drift with the scroll. */
+  max-height: 24rem;
+  overflow: auto;
+  /* The small bottom inset keeps the horizontal thumb off the last line's
+     descenders now that scrollbars are visible. */
+  padding: 0 1.25rem 0.375rem;
+  /* Thin theme-colored scrollbars, matching the sidebar treatment, so a
+     height-capped block reads as scrollable instead of simply ending.
+     Safari before 18.2 supports neither property and falls back to the
+     platform-default scrollbar — acceptable, since macOS overlays it. */
+  scrollbar-color: var(--blume-border) transparent;
+  scrollbar-width: thin;
+}
+
+/* The dark border token is too close to the page background to read as a
+   scrollbar thumb; derive a brighter one from the muted foreground instead. */
+:root[data-theme="dark"]
+  .prose
+  :where(pre:not(.twoslash, .twoslash pre, blume-panel-tabs *) > code) {
+  scrollbar-color: color-mix(in oklab, var(--blume-muted-foreground) 55%, transparent)
+    transparent;
 }
 
 /* Word wrap (markdown.code.wrap): long lines wrap instead of scrolling. The
@@ -532,6 +542,10 @@ pre.blume-source {
 
 pre.blume-source > code {
   flex: 1;
+  /* The pane's measured height is authoritative (it can exceed the prose
+     24rem cap), so undo the generic prose max-height: the code must fill
+     whatever height the tab was given. */
+  max-height: none;
   min-height: 0;
   overflow: auto;
 }
@@ -713,6 +727,13 @@ pre:has(.line.focused):hover .line:not(.focused) {
     display: none !important;
   }
 
+  /* Paper can't scroll: uncap the code scroller so long blocks print in
+     full, same as tab panels are force-expanded below. */
+  .prose :where(pre:not(.twoslash, .twoslash pre, blume-panel-tabs *) > code) {
+    max-height: none;
+    overflow: visible;
+  }
+
   #blume-content {
     padding: 0 !important;
   }