blume 2.0.3 → 2.1.1

package/src/search/documents.ts

@@ -54,6 +54,10 @@ export interface SearchDocument {
    * Filterable through the MCP tools' `filters` input.
    */
   facets?: Record<string, string>;
+  /** Frontmatter `search.boost`: the page's relevance multiplier, when set. */
+  boost?: number;
+  /** Frontmatter `search.keywords`: extra terms the page is found by. */
+  keywords?: string[];
 }
 
 /**
@@ -77,6 +81,13 @@ export interface SearchRecord {
   version: string;
   /** Single faceting tag (the first frontmatter tag, when present). */
   tag?: string;
+  /**
+   * The page's relevance multiplier, 1 unless `search.boost` sets it: a
+   * ranking attribute every record carries, so the backend can sort by it.
+   */
+  boost: number;
+  /** Extra terms the page is found by (`search.keywords`). */
+  keywords?: string[];
 }
 
 // Tag-shaped only: a name (or closing slash/fragment) right after `<`, and no
@@ -414,6 +425,17 @@ const pageBody = async (
   });
 };
 
+/** The ranking a page asks for: its `search.boost` and `search.keywords`. */
+const rankingFields = (
+  search: PageRecord["meta"]["search"] | undefined
+): Pick<SearchDocument, "boost" | "keywords"> => {
+  const { boost, keywords } = search ?? {};
+  return {
+    ...(boost !== undefined && boost !== 1 && { boost }),
+    ...(keywords && keywords.length > 0 && { keywords }),
+  };
+};
+
 /**
  * Build search documents from the content graph. Only indexable pages are
  * included (per the route manifest), and content comes from the source files,
@@ -531,7 +553,7 @@ export const buildSearchDocuments = async (
       if (facets) {
         document.facets = facets;
       }
-      return document;
+      return { ...document, ...rankingFields(page?.meta?.search) };
     })
   );
 };
@@ -539,13 +561,15 @@ export const buildSearchDocuments = async (
 /**
  * Map per-page search documents to the flat record shape hosted backends
  * ingest. One record per page, keyed by route; the first tag becomes the
- * faceting `tag`.
+ * faceting `tag`, and every record carries a `boost` (1 by default).
  */
 export const toSearchRecords = (documents: SearchDocument[]): SearchRecord[] =>
   documents.map((doc) => ({
     _id: doc.route,
+    boost: doc.boost ?? 1,
     content: doc.content,
     description: doc.description,
+    ...(doc.keywords && { keywords: doc.keywords }),
     locale: doc.locale,
     tag: doc.tags?.[0],
     title: doc.title,

package/src/search/orama-index.ts

@@ -1,8 +1,16 @@
-import { create, insertMultiple, search } from "@orama/orama";
+import {
+  components,
+  create,
+  insert,
+  insertMultiple,
+  search,
+} from "@orama/orama";
 import type {
   AnyOrama,
   EnumArrComparisonOperator,
   EnumComparisonOperator,
+  Result,
+  SearchParamsFullText,
   Tokenizer,
 } from "@orama/orama";
 
@@ -31,6 +39,10 @@ export interface OramaDoc {
    * but not indexed, so they ride along on the returned document untouched. */
   breadcrumb?: string[];
   section?: string;
+  /** The page's relevance multiplier (`search.boost`); stored, not indexed. */
+  boost?: number;
+  /** Extra terms the page is found by (`search.keywords`). */
+  keywords?: string[];
 }
 
 const SCHEMA = {
@@ -42,6 +54,7 @@ const SCHEMA = {
   // serves every project's facet keys, with `containsAll` matching a filter
   // set. Derived from `facets` at insert time.
   facetTerms: "enum[]",
+  keywords: "string[]",
   locale: "enum",
   route: "string",
   title: "string",
@@ -52,8 +65,21 @@ const SCHEMA = {
 const toFacetTerms = (facets: Record<string, string>): string[] =>
   Object.entries(facets).map(([key, value]) => `${key}:${value}`);
 
-/** Title and description outrank body text, matching the search dialog. */
-const BOOST = { description: 2, title: 3 };
+/**
+ * Title and description outrank body text, matching the search dialog; a
+ * page's keywords count as much as its title, since its author chose them.
+ */
+const BOOST = { description: 2, keywords: 3, title: 3 };
+
+/** Databases holding a boosted page, which rank by boosted score. */
+const boostedDatabases = new WeakSet<AnyOrama>();
+
+/** A search match as Orama's sort function sees it: id, score, document. */
+type RankedMatch = [unknown, number, OramaDoc];
+
+/** Rank matches by relevance times the page's `search.boost`. */
+const byBoostedScore = (a: RankedMatch, b: RankedMatch): number =>
+  b[1] * (b[2].boost ?? 1) - a[1] * (a[2].boost ?? 1);
 
 /**
  * The script whose text Orama's default tokenizer keeps mostly intact. Its
@@ -296,32 +322,92 @@ const segmentingTokenizer = (locale?: string): Tokenizer | undefined => {
   };
 };
 
+/**
+ * On a Latin-default index holding pages in non-Latin locales, the tokenizer
+ * each of those locales' pages was indexed with, keyed by locale code. A query
+ * reads it to tokenize its term the way the pages it searches were.
+ */
+const translationTokenizers = new WeakMap<AnyOrama, Map<string, Tokenizer>>();
+
+/** A segmenting tokenizer for each document locale Orama's can't serve. */
+const localeTokenizers = (documents: OramaDoc[]): Map<string, Tokenizer> => {
+  const tokenizers = new Map<string, Tokenizer>();
+  for (const code of new Set(documents.map((doc) => doc.locale ?? ""))) {
+    const tokenizer = segmentingTokenizer(code);
+    if (tokenizer) {
+      tokenizers.set(code, tokenizer);
+    }
+  }
+  return tokenizers;
+};
+
+/**
+ * One tokenizer that hands each text to its locale's: Orama passes the
+ * language given to `insert` and `search` through to `tokenize`, so a page
+ * inserted under `ja` and a query searched under `ja` both reach the Japanese
+ * tokenizer, and everything else reaches Orama's own, exactly as on an index
+ * with no translations.
+ */
+const dispatchingTokenizer = (
+  tokenizers: Map<string, Tokenizer>
+): Tokenizer => {
+  const standard = components.tokenizer.createTokenizer();
+  return {
+    language: standard.language,
+    normalizationCache: standard.normalizationCache,
+    tokenize: (raw, language, prop, withCache) =>
+      // Orama's tokenizer rejects a language other than its own, so the
+      // locale that picked it is not passed on.
+      (tokenizers.get(language ?? "") ?? standard).tokenize(
+        raw,
+        undefined,
+        prop,
+        withCache
+      ),
+  };
+};
+
 /**
  * Build an in-memory Orama full-text index from search documents. Shared by the
  * Orama client loader (browser), the MCP server, and assistant grounding (Node),
  * so ranking is identical wherever docs are queried. `locale` — the site's
  * `i18n.defaultLocale` — swaps in a word-segmenting tokenizer for every
  * non-Latin script, all of which Orama's default tokenizer reduces to zero
- * tokens; the tokenizer belongs to the database, so on a mixed-locale site it
- * applies to every document. That is safe in one direction only: Latin words
+ * tokens. A non-Latin default's tokenizer serves every page: Latin words
  * survive segmentation intact, so English pages on a segmented index stay
- * searchable, but non-Latin translations on a Latin-default index still
- * collapse to zero tokens.
+ * searchable. A Latin default keeps Orama's tokenizer for its Latin-script
+ * pages, and each page in a non-Latin locale (the Japanese and Hindi
+ * translations of an English site) is indexed with its own locale's tokenizer.
  */
 export const buildOramaIndex = async (
   documents: OramaDoc[],
   locale?: string
 ): Promise<AnyOrama> => {
-  const tokenizer = segmentingTokenizer(locale);
-  const db = tokenizer
-    ? create({ components: { tokenizer }, schema: SCHEMA })
-    : create({ schema: SCHEMA });
-  await insertMultiple(
-    db,
-    documents.map((doc) =>
-      doc.facets ? { ...doc, facetTerms: toFacetTerms(doc.facets) } : doc
-    )
+  const rows = documents.map((doc) =>
+    doc.facets ? { ...doc, facetTerms: toFacetTerms(doc.facets) } : doc
   );
+  const tokenizer = segmentingTokenizer(locale);
+  const translations = tokenizer ? undefined : localeTokenizers(documents);
+  let db: AnyOrama;
+  if (translations && translations.size > 0) {
+    db = create({
+      components: { tokenizer: dispatchingTokenizer(translations) },
+      schema: SCHEMA,
+    });
+    for (const row of rows) {
+      // oxlint-disable-next-line no-await-in-loop -- one at a time, in order, so each page is tokenized under its own locale and ties still rank in document order.
+      await insert(db, row, row.locale);
+    }
+    translationTokenizers.set(db, translations);
+  } else {
+    db = tokenizer
+      ? create({ components: { tokenizer }, schema: SCHEMA })
+      : create({ schema: SCHEMA });
+    await insertMultiple(db, rows);
+  }
+  if (documents.some((doc) => (doc.boost ?? 1) !== 1)) {
+    boostedDatabases.add(db);
+  }
   return db;
 };
 
@@ -358,17 +444,56 @@ interface OramaWhereClause {
     | undefined;
 }
 
+/** A ranked match, scored and carrying its document. */
+type Hit = Result<OramaDoc>;
+
+/**
+ * One search, with the term tokenized under `language`. On a bigrammed
+ * tokenizer the strict pass runs first: a term is only meant to match where
+ * its bigrams sit together, and scoring them independently lets a page
+ * sharing a couple of windows outrank the page the term is about. Terms
+ * spanning several words rarely appear in full on one page, so an empty
+ * strict result falls back to the default pass rather than reporting no
+ * matches.
+ */
+const rankedHits = async (
+  db: AnyOrama,
+  params: SearchParamsFullText<AnyOrama, OramaDoc>,
+  language: string | undefined,
+  bigrammed: boolean
+): Promise<Hit[]> => {
+  // The result-document generic is OramaDoc because `buildOramaIndex` is the
+  // only writer to this database and inserts OramaDoc records (plus the
+  // derived `facetTerms`).
+  const strict = bigrammed
+    ? await search<AnyOrama, OramaDoc>(
+        db,
+        { ...params, threshold: ALL_TOKENS },
+        language
+      )
+    : undefined;
+  const found =
+    strict && strict.hits.length > 0
+      ? strict
+      : await search<AnyOrama, OramaDoc>(db, params, language);
+  return found.hits;
+};
+
+/** A hit's rank across searches: its relevance times the page's boost. */
+const boostedScore = (hit: Hit): number =>
+  hit.score * (hit.document.boost ?? 1);
+
 /**
  * Query the index, returning the matching documents (highest-ranked first).
  * `filters` narrows results by exact `where` matches on the enum fields:
  * `locale` to one language, `contentTypes` to a set of page types, `facets`
  * to documents carrying every requested `key:value` term.
  *
- * On a bigrammed index the strict pass runs first: a term is only meant to
- * match where its bigrams sit together, and scoring them independently lets a
- * page sharing a couple of windows outrank the page the term is about. Terms
- * spanning several words rarely appear in full on one page, so an empty strict
- * result falls back to the default pass rather than reporting no matches.
+ * On an index whose translations have tokenizers of their own, a query scoped
+ * to a locale is tokenized with that locale's tokenizer. An unscoped query
+ * searches each locale's pages with its own tokenizer, and the Latin-script
+ * pages with Orama's, then merges the matches by score: every search runs over
+ * the one index, so their scores share its term statistics.
  */
 export const queryOramaIndex = async (
   db: AnyOrama,
@@ -391,24 +516,56 @@ export const queryOramaIndex = async (
   if (facetTerms.length > 0) {
     where.facetTerms = { containsAll: facetTerms };
   }
+  // Boosts multiply every match's score before the limit applies (Orama
+  // sorts the whole match set with a sort function), so a boosted page can
+  // rise past matches outside the first page.
   const unfiltered = {
     boost: BOOST,
     limit,
-    properties: ["title", "description", "content"],
+    properties: ["title", "keywords", "description", "content"],
+    ...(boostedDatabases.has(db) && { sortBy: byBoostedScore }),
     term,
   };
   const params =
     Object.keys(where).length > 0 ? { ...unfiltered, where } : unfiltered;
-  const bigrammed = isBigramLanguage(db.tokenizer?.language ?? "");
-  // The result-document generic is OramaDoc because `buildOramaIndex` is the
-  // only writer to this database and inserts OramaDoc records (plus the
-  // derived `facetTerms`).
-  const strict = bigrammed
-    ? await search<AnyOrama, OramaDoc>(db, { ...params, threshold: ALL_TOKENS })
-    : undefined;
-  const found =
-    strict && strict.hits.length > 0
-      ? strict
-      : await search<AnyOrama, OramaDoc>(db, params);
-  return found.hits.map((hit) => hit.document);
+  const translations = translationTokenizers.get(db);
+  if (!translations) {
+    const bigrammed = isBigramLanguage(db.tokenizer?.language ?? "");
+    const hits = await rankedHits(db, params, undefined, bigrammed);
+    return hits.map((hit) => hit.document);
+  }
+  if (filters?.locale) {
+    const { language = "" } = translations.get(filters.locale) ?? {};
+    const bigrammed = isBigramLanguage(language);
+    const hits = await rankedHits(db, params, filters.locale, bigrammed);
+    return hits.map((hit) => hit.document);
+  }
+  // `nin` matches only pages with a locale, which every Blume-built index
+  // gives each page. (Orama's `not` would admit locale-less pages, but it
+  // drops the sibling filters beside it.)
+  const searches = [
+    rankedHits(
+      db,
+      {
+        ...params,
+        where: { ...where, locale: { nin: [...translations.keys()] } },
+      },
+      undefined,
+      false
+    ),
+    ...[...translations].map(([code, tokenizer]) =>
+      rankedHits(
+        db,
+        { ...params, where: { ...where, locale: { eq: code } } },
+        code,
+        isBigramLanguage(tokenizer.language)
+      )
+    ),
+  ];
+  const found = await Promise.all(searches);
+  return found
+    .flat()
+    .toSorted((a, b) => boostedScore(b) - boostedScore(a))
+    .slice(0, limit)
+    .map((hit) => hit.document);
 };

package/src/search/pagefind-ranking.ts

@@ -0,0 +1,39 @@
+/**
+ * A page's `search.boost` and `search.keywords`, as Pagefind reads them from
+ * the built page. Pagefind has no page-level boost, only element weights, so
+ * the page's body carries its boost as its weight: a boosted page's matches
+ * count for more and it ranks higher. Pagefind damps repeated matches, so the
+ * lift is smaller than the full multiplier: a boost of 5 scored its page about
+ * 1.5 to 2 times higher in testing. A boost past Pagefind's largest weight,
+ * 10, counts as 10.
+ * Keywords go on an empty element as an indexed attribute, found like text
+ * and never shown.
+ */
+
+/** The largest weight Pagefind takes. */
+const MAX_WEIGHT = 10;
+
+/** The attribute a page's keywords ride on. */
+export const KEYWORDS_ATTRIBUTE = "data-blume-keywords";
+
+/** What the page body carries for Pagefind. */
+export interface PagefindRanking {
+  /** `data-pagefind-weight` for the body, when the page is boosted. */
+  weight?: string;
+  /** The keywords, space-separated, when the page sets any. */
+  keywords?: string;
+}
+
+export const pagefindRanking = (search: {
+  boost?: number;
+  keywords?: string[];
+}): PagefindRanking => {
+  const { boost, keywords } = search;
+  return {
+    ...(boost !== undefined &&
+      boost !== 1 && {
+        weight: String(Math.min(boost, MAX_WEIGHT)),
+      }),
+    ...(keywords && keywords.length > 0 && { keywords: keywords.join(" ") }),
+  };
+};

package/src/search/source-pages.ts

@@ -0,0 +1,29 @@
+import { relative } from "pathe";
+
+import type { BlumeProject } from "../core/project-graph.ts";
+
+/** Where a page is served, and its title: what a search hit links to. */
+export interface SourcePage {
+  title: string;
+  url: string;
+}
+
+/**
+ * Each search-indexable page's source file, relative to the project root,
+ * paired with the page it renders. The Mixedbread endpoint links a hit to its
+ * page through the path `mxbai store sync` recorded for the chunk's file,
+ * which is relative to wherever the CLI ran: the project root, or a directory
+ * above it in a monorepo. Pages from a source without local files have no
+ * path to match and are left out.
+ */
+export const sourcePages = (project: BlumeProject): [string, SourcePage][] =>
+  project.manifest.routes.flatMap((route): [string, SourcePage][] =>
+    route.indexable && route.sourcePath
+      ? [
+          [
+            relative(project.context.root, route.sourcePath),
+            { title: route.title, url: route.path },
+          ],
+        ]
+      : []
+  );

package/src/search/sync/algolia.ts

@@ -16,10 +16,94 @@ export type AlgoliaSyncConfig = Pick<AlgoliaOptions, "appId" | "indexName">;
  */
 const FILTER_ATTRIBUTES = ["locale", "version"] as const;
 
+/** The custom ranking rule that orders close matches by `search.boost`. */
+const BOOST_RANKING = "desc(boost)";
+
 /** A faceting declaration's attribute: `filterOnly(locale)` → `locale`. */
 const facetAttribute = (declaration: string): string =>
   declaration.replace(/^\w+\((?<name>.*)\)$/u, "$<name>");
 
+/**
+ * The largest record Algolia accepts on its Build and Grow plans, in bytes of
+ * the record's JSON. One larger record fails the whole upload.
+ */
+const MAX_RECORD_BYTES = 10_000;
+
+/**
+ * A record as uploaded: the search record's fields plus Algolia's `objectID`.
+ * (Mapped rather than the interface itself, which the SDK's
+ * `Record<string, unknown>` objects don't accept.)
+ */
+type AlgoliaObject = Pick<SearchRecord, keyof SearchRecord> & {
+  objectID: string;
+};
+
+/** A string's or record's size as Algolia measures it: UTF-8 bytes of its JSON. */
+const jsonBytes = (value: string | AlgoliaObject): number =>
+  Buffer.byteLength(JSON.stringify(value), "utf-8");
+
+/** The JSON of the widest single character: a control character's `\u` escape. */
+const WIDEST_CHARACTER = jsonBytes("\u0000");
+
+/** The length of `text`'s longest prefix whose JSON string fits `budget`. */
+const fittingLength = (text: string, budget: number): number => {
+  let size = jsonBytes("");
+  let end = 0;
+  [...text].some((character) => {
+    size += jsonBytes(character) - jsonBytes("");
+    if (size > budget) {
+      return true;
+    }
+    end += character.length;
+    return false;
+  });
+  return end;
+};
+
+/**
+ * Cut `text` into pieces whose JSON strings fit `budget` bytes each, at the
+ * last space that fits so no word is split; a run with no space in reach (an
+ * unspaced script, a long URL) is cut between characters.
+ */
+const splitText = (text: string, budget: number): string[] => {
+  const pieces: string[] = [];
+  let rest = text;
+  while (jsonBytes(rest) > budget) {
+    const end = fittingLength(rest, budget);
+    const space = rest.lastIndexOf(" ", end);
+    const cut = space > 0 ? space : end;
+    pieces.push(rest.slice(0, cut));
+    rest = rest.slice(cut).trimStart();
+  }
+  pieces.push(rest);
+  return pieces;
+};
+
+/**
+ * A page's record, split to fit Algolia's cap the way Algolia recommends for
+ * long documents: every piece carries the page's title, description, URL and
+ * facets with a stretch of its body, and the index's `distinct` on `url`
+ * shows the page once, at its best-matching piece. The first piece keeps the
+ * page's own `objectID`; the rest add `#1`, `#2`… A record whose other fields
+ * leave no room for a single character of body can't be split to fit, and
+ * uploads whole for Algolia to reject by name.
+ */
+const toObjects = (record: SearchRecord): AlgoliaObject[] => {
+  const whole = { ...record, objectID: record._id };
+  const budget =
+    MAX_RECORD_BYTES -
+    jsonBytes({ ...whole, content: "", objectID: `${record._id}#000` }) +
+    jsonBytes("");
+  if (jsonBytes(whole) <= MAX_RECORD_BYTES || budget < WIDEST_CHARACTER) {
+    return [whole];
+  }
+  return splitText(record.content, budget).map((content, index) => ({
+    ...record,
+    content,
+    objectID: index === 0 ? record._id : `${record._id}#${index}`,
+  }));
+};
+
 /**
  * Upload the search records to Algolia. Uses the admin key from
  * `ALGOLIA_ADMIN_API_KEY` (never the adapter options, which hold only the
@@ -27,9 +111,12 @@ const facetAttribute = (declaration: string): string =>
  *
  * Uses `replaceAllObjects`, which atomically replaces the index contents, so
  * pages deleted or renamed since the last sync don't linger as stale search
- * hits that 404 when clicked. Then adds `filterOnly(locale)` and
+ * hits that 404 when clicked. A page too long for one record is split across
+ * several (see {@link toObjects}). Then adds `filterOnly(locale)` and
  * `filterOnly(version)` to the index's `attributesForFaceting`, keeping any
- * the site declared itself, so the dialog's locale and version filters match.
+ * the site declared itself, so the dialog's locale and version filters match,
+ * and, once a page is split, `url` as the index's `attributeForDistinct`
+ * unless the site set its own.
  */
 export const syncAlgolia = async (
   records: SearchRecord[],
@@ -44,16 +131,26 @@ export const syncAlgolia = async (
   const { algoliasearch }: typeof AlgoliaSdk = nodeRequire("algoliasearch");
   const client = algoliasearch(config.appId, adminKey);
   const { indexName } = config;
-  await client.replaceAllObjects({
-    indexName,
-    objects: records.map((record) => ({ ...record, objectID: record._id })),
-  });
-  const { attributesForFaceting = [] } = await client.getSettings({
-    indexName,
-  });
+  const objects = records.flatMap(toObjects);
+  await client.replaceAllObjects({ indexName, objects });
+  const {
+    attributeForDistinct,
+    attributesForFaceting = [],
+    customRanking = [],
+  } = await client.getSettings({ indexName });
   const declared = new Set(attributesForFaceting.map(facetAttribute));
   const missing = FILTER_ATTRIBUTES.filter((name) => !declared.has(name));
-  if (missing.length === 0) {
+  // Every record carries `boost` (search.boost, 1 by default): ranking by it
+  // after textual relevance puts boosted pages first among close matches.
+  // A ranking the site set in the dashboard keeps its place ahead of it.
+  const ranksByBoost = customRanking.some((rule) =>
+    /^(?:asc|desc)\(boost\)$/u.test(rule)
+  );
+  // A split page's records share its `url`: distinct on it lists the page
+  // once, at its best-matching record.
+  const setsDistinct =
+    objects.length > records.length && attributeForDistinct === undefined;
+  if (missing.length === 0 && ranksByBoost && !setsDistinct) {
     return;
   }
   const { taskID } = await client.setSettings({
@@ -63,6 +160,10 @@ export const syncAlgolia = async (
         ...attributesForFaceting,
         ...missing.map((name) => `filterOnly(${name})`),
       ],
+      customRanking: ranksByBoost
+        ? customRanking
+        : [...customRanking, BOOST_RANKING],
+      ...(setsDistinct && { attributeForDistinct: "url", distinct: true }),
     },
   });
   await client.waitForTask({ indexName, taskID });