@jxsuite/search 0.2.4 → 0.2.6

package/README.md

@@ -1,6 +1,6 @@
 # `@jxsuite/search`
 
-> Full-text site search built at compile time from content collections — no server, no third-party
+> Full-text site search built at compile time from content collections. No server, no third-party
 > service.
 
 ## Overview
@@ -9,10 +9,10 @@
 index file next to the pages. In the browser, a headless MiniSearch client fetches that file once
 and answers queries synchronously. Two classes, declared in `jx-extension.json`:
 
-- **`SearchIndex`** — owner of the project.json `search` section. `projectData` normalizes the
+- **`SearchIndex`**: owner of the project.json `search` section. `projectData` normalizes the
   section into `_project.search`; `emit` (specs/extensions.md §8.4) builds the index from the
   already-loaded content collections and hands it to the host to write.
-- **`Search`** — a state class that **lowers** to core defs (specs/extensions.md §8.3): its static
+- **`Search`** is a state class that **lowers** to core defs (specs/extensions.md §8.3): its static
   `lower` rewrites a `timing: "client"` def into a core `$prototype: "Function"` computed, so the
   only extension code reaching the browser is the client bundle.
 
@@ -45,7 +45,7 @@ collection, and rejects unknown keys. `engine` is an enum whose only member toda
 `@jxsuite/parser` is what loads the content collections this indexes. Defaults per collection:
 `fields` `["title", "heading", "text"]`, `sections` `true`, `sectionDepth` `3`; `output` defaults
 to `/search-index.json`. `basePath` is normalized to leading **and** trailing slashes, so `"blog"`
-becomes `"/blog/"`. This is the whole configuration surface — there is no CLI of its own.
+becomes `"/blog/"`. This is the whole configuration surface. There is no CLI of its own.
 
 One default does not come from the fragment: `boost` falls back to `{}` in `normalizeSearchConfig`
 (`src/shared.ts`), and the fragment declares no default for it. The schema is the source of truth
@@ -53,8 +53,8 @@ for the section's shape and for every other default.
 
 ## Build-time emission (`SearchIndex`)
 
-`emit` reads `ctx.sections.content` — the collections the parser already loaded, keyed by name —
-and never re-reads source files. It returns exactly one record, `{ path: <output>, content }`,
+`emit` reads `ctx.sections.content` (the collections the parser already loaded, keyed by name) and
+never re-reads source files. It returns exactly one record, `{ path: <output>, content }`,
 which the compiler writes under `outDir` (step 6e of the site build). The content is one envelope:
 
 ```json
@@ -62,7 +62,7 @@ which the compiler writes under `outDir` (step 6e of the site build). The conten
 ```
 
 `fields` is the union of every configured collection's `fields` and `boost` their merged boost
-maps — baked in so the client needs no configuration of its own. Both cover only the collections
+maps, baked in so the client needs no configuration of its own. Both cover only the collections
 that actually loaded: a name matching no content collection is skipped before its fields and boosts
 are merged, so a typo silently narrows the index rather than failing the build.
 
@@ -72,11 +72,11 @@ Each content entry yields a **page document** (`heading: ""`, the entry's full e
 entry id when frontmatter has no string `title`; `description` falls back to `""`. Entry URLs
 honour `build.trailingSlash` (default `"always"`).
 
-Localized collections are indexed per locale: each entry is indexed at the URL its own locale routes
-to — under the default `prefix-except-default` routing the default locale stays unprefixed and only
-the others carry a `/<locale>` prefix; under `prefix-always` every locale carries one — the document
-id becomes `<collection>:<locale>:<entry-id>`, and the document carries a canonical `locale` tag. A
-project with no `i18n` emits documents with no `locale` field at all.
+Localized collections are indexed per locale. Each entry is indexed at the URL its own locale routes
+to: under the default `prefix-except-default` routing the default locale stays unprefixed and only
+the others carry a `/<locale>` prefix, while under `prefix-always` every locale carries one. The
+document id becomes `<collection>:<locale>:<entry-id>`, and the document carries a canonical
+`locale` tag. A project with no `i18n` emits documents with no `locale` field at all.
 
 ## The `Search` state class
 
@@ -96,8 +96,8 @@ project with no `i18n` emits documents with no `locale` field at all.
 
 `lower` emits `{ $bundle: ["npm:@jxsuite/search/client"], $prototype: "Function", timing: "client",
 body }`. The body lazy-imports the client once per page onto `globalThis.__jxSearch`, preloads the
-index, flips `state.__jxSearchReady` — reading that flag is what makes the computed re-run when the
-client finishes loading — and returns results only once `isReady()`. The bundle URL is the
+index, flips `state.__jxSearchReady`, and returns results only once `isReady()`. Reading that flag
+is what makes the computed re-run when the client finishes loading. The bundle URL is the
 deterministic, hash-free `/assets/jxsuite-search-client.js`, derived from
 `sidecarAssetPath("npm:@jxsuite/search/client")` in `@jxsuite/schema/asset-paths`, the same mapping
 the compiler's sidecar bundler uses, so neither side depends on the other.
@@ -105,14 +105,13 @@ the compiler's sidecar bundler uses, so neither side depends on the other.
 ## The client (`@jxsuite/search/client`)
 
 MiniSearch inside (`fuzzy: 0.2`, `prefix: true`), no UI. Core API: `preload(indexUrl?)`,
-`isReady()`, and a synchronous `query(q, { limit, group, pageCap, locale })` — `limit` 8, `group`
-true, `pageCap` 3 (flat mode). That `limit` default governs a direct `query()` call only: the
-`$src` helper `runSearch` below always passes a fixed `UI_LIMIT` of 20. Grouped mode returns
-`SearchResultGroup[]`: a page with its matching sections, sorted by best score. Flat mode returns
-presentation-ready `SearchResult[]` carrying
-`crumbs` (title-cased slug ancestors, plus the page title on a section row), `titleTokens` and
-`excerptTokens` — arrays of `{ t, m }` runs where `m: true` marks a match, so renderers highlight
-without injecting HTML.
+`isReady()`, and a synchronous `query(q, { limit, group, pageCap, locale })` with defaults `limit`
+8, `group` true, `pageCap` 3 (flat mode). That `limit` default governs a direct `query()` call
+only: the `$src` helper `runSearch` below always passes a fixed `UI_LIMIT` of 20. Grouped mode
+returns `SearchResultGroup[]`: a page with its matching sections, sorted by best score. Flat mode
+returns presentation-ready `SearchResult[]` carrying `crumbs` (title-cased slug ancestors, plus the
+page title on a section row), `titleTokens` and `excerptTokens`. Those two are arrays of
+`{ t, m }` runs where `m: true` marks a match, so renderers highlight without injecting HTML.
 
 Compiled components do not go through lowering, so inside a component you use the `$src` state
 conventions instead of the `Search` prototype (`sites/jxsuite.com/components/site-search.json` is
@@ -144,40 +143,40 @@ other name gets the event.
 
 ## Surprises
 
-- **A collection name that is not a loaded content collection warns and is skipped** — the build
+- **A collection name that is not a loaded content collection warns and is skipped**. The build
   still succeeds and the envelope still emits, possibly with zero documents. A typo is therefore a
   silently empty search box.
 - **A heading with no `id` never starts a section document.** Heading ids come from the parser
   (specs/parser.md §3.2) and are what make the `#anchor` land on the rendered page; a heading
   without one folds into the enclosing section, as does anything under a heading deeper than
-  `sectionDepth`. Text before the first heading belongs to no section — the page document covers it.
-- **`emit` only runs when the project declares a non-empty `search` section** — the compiler skips
-  section-owner emitters whose value is `null` or `{}` (specs/extensions.md §8.4, "Gating").
+  `sectionDepth`. Text before the first heading belongs to no section; the page document covers it.
+- **`emit` only runs when the project declares a non-empty `search` section**, because the compiler
+  skips section-owner emitters whose value is `null` or `{}` (specs/extensions.md §8.4, "Gating").
 - **The `public/` copy runs after `emit`**, so a file in `public/` at the same path shadows the
-  emitted index — the same semantics as `sitemap.xml`.
+  emitted index (the same semantics as `sitemap.xml`).
 - **Omitting `locale` is not the same as `locale: null`.** Omitted means the page's own language,
   read from `<html lang>`; `null` searches every language. Both `lower` and `resolve` drop the key
   rather than serialize `undefined`. A document with no `locale` answers every search, and region
   is ignored when tags differ only there (`fr` matches `fr-CA`).
-- **`Search.resolve()` returns `[]` outside the browser**, with a warning — search is a client-side
+- **`Search.resolve()` returns `[]` outside the browser**, with a warning. Search is a client-side
   interaction, so compiler-timing bakes and dev-server resolve proxying degrade rather than fail.
-- **The index is one file per site and the client is a module-level singleton** — the first
+- **The index is one file per site and the client is a module-level singleton**, so the first
   `preload` wins. Multiple collections share the one index; results carry their `collection` name.
   A failed load clears the memo so a later call can retry.
 - **First-party (`@jxsuite/*`) schema fragments are read from the host, not the project's
-  `node_modules`** — schema composition routes every `@jxsuite/*` `.json` ref through host
+  `node_modules`**: schema composition routes every `@jxsuite/*` `.json` ref through host
   resolution and throws when the host cannot resolve it, so installing this package into a project
   alone does not make its `search` section validate. A third-party extension's fragment still
   resolves project-first.
 - **Class names are global across extensions.** `Search` and `SearchIndex` occupy the
   `$prototype`-visible namespace, not a package-scoped one.
 - **`bun run build` bundles only `search-index.ts` and `search-state.ts` into `dist/`.** The client
-  is deliberately not built here — the site build bundles it from source through the sidecar
+  is deliberately not built here, because the site build bundles it from source through the sidecar
   bundler.
 
 ## Versioning
 
-Published to npm as `@jxsuite/search` — TypeScript source, like every `@jxsuite` package, following
+Published to npm as `@jxsuite/search`, TypeScript source like every `@jxsuite` package, following
 the monorepo's release train. Its runtime dependencies are `@jxsuite/schema` and `minisearch`;
-extensions may depend on core packages but never the reverse — no core package lists this one as a
+extensions may depend on core packages but never the reverse: no core package lists this one as a
 runtime dependency or imports it from `src/`, and `scripts/check-dep-rules.ts` enforces that.

package/dist/search-index.js

@@ -193,8 +193,9 @@ function collectText(children) {
   return parts;
 }
 var HEADING_TAG = /^h([1-6])$/;
-function splitSections(children, sectionDepth) {
+function splitEntry(children, sectionDepth) {
   const sections = [];
+  const preambleParts = [];
   let current = null;
   for (const node of children ?? []) {
     const heading = headingOf(node);
@@ -205,20 +206,24 @@ function splitSections(children, sectionDepth) {
       current = { ...heading, parts: [], text: "" };
       continue;
     }
-    if (current && typeof node === "string") {
-      current.parts.push(node);
-    } else if (current && typeof node === "object" && node !== null) {
-      current.parts.push(jxTreeToText([node]));
+    const parts = current ? current.parts : preambleParts;
+    if (typeof node === "string") {
+      parts.push(node);
+    } else if (typeof node === "object" && node !== null) {
+      parts.push(jxTreeToText([node]));
     }
   }
   if (current) {
     sections.push(finishSection(current));
   }
-  return sections;
+  return { preamble: normalizeText(preambleParts), sections };
 }
 function finishSection(section) {
   const { heading, anchor, depth } = section;
-  return { anchor, depth, heading, text: section.parts.join(" ").replaceAll(/\s+/g, " ").trim() };
+  return { anchor, depth, heading, text: normalizeText(section.parts) };
+}
+function normalizeText(parts) {
+  return parts.join(" ").replaceAll(/\s+/g, " ").trim();
 }
 function headingOf(node) {
   if (typeof node !== "object" || node === null || !node.tagName) {
@@ -266,6 +271,8 @@ var SearchIndex = {
         const description = typeof data.description === "string" ? data.description : "";
         const children = entry.$children;
         const documentId = locale === undefined ? `${name}:${entry.id}` : `${name}:${locale}:${entry.id}`;
+        const split = collection.sections ? splitEntry(children, collection.sectionDepth) : undefined;
+        const sections = split?.sections ?? [];
         documents.push({
           id: documentId,
           collection: name,
@@ -274,23 +281,21 @@ var SearchIndex = {
           title,
           description,
           heading: "",
-          text: jxTreeToText(children),
+          text: sections.length > 0 && split ? split.preamble : jxTreeToText(children),
           ...locale === undefined ? {} : { locale }
         });
-        if (collection.sections) {
-          for (const section of splitSections(children, collection.sectionDepth)) {
-            documents.push({
-              id: `${documentId}#${section.anchor}`,
-              collection: name,
-              slug: entry.id,
-              url: `${url}#${section.anchor}`,
-              title,
-              description,
-              heading: section.heading,
-              text: section.text,
-              ...locale === undefined ? {} : { locale }
-            });
-          }
+        for (const section of sections) {
+          documents.push({
+            id: `${documentId}#${section.anchor}`,
+            collection: name,
+            slug: entry.id,
+            url: `${url}#${section.anchor}`,
+            title,
+            description,
+            heading: section.heading,
+            text: section.text,
+            ...locale === undefined ? {} : { locale }
+          });
         }
       }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@jxsuite/search",
-  "version": "0.2.4",
+  "version": "0.2.6",
   "description": "Static site search for Jx: build-time index emission from content collections and a headless MiniSearch browser client",
   "license": "MIT",
   "repository": {
@@ -33,10 +33,10 @@
     "build": "bun run prepare",
     "test": "bun test --isolate",
     "test:coverage": "bun test --isolate --coverage",
-    "upgrade": "bunx npm-check-updates -u && bun install"
+    "upgrade": "bun update --latest && bun install"
   },
   "dependencies": {
-    "@jxsuite/schema": "^1.9.0",
+    "@jxsuite/schema": "^2.1.0",
     "minisearch": "^7.2.0"
   },
   "devDependencies": {

package/src/client.ts

@@ -99,6 +99,14 @@ interface ClientState {
 
 const state: ClientState = { indexUrl: "/search-index.json", loading: null, mini: null };
 
+/**
+ * Documents indexed per chunk before yielding the main thread.
+ *
+ * Small enough that one chunk stays well inside a frame on a low-end phone, large enough that the
+ * per-chunk scheduling overhead stays noise against the tokenising itself.
+ */
+const INDEX_CHUNK_SIZE = 50;
+
 /** `text` is stored so flat results can carry a highlighted excerpt of the body. */
 const STORE_FIELDS = [
   "collection",
@@ -291,7 +299,14 @@ export function preload(indexUrl?: string): Promise<void> {
       },
       storeFields: STORE_FIELDS,
     });
-    mini.addAll(envelope.documents);
+    /*
+     * `addAll` is synchronous: it tokenises the entire corpus in one uninterruptible task, which on
+     * jxsuite.com's 1,342-document index measured 181 ms on a desktop and surfaced as a ~1.2 s
+     * "Unattributable" long task under Lighthouse's mobile CPU throttle — the single largest
+     * contributor to Total Blocking Time. `addAllAsync` does identical work and yields between
+     * chunks, so the same milliseconds no longer block paint or input.
+     */
+    await mini.addAllAsync(envelope.documents, { chunkSize: INDEX_CHUNK_SIZE });
     state.mini = mini;
   })().catch((error: unknown) => {
     state.loading = null;

package/src/search-index.ts

@@ -11,7 +11,7 @@
  */
 
 import { canonicalizeLocale, localeUrlPrefix, resolveI18n } from "@jxsuite/schema/locale";
-import { entryUrl, jxTreeToText, normalizeSearchConfig, splitSections } from "./shared.ts";
+import { entryUrl, jxTreeToText, normalizeSearchConfig, splitEntry } from "./shared.ts";
 import type { NormalizedSearchConfig } from "./shared.ts";
 import type { ContentLoaderEntry, JxElement, ProjectConfig } from "@jxsuite/schema/types";
 
@@ -123,6 +123,20 @@ export const SearchIndex = {
         const documentId =
           locale === undefined ? `${name}:${entry.id}` : `${name}:${locale}:${entry.id}`;
 
+        /*
+         * The preamble and the sections partition the entry, so the page document carries only the
+         * preamble when sections were produced — emitting the full text as well indexed the whole
+         * corpus twice, doubling both the bytes a visitor downloads and the tokenising their main
+         * thread does before search can answer.
+         *
+         * An entry that yielded no sections keeps its full text: it has no headings, or none with
+         * an id, and nothing else would index it.
+         */
+        const split = collection.sections
+          ? splitEntry(children, collection.sectionDepth)
+          : undefined;
+        const sections = split?.sections ?? [];
+
         documents.push({
           id: documentId,
           collection: name,
@@ -131,24 +145,22 @@ export const SearchIndex = {
           title,
           description,
           heading: "",
-          text: jxTreeToText(children),
+          text: sections.length > 0 && split ? split.preamble : jxTreeToText(children),
           ...(locale === undefined ? {} : { locale }),
         });
 
-        if (collection.sections) {
-          for (const section of splitSections(children, collection.sectionDepth)) {
-            documents.push({
-              id: `${documentId}#${section.anchor}`,
-              collection: name,
-              slug: entry.id,
-              url: `${url}#${section.anchor}`,
-              title,
-              description,
-              heading: section.heading,
-              text: section.text,
-              ...(locale === undefined ? {} : { locale }),
-            });
-          }
+        for (const section of sections) {
+          documents.push({
+            id: `${documentId}#${section.anchor}`,
+            collection: name,
+            slug: entry.id,
+            url: `${url}#${section.anchor}`,
+            title,
+            description,
+            heading: section.heading,
+            text: section.text,
+            ...(locale === undefined ? {} : { locale }),
+          });
         }
       }
     }

package/src/shared.ts

@@ -114,18 +114,30 @@ export interface ContentSection {
 
 const HEADING_TAG = /^h([1-6])$/;
 
+/** A content entry split into the text before its first section and the sections themselves. */
+export interface SplitEntry {
+  /** Text before the first section-starting heading. Empty when the entry opens with one. */
+  preamble: string;
+  sections: ContentSection[];
+}
+
 /**
- * Split a content entry's rendered tree into heading sections for section-level search documents. A
- * heading of depth ≤ `sectionDepth` (with an assigned `id` — see parser.md §3.2) starts a section;
- * its text runs until the next section-starting heading. Content before the first heading belongs
- * to no section (the page-level document already indexes the full text). Deeper headings' text
- * stays inside the enclosing section.
+ * Split a content entry's rendered tree into a preamble and heading sections. A heading of depth ≤
+ * `sectionDepth` (with an assigned `id` — see parser.md §3.2) starts a section; its text runs until
+ * the next section-starting heading. Deeper headings' text stays inside the enclosing section.
+ *
+ * The preamble is returned rather than discarded because together the two **partition** the entry:
+ * a page document that also carried the full text stored the whole corpus a second time. On
+ * jxsuite.com that was 922,007 characters of page text against 899,502 of section text, of which
+ * only ~22,505 (2.4%) was preamble no section already covered — an index twice the size it needed
+ * to be, and twice the parse and tokenise cost on every visitor's main thread.
  */
-export function splitSections(
+export function splitEntry(
   children: (JxElement | string)[] | undefined,
   sectionDepth: number,
-): ContentSection[] {
+): SplitEntry {
   const sections: ContentSection[] = [];
+  const preambleParts: string[] = [];
   let current: (ContentSection & { parts: string[] }) | null = null;
 
   for (const node of children ?? []) {
@@ -137,21 +149,35 @@ export function splitSections(
       current = { ...heading, parts: [], text: "" };
       continue;
     }
-    if (current && typeof node === "string") {
-      current.parts.push(node);
-    } else if (current && typeof node === "object" && node !== null) {
-      current.parts.push(jxTreeToText([node]));
+    const parts = current ? current.parts : preambleParts;
+    if (typeof node === "string") {
+      parts.push(node);
+    } else if (typeof node === "object" && node !== null) {
+      parts.push(jxTreeToText([node]));
     }
   }
   if (current) {
     sections.push(finishSection(current));
   }
-  return sections;
+  return { preamble: normalizeText(preambleParts), sections };
+}
+
+/** {@link splitEntry}, sections only. */
+export function splitSections(
+  children: (JxElement | string)[] | undefined,
+  sectionDepth: number,
+): ContentSection[] {
+  return splitEntry(children, sectionDepth).sections;
 }
 
 function finishSection(section: ContentSection & { parts: string[] }): ContentSection {
   const { heading, anchor, depth } = section;
-  return { anchor, depth, heading, text: section.parts.join(" ").replaceAll(/\s+/g, " ").trim() };
+  return { anchor, depth, heading, text: normalizeText(section.parts) };
+}
+
+/** Join collected text parts into one whitespace-normalized string. */
+function normalizeText(parts: readonly string[]): string {
+  return parts.join(" ").replaceAll(/\s+/g, " ").trim();
 }
 
 function headingOf(