@dogsbay/search 0.2.0-beta.133 → 0.2.0-beta.134

package/README.md

@@ -54,6 +54,8 @@ if (page) index.add(page.doc, page.parts);
 for (const [file, content] of index.emitFiles()) write(`_search/${file}`, content);
 ```
 
+On a large site, call `index.takeReadyFiles()` after each `add` and write what it yields: each fragment file is handed over as soon as it is complete, so the builder does not hold every page's excerpt to the end. `emitFiles()` then writes the rest. The builder keeps its postings in typed arrays (about 4 bytes a number): dogsbay-ssg measured OpenShift's 1,799 pages building with search on in a 128 MB Worker heap, which the per-term arrays did not allow. Add every page before calling `emitFiles()`: `add()` after it throws.
+
 - `@dogsbay/search/index`: `SearchIndexBuilder`, and `extractText()` for a dogsbay document tree. It reads API reference nodes too: an OpenAPI operation's method and path, summary, description, parameters and schema properties (nested ones included); autodoc symbol names (as headings) and docstrings; and an embedded spec's descriptions. Code (signatures, source, samples) is skipped, as code blocks are.
 - `@dogsbay/search/html`: `extractHtml()`.
 - `@dogsbay/search/query`: `search()`, `parseQuery()`, `scriptData()`, `memoryData()`.

package/dist/index.d.ts

@@ -54,17 +54,41 @@ declare class SearchIndexBuilder {
     /** Token counts per document and field (title, headings, body), for length normalisation. */
     private lengths;
     /**
-     * term → flat [doc, counts, doc, counts, …]. Documents arrive in order, so
-     * each page's counts are summed locally and appended once. Flat arrays
-     * cost a fraction of a Map per term, which matters inside a Worker.
+     * The postings as one append-only log of (term id, counts) pairs in Int32Array chunks, in
+     * document order, and where each document's pairs start. An array per term cost about 16
+     * bytes a number with its header, Map entry and growth slack: on OpenShift (1,799 pages,
+     * 49,014 terms) 20.7 MB of a 128 MB Worker, which search-on could not build in. The log is
+     * 4 bytes a number; `emitFiles` groups it by term once (request from dogsbay-ssg, 2026-10-22).
      */
-    private postings;
+    private chunks;
+    /** Numbers written to the log. */
+    private used;
+    /** Where each document's pairs start in the log. */
+    private docStart;
+    /** term → id, in first-seen order (shard grouping and key order follow it). */
+    private termIds;
+    /** Postings per term id. */
+    private counts;
+    /** Fragment blocks already handed over by `takeReadyFiles`. */
+    private fragmentsOut;
+    /** Set once `emitFiles` starts: the log is grouped then, and pages can no longer be added. */
+    private emitted;
+    /** The log grouped by term ([doc, counts, …] per term id from `offset[id]`), kept until every shard is written. */
+    private grouped?;
     add(doc: SearchDoc, parts: {
         title: string;
         headings: string[];
         body: string;
         keywords?: string[];
     }): void;
+    /**
+     * The fragment files complete so far (every BLOCK documents make one), each handed over once
+     * and then let go: a caller that drains this after each `add` does not hold every page's
+     * fragment to the end of the build (2.5M characters on OpenShift). `emitFiles` writes the rest.
+     */
+    takeReadyFiles(): Generator<[string, string]>;
+    /** Fragment block `b`. */
+    private fragmentFile;
     /** The stored score of one posting: saturated, length-normalised field scores, weighted and summed. */
     private scorer;
     get size(): number;
@@ -82,6 +106,9 @@ declare class SearchIndexBuilder {
      * one shard per term prefix. Each shard is built, serialised and handed
      * over before the next, and its postings are dropped, so writing the index
      * needs memory for one shard rather than for every shard and its JSON.
+     * The postings log is grouped by term first, each chunk let go once read.
+     * A call stopped part way leaves the shards not yet written to the next call;
+     * no page can be added once it has started.
      */
     emitFiles(): Generator<[string, string]>;
 }

package/dist/index.js

@@ -18,6 +18,7 @@ const MAX_DEPTH = 3;
 function keyDepth(k) {
   return k.replace(/_[0-9a-f]+_/g, "x").length;
 }
+const CHUNK = 1 << 18;
 class SearchIndexBuilder {
   constructor(opts = {}) {
     this.opts = opts;
@@ -27,12 +28,29 @@ class SearchIndexBuilder {
   /** Token counts per document and field (title, headings, body), for length normalisation. */
   lengths = [];
   /**
-   * term → flat [doc, counts, doc, counts, …]. Documents arrive in order, so
-   * each page's counts are summed locally and appended once. Flat arrays
-   * cost a fraction of a Map per term, which matters inside a Worker.
+   * The postings as one append-only log of (term id, counts) pairs in Int32Array chunks, in
+   * document order, and where each document's pairs start. An array per term cost about 16
+   * bytes a number with its header, Map entry and growth slack: on OpenShift (1,799 pages,
+   * 49,014 terms) 20.7 MB of a 128 MB Worker, which search-on could not build in. The log is
+   * 4 bytes a number; `emitFiles` groups it by term once (request from dogsbay-ssg, 2026-10-22).
    */
-  postings = /* @__PURE__ */ new Map();
+  chunks = [];
+  /** Numbers written to the log. */
+  used = 0;
+  /** Where each document's pairs start in the log. */
+  docStart = [];
+  /** term → id, in first-seen order (shard grouping and key order follow it). */
+  termIds = /* @__PURE__ */ new Map();
+  /** Postings per term id. */
+  counts = new Int32Array(1024);
+  /** Fragment blocks already handed over by `takeReadyFiles`. */
+  fragmentsOut = 0;
+  /** Set once `emitFiles` starts: the log is grouped then, and pages can no longer be added. */
+  emitted = false;
+  /** The log grouped by term ([doc, counts, …] per term id from `offset[id]`), kept until every shard is written. */
+  grouped;
   add(doc, parts) {
+    if (this.emitted) throw new Error("SearchIndexBuilder: add() after emitFiles(): the index has been written");
     const id = this.docs.length;
     this.docs.push({
       title: detach(doc.title),
@@ -57,12 +75,45 @@ class SearchIndexBuilder {
     for (const h of parts.headings) count(h, B_HEAD, 15, 1);
     count(parts.body, 1, 4095, 2);
     this.lengths.push(len[0], len[1], len[2]);
+    this.docStart.push(this.used);
     for (const [t, v] of counts) {
-      const list = this.postings.get(t);
-      if (list) list.push(id, v);
-      else this.postings.set(detach(t), [id, v]);
+      let term = this.termIds.get(t);
+      if (term === void 0) {
+        term = this.termIds.size;
+        this.termIds.set(detach(t), term);
+        if (term === this.counts.length) {
+          const grown = new Int32Array(this.counts.length * 2);
+          grown.set(this.counts);
+          this.counts = grown;
+        }
+      }
+      this.counts[term]++;
+      const at = this.used % CHUNK;
+      if (at === 0) this.chunks.push(new Int32Array(CHUNK));
+      const chunk = this.chunks[this.chunks.length - 1];
+      chunk[at] = term;
+      chunk[at + 1] = v;
+      this.used += 2;
     }
   }
+  /**
+   * The fragment files complete so far (every BLOCK documents make one), each handed over once
+   * and then let go: a caller that drains this after each `add` does not hold every page's
+   * fragment to the end of the build (2.5M characters on OpenShift). `emitFiles` writes the rest.
+   */
+  *takeReadyFiles() {
+    if (this.emitted) return;
+    while ((this.fragmentsOut + 1) * BLOCK <= this.docs.length) {
+      const b = this.fragmentsOut++;
+      yield this.fragmentFile(b);
+      for (let i = b * BLOCK; i < (b + 1) * BLOCK; i++) this.docs[i].fragment = "";
+    }
+  }
+  /** Fragment block `b`. */
+  fragmentFile(b) {
+    const part = this.docs.slice(b * BLOCK, (b + 1) * BLOCK);
+    return [file.fragment(b), wrapJson(key.fragment(b), `[${part.map((d) => d.fragment).join(",")}]`)];
+  }
   /** The stored score of one posting: saturated, length-normalised field scores, weighted and summed. */
   scorer() {
     const n = this.docs.length || 1;
@@ -96,7 +147,7 @@ class SearchIndexBuilder {
    */
   shardTerms() {
     const out = /* @__PURE__ */ new Map();
-    const size = (terms) => terms.reduce((n, t) => n + t.length + 4 + this.postings.get(t).length * 4, 0);
+    const size = (terms) => terms.reduce((n, t) => n + t.length + 4 + this.counts[this.termIds.get(t)] * 2 * 4, 0);
     const place = (terms, depth) => {
       const groups = /* @__PURE__ */ new Map();
       for (const t of terms) {
@@ -113,7 +164,7 @@ class SearchIndexBuilder {
         } else out.set(k, g);
       }
     };
-    place([...this.postings.keys()], 1);
+    place([...this.termIds.keys()], 1);
     return out;
   }
   /** Emit the index files, keyed by path relative to the unit's `_dogsbay/search/` dir. */
@@ -125,8 +176,12 @@ class SearchIndexBuilder {
    * one shard per term prefix. Each shard is built, serialised and handed
    * over before the next, and its postings are dropped, so writing the index
    * needs memory for one shard rather than for every shard and its JSON.
+   * The postings log is grouped by term first, each chunk let go once read.
+   * A call stopped part way leaves the shards not yet written to the next call;
+   * no page can be added once it has started.
    */
   *emitFiles() {
+    this.emitted = true;
     const byShard = this.shardTerms();
     const depth = Math.max(1, ...[...byShard.keys()].map((k) => keyDepth(k)));
     const score = this.scorer();
@@ -147,24 +202,46 @@ class SearchIndexBuilder {
     });
     const meta = { format: FORMAT, depth, count: this.docs.length, shards: [...byShard.keys()].sort(), sections, tags, block: BLOCK, lang: this.opts.lang, docs };
     yield [file.docs, wrap(key.docs, meta)];
-    for (let b = 0; b * BLOCK < this.docs.length; b++) {
-      const part = this.docs.slice(b * BLOCK, (b + 1) * BLOCK);
-      yield [file.fragment(b), wrapJson(key.fragment(b), `[${part.map((d) => d.fragment).join(",")}]`)];
+    for (let b = this.fragmentsOut; b * BLOCK < this.docs.length; b++) yield this.fragmentFile(b);
+    if (!this.grouped) {
+      const n = this.termIds.size;
+      const offset2 = new Int32Array(n + 1);
+      for (let i = 0; i < n; i++) offset2[i + 1] = offset2[i] + this.counts[i] * 2;
+      const flat2 = new Int32Array(this.used);
+      const fill = offset2.slice(0, n);
+      let doc = -1;
+      for (let p = 0; p < this.used; p += 2) {
+        while (doc + 1 < this.docStart.length && this.docStart[doc + 1] <= p) doc++;
+        const c = p / CHUNK | 0;
+        const at = p % CHUNK;
+        const chunk = this.chunks[c];
+        const term = chunk[at];
+        flat2[fill[term]] = doc;
+        flat2[fill[term] + 1] = chunk[at + 1];
+        fill[term] += 2;
+        if (at + 2 === CHUNK) this.chunks[c] = void 0;
+      }
+      this.chunks = [];
+      this.docStart = [];
+      this.used = 0;
+      this.grouped = { flat: flat2, offset: offset2 };
     }
+    const { flat, offset } = this.grouped;
     for (const [k, terms] of byShard) {
       const data = {};
       for (const term of terms) {
-        const flat = this.postings.get(term);
+        const id = this.termIds.get(term);
         const pairs = [];
-        for (let i = 0; i < flat.length; i += 2) pairs.push([flat[i], score(flat[i], flat[i + 1])]);
+        for (let i = offset[id]; i < offset[id + 1]; i += 2) pairs.push([flat[i], score(flat[i], flat[i + 1])]);
         pairs.sort((a, b) => b[1] - a[1]);
         const out = [];
         for (const [d, sc] of pairs) out.push(d, sc);
         data[term] = out;
-        this.postings.delete(term);
+        this.termIds.delete(term);
       }
       yield [file.shard(k), wrap(key.shard(k), data)];
     }
+    this.grouped = void 0;
   }
 }
 const str = (v) => typeof v === "string" ? v : "";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dogsbay/search",
-  "version": "0.2.0-beta.133",
+  "version": "0.2.0-beta.134",
   "description": "Static search for docs sites: builds in a Worker, queries from file://",
   "type": "module",
   "main": "./dist/entry.js",
@@ -42,7 +42,7 @@
   ],
   "dependencies": {
     "stemmer": "^2.0.1",
-    "@dogsbay/types": "0.2.0-beta.133"
+    "@dogsbay/types": "0.2.0-beta.134"
   },
   "devDependencies": {
     "@playwright/test": "1.61.1",
@@ -52,7 +52,7 @@
     "tsup": "^8.5.1",
     "typescript": "^5.9.3",
     "vitest": "^4.1.10",
-    "@dogsbay/format-openapi": "0.2.0-beta.133"
+    "@dogsbay/format-openapi": "0.2.0-beta.134"
   },
   "license": "MIT",
   "repository": {