crbro-memory 2.10.0 → 2.11.0

package/dist/search/semantic.js

@@ -32,7 +32,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.SemanticIndex = exports.SEMANTIC_DTYPE_DEFAULT = exports.SEMANTIC_MODEL_DEFAULT = void 0;
+exports.SemanticIndex = exports.SEMANTIC_MODEL_FILES = exports.SEMANTIC_RUNTIME = exports.SEMANTIC_DTYPE_DEFAULT = exports.SEMANTIC_MODEL_DEFAULT = void 0;
+exports.verifyModelFiles = verifyModelFiles;
 exports.semanticModel = semanticModel;
 exports.semanticDtype = semanticDtype;
 exports.semanticEnabled = semanticEnabled;
@@ -43,6 +44,7 @@ exports.semanticStatus = semanticStatus;
 const node_module_1 = require("node:module");
 const node_url_1 = require("node:url");
 const node_fs_1 = require("node:fs");
+const node_crypto_1 = require("node:crypto");
 const node_path_1 = __importDefault(require("node:path"));
 const node_os_1 = __importDefault(require("node:os"));
 /**
@@ -54,6 +56,81 @@ const node_os_1 = __importDefault(require("node:os"));
  */
 exports.SEMANTIC_MODEL_DEFAULT = 'Xenova/multilingual-e5-small';
 exports.SEMANTIC_DTYPE_DEFAULT = 'q8';
+/**
+ * The runtime `semantic install` puts in the semantic home, pinned to an
+ * exact version (2.11; it was the floating range `@3`, so every install could
+ * pull a different release of a package that runs install scripts).
+ */
+exports.SEMANTIC_RUNTIME = '@huggingface/transformers@3.8.1';
+/**
+ * The default model, pinned by content (2.11). The Hub revision is a moving
+ * `main`; these are the SHA-256 of the files the benchmarks were measured
+ * with (downloaded 2026-09-03). A model whose files differ is refused, never
+ * used: the layer reports why and recall stays keyword-only. Only checked for
+ * the default model and dtype; CRBRO_SEMANTIC_MODEL users pick their own.
+ */
+exports.SEMANTIC_MODEL_FILES = {
+    'config.json': { sha256: 'cb99455288675345e1a4f411438d5d0adbba5fbd3a67ea4fb03c015433b996c1', bytes: 658 },
+    'tokenizer.json': { sha256: '0b44a9d7b51c3c62626640cda0e2c2f70fdacdc25bbbd68038369d14ebdf4c39', bytes: 17_082_730 },
+    'tokenizer_config.json': { sha256: 'a1d6bc8734a6f635dc158508bef000f8e2e5a759c7d92f984b2c86e5ff53425b', bytes: 443 },
+    'onnx/model_quantized.onnx': { sha256: 'f80102d3f2a1229f387d3c81909990d8945513e347b0eab049f7de3c6f98c193', bytes: 118_308_185 },
+};
+async function sha256Of(file) {
+    const h = (0, node_crypto_1.createHash)('sha256');
+    await new Promise((resolve, reject) => {
+        (0, node_fs_1.createReadStream)(file).on('data', d => h.update(d)).on('end', () => resolve()).on('error', reject);
+    });
+    return h.digest('hex');
+}
+/**
+ * Check the cached files of the default model against SEMANTIC_MODEL_FILES.
+ * Returns null when they match (or the model is not the default one), else
+ * what is wrong.
+ */
+async function verifyModelFiles(home = semanticHome()) {
+    if (semanticModel() !== exports.SEMANTIC_MODEL_DEFAULT || semanticDtype() !== exports.SEMANTIC_DTYPE_DEFAULT)
+        return null;
+    const dir = node_path_1.default.join(home, 'models', ...exports.SEMANTIC_MODEL_DEFAULT.split('/'));
+    // Each file is hashed once per version of it: one whose size and date are
+    // those of the last successful check is not read again. Hashing the 135 MB
+    // cost ~140 ms at every process start, before the first semantic answer.
+    const marca = node_path_1.default.join(dir, VERIFIED_FILE);
+    let vistos = {};
+    try {
+        vistos = JSON.parse(await node_fs_1.promises.readFile(marca, 'utf8')) || {};
+    }
+    catch { /* first check */ }
+    let nuevos = false;
+    for (const [rel, want] of Object.entries(exports.SEMANTIC_MODEL_FILES)) {
+        const file = node_path_1.default.join(dir, ...rel.split('/'));
+        let st;
+        try {
+            st = await node_fs_1.promises.stat(file);
+        }
+        catch {
+            return `model file missing: ${rel} (run \`npx crbro-memory semantic install\`)`;
+        }
+        const mismatch = `model file ${rel} does not match the pinned hash; refusing to use it (delete ${dir} and run \`npx crbro-memory semantic install\`)`;
+        if (st.size !== want.bytes)
+            return mismatch;
+        const v = vistos[rel];
+        if (v && v.size === st.size && v.mtime === st.mtimeMs && v.sha256 === want.sha256)
+            continue;
+        if ((await sha256Of(file)) !== want.sha256)
+            return mismatch;
+        vistos[rel] = { size: st.size, mtime: st.mtimeMs, sha256: want.sha256 };
+        nuevos = true;
+    }
+    if (nuevos) {
+        try {
+            await node_fs_1.promises.writeFile(marca, JSON.stringify(vistos));
+        }
+        catch { /* checked again next time */ }
+    }
+    return null;
+}
+/** Where verifyModelFiles remembers the files it already hashed. */
+const VERIFIED_FILE = '.crbro-verified.json';
 function semanticModel() { return process.env.CRBRO_SEMANTIC_MODEL || exports.SEMANTIC_MODEL_DEFAULT; }
 function semanticDtype() { return process.env.CRBRO_SEMANTIC_DTYPE || exports.SEMANTIC_DTYPE_DEFAULT; }
 /**
@@ -75,6 +152,10 @@ const QUERY_PREFIX = 'query: ';
 const PASSAGE_PREFIX = 'passage: ';
 /** Longer passages are truncated by the tokenizer anyway; cap the work. */
 const MAX_CHARS = 2000;
+/** A background pass is committed in slices of this many passages (2.11). */
+const UPSERT_SLICE = 64;
+/** ...and saved this often while it runs, so a process that leaves keeps what it did. */
+const UPSERT_SAVE_MS = 30_000;
 /** Off when CRBRO_SEMANTIC says so, forced on when it says so, else on exactly when the runtime is installed. */
 function semanticEnabled() {
     const mode = semanticMode();
@@ -118,6 +199,7 @@ function semanticStatus() {
 }
 class SemanticIndex {
     dir;
+    options;
     ids = [];
     /** Vector width, read from the first embedding (384 for e5-small). */
     dim = 0;
@@ -130,8 +212,48 @@ class SemanticIndex {
     broken = null;
     /** Upserts run one at a time: two overlapping ones would both extend the same array. */
     queue = Promise.resolve();
-    constructor(dir) {
+    /** Bumped on every change of the vectors, so a write knows whether it caught up. */
+    version = 0;
+    /**
+     * Queries waiting for the model (2.11). A background pass (thousands of
+     * passages after an upgrade split long entries into parts) pauses between
+     * passages while any is waiting: a recall used to queue behind it, ~2 s
+     * each, for the minutes the pass lasted.
+     */
+    foreground = 0;
+    foregroundWaiters = [];
+    /** Passages a running pass still has to embed. */
+    left = 0;
+    /** A large pass is running: more than one slice of passages still to embed. */
+    backfilling() { return this.left > UPSERT_SLICE; }
+    /**
+     * Something in the foreground (a search, a query) is running: background
+     * passages wait until every hold is released. Returns the release.
+     */
+    hold() {
+        this.foreground++;
+        let released = false;
+        return () => {
+            if (released)
+                return;
+            released = true;
+            this.foreground--;
+            if (this.foreground === 0) {
+                const w = this.foregroundWaiters;
+                this.foregroundWaiters = [];
+                for (const f of w)
+                    f();
+            }
+        };
+    }
+    /**
+     * allowDownload: only `semantic install` may fetch the model from the Hub
+     * (2.11). A server never downloads during a query: a missing model makes
+     * the layer report why and recall stays keyword-only.
+     */
+    constructor(dir, options = {}) {
         this.dir = dir;
+        this.options = options;
     }
     metaPath() { return node_path_1.default.join(this.dir, 'vectors.meta.json'); }
     dataPath() { return node_path_1.default.join(this.dir, 'vectors.f32'); }
@@ -153,11 +275,14 @@ class SemanticIndex {
     async load() {
         try {
             const meta = JSON.parse(await node_fs_1.promises.readFile(this.metaPath(), 'utf8'));
-            if (meta.model !== semanticModel() || !(meta.dim > 0))
+            // Vectors of another model or another quantisation are not comparable.
+            if (meta.model !== semanticModel() || (meta.dtype && meta.dtype !== semanticDtype()) || !(meta.dim > 0))
                 return;
             const buf = await node_fs_1.promises.readFile(this.dataPath());
             const expected = meta.ids.length * meta.dim * 4;
-            if (buf.byteLength !== expected)
+            // A torn pair (a crash between the two renames) does not match: start
+            // empty and let the index refill what is missing (2.11).
+            if (buf.byteLength !== expected || (meta.bytes !== undefined && meta.bytes !== buf.byteLength))
                 return;
             this.vectors = new Float32Array(buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength));
             this.ids = meta.ids;
@@ -171,15 +296,20 @@ class SemanticIndex {
     async persist() {
         if (!this.dirty)
             return;
+        // What is written is this moment's pair; a vector added while the files
+        // are written keeps the layer dirty for the next write.
+        const version = this.version;
+        const vectors = this.vectors;
+        const meta = { model: semanticModel(), dtype: semanticDtype(), dim: this.dim, ids: [...this.ids], bytes: vectors.byteLength };
         await node_fs_1.promises.mkdir(this.dir, { recursive: true });
-        const meta = { model: semanticModel(), dtype: semanticDtype(), dim: this.dim, ids: this.ids };
         const tmpMeta = this.metaPath() + '.tmp';
         const tmpData = this.dataPath() + '.tmp';
-        await node_fs_1.promises.writeFile(tmpData, Buffer.from(this.vectors.buffer, this.vectors.byteOffset, this.vectors.byteLength));
+        await node_fs_1.promises.writeFile(tmpData, Buffer.from(vectors.buffer, vectors.byteOffset, vectors.byteLength));
         await node_fs_1.promises.writeFile(tmpMeta, JSON.stringify(meta));
         await node_fs_1.promises.rename(tmpData, this.dataPath());
         await node_fs_1.promises.rename(tmpMeta, this.metaPath());
-        this.dirty = false;
+        if (this.version === version)
+            this.dirty = false;
     }
     /** Load transformers.js from the semantic home and build the extractor. */
     async model() {
@@ -198,8 +328,23 @@ class SemanticIndex {
                 const tf = mod.pipeline ? mod : mod.default;
                 tf.env.cacheDir = node_path_1.default.join(semanticHome(), 'models');
                 tf.env.allowLocalModels = true;
-                tf.env.allowRemoteModels = true;
+                tf.env.allowRemoteModels = this.options.allowDownload === true;
+                if (!this.options.allowDownload) {
+                    const wrong = await verifyModelFiles();
+                    if (wrong) {
+                        this.broken = wrong;
+                        throw new Error(wrong);
+                    }
+                }
                 this.extractor = await tf.pipeline('feature-extraction', semanticModel(), { dtype: semanticDtype() });
+                if (this.options.allowDownload) {
+                    const wrong = await verifyModelFiles();
+                    if (wrong) {
+                        this.extractor = null;
+                        this.broken = wrong;
+                        throw new Error(wrong);
+                    }
+                }
                 return this.extractor;
             })().catch(err => {
                 this.broken = this.broken || `semantic model failed to load: ${err instanceof Error ? err.message : String(err)}`;
@@ -215,6 +360,9 @@ class SemanticIndex {
         const out = [];
         const BATCH = batchSize();
         for (let i = 0; i < texts.length; i += BATCH) {
+            // Background work gives way to a query that is waiting (2.11).
+            while (kind === 'passage' && this.foreground > 0)
+                await new Promise(r => this.foregroundWaiters.push(r));
             const batch = texts.slice(i, i + BATCH).map(t => prefix + t.slice(0, MAX_CHARS));
             const res = await extractor(batch, { pooling: 'mean', normalize: true });
             // res.data is a flat typed array of batch × dim.
@@ -243,23 +391,57 @@ class SemanticIndex {
         const nuevos = entries.filter(e => e.text && !this.pos.has(e.id));
         if (nuevos.length === 0)
             return 0;
-        const vecs = await this.embed(nuevos.map(e => e.text), 'passage');
+        // In slices, each one usable at once and the whole saved every half
+        // minute (2.11): a pass of thousands used to land all at the end, so a
+        // process that left halfway — a classic session — kept nothing, and the
+        // next one started over.
+        let guardado = Date.now();
+        this.left = nuevos.length;
+        try {
+            for (let i = 0; i < nuevos.length; i += UPSERT_SLICE) {
+                const tramo = nuevos.slice(i, i + UPSERT_SLICE);
+                const vecs = await this.embed(tramo.map(e => e.text), 'passage');
+                this.append(tramo, vecs);
+                this.left = Math.max(0, nuevos.length - i - tramo.length);
+                if (this.left > 0 && Date.now() - guardado > UPSERT_SAVE_MS) {
+                    await this.persist().catch(() => undefined);
+                    guardado = Date.now();
+                }
+            }
+        }
+        finally {
+            this.left = 0;
+        }
+        return nuevos.length;
+    }
+    append(tramo, vecs) {
         // The base offset is captured once: ids.push inside the loop would
         // otherwise move it with every vector. Found as a RangeError, silently
         // swallowed upstream — which is why the first benchmark run showed no
         // change at all. Measure the effect, not the artefact.
         const base = this.ids.length;
-        const grown = new Float32Array((base + nuevos.length) * this.dim);
-        grown.set(this.vectors, 0);
-        for (let i = 0; i < nuevos.length; i++) {
-            grown.set(vecs[i], (base + i) * this.dim);
-            this.pos.set(nuevos[i].id, base + i);
-            this.ids.push(nuevos[i].id);
+        const need = (base + tramo.length) * this.dim;
+        // Room grows by half each time (2.11): a pass that added 64 vectors at a
+        // time reallocated and copied the whole array each slice — tens of MB
+        // every couple of seconds, and the collector paid for it in pauses.
+        let store = this.vectors.buffer.byteLength / 4 >= need && this.vectors.byteOffset === 0
+            ? new Float32Array(this.vectors.buffer)
+            : null;
+        if (!store) {
+            store = new Float32Array(Math.max(need, Math.ceil((this.vectors.length || 0) * 1.5), 256 * this.dim));
+            store.set(this.vectors, 0);
         }
-        this.vectors = grown;
+        for (let i = 0; i < tramo.length; i++) {
+            store.set(vecs[i], (base + i) * this.dim);
+            this.pos.set(tramo[i].id, base + i);
+            this.ids.push(tramo[i].id);
+        }
+        this.vectors = store.subarray(0, need);
+        this.version++;
         this.dirty = true;
-        return nuevos.length;
     }
+    /** Every id holding a vector (to compare with the index: B14). */
+    idList() { return this.ids; }
     /** Drop vectors for ids that left the index. */
     remove(ids) {
         const gone = new Set();
@@ -281,6 +463,7 @@ class SemanticIndex {
         this.ids = keepIds;
         this.vectors = keep;
         this.pos = new Map(keepIds.map((id, i) => [id, i]));
+        this.version++;
         this.dirty = true;
         return gone.size;
     }
@@ -288,7 +471,14 @@ class SemanticIndex {
     async query(text, k) {
         if (this.ids.length === 0)
             return [];
-        const [q] = await this.embed([text], 'query');
+        const release = this.hold();
+        let q;
+        try {
+            [q] = await this.embed([text], 'query');
+        }
+        finally {
+            release();
+        }
         const scores = new Float32Array(this.ids.length);
         for (let i = 0; i < this.ids.length; i++) {
             let dot = 0;

package/dist/search/tokenize.js

@@ -11,6 +11,8 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.STOPWORDS = void 0;
 exports.fold = fold;
 exports.tokenize = tokenize;
+exports.indexTokens = indexTokens;
+exports.createIndexTokenizer = createIndexTokenizer;
 exports.queryTerms = queryTerms;
 exports.synonymsEnabled = synonymsEnabled;
 exports.variants = variants;
@@ -59,6 +61,48 @@ function tokenize(text) {
         .split(/[^a-z0-9_]+/)
         .filter(Boolean);
 }
+/**
+ * The terms a stored text is indexed under (2.11). Exactly the query side's
+ * fold and split — until 2.10 the index used Orama's English splitter, which
+ * cuts BEFORE folding and keeps only à è é ì ò ó ù inside a word, so
+ * "página" was stored as "p" + "gina" and "diseño" as "dise" + "o" while the
+ * query asked for "pagina" and "diseno": 72 % of the entries of a real
+ * Spanish brain were unreachable by any word with á, í, ú, ñ or ü. The same
+ * splitter kept "crbro-memory" as one token, so "memory" found nothing; the
+ * hyphen now splits on both sides. A snake_case identifier is kept whole (a
+ * query keeps it whole too) and its parts are added, so "lista" also finds
+ * "ocultar_de_lista".
+ */
+function indexTokens(text) {
+    const out = [];
+    for (const t of tokenize(text)) {
+        out.push(t);
+        if (t.includes('_'))
+            for (const p of t.split('_'))
+                if (p)
+                    out.push(p);
+    }
+    return out;
+}
+/**
+ * An Orama tokenizer component built on the functions above. Orama calls it
+ * with the property name when it indexes a document and without one when it
+ * tokenizes a search term: documents get indexTokens(), terms tokenize(), so
+ * both sides share one normalisation by construction. Tokens are
+ * de-duplicated per text, like Orama's own tokenizer does by default, so
+ * BM25 sees the same term frequencies it always saw.
+ */
+function createIndexTokenizer() {
+    return {
+        language: 'english',
+        normalizationCache: new Map(),
+        tokenize(raw, _language, prop) {
+            if (typeof raw !== 'string')
+                return [raw];
+            return [...new Set(prop ? indexTokens(raw) : tokenize(raw))];
+        },
+    };
+}
 /**
  * Tokenize a search query: drop stopwords and 1-char noise, dedupe,
  * and keep the original order. If everything was a stopword, fall back