@nxuss/lemma 1.38.0 → 1.40.0

package/dist/cjs/subconscious/TheBrainV2.js

@@ -14,7 +14,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.TheBrainV2 = exports.MAX_DERIVED_DEPTH = exports.BloomFilter = void 0;
+exports.TheBrainV2 = exports.MAX_DERIVED_DEPTH = exports.BloomFilter = exports.TOKENIZER_VERSION = void 0;
 exports.resetBrainKeyCache = resetBrainKeyCache;
 exports.listSnapshotNames = listSnapshotNames;
 exports.mergeEntryMaps = mergeEntryMaps;
@@ -658,6 +658,14 @@ function tokenize(text) {
         'cada', 'otra', 'otro', 'otras', 'otros', 'entre', 'sobre', 'sin', 'ya', 'tan', 'mas',
         'lo', 'le', 'les', 'nos', 'os', 'ni', 'sino', 'porque', 'cuando', 'donde', 'quien',
         'cual', 'cuales', 'sea', 'ha', 'han', 'hemos', 'habia', 'habían', 'esta', 'están',
+        // Conversational filler. Prompts are written to a person ("can you please help me…",
+        // "necesito que realmente…"), and these words outweighed the topic when measuring how much
+        // of a prompt a memory covers. They name no subject a memory could be about.
+        'please', 'help', 'know', 'want', 'like', 'just', 'really', 'actually', 'maybe', 'also', 'let',
+        'me', 'my', 'we', 'our', 'us', 'you', 'your', 'i', 'im', 'get', 'got', 'make', 'sure', 'thing', 'things',
+        'por', 'favor', 'necesito', 'quiero', 'puedes', 'podemos', 'podrias', 'haz', 'hacer', 'dime', 'ayuda',
+        'ayudame', 'realmente', 'actualmente', 'tambien', 'solo', 'algo', 'cosa', 'cosas', 'nos', 'tu', 'yo',
+        'me', 'mi', 'mis', 'te', 'ver', 'bien',
     ]);
     const terms = normalized
         .split(/\s+/)
@@ -679,7 +687,25 @@ function tokenize(text) {
     const synonymTags = terms
         .map(t => SYNONYM_CLUSTERS.get(t))
         .filter((c) => c !== undefined);
-    return [...new Set([...terms, ...bigrams, ...stems, ...synonymTags])];
+    // 8. English/Spanish cognates share a root once their suffix is dropped: "description" and
+    // "descripcion", "configuration"/"configuracion", "velocity"/"velocidad", "automatic"/
+    // "automatico". Same additive rule as stems and synonyms.
+    const cognateTags = terms.map(cognate).filter((c) => c !== undefined);
+    return [...new Set([...terms, ...bigrams, ...stems, ...synonymTags, ...cognateTags])];
+}
+/**
+ * Bump whenever tokenize() changes what it emits. Terms are computed once, at store time, and
+ * persisted, so a tokenizer improvement (stems, diacritics, synonym groups) used to reach new
+ * memories only — every memory already in the Brain kept the vocabulary it was stored with.
+ * rebuildIndex() re-tokenizes any entry indexed under another version.
+ */
+exports.TOKENIZER_VERSION = 6;
+/** (Re)computes an entry's index terms with the current tokenizer. */
+function indexTerms(entry) {
+    const terms = tokenize(`${entry.query} ${entry.response || ''}`);
+    entry.terms = [...new Set(terms)];
+    entry.termFreq = termFrequencies(terms);
+    entry.tokv = exports.TOKENIZER_VERSION;
 }
 // Groups of interchangeable dev-vocabulary terms (bilingual). Each member maps to a shared
 // `syn_<canonical>` token so any two synonyms in the same group become a shared index bucket
@@ -689,14 +715,43 @@ function tokenize(text) {
 const SYNONYM_GROUPS = [
     ['jwt', 'token', 'session', 'auth', 'login', 'autenticacion', 'sesion', 'autenticar'],
     ['timeout', 'expiration', 'expiry', 'vencimiento', 'caducidad', 'expirar', 'vencer'],
-    ['bug', 'error', 'fallo', 'falla', 'issue', 'defect', 'defecto'],
+    ['bug', 'error', 'fallo', 'falla', 'issue', 'defect', 'defecto', 'fail', 'fails', 'failing', 'failed', 'fallar', 'fallaba'],
     ['fix', 'patch', 'solucion', 'solucionar', 'arreglar', 'corregir', 'hotfix'],
     ['delete', 'remove', 'eliminar', 'borrar', 'quitar'],
     ['config', 'configuration', 'configuracion', 'settings', 'ajustes'],
     ['deploy', 'deployment', 'despliegue', 'desplegar', 'release'],
-    ['crash', 'exception', 'excepcion', 'throw', 'falla'],
+    ['crash', 'exception', 'excepcion', 'throw'],
     ['slow', 'lento', 'lentitud', 'latency', 'latencia', 'performance', 'rendimiento'],
     ['test', 'tests', 'testing', 'prueba', 'pruebas', 'spec'],
+    // Everyday words a bilingual corpus splits across languages: a Spanish memory and an English
+    // question about the same thing otherwise share no term at all.
+    ['change', 'changes', 'changed', 'cambio', 'cambios', 'cambiar', 'cambia', 'cambiado'],
+    ['active', 'activo', 'activos', 'activa', 'activas', 'enabled', 'habilitado'],
+    ['file', 'files', 'archivo', 'archivos', 'fichero'],
+    ['search', 'searches', 'buscar', 'busqueda', 'busca', 'find'],
+    ['memory', 'memories', 'memoria', 'memorias'],
+    ['write', 'writes', 'escribir', 'escribe', 'escritura'],
+    ['read', 'reads', 'leer', 'lee', 'lectura', 'lecturas'],
+    ['build', 'compile', 'compilar', 'compila', 'compilacion'],
+    ['publish', 'published', 'publicar', 'publicado', 'publicada', 'publicacion'],
+    ['comment', 'comments', 'comentario', 'comentarios'],
+    ['string', 'strings', 'cadena', 'cadenas'],
+    ['server', 'servidor'],
+    ['tool', 'tools', 'herramienta', 'herramientas'],
+    ['project', 'projects', 'proyecto', 'proyectos'],
+    ['broken', 'breaks', 'roto', 'rota', 'rompe', 'romper', 'rompia'],
+    ['missing', 'falta', 'faltante', 'faltaba'],
+    ['ignore', 'ignored', 'ignores', 'ignora', 'ignorar', 'ignorado', 'ignoraba'],
+    ['update', 'updated', 'actualizar', 'actualiza', 'actualizacion'],
+    ['stale', 'outdated', 'desactualizado', 'desactualizada', 'obsoleto', 'obsoleta'],
+    ['restart', 'reconnect', 'reiniciar', 'reconectar'],
+    ['wrong', 'incorrect', 'incorrecto', 'incorrecta', 'erroneo'],
+    // Money vocabulary: "¿por qué el saldo sale negativo?" shared only "negativo" with the memory
+    // that answers it ("why the ledger was negative") — and that with every "falso negativo".
+    ['ledger', 'saldo', 'balance', 'bitacora'],
+    ['negative', 'negativo', 'negativa', 'negativos', 'negativas'],
+    ['cost', 'costs', 'costo', 'costos', 'gasto', 'gastos', 'spent', 'spend', 'charge', 'charged', 'cobro', 'cobrar', 'cobrado', 'cobraba'],
+    ['savings', 'saving', 'saved', 'ahorro', 'ahorros', 'ahorrar', 'ahorra'],
 ];
 const SYNONYM_CLUSTERS = new Map();
 for (const group of SYNONYM_GROUPS) {
@@ -704,6 +759,24 @@ for (const group of SYNONYM_GROUPS) {
     for (const member of group)
         SYNONYM_CLUSTERS.set(member, canonical);
 }
+const COGNATE_RULES = [
+    [/(?:tions?|ciones|cion)$/, 'tion'],
+    [/(?:ities|ity|idades|idad)$/, 'ity'],
+    [/(?:ically|icamente)$/, 'ic'],
+    [/(?:ics?|icos?|icas?)$/, 'ic'],
+];
+/** `cog_<root><suffix>` for a word with an English/Spanish cognate suffix, else undefined. */
+function cognate(term) {
+    if (term.length < 6 || term.includes('_'))
+        return undefined;
+    for (const [re, kind] of COGNATE_RULES) {
+        if (re.test(term)) {
+            const root = term.replace(re, '');
+            return root.length >= 3 ? `cog_${root}_${kind}` : undefined;
+        }
+    }
+    return undefined;
+}
 // Suffix-stripping rules, longest/most-specific match first. Additive expansion only —
 // this is not a correctness-critical stemmer, so mild over-stemming (merging two distinct
 // words into one stem) is an acceptable recall/precision trade: it can only ever add a
@@ -873,6 +946,31 @@ function hashSymbolsForFreshness(refs) {
     }
     return hashes;
 }
+/**
+ * Splits symbol refs into the ones the resolver can actually locate and the ones it cannot
+ * (a typo, a rename, a file that is not code). An unlocatable symbol hashes MISSING when it is
+ * stored and MISSING again when it is verified, so MISSING === MISSING and the memory could
+ * never go stale — the worst way for freshness to fail, because the reply still says
+ * "tracking N symbols". Callers track the whole file for those instead and say so.
+ */
+function partitionSymbolRefs(refs) {
+    const resolved = [];
+    const unresolved = [];
+    for (const ref of refs) {
+        const hash = defaultLocalResolver.hashSymbol(path_1.default.resolve(ref.filePath), ref.symbolName);
+        (hash && hash.raw !== 'MISSING' ? resolved : unresolved).push(ref);
+    }
+    return { resolved, unresolved };
+}
+/** `files` minus those that contain one of the tracked `symbols`; `dropped` lists what was removed. */
+function dropFilesCoveredBySymbols(files, symbols) {
+    const covered = new Set(symbols.map((r) => path_1.default.resolve(r.filePath)));
+    const kept = [];
+    const dropped = [];
+    for (const f of files ?? [])
+        (covered.has(path_1.default.resolve(f)) ? dropped : kept).push(f);
+    return { files: kept, dropped };
+}
 /**
  * Fase D companion to hashSymbolsForFreshness: same symbols, hashed after
  * normalizeForFreshness strips comments/whitespace. Always computed alongside the raw hash
@@ -1265,6 +1363,13 @@ class TheBrainV2 {
         this.diskStamp = null;
         /** id -> ISO deletion time, for ids that must not come back through a merge. */
         this.tombstones = new Map();
+        /**
+         * Ids a restore brought back that the on-disk meta.json still lists as deleted. save() absorbs
+         * the peers' tombstones first, which re-added the one restore had just removed and dropped the
+         * entry again: "Restored memory…" was followed by the memory vanishing at the next save. Skipped
+         * by the absorb until a save has written meta.json without them.
+         */
+        this.revived = new Set();
         /**
          * Last integrity incident on the corpus files (corrupt main file, undecryptable
          * envelope, skipped sync). Set by load()/save()/syncIfChanged(), never cleared
@@ -1317,7 +1422,9 @@ class TheBrainV2 {
             // index.json. It is a pure derivation of the entries, so reconstructing it is the
             // only way it can never disagree with them — and a stale or half-written index.json
             // used to be able to hide entries from search entirely while they sat on disk intact.
-            this.rebuildIndex();
+            // Entries indexed by an older tokenizer were re-tokenized: write that back once.
+            if (this.rebuildIndex() > 0)
+                this.scheduleSave();
             this.recalcAvgDocLength();
         }
         catch (err) {
@@ -1343,7 +1450,12 @@ class TheBrainV2 {
         // A merge can replace an id's entry with a peer's newer version whose query differs, so
         // the memo is rebuilt alongside the index rather than trusted across one.
         this.queryTermCache.clear();
+        let retokenized = 0;
         for (const entry of this.entries.values()) {
+            if (entry.tokv !== exports.TOKENIZER_VERSION) {
+                indexTerms(entry);
+                retokenized++;
+            }
             for (const term of entry.terms) {
                 let ids = this.invertedIndex.get(term);
                 if (!ids) {
@@ -1353,6 +1465,7 @@ class TheBrainV2 {
                 ids.add(entry.id);
             }
         }
+        return retokenized;
     }
     /**
      * Pull in anything another process wrote since we last touched the corpus.
@@ -1511,6 +1624,7 @@ class TheBrainV2 {
             }));
             this.diskStamp = stampOf(ENTRIES_FILE);
             this.dirty = false;
+            this.revived.clear(); // meta.json no longer lists them
         }
         catch { /* fail silently */ }
         finally {
@@ -1529,6 +1643,8 @@ class TheBrainV2 {
      */
     absorbPeerTombstones() {
         for (const [id, at] of readTombstonesFile()) {
+            if (this.revived.has(id))
+                continue;
             if (!this.tombstones.has(id))
                 this.tombstones.set(id, at);
             const entry = this.entries.get(id);
@@ -1607,6 +1723,24 @@ class TheBrainV2 {
                 };
             }
         }
+        let unresolvedSymbols;
+        let narrowedFiles;
+        if (symbolRefs && symbolRefs.length > 0 && !options.evidence?.symbols) {
+            const split = partitionSymbolRefs(symbolRefs);
+            if (split.unresolved.length > 0) {
+                unresolvedSymbols = split.unresolved;
+                symbolRefs = split.resolved;
+                filePaths = [...new Set([...(filePaths ?? []), ...split.unresolved.map((r) => r.filePath)])];
+            }
+            // A file whose symbols are tracked is not also tracked whole: the symbol is the narrower
+            // claim, and the whole file would stale the memory on every unrelated edit to it (callers
+            // routinely pass both: the file the symbol lives in, and the symbol).
+            const narrowed = dropFilesCoveredBySymbols(filePaths, split.resolved);
+            if (narrowed.dropped.length > 0) {
+                filePaths = narrowed.files;
+                narrowedFiles = narrowed.dropped;
+            }
+        }
         // Conflict check: another entry about the exact same symbol, with the opposite
         // outcome, is worth flagging before this one is even written — two memories about one
         // function disagreeing on whether an approach works is a trap for whoever searches next.
@@ -1651,6 +1785,7 @@ class TheBrainV2 {
             response: response.trim(),
             terms: [...new Set(terms)],
             termFreq,
+            tokv: exports.TOKENIZER_VERSION,
             provider,
             timestamp: options.timestamp || new Date().toISOString(),
             hits: 0,
@@ -1687,7 +1822,7 @@ class TheBrainV2 {
         this.recalcAvgDocLength();
         this.evictIfOverCapacity();
         this.scheduleSave();
-        return { stored: true, reason: 'Stored successfully', id, conflicts, textConflicts, redactions };
+        return { stored: true, reason: 'Stored successfully', id, conflicts, textConflicts, redactions, ...(unresolvedSymbols ? { unresolvedSymbols } : {}), ...(narrowedFiles ? { narrowedFiles } : {}) };
     }
     /**
      * Evict the lowest-value entries once the Brain is over capacity. Value = hits (proven
@@ -1762,17 +1897,15 @@ class TheBrainV2 {
         }
         // 2b. Scope to the calling project. Entries with no projectId predate scoping and stay
         // visible everywhere — dropping a user's existing memories to enforce a new field would
-        // look exactly like the Brain losing data. If scoping leaves nothing, fall back to the
-        // unscoped candidates rather than reporting a miss the Brain could have answered.
-        if (options.projectId) {
-            const scoped = new Set([...candidateIds].filter((id) => {
+        // look exactly like the Brain losing data — but they are labelled 'unscoped' in the result.
+        // Another project's memories are never returned unless the caller asks for allProjects:
+        // this used to fall back to every project's candidates whenever the caller's own had none,
+        // so a project with an empty Brain was answered from someone else's.
+        if (options.projectId && !options.allProjects) {
+            for (const id of [...candidateIds]) {
                 const owner = this.entries.get(id)?.projectId;
-                return owner === undefined || owner === options.projectId;
-            }));
-            if (scoped.size > 0 || options.strictProject) {
-                candidateIds.clear();
-                for (const id of scoped)
-                    candidateIds.add(id);
+                if (owner !== undefined && owner !== options.projectId)
+                    candidateIds.delete(id);
             }
         }
         if (candidateIds.size === 0) {
@@ -1918,6 +2051,11 @@ class TheBrainV2 {
                 ...(r.entry.outcome ? { outcome: r.entry.outcome } : {}),
                 ...(claims && claims.length > 0 ? { claims } : {}),
                 ...(r.entry.domain ? { domain: r.entry.domain } : {}),
+                ...(r.entry.projectId === undefined
+                    ? { scope: 'unscoped' }
+                    : options.projectId && r.entry.projectId !== options.projectId
+                        ? { scope: 'other-project', ownerProject: r.entry.projectId }
+                        : {}),
                 ...(cosmeticChanges && cosmeticChanges.length > 0 ? { cosmeticChanges } : {}),
             };
         });
@@ -2007,6 +2145,36 @@ class TheBrainV2 {
         }
         return results;
     }
+    /**
+     * Give memories stored before projects existed to the project their evidence lives in.
+     *
+     * An unscoped memory is visible to every project, so one project's notes surface in another's
+     * searches. When all the code it tracks sits inside `root`, there is no doubt whose it is, and
+     * it is stamped with `projectId`. A memory that tracks nothing, or tracks files outside `root`,
+     * is left alone: nothing proves who it belongs to, and guessing would hide it from the project
+     * that does own it. Returns how many were adopted.
+     */
+    adoptUnscopedByEvidence(projectId, root) {
+        const base = root.endsWith(path_1.default.sep) ? root : root + path_1.default.sep;
+        let adopted = 0;
+        for (const entry of this.entries.values()) {
+            if (entry.projectId !== undefined)
+                continue;
+            const tracked = [
+                ...Object.keys(entry.fileHashes || {}),
+                ...Object.keys(entry.symbolHashes || {}).map((k) => k.substring(0, k.lastIndexOf('::'))),
+            ];
+            if (tracked.length === 0)
+                continue;
+            if (!tracked.every((abs) => path_1.default.resolve(abs).startsWith(base)))
+                continue;
+            entry.projectId = projectId;
+            adopted++;
+        }
+        if (adopted > 0)
+            this.scheduleSave();
+        return adopted;
+    }
     embeddingSidecar() {
         if (!this.sidecar)
             this.sidecar = new BrainEmbeddings_1.EmbeddingSidecar(EMBEDDINGS_FILE);
@@ -2120,6 +2288,21 @@ class TheBrainV2 {
      * whether a reuse happened. Unknown ids fail silently (a peer may have forgotten the
      * entry since the search ranked it); savings attribution must never break a search.
      */
+    /** Inverse document frequency of one index term over the whole corpus (BM25's formula). */
+    termIdf(term) {
+        this.syncIfChanged();
+        const N = this.entries.size;
+        const df = this.invertedIndex.get(term)?.size ?? 0;
+        return Math.log((N - df + 0.5) / (df + 0.5) + 1);
+    }
+    /** One reuse of a memory the agent chose and read (see search_memory's shortlist). */
+    recordReuse(id) {
+        const entry = this.entries.get(id);
+        if (!entry)
+            return;
+        entry.hits++;
+        this.scheduleSave();
+    }
     creditSavings(id, tokens) {
         if (!Number.isFinite(tokens) || tokens <= 0)
             return;
@@ -2252,6 +2435,7 @@ class TheBrainV2 {
         const terms = tokenize(keeper.query + ' ' + keeper.response);
         keeper.terms = [...new Set(terms)];
         keeper.termFreq = termFrequencies(terms);
+        keeper.tokv = exports.TOKENIZER_VERSION;
         keeper.charCount = keeper.query.length + keeper.response.length;
         this.entries.set(keepId, keeper);
         for (const term of keeper.terms) {
@@ -2363,8 +2547,10 @@ class TheBrainV2 {
             this.bloom.add(entry.query.trim().toLowerCase().substring(0, 200));
         }
         // A restored id was deliberately brought back — its old tombstone must not suppress it.
-        for (const id of this.entries.keys())
-            this.tombstones.delete(id);
+        for (const id of this.entries.keys()) {
+            if (this.tombstones.delete(id))
+                this.revived.add(id);
+        }
         this.scheduleSave();
         return { ok: true, restored: entries.size, message: `Restored ${entries.size} memories from snapshot "${name}". The pre-restore state was snapshotted, so this is undoable too.` };
     }
@@ -2447,6 +2633,7 @@ class TheBrainV2 {
         }
         this.bloom.add(revived.query.trim().toLowerCase().substring(0, 200));
         this.tombstones.delete(id);
+        this.revived.add(id);
         this.recalcAvgDocLength();
         this.scheduleSave();
         return { ok: true, message: `Restored memory "${id}" from the trash (${revived.hits || 0} prior reuse(s) kept). Its freshness will be re-judged on the next search.` };
@@ -2541,12 +2728,16 @@ class TheBrainV2 {
         if (hasNewTracking) {
             // Re-point: the new tracking replaces the old wholesale rather than merging, because
             // a moved symbol's old path must stop counting against the entry forever.
-            entry.fileHashes = opts.filePaths?.length ? hashFilesForFreshness(opts.filePaths) : undefined;
-            entry.symbolHashes = opts.symbols?.length ? hashSymbolsForFreshness(opts.symbols) : undefined;
-            entry.symbolNormalizedHashes = opts.symbols?.length ? hashSymbolsNormalizedForFreshness(opts.symbols) : undefined;
-            for (const p of opts.filePaths || [])
+            // A symbol the resolver cannot locate would be "fresh" forever (MISSING === MISSING), so
+            // its file is tracked whole instead, and the label says file, not symbol.
+            const split = partitionSymbolRefs(opts.symbols ?? []);
+            const files = dropFilesCoveredBySymbols([...new Set([...(opts.filePaths ?? []), ...split.unresolved.map((r) => r.filePath)])], split.resolved).files;
+            entry.fileHashes = files.length ? hashFilesForFreshness(files) : undefined;
+            entry.symbolHashes = split.resolved.length ? hashSymbolsForFreshness(split.resolved) : undefined;
+            entry.symbolNormalizedHashes = split.resolved.length ? hashSymbolsNormalizedForFreshness(split.resolved) : undefined;
+            for (const p of files)
                 retracked.push(rel(path_1.default.resolve(p)));
-            for (const s of opts.symbols || [])
+            for (const s of split.resolved)
                 retracked.push(`${rel(path_1.default.resolve(s.filePath))}::${s.symbolName}`);
         }
         else {
@@ -2564,10 +2755,19 @@ class TheBrainV2 {
                     const sep = key.lastIndexOf('::');
                     return { filePath: key.substring(0, sep), symbolName: key.substring(sep + 2) };
                 });
-                entry.symbolHashes = hashSymbolsForFreshness(refs);
-                entry.symbolNormalizedHashes = hashSymbolsNormalizedForFreshness(refs);
+                // A symbol that no longer resolves keeps the hash it was stored with: re-hashing it would
+                // write MISSING, which is what it will read as forever, and bless a deleted symbol.
+                const gone = new Set(partitionSymbolRefs(refs).unresolved.map((r) => `${path_1.default.resolve(r.filePath)}::${r.symbolName}`));
+                const rehashed = hashSymbolsForFreshness(refs);
+                const rehashedNorm = hashSymbolsNormalizedForFreshness(refs);
+                for (const key of gone) {
+                    rehashed[key] = entry.symbolHashes[key] ?? rehashed[key];
+                    rehashedNorm[key] = entry.symbolNormalizedHashes?.[key] ?? rehashedNorm[key];
+                }
+                entry.symbolHashes = rehashed;
+                entry.symbolNormalizedHashes = rehashedNorm;
                 for (const r of refs)
-                    retracked.push(`${rel(r.filePath)}::${r.symbolName}`);
+                    retracked.push(`${rel(r.filePath)}::${r.symbolName}${gone.has(`${path_1.default.resolve(r.filePath)}::${r.symbolName}`) ? ' (no longer found — stays stale)' : ''}`);
             }
             if (entry.claims) {
                 for (const claim of entry.claims) {
@@ -2695,6 +2895,7 @@ class TheBrainV2 {
         const terms = tokenize(entry.query + ' ' + entry.response);
         entry.terms = [...new Set(terms)];
         entry.termFreq = termFrequencies(terms);
+        entry.tokv = exports.TOKENIZER_VERSION;
         entry.charCount = entry.query.length + entry.response.length;
         // Claims supplied in this patch were just hashed by buildClaims; the helper re-anchors the
         // ones carried over, so either way the entry's evidence describes the code as it is now.