@nxuss/lemma 1.38.0 → 1.39.0

package/dist/esm/subconscious/TheBrainV2.js

@@ -631,6 +631,14 @@ export function tokenize(text) {
         'cada', 'otra', 'otro', 'otras', 'otros', 'entre', 'sobre', 'sin', 'ya', 'tan', 'mas',
         'lo', 'le', 'les', 'nos', 'os', 'ni', 'sino', 'porque', 'cuando', 'donde', 'quien',
         'cual', 'cuales', 'sea', 'ha', 'han', 'hemos', 'habia', 'habían', 'esta', 'están',
+        // Conversational filler. Prompts are written to a person ("can you please help me…",
+        // "necesito que realmente…"), and these words outweighed the topic when measuring how much
+        // of a prompt a memory covers. They name no subject a memory could be about.
+        'please', 'help', 'know', 'want', 'like', 'just', 'really', 'actually', 'maybe', 'also', 'let',
+        'me', 'my', 'we', 'our', 'us', 'you', 'your', 'i', 'im', 'get', 'got', 'make', 'sure', 'thing', 'things',
+        'por', 'favor', 'necesito', 'quiero', 'puedes', 'podemos', 'podrias', 'haz', 'hacer', 'dime', 'ayuda',
+        'ayudame', 'realmente', 'actualmente', 'tambien', 'solo', 'algo', 'cosa', 'cosas', 'nos', 'tu', 'yo',
+        'me', 'mi', 'mis', 'te', 'ver', 'bien',
     ]);
     const terms = normalized
         .split(/\s+/)
@@ -652,7 +660,25 @@ export function tokenize(text) {
     const synonymTags = terms
         .map(t => SYNONYM_CLUSTERS.get(t))
         .filter((c) => c !== undefined);
-    return [...new Set([...terms, ...bigrams, ...stems, ...synonymTags])];
+    // 8. English/Spanish cognates share a root once their suffix is dropped: "description" and
+    // "descripcion", "configuration"/"configuracion", "velocity"/"velocidad", "automatic"/
+    // "automatico". Same additive rule as stems and synonyms.
+    const cognateTags = terms.map(cognate).filter((c) => c !== undefined);
+    return [...new Set([...terms, ...bigrams, ...stems, ...synonymTags, ...cognateTags])];
+}
+/**
+ * Bump whenever tokenize() changes what it emits. Terms are computed once, at store time, and
+ * persisted, so a tokenizer improvement (stems, diacritics, synonym groups) used to reach new
+ * memories only — every memory already in the Brain kept the vocabulary it was stored with.
+ * rebuildIndex() re-tokenizes any entry indexed under another version.
+ */
+export const TOKENIZER_VERSION = 6;
+/** (Re)computes an entry's index terms with the current tokenizer. */
+function indexTerms(entry) {
+    const terms = tokenize(`${entry.query} ${entry.response || ''}`);
+    entry.terms = [...new Set(terms)];
+    entry.termFreq = termFrequencies(terms);
+    entry.tokv = TOKENIZER_VERSION;
 }
 // Groups of interchangeable dev-vocabulary terms (bilingual). Each member maps to a shared
 // `syn_<canonical>` token so any two synonyms in the same group become a shared index bucket
@@ -662,14 +688,43 @@ export function tokenize(text) {
 const SYNONYM_GROUPS = [
     ['jwt', 'token', 'session', 'auth', 'login', 'autenticacion', 'sesion', 'autenticar'],
     ['timeout', 'expiration', 'expiry', 'vencimiento', 'caducidad', 'expirar', 'vencer'],
-    ['bug', 'error', 'fallo', 'falla', 'issue', 'defect', 'defecto'],
+    ['bug', 'error', 'fallo', 'falla', 'issue', 'defect', 'defecto', 'fail', 'fails', 'failing', 'failed', 'fallar', 'fallaba'],
     ['fix', 'patch', 'solucion', 'solucionar', 'arreglar', 'corregir', 'hotfix'],
     ['delete', 'remove', 'eliminar', 'borrar', 'quitar'],
     ['config', 'configuration', 'configuracion', 'settings', 'ajustes'],
     ['deploy', 'deployment', 'despliegue', 'desplegar', 'release'],
-    ['crash', 'exception', 'excepcion', 'throw', 'falla'],
+    ['crash', 'exception', 'excepcion', 'throw'],
     ['slow', 'lento', 'lentitud', 'latency', 'latencia', 'performance', 'rendimiento'],
     ['test', 'tests', 'testing', 'prueba', 'pruebas', 'spec'],
+    // Everyday words a bilingual corpus splits across languages: a Spanish memory and an English
+    // question about the same thing otherwise share no term at all.
+    ['change', 'changes', 'changed', 'cambio', 'cambios', 'cambiar', 'cambia', 'cambiado'],
+    ['active', 'activo', 'activos', 'activa', 'activas', 'enabled', 'habilitado'],
+    ['file', 'files', 'archivo', 'archivos', 'fichero'],
+    ['search', 'searches', 'buscar', 'busqueda', 'busca', 'find'],
+    ['memory', 'memories', 'memoria', 'memorias'],
+    ['write', 'writes', 'escribir', 'escribe', 'escritura'],
+    ['read', 'reads', 'leer', 'lee', 'lectura', 'lecturas'],
+    ['build', 'compile', 'compilar', 'compila', 'compilacion'],
+    ['publish', 'published', 'publicar', 'publicado', 'publicada', 'publicacion'],
+    ['comment', 'comments', 'comentario', 'comentarios'],
+    ['string', 'strings', 'cadena', 'cadenas'],
+    ['server', 'servidor'],
+    ['tool', 'tools', 'herramienta', 'herramientas'],
+    ['project', 'projects', 'proyecto', 'proyectos'],
+    ['broken', 'breaks', 'roto', 'rota', 'rompe', 'romper', 'rompia'],
+    ['missing', 'falta', 'faltante', 'faltaba'],
+    ['ignore', 'ignored', 'ignores', 'ignora', 'ignorar', 'ignorado', 'ignoraba'],
+    ['update', 'updated', 'actualizar', 'actualiza', 'actualizacion'],
+    ['stale', 'outdated', 'desactualizado', 'desactualizada', 'obsoleto', 'obsoleta'],
+    ['restart', 'reconnect', 'reiniciar', 'reconectar'],
+    ['wrong', 'incorrect', 'incorrecto', 'incorrecta', 'erroneo'],
+    // Money vocabulary: "¿por qué el saldo sale negativo?" shared only "negativo" with the memory
+    // that answers it ("why the ledger was negative") — and that with every "falso negativo".
+    ['ledger', 'saldo', 'balance', 'bitacora'],
+    ['negative', 'negativo', 'negativa', 'negativos', 'negativas'],
+    ['cost', 'costs', 'costo', 'costos', 'gasto', 'gastos', 'spent', 'spend', 'charge', 'charged', 'cobro', 'cobrar', 'cobrado', 'cobraba'],
+    ['savings', 'saving', 'saved', 'ahorro', 'ahorros', 'ahorrar', 'ahorra'],
 ];
 const SYNONYM_CLUSTERS = new Map();
 for (const group of SYNONYM_GROUPS) {
@@ -677,6 +732,24 @@ for (const group of SYNONYM_GROUPS) {
     for (const member of group)
         SYNONYM_CLUSTERS.set(member, canonical);
 }
+const COGNATE_RULES = [
+    [/(?:tions?|ciones|cion)$/, 'tion'],
+    [/(?:ities|ity|idades|idad)$/, 'ity'],
+    [/(?:ically|icamente)$/, 'ic'],
+    [/(?:ics?|icos?|icas?)$/, 'ic'],
+];
+/** `cog_<root><suffix>` for a word with an English/Spanish cognate suffix, else undefined. */
+function cognate(term) {
+    if (term.length < 6 || term.includes('_'))
+        return undefined;
+    for (const [re, kind] of COGNATE_RULES) {
+        if (re.test(term)) {
+            const root = term.replace(re, '');
+            return root.length >= 3 ? `cog_${root}_${kind}` : undefined;
+        }
+    }
+    return undefined;
+}
 // Suffix-stripping rules, longest/most-specific match first. Additive expansion only —
 // this is not a correctness-critical stemmer, so mild over-stemming (merging two distinct
 // words into one stem) is an acceptable recall/precision trade: it can only ever add a
@@ -1289,7 +1362,9 @@ export class TheBrainV2 {
             // index.json. It is a pure derivation of the entries, so reconstructing it is the
             // only way it can never disagree with them — and a stale or half-written index.json
             // used to be able to hide entries from search entirely while they sat on disk intact.
-            this.rebuildIndex();
+            // Entries indexed by an older tokenizer were re-tokenized: write that back once.
+            if (this.rebuildIndex() > 0)
+                this.scheduleSave();
             this.recalcAvgDocLength();
         }
         catch (err) {
@@ -1315,7 +1390,12 @@ export class TheBrainV2 {
         // A merge can replace an id's entry with a peer's newer version whose query differs, so
         // the memo is rebuilt alongside the index rather than trusted across one.
         this.queryTermCache.clear();
+        let retokenized = 0;
         for (const entry of this.entries.values()) {
+            if (entry.tokv !== TOKENIZER_VERSION) {
+                indexTerms(entry);
+                retokenized++;
+            }
             for (const term of entry.terms) {
                 let ids = this.invertedIndex.get(term);
                 if (!ids) {
@@ -1325,6 +1405,7 @@ export class TheBrainV2 {
                 ids.add(entry.id);
             }
         }
+        return retokenized;
     }
     /**
      * Pull in anything another process wrote since we last touched the corpus.
@@ -1623,6 +1704,7 @@ export class TheBrainV2 {
             response: response.trim(),
             terms: [...new Set(terms)],
             termFreq,
+            tokv: TOKENIZER_VERSION,
             provider,
             timestamp: options.timestamp || new Date().toISOString(),
             hits: 0,
@@ -1734,17 +1816,15 @@ export class TheBrainV2 {
         }
         // 2b. Scope to the calling project. Entries with no projectId predate scoping and stay
         // visible everywhere — dropping a user's existing memories to enforce a new field would
-        // look exactly like the Brain losing data. If scoping leaves nothing, fall back to the
-        // unscoped candidates rather than reporting a miss the Brain could have answered.
-        if (options.projectId) {
-            const scoped = new Set([...candidateIds].filter((id) => {
+        // look exactly like the Brain losing data — but they are labelled 'unscoped' in the result.
+        // Another project's memories are never returned unless the caller asks for allProjects:
+        // this used to fall back to every project's candidates whenever the caller's own had none,
+        // so a project with an empty Brain was answered from someone else's.
+        if (options.projectId && !options.allProjects) {
+            for (const id of [...candidateIds]) {
                 const owner = this.entries.get(id)?.projectId;
-                return owner === undefined || owner === options.projectId;
-            }));
-            if (scoped.size > 0 || options.strictProject) {
-                candidateIds.clear();
-                for (const id of scoped)
-                    candidateIds.add(id);
+                if (owner !== undefined && owner !== options.projectId)
+                    candidateIds.delete(id);
             }
         }
         if (candidateIds.size === 0) {
@@ -1890,6 +1970,11 @@ export class TheBrainV2 {
                 ...(r.entry.outcome ? { outcome: r.entry.outcome } : {}),
                 ...(claims && claims.length > 0 ? { claims } : {}),
                 ...(r.entry.domain ? { domain: r.entry.domain } : {}),
+                ...(r.entry.projectId === undefined
+                    ? { scope: 'unscoped' }
+                    : options.projectId && r.entry.projectId !== options.projectId
+                        ? { scope: 'other-project', ownerProject: r.entry.projectId }
+                        : {}),
                 ...(cosmeticChanges && cosmeticChanges.length > 0 ? { cosmeticChanges } : {}),
             };
         });
@@ -1979,6 +2064,36 @@ export class TheBrainV2 {
         }
         return results;
     }
+    /**
+     * Give memories stored before projects existed to the project their evidence lives in.
+     *
+     * An unscoped memory is visible to every project, so one project's notes surface in another's
+     * searches. When all the code it tracks sits inside `root`, there is no doubt whose it is, and
+     * it is stamped with `projectId`. A memory that tracks nothing, or tracks files outside `root`,
+     * is left alone: nothing proves who it belongs to, and guessing would hide it from the project
+     * that does own it. Returns how many were adopted.
+     */
+    adoptUnscopedByEvidence(projectId, root) {
+        const base = root.endsWith(path.sep) ? root : root + path.sep;
+        let adopted = 0;
+        for (const entry of this.entries.values()) {
+            if (entry.projectId !== undefined)
+                continue;
+            const tracked = [
+                ...Object.keys(entry.fileHashes || {}),
+                ...Object.keys(entry.symbolHashes || {}).map((k) => k.substring(0, k.lastIndexOf('::'))),
+            ];
+            if (tracked.length === 0)
+                continue;
+            if (!tracked.every((abs) => path.resolve(abs).startsWith(base)))
+                continue;
+            entry.projectId = projectId;
+            adopted++;
+        }
+        if (adopted > 0)
+            this.scheduleSave();
+        return adopted;
+    }
     embeddingSidecar() {
         if (!this.sidecar)
             this.sidecar = new EmbeddingSidecar(EMBEDDINGS_FILE);
@@ -2092,6 +2207,21 @@ export class TheBrainV2 {
      * whether a reuse happened. Unknown ids fail silently (a peer may have forgotten the
      * entry since the search ranked it); savings attribution must never break a search.
      */
+    /** Inverse document frequency of one index term over the whole corpus (BM25's formula). */
+    termIdf(term) {
+        this.syncIfChanged();
+        const N = this.entries.size;
+        const df = this.invertedIndex.get(term)?.size ?? 0;
+        return Math.log((N - df + 0.5) / (df + 0.5) + 1);
+    }
+    /** One reuse of a memory the agent chose and read (see search_memory's shortlist). */
+    recordReuse(id) {
+        const entry = this.entries.get(id);
+        if (!entry)
+            return;
+        entry.hits++;
+        this.scheduleSave();
+    }
     creditSavings(id, tokens) {
         if (!Number.isFinite(tokens) || tokens <= 0)
             return;
@@ -2224,6 +2354,7 @@ export class TheBrainV2 {
         const terms = tokenize(keeper.query + ' ' + keeper.response);
         keeper.terms = [...new Set(terms)];
         keeper.termFreq = termFrequencies(terms);
+        keeper.tokv = TOKENIZER_VERSION;
         keeper.charCount = keeper.query.length + keeper.response.length;
         this.entries.set(keepId, keeper);
         for (const term of keeper.terms) {
@@ -2667,6 +2798,7 @@ export class TheBrainV2 {
         const terms = tokenize(entry.query + ' ' + entry.response);
         entry.terms = [...new Set(terms)];
         entry.termFreq = termFrequencies(terms);
+        entry.tokv = TOKENIZER_VERSION;
         entry.charCount = entry.query.length + entry.response.length;
         // Claims supplied in this patch were just hashed by buildClaims; the helper re-anchors the
         // ones carried over, so either way the entry's evidence describes the code as it is now.