noormme 1.0.6 → 1.2.0

package/dist/esm/agentic/improvement/KnowledgeDistiller.js

@@ -9,6 +9,7 @@ export class KnowledgeDistiller {
     config;
     knowledgeTable;
     linksTable;
+    bloomFilter = new Set();
     constructor(db, // accepting any Kysely but casting internally for our specific tables
     config = {}) {
         this.db = db;
@@ -26,6 +27,28 @@ export class KnowledgeDistiller {
      * Add or update a knowledge item with robust merging and transaction support.
      */
     async distill(entity, fact, confidence, sourceSessionId, tags = [], metadata = {}, source = 'assistant') {
+        // Pass 6: Bloom filter Heuristic
+        // Lightweight de-duplication for ultra-scale throughput.
+        // If the fact matches a very recent pattern, skip expensive DB checks.
+        const factHash = this.bloomHash(`${entity}:${fact}`);
+        if (this.bloomFilter.has(factHash)) {
+            // Likely a duplicate, but we'll do a quick check if it's "verified" to avoid re-distilling
+            console.log(`[KnowledgeDistiller] Bloom filter hit for ${entity}. Skipping redundant distillation.`);
+            const quickMatch = await this.db
+                .selectFrom(this.knowledgeTable)
+                .select(['id', 'status'])
+                .where('entity', '=', entity)
+                .where('fact', '=', fact)
+                .executeTakeFirst();
+            if (quickMatch &&
+                quickMatch.status === 'verified' &&
+                source !== 'user') {
+                return (await this.getKnowledgeByEntity(entity)).find((k) => k.fact === fact);
+            }
+        }
+        this.bloomFilter.add(factHash);
+        if (this.bloomFilter.size > 1000)
+            this.bloomFilter.clear(); // Simple rolling window
         return await this.db.transaction().execute(async (trx) => {
             // Check for exact match first
             const existing = await trx
@@ -36,10 +59,14 @@ export class KnowledgeDistiller {
                 .executeTakeFirst();
             if (existing) {
                 // Merge tags
-                const existingTags = existing.tags ? JSON.parse(existing.tags) : [];
+                const existingTags = existing.tags
+                    ? JSON.parse(existing.tags)
+                    : [];
                 const mergedTags = Array.from(new Set([...existingTags, ...tags]));
                 // Merge metadata
-                const existingMeta = existing.metadata ? JSON.parse(existing.metadata) : {};
+                const existingMeta = existing.metadata
+                    ? JSON.parse(existing.metadata)
+                    : {};
                 // Hallucination Prevention: Tracking contributing sessions
                 const sessions = new Set(existingMeta.sessions || []);
                 if (sourceSessionId)
@@ -48,7 +75,7 @@ export class KnowledgeDistiller {
                     ...existingMeta,
                     ...metadata,
                     sessions: Array.from(sessions),
-                    session_count: sessions.size
+                    session_count: sessions.size,
                 };
                 // Source weighting: User verification boosts confidence faster
                 const boost = source === 'user' ? 0.2 : 0.05;
@@ -66,7 +93,7 @@ export class KnowledgeDistiller {
                     tags: JSON.stringify(mergedTags),
                     metadata: JSON.stringify(mergedMeta),
                     updated_at: new Date(),
-                    source_session_id: sourceSessionId ?? existing.source_session_id
+                    source_session_id: sourceSessionId ?? existing.source_session_id,
                 })
                     .where('id', '=', existing.id)
                     .returningAll()
@@ -80,7 +107,7 @@ export class KnowledgeDistiller {
                 ...metadata,
                 source,
                 sessions: sourceSessionId ? [sourceSessionId] : [],
-                session_count: sourceSessionId ? 1 : 0
+                session_count: sourceSessionId ? 1 : 0,
             };
             const created = await trx
                 .insertInto(this.knowledgeTable)
@@ -93,7 +120,7 @@ export class KnowledgeDistiller {
                 tags: JSON.stringify(tags),
                 metadata: JSON.stringify(initialMeta),
                 created_at: new Date(),
-                updated_at: new Date()
+                updated_at: new Date(),
             })
                 .returningAll()
                 .executeTakeFirstOrThrow();
@@ -115,7 +142,9 @@ export class KnowledgeDistiller {
             .executeTakeFirst();
         if (!existing)
             return null;
-        const metadata = typeof existing.metadata === 'string' ? JSON.parse(existing.metadata) : (existing.metadata || {});
+        const metadata = typeof existing.metadata === 'string'
+            ? JSON.parse(existing.metadata)
+            : existing.metadata || {};
         // Hallucination Guard: Cap confidence for non-user sources until cross-session verification
         let maxConfidence = 1.0;
         if (metadata.source !== 'user' && (metadata.session_count || 0) < 3) {
@@ -133,7 +162,7 @@ export class KnowledgeDistiller {
             .set({
             confidence: newConfidence,
             status: newStatus,
-            updated_at: new Date()
+            updated_at: new Date(),
         })
             .where('id', '=', id)
             .returningAll()
@@ -151,14 +180,14 @@ export class KnowledgeDistiller {
             .where('entity', '=', entity)
             .orderBy('confidence', 'desc');
         const items = (await query.execute());
-        const parsed = items.map(i => this.parseKnowledge(i));
+        const parsed = items.map((i) => this.parseKnowledge(i));
         let filtered = parsed;
         if (filterTags && filterTags.length > 0) {
-            filtered = parsed.filter(item => item.tags?.some(tag => filterTags.includes(tag)));
+            filtered = parsed.filter((item) => item.tags?.some((tag) => filterTags.includes(tag)));
         }
         // Record hits asynchronously
         for (const item of filtered) {
-            this.recordHit(item.id).catch(err => console.error(`[KnowledgeDistiller] Failed to record hit for ${item.id}:`, err));
+            this.recordHit(item.id).catch((err) => console.error(`[KnowledgeDistiller] Failed to record hit for ${item.id}:`, err));
         }
         return filtered;
     }
@@ -175,14 +204,14 @@ export class KnowledgeDistiller {
             return;
         const metadata = typeof existing.metadata === 'string'
             ? JSON.parse(existing.metadata)
-            : (existing.metadata || {});
+            : existing.metadata || {};
         metadata.hit_count = (metadata.hit_count || 0) + 1;
         metadata.last_retrieved_at = new Date().toISOString();
         await this.db
             .updateTable(this.knowledgeTable)
             .set({
             metadata: JSON.stringify(metadata),
-            updated_at: new Date()
+            updated_at: new Date(),
         })
             .where('id', '=', id)
             .execute();
@@ -192,7 +221,7 @@ export class KnowledgeDistiller {
             .values({
             metric_name: `entity_hit_${existing.entity}`,
             metric_value: 1,
-            created_at: new Date()
+            created_at: new Date(),
         })
             .execute();
     }
@@ -209,7 +238,7 @@ export class KnowledgeDistiller {
         const stn = Math.min(1.0, hitCount / ageInDays);
         // Source Multiplier: User verified facts get 1.0, Assistant-generated 0.7
         const sourceMult = metadata.source === 'user' ? 1.0 : 0.7;
-        return (item.confidence * 0.4) + (stn * 0.4) + (sourceMult * 0.2);
+        return item.confidence * 0.4 + stn * 0.4 + sourceMult * 0.2;
     }
     /**
      * Challenge existing knowledge with new evidence.
@@ -236,12 +265,18 @@ export class KnowledgeDistiller {
                 let newStatus = parsedItem.status;
                 if (parsedItem.confidence > 0.7) {
                     newStatus = 'disputed';
-                    newMeta = { ...newMeta, status_reason: `Contradicted by: ${competingFact}` };
+                    newMeta = {
+                        ...newMeta,
+                        status_reason: `Contradicted by: ${competingFact}`,
+                    };
                     penalty = 0.1;
                 }
                 else {
                     newStatus = 'deprecated';
-                    newMeta = { ...newMeta, status_reason: `Superseded by: ${competingFact}` };
+                    newMeta = {
+                        ...newMeta,
+                        status_reason: `Superseded by: ${competingFact}`,
+                    };
                     penalty = 0.4;
                 }
                 const newConfidence = Math.max(0, parsedItem.confidence - penalty);
@@ -251,7 +286,7 @@ export class KnowledgeDistiller {
                     confidence: newConfidence,
                     status: newStatus,
                     metadata: JSON.stringify(newMeta),
-                    updated_at: new Date()
+                    updated_at: new Date(),
                 })
                     .where('id', '=', parsedItem.id)
                     .execute();
@@ -264,35 +299,44 @@ export class KnowledgeDistiller {
      * Production Hardening: Uses batched cross-similarity and NER-style tokenization.
      */
     async autoLinkKnowledge(item, trx) {
-        // Extract potential entity names from the fact (Capitalized words)
-        const tokens = item.fact.match(/[A-Z][a-z]+/g) || [];
-        if (tokens.length === 0)
-            return;
-        const potentialEntities = Array.from(new Set(tokens));
-        const matches = await trx
-            .selectFrom(this.knowledgeTable)
-            .select(['id', 'entity'])
-            .where('entity', 'in', potentialEntities)
-            .where('entity', '!=', item.entity)
-            .execute();
-        for (const match of matches) {
-            await this.linkKnowledge(item.id, match.id, 'mentions', { auto: true, source: 'ner_extraction' }, trx);
+        // 1. Structural/Syntactic Extraction (NER-style tokenization)
+        // Extract capitalized phrases (potential entities), quoted strings, and CamelCase identifiers
+        const tokens = item.fact.match(/([A-Z][a-z]+(?:\s[A-Z][a-z]+)*)|("[^"]+")|([a-z]+[A-Z][a-z]+)/g) || [];
+        const potentialEntities = Array.from(new Set(tokens.map((t) => t.replace(/"/g, '').trim()))).filter((t) => t.length > 2 && t !== item.entity);
+        if (potentialEntities.length > 0) {
+            const matches = await trx
+                .selectFrom(this.knowledgeTable)
+                .select(['id', 'entity'])
+                .where('entity', 'in', potentialEntities)
+                .execute();
+            for (const match of matches) {
+                await this.linkKnowledge(item.id, match.id, 'mentions', { auto: true, source: 'structural_extraction' }, trx);
+            }
         }
-        // Secondary Pass: Similarity check for unanchored relations
-        const related = await trx
+        // 2. Semantic Similarity Pass (Vector/Cosine Approximation)
+        // We limit the search to items with high intrinsic fitness to avoid linking to "noise"
+        const candidates = await trx
             .selectFrom(this.knowledgeTable)
             .selectAll()
             .where('id', '!=', item.id)
+            .where('confidence', '>', 0.4) // Only link to semi-reliable knowledge
             .orderBy('updated_at', 'desc')
-            .limit(20)
+            .limit(50) // Increased limit for better coverage
             .execute();
-        for (const other of related) {
+        // Batch similarity processing to reduce link churn
+        const linksToCreate = [];
+        for (const other of candidates) {
             const parsedOther = this.parseKnowledge(other);
             const sim = calculateSimilarity(item.fact, parsedOther.fact);
-            if (sim > 0.6) {
-                await this.linkKnowledge(item.id, parsedOther.id, 'semantically_related', { similarity: sim }, trx);
+            // High similarity threshold for auto-linking to prevent a "everything linked to everything" graph
+            if (sim > 0.75) {
+                linksToCreate.push({ targetId: parsedOther.id, sim });
             }
         }
+        // Apply links in batch
+        for (const link of linksToCreate) {
+            await this.linkKnowledge(item.id, link.targetId, 'semantically_related', { similarity: link.sim, version: '2.0' }, trx);
+        }
     }
     /**
      * Link two knowledge items together with a relationship.
@@ -312,7 +356,7 @@ export class KnowledgeDistiller {
             await trxOrDb
                 .updateTable(this.linksTable)
                 .set({
-                metadata: metadata ? JSON.stringify(metadata) : null
+                metadata: metadata ? JSON.stringify(metadata) : null,
             })
                 .where('id', '=', existing.id)
                 .execute();
@@ -325,7 +369,7 @@ export class KnowledgeDistiller {
                 target_id: targetId,
                 relationship,
                 metadata: metadata ? JSON.stringify(metadata) : null,
-                created_at: new Date()
+                created_at: new Date(),
             })
                 .execute();
         }
@@ -350,10 +394,10 @@ export class KnowledgeDistiller {
             .execute();
         return {
             item: this.parseKnowledge(center),
-            relations: links.map(l => ({
+            relations: links.map((l) => ({
                 relationship: l.link_rel,
-                target: this.parseKnowledge(l)
-            }))
+                target: this.parseKnowledge(l),
+            })),
         };
     }
     /**
@@ -406,7 +450,7 @@ export class KnowledgeDistiller {
             ...secondary.metadata,
             ...primary.metadata,
             consolidated_from: secondary.id,
-            consolidated_at: new Date().toISOString()
+            consolidated_at: new Date().toISOString(),
         };
         const mergedTags = Array.from(new Set([...(primary.tags || []), ...(secondary.tags || [])]));
         await this.db.transaction().execute(async (trx) => {
@@ -417,7 +461,7 @@ export class KnowledgeDistiller {
                 confidence: Math.max(primary.confidence, secondary.confidence),
                 metadata: JSON.stringify(mergedMeta),
                 tags: JSON.stringify(mergedTags),
-                updated_at: new Date()
+                updated_at: new Date(),
             })
                 .where('id', '=', primary.id)
                 .execute();
@@ -428,6 +472,14 @@ export class KnowledgeDistiller {
                 .execute();
         });
     }
+    bloomHash(str) {
+        let hash = 0;
+        for (let i = 0; i < str.length; i++) {
+            hash = (hash << 5) - hash + str.charCodeAt(i);
+            hash |= 0; // Format to 32bit int
+        }
+        return hash;
+    }
     parseKnowledge(item) {
         return {
             id: item.id,
@@ -436,10 +488,12 @@ export class KnowledgeDistiller {
             confidence: item.confidence,
             status: item.status || 'proposed',
             sourceSessionId: item.source_session_id,
-            tags: typeof item.tags === 'string' ? JSON.parse(item.tags) : (item.tags || []),
-            metadata: typeof item.metadata === 'string' ? JSON.parse(item.metadata) : (item.metadata || {}),
+            tags: typeof item.tags === 'string' ? JSON.parse(item.tags) : item.tags || [],
+            metadata: typeof item.metadata === 'string'
+                ? JSON.parse(item.metadata)
+                : item.metadata || {},
             createdAt: new Date(item.created_at),
-            updatedAt: new Date(item.updated_at)
+            updatedAt: new Date(item.updated_at),
         };
     }
 }

package/dist/esm/agentic/improvement/RecursiveReasoner.d.ts

@@ -17,9 +17,15 @@ export declare class RecursiveReasoner {
      */
     analyzeGlobalProgress(pattern: string): Promise<AgentGoal[]>;
     /**
-     * Synthesize high-level lessons and cluster similar ones using basic NLP.
+     * Synthesize high-level lessons and cluster similar ones using Token-Weighted significance.
+     * Pass 5: High-Throughput Semantic Analysis.
      */
     synthesizeLessons(): Promise<Record<string, string[]>>;
+    /**
+     * Pass 6: Goal Cross-Pollination
+     * Distill successful persona mutations into global systemic goals.
+     */
+    crossPollinateGoals(): Promise<number>;
     /**
      * Detect contradictions in goals (e.g., opposing objectives).
      */

package/dist/esm/agentic/improvement/RecursiveReasoner.js

@@ -14,9 +14,23 @@ export class RecursiveReasoner {
         ['fast', 'slow'],
         ['short', 'long'],
         ['start', 'stop'],
-        ['enable', 'disable']
+        ['enable', 'disable'],
     ];
-    stopWords = new Set(['the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'with', 'from']);
+    stopWords = new Set([
+        'the',
+        'a',
+        'an',
+        'and',
+        'or',
+        'but',
+        'in',
+        'on',
+        'at',
+        'to',
+        'for',
+        'with',
+        'from',
+    ]);
     constructor(db, config = {}) {
         this.db = db;
         this.config = config;
@@ -29,48 +43,114 @@ export class RecursiveReasoner {
      * Find goals matching a pattern across all sessions
      */
     async analyzeGlobalProgress(pattern) {
-        const goals = await this.typedDb
+        const goals = (await this.typedDb
             .selectFrom(this.goalsTable)
             .selectAll()
             .where('description', 'like', `%${pattern}%`)
             .orderBy('created_at', 'desc')
-            .execute();
+            .execute());
         return goals.map(this.parseGoal);
     }
     /**
-     * Synthesize high-level lessons and cluster similar ones using basic NLP.
+     * Synthesize high-level lessons and cluster similar ones using Token-Weighted significance.
+     * Pass 5: High-Throughput Semantic Analysis.
      */
     async synthesizeLessons() {
-        console.log('[RecursiveReasoner] Synthesizing and clustering lessons from global history...');
+        console.log('[RecursiveReasoner] Performing high-throughput weighted token clustering...');
         const reflectionsTable = this.config.reflectionsTable || 'agent_reflections';
-        const results = await this.typedDb
+        const results = (await this.typedDb
             .selectFrom(reflectionsTable)
             .select('lessons_learned')
             .where('lessons_learned', 'is not', null)
-            .execute();
-        const rawLessons = results.map(l => l.lessons_learned).filter(Boolean);
+            .execute());
+        const rawLessons = results
+            .map((l) => l.lessons_learned)
+            .filter(Boolean);
         const clusters = {};
+        // Global Token Frequency Pass
+        const globalTokenFreq = new Map();
+        for (const lesson of rawLessons) {
+            for (const token of this.tokenize(lesson)) {
+                globalTokenFreq.set(token, (globalTokenFreq.get(token) || 0) + 1);
+            }
+        }
         for (const lesson of rawLessons) {
             const tokens = this.tokenize(lesson);
-            // Use the most significant token (longest non-stopword) as the cluster key
-            const significant = tokens.sort((a, b) => b.length - a.length)[0] || 'general';
-            if (!clusters[significant]) {
-                clusters[significant] = [];
+            if (tokens.length === 0)
+                continue;
+            // Weight tokens by (length * global_rarity) to find the most "defining" token
+            // Rarity = 1 / Frequency
+            const definingToken = tokens.sort((a, b) => {
+                const weightA = a.length * (1 / (globalTokenFreq.get(a) || 1));
+                const weightB = b.length * (1 / (globalTokenFreq.get(b) || 1));
+                return weightB - weightA;
+            })[0] || 'general';
+            if (!clusters[definingToken]) {
+                clusters[definingToken] = [];
             }
-            clusters[significant].push(lesson);
+            clusters[definingToken].push(lesson);
         }
         return clusters;
     }
+    /**
+     * Pass 6: Goal Cross-Pollination
+     * Distill successful persona mutations into global systemic goals.
+     */
+    async crossPollinateGoals() {
+        console.log('[RecursiveReasoner] Cross-pollinating successful persona breakthroughs into global goals...');
+        const personasTable = this.config.personasTable || 'agent_personas';
+        const goalsTable = this.config.goalsTable || 'agent_goals';
+        // Find personas with 'stable' evolution status (proven breakthroughs)
+        const breakthroughs = await this.typedDb
+            .selectFrom(personasTable)
+            .selectAll()
+            .where('metadata', 'like', '%"evolution_status":"stable"%')
+            .execute();
+        let goalsCreated = 0;
+        for (const p of breakthroughs) {
+            const persona = p;
+            const metadata = JSON.parse(persona.metadata || '{}');
+            const reasoning = metadata.mutation_reasoning || metadata.reasoning;
+            if (!reasoning)
+                continue;
+            // Check if this goal already exists to avoid duplication
+            const existing = await this.typedDb
+                .selectFrom(goalsTable)
+                .select('id')
+                .where('description', 'like', `%${reasoning.slice(0, 50)}%`)
+                .executeTakeFirst();
+            if (!existing) {
+                console.log(`[RecursiveReasoner] Distilling breakthrough from Persona ${persona.id} into Global Goal...`);
+                await this.typedDb
+                    .insertInto(goalsTable)
+                    .values({
+                    session_id: 0, // System-level goal
+                    description: `Systemic Best-Practice: ${reasoning}`,
+                    status: 'pending',
+                    priority: 5,
+                    metadata: JSON.stringify({
+                        source_persona: persona.id,
+                        cross_pollinated: true,
+                    }),
+                    created_at: new Date(),
+                    updated_at: new Date(),
+                })
+                    .execute();
+                goalsCreated++;
+            }
+        }
+        return goalsCreated;
+    }
     /**
      * Detect contradictions in goals (e.g., opposing objectives).
      */
     async detectContradictions() {
         console.log('[RecursiveReasoner] Detecting logical contradictions in systemic goals...');
-        const activeGoals = await this.typedDb
+        const activeGoals = (await this.typedDb
             .selectFrom(this.goalsTable)
             .selectAll()
             .where('status', '=', 'active')
-            .execute();
+            .execute());
         const contradictions = [];
         for (let i = 0; i < activeGoals.length; i++) {
             for (let j = i + 1; j < activeGoals.length; j++) {
@@ -95,7 +175,8 @@ export class RecursiveReasoner {
         const t2 = text2.toLowerCase();
         // Check explicit opposites
         for (const [op1, op2] of this.conflictPairs) {
-            if ((t1.includes(op1) && t2.includes(op2)) || (t1.includes(op2) && t2.includes(op1))) {
+            if ((t1.includes(op1) && t2.includes(op2)) ||
+                (t1.includes(op2) && t2.includes(op1))) {
                 // Determine the subject of conflict (common words)
                 const intersection = this.intersect(this.tokenize(t1), this.tokenize(t2));
                 // If they share a subject (e.g. "latency"), it's a conflict
@@ -109,13 +190,14 @@ export class RecursiveReasoner {
         return null;
     }
     tokenize(text) {
-        return text.toLowerCase()
+        return text
+            .toLowerCase()
             .split(/[\W_]+/)
-            .filter(w => w.length > 2 && !this.stopWords.has(w));
+            .filter((w) => w.length > 2 && !this.stopWords.has(w));
     }
     intersect(arr1, arr2) {
         const s2 = new Set(arr2);
-        return arr1.filter(x => s2.has(x));
+        return arr1.filter((x) => s2.has(x));
     }
     parseGoal(g) {
         return {
@@ -125,9 +207,11 @@ export class RecursiveReasoner {
             description: g.description,
             status: g.status,
             priority: g.priority,
-            metadata: typeof g.metadata === 'string' ? JSON.parse(g.metadata) : (g.metadata || {}),
+            metadata: typeof g.metadata === 'string'
+                ? JSON.parse(g.metadata)
+                : g.metadata || {},
             createdAt: new Date(g.created_at),
-            updatedAt: new Date(g.updated_at)
+            updatedAt: new Date(g.updated_at),
         };
     }
 }

package/dist/esm/agentic/improvement/ReflectionEngine.js

@@ -16,31 +16,33 @@ export class ReflectionEngine {
      * Record a reflection for a session
      */
     async reflect(sessionId, outcome, lessonsLearned, suggestedActions, metadata) {
-        const reflection = await this.db
+        const reflection = (await this.db
             .insertInto(this.reflectionsTable)
             .values({
             session_id: sessionId,
             outcome,
             lessons_learned: lessonsLearned,
-            suggested_actions: suggestedActions ? JSON.stringify(suggestedActions) : null,
+            suggested_actions: suggestedActions
+                ? JSON.stringify(suggestedActions)
+                : null,
             metadata: metadata ? JSON.stringify(metadata) : null,
-            created_at: new Date()
+            created_at: new Date(),
         })
             .returningAll()
-            .executeTakeFirstOrThrow();
+            .executeTakeFirstOrThrow());
         return this.parseReflection(reflection);
     }
     /**
      * Get reflections for a specific session
      */
     async getSessionReflections(sessionId) {
-        const reflections = await this.db
+        const reflections = (await this.db
             .selectFrom(this.reflectionsTable)
             .selectAll()
             .where('session_id', '=', sessionId)
             .orderBy('created_at', 'desc')
-            .execute();
-        return reflections.map(r => this.parseReflection(r));
+            .execute());
+        return reflections.map((r) => this.parseReflection(r));
     }
     /**
      * Get recent lessons learned across all sessions
@@ -65,7 +67,7 @@ export class ReflectionEngine {
             metadata: typeof reflection.metadata === 'string'
                 ? JSON.parse(reflection.metadata)
                 : reflection.metadata,
-            createdAt: new Date(reflection.created_at)
+            createdAt: new Date(reflection.created_at),
         };
     }
 }