mark-improving-agent 2.3.2 → 2.3.3

package/VERSION

@@ -1 +1 @@
-2.3.2
+2.3.3

package/dist/core/memory/index.js

@@ -10,3 +10,4 @@ export * from './adaptive-rag.js';
 export { createContextFragmentationEngine } from './context-fragmentation.js';
 export { createHybridSearchEngine, createBM25Index, bm25Score, normalizeBM25Scores, DEFAULT_HYBRID_CONFIG } from './hybrid-search.js';
 export { createPatternRecognizer } from './pattern-recognizer.js';
+export { createMemoryObserver } from './observer.js';

package/dist/core/memory/observer.js

@@ -0,0 +1,350 @@
+/**
+ * Memory Observer - Silent Background Memory Writer
+ *
+ * Implements the Mnemostroma-inspired Observer pattern: the AI agent NEVER writes
+ * memory directly. Instead, this Observer sidecar silently watches all I/O,
+ * extracts entities, embeds, scores, and indexes content automatically.
+ *
+ * Key principles:
+ * - AI never writes memory — Observer does it silently
+ * - Dual async pipeline: Observer (write) + Content Branch (versioned artifacts)
+ * - Memory Hulling: separates conversational noise from extractable kernels
+ * - 20ms hot buffer retrieval, ~50ms full extraction latency
+ *
+ * Based on: GG-QandV/mnemostroma (https://github.com/GG-QandV/mnemostroma)
+ *
+ * @module core/memory
+ * @fileoverview Observer pattern for silent memory writing
+ */
+import { randomUUID } from 'crypto';
+import { createLogger } from '../../utils/logger.js';
+import { createEmbedder, cosineSimilarity } from './embedder.js';
+const logger = createLogger('[MemoryObserver]');
+const DEFAULT_OBSERVER_CONFIG = {
+    importanceThreshold: 0.3,
+    hotBufferSize: 50,
+    autoExtract: true,
+    debounceMs: 100,
+    maxShellAge: 5 * 60 * 1000, // 5 minutes
+    embedder: createEmbedder({ dimensions: 128 }),
+};
+/**
+ * Noise patterns that indicate conversational shell (not worth storing)
+ */
+const SHELL_PATTERNS = [
+    /^(hi|hello|hey|what's up|howdy)\b/i,
+    /^(thanks?|thank you|thx|ty|much appreciated)\b/i,
+    /^(okay|ok|okk|kk|sure|yes|no|nah|yeah|yep|nope)\b/i,
+    /^(lol|lmao|rofl|haha|heh)\b/i,
+    /^(bye|goodbye|see you|cya|good night|night)\b/i,
+    /^(sorry|apologies|my bad|whoops)\b/i,
+    /^(please|pls|plz|could you|would you)\b/i,
+    /^(interesting|cool|nice|awesome|great|good)\b/i,
+    /^(i see|i understand|i get it|got it|understood)\b/i,
+    /^[\s.,!?;:]*$/,
+    /^(oh|ah|uh|um|hmm|well)\b/i,
+    /^(let me know|feel free|take care|talk later)\b/i,
+];
+/**
+ * Anchor patterns that indicate important kernels (decisions, deadlines, facts)
+ */
+const ANCHOR_PATTERNS = [
+    /\b(decided|decision|agreed|chosen|selected|picked|settled)\b/i,
+    /\b(must|have to|need to|required|mandatory|essential)\b/i,
+    /\b(deadline|due|by|before|after|until|timing)\b/i,
+    /\b(never|always|don't|do not|must not|forbidden)\b/i,
+    /\b(important|critical|key|vital|priority)\b/i,
+    /\b(remember|forget|keep in mind|note that)\b/i,
+    /\b(because|since|reason|why|therefore|thus)\b/i,
+    /\b(fact|true|real|actually|indeed|in fact)\b/i,
+    /\b(but|however|although|except|except for)\b/i,
+    /\b(name|date|location|price|cost|amount)\b/i,
+    /\b(user|client|customer|they|their)\b.*\b(want|need|prefer|like)\b/i,
+];
+/**
+ * Kernel type classifiers
+ */
+function classifyKernel(content, context) {
+    const lower = content.toLowerCase();
+    const fullText = [content, ...context].join(' ').toLowerCase();
+    if (/deadline|due|by |before |after |timing/i.test(lower))
+        return 'constraint';
+    if (/decided|agreed|chosen|picked|selected/i.test(lower))
+        return 'decision';
+    if (/must|have to|need to|required|mandatory/i.test(lower))
+        return 'rule';
+    if (/fact|true|actual|indeed|in fact|real/i.test(fullText))
+        return 'fact';
+    if (/remember|forget|note|keep in mind/i.test(lower))
+        return 'context';
+    if (/name|date|location|price|amount/i.test(lower))
+        return 'entity';
+    return 'context';
+}
+/**
+ * Score importance based on content features
+ */
+function scoreImportance(content, isAnchor) {
+    let score = 0.5;
+    if (isAnchor)
+        score += 0.2;
+    const words = content.split(/\s+/).length;
+    if (words >= 5 && words <= 50)
+        score += 0.15;
+    else if (words > 100)
+        score -= 0.1;
+    if (/\d+/.test(content))
+        score += 0.1;
+    if (/[A-Z][a-z]+\s+[A-Z][a-z]+/.test(content))
+        score += 0.1;
+    if (/\?$/.test(content))
+        score += 0.05;
+    if (/!$/.test(content))
+        score += 0.05;
+    return Math.max(0, Math.min(1, score));
+}
+/**
+ * Check if content is shell (noise)
+ */
+function isShell(content) {
+    const trimmed = content.trim();
+    if (trimmed.length < 3)
+        return true;
+    for (const pattern of SHELL_PATTERNS) {
+        if (pattern.test(trimmed))
+            return true;
+    }
+    return false;
+}
+/**
+ * Extract anchors from content
+ */
+function extractAnchors(content) {
+    const anchors = [];
+    for (const pattern of ANCHOR_PATTERNS) {
+        const match = content.match(pattern);
+        if (match) {
+            anchors.push(match[0]);
+        }
+    }
+    return anchors;
+}
+function buildMemoryObserver(config = {}) {
+    const cfg = { ...DEFAULT_OBSERVER_CONFIG, ...config };
+    const hotBuffer = [];
+    const anchorIndex = new Map();
+    const tagIndex = new Map();
+    let kernelsExtracted = 0;
+    let shellDiscarded = 0;
+    let lastFlush = null;
+    let sessionId = randomUUID();
+    let previousSessionId = null;
+    /**
+     * Hull content — separate kernels from shell
+     */
+    function hull(content, context = []) {
+        const kernels = [];
+        const shell = [];
+        const sentences = content.split(/[.!?]+/).map(s => s.trim()).filter(s => s.length > 2);
+        for (const sentence of sentences) {
+            if (isShell(sentence)) {
+                shell.push(sentence);
+                shellDiscarded++;
+                continue;
+            }
+            const anchors = extractAnchors(sentence);
+            const isAnchor = anchors.length > 0;
+            const importance = scoreImportance(sentence, isAnchor);
+            if (importance >= cfg.importanceThreshold) {
+                const type = classifyKernel(sentence, context);
+                const kernel = {
+                    type,
+                    content: sentence,
+                    importance,
+                    tags: [type],
+                    anchors,
+                    sourceExcerpt: sentence.slice(0, 200),
+                };
+                if (isAnchor) {
+                    kernel.tags.push('anchor');
+                }
+                kernels.push(kernel);
+                kernelsExtracted++;
+                for (const anchor of anchors) {
+                    anchorIndex.set(anchor, {
+                        content: sentence,
+                        type,
+                        timestamp: Date.now(),
+                    });
+                }
+                for (const tag of kernel.tags) {
+                    if (!tagIndex.has(tag)) {
+                        tagIndex.set(tag, []);
+                    }
+                    tagIndex.get(tag).push(kernel);
+                }
+            }
+            else {
+                shell.push(sentence);
+                shellDiscarded++;
+            }
+        }
+        return { kernels, shell };
+    }
+    function maintainHotBuffer() {
+        while (hotBuffer.length > cfg.hotBufferSize) {
+            let minIdx = 0;
+            let minImportance = Infinity;
+            for (let i = 0; i < hotBuffer.length; i++) {
+                if (hotBuffer[i].importance < minImportance) {
+                    minImportance = hotBuffer[i].importance;
+                    minIdx = i;
+                }
+            }
+            hotBuffer.splice(minIdx, 1);
+        }
+    }
+    async function semanticSearch(query, limit = 5) {
+        try {
+            const queryEmbedding = cfg.embedder.embed(query);
+            const results = [];
+            for (const kernel of hotBuffer) {
+                const kernelEmbedding = cfg.embedder.embed(kernel.content);
+                const similarity = cosineSimilarity(queryEmbedding, kernelEmbedding);
+                if (similarity > 0.3) {
+                    results.push({ content: kernel.content, score: similarity });
+                }
+            }
+            results.sort((a, b) => b.score - a.score);
+            return results.slice(0, limit);
+        }
+        catch (err) {
+            logger.warn('Semantic search failed, falling back to keyword', { error: err });
+            const queryLower = query.toLowerCase();
+            return hotBuffer
+                .filter(k => k.content.toLowerCase().includes(queryLower))
+                .slice(0, limit)
+                .map(k => ({ content: k.content, score: k.importance }));
+        }
+    }
+    return {
+        observe(content, stream = 'semantic', context) {
+            const start = Date.now();
+            const { kernels, shell } = hull(content, context);
+            hotBuffer.push(...kernels);
+            maintainHotBuffer();
+            const processingMs = Date.now() - start;
+            logger.debug(`Observed [${stream}]: ${kernels.length} kernels extracted, ${shell.length} shell discarded (${processingMs}ms)`);
+            return {
+                id: randomUUID(),
+                stream,
+                kernels,
+                shell,
+                timestamp: Date.now(),
+                processingMs,
+            };
+        },
+        async flush(tier = 'learned') {
+            if (hotBuffer.length === 0)
+                return [];
+            const entries = [];
+            for (const kernel of hotBuffer) {
+                const entry = {
+                    id: randomUUID(),
+                    tier,
+                    content: kernel.content,
+                    importance: kernel.importance,
+                    tags: kernel.tags,
+                    timestamp: Date.now(),
+                    accessCount: 0,
+                    lastAccessed: Date.now(),
+                    source: 'self',
+                };
+                entries.push(entry);
+            }
+            lastFlush = Date.now();
+            logger.info(`Flushed ${entries.length} kernels to ${tier} tier`);
+            return entries;
+        },
+        async ctx_semantic(query, limit = 5) {
+            return semanticSearch(query, limit);
+        },
+        ctx_anchors(limit = 20) {
+            const results = Array.from(anchorIndex.entries())
+                .map(([, data]) => ({
+                content: data.content,
+                type: data.type,
+                timestamp: data.timestamp,
+            }))
+                .sort((a, b) => b.timestamp - a.timestamp);
+            return results.slice(0, limit);
+        },
+        async ctx_search(tags, limit = 10) {
+            const matchingKernels = [];
+            for (const tag of tags) {
+                const kernels = tagIndex.get(tag) || [];
+                matchingKernels.push(...kernels);
+            }
+            const seen = new Set();
+            const unique = matchingKernels.filter(k => {
+                if (seen.has(k.content))
+                    return false;
+                seen.add(k.content);
+                return true;
+            });
+            return unique.slice(0, limit).map(k => ({
+                id: randomUUID(),
+                tier: 'learned',
+                content: k.content,
+                importance: k.importance,
+                tags: k.tags,
+                timestamp: Date.now(),
+                accessCount: 0,
+                lastAccessed: Date.now(),
+            }));
+        },
+        ctx_bridge(prevSessionId) {
+            const decisions = Array.from(anchorIndex.entries())
+                .filter(([, data]) => data.type === 'decision')
+                .map(([, data]) => data.content);
+            const constraints = Array.from(anchorIndex.entries())
+                .filter(([, data]) => data.type === 'constraint' || data.type === 'rule')
+                .map(([, data]) => data.content);
+            const importantFacts = Array.from(anchorIndex.entries())
+                .filter(([, data]) => data.type === 'fact')
+                .map(([, data]) => ({
+                fact: data.content,
+                certainty: 0.8,
+            }));
+            const contextSummary = hotBuffer.length > 0
+                ? hotBuffer.slice(0, 5).map(k => k.content).join(' ')
+                : '';
+            const packet = {
+                sessionId,
+                previousSessionId: prevSessionId || previousSessionId,
+                context: contextSummary,
+                decisions,
+                constraints,
+                pendingTasks: [],
+                importantFacts,
+                timestamp: Date.now(),
+                agentVersion: '2.3.3',
+            };
+            previousSessionId = sessionId;
+            sessionId = randomUUID();
+            return packet;
+        },
+        getStats() {
+            return {
+                hotBufferSize: hotBuffer.length,
+                kernelsExtracted,
+                shellDiscarded,
+                lastFlush,
+            };
+        },
+    };
+}
+export function createMemoryObserver(config) {
+    logger.info('Creating Memory Observer (Mnemostroma-inspired)');
+    return buildMemoryObserver(config);
+}

package/dist/version.js

@@ -1 +1 @@
-export const VERSION = '2.3.2';
+export const VERSION = '2.3.3';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mark-improving-agent",
-  "version": "2.3.2",
+  "version": "2.3.3",
   "description": "Self-evolving AI agent with permanent memory, identity continuity, and self-evolution — for AI agents that need to remember, learn, and evolve across sessions",
   "type": "module",
   "main": "./dist/index.js",