mark-improving-agent 2.3.4 → 2.3.5

package/VERSION

@@ -1 +1 @@
-2.3.4
+2.3.5

package/dist/core/memory/binary-vector.js

@@ -160,7 +160,7 @@ export function addToBinaryIndex(index, vectors, ids, metadata) {
     return {
         vectors: newVectors,
         indices: [...index.indices, ...ids.map((_, i) => index.indices.length + i)],
-        metadata: [...index.metadata, ...metadata || ids.map(() => ({ id: '', importance: 0.5, timestamp: Date.now() }))],
+        metadata: [...index.metadata, ...metadata || ids.map((id, i) => ({ id, importance: 0.5, timestamp: Date.now() }))],
         config: index.config,
     };
 }

package/dist/core/memory/budget-retrieval.js

@@ -0,0 +1,420 @@
+/**
+ * Budget-Aware Memory Retrieval System
+ *
+ * A deterministic, token-budget-conscious memory retrieval system that:
+ * - Respects max token budgets during retrieval
+ * - Provides sub-millisecond query latency
+ * - Ensures identical queries return identical results
+ * - Supports semantic caching for similar queries
+ *
+ * Based on: Doorman11991/budget-aware-mcp (https://github.com/Doorman11991/budget-aware-mcp)
+ *
+ * Key concepts:
+ * - Hop-based graph walk: expand outward from anchor symbols hop by hop
+ * - Token budget: retrieval stops when budget is exhausted
+ * - Deterministic ordering: alphabetical within each hop level
+ * - Semantic cache: similar queries hit cache instantly
+ *
+ * @module core/memory
+ * @fileoverview Budget-aware memory retrieval with token control
+ */
+import { createLogger } from '../../utils/logger.js';
+import { cosineSimilarity } from './embedder.js';
+const logger = createLogger('[BudgetRetrieval]');
+/**
+ * Trigram-based similarity for semantic caching
+ */
+function extractTrigrams(text) {
+    const normalized = text.toLowerCase().replace(/[^a-z0-9\s]/g, ' ');
+    const words = normalized.split(/\s+/).filter(w => w.length > 0);
+    const trigrams = [];
+    for (const word of words) {
+        if (word.length >= 3) {
+            for (let i = 0; i <= word.length - 3; i++) {
+                trigrams.push(word.slice(i, i + 3));
+            }
+        }
+    }
+    return trigrams;
+}
+/**
+ * Jaccard similarity between two trigram sets
+ */
+function trigramSimilarity(a, b) {
+    if (a.length === 0 || b.length === 0)
+        return 0;
+    const setA = new Set(a);
+    const setB = new Set(b);
+    const intersection = new Set([...setA].filter(x => setB.has(x)));
+    const union = new Set([...setA, ...setB]);
+    return intersection.size / union.size;
+}
+/**
+ * Estimate tokens for a memory entry (rough approximation)
+ */
+function estimateTokens(entry) {
+    // Rough estimate: ~4 characters per token for typical English text
+    // Plus overhead for metadata
+    const contentTokens = Math.ceil(entry.content.length / 4);
+    const metadataTokens = 20; // tags, id, timestamps, etc.
+    return contentTokens + metadataTokens;
+}
+/**
+ * Create a deterministic hash from query
+ */
+function hashQuery(query) {
+    const parts = [
+        query.query.toLowerCase().trim(),
+        query.anchorId || '',
+        query.tier || '',
+        (query.maxTokens || 0).toString(),
+        (query.tags || []).sort().join(','),
+        (query.importanceThreshold || 0).toString()
+    ];
+    // Simple hash function for determinism
+    const str = parts.join('|');
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        const char = str.charCodeAt(i);
+        hash = ((hash << 5) - hash) + char;
+        hash = hash & hash; // Convert to 32-bit integer
+    }
+    return Math.abs(hash).toString(36);
+}
+/**
+ * Alphabetical sort for deterministic ordering within hop levels
+ */
+function deterministicSort(entries) {
+    return entries.sort((a, b) => {
+        // First by hop level
+        if (a.hopLevel !== b.hopLevel) {
+            return a.hopLevel - b.hopLevel;
+        }
+        // Then alphabetically by content
+        return a.entry.content.localeCompare(b.entry.content);
+    });
+}
+/**
+ * Default configuration
+ */
+export const DEFAULT_BUDGET_CONFIG = {
+    maxTokens: 4000,
+    tokensPerEntry: 200,
+    maxHops: 3,
+    enableCache: true,
+    cacheTTLMs: 5 * 60 * 1000, // 5 minutes
+    cacheSimilarityThreshold: 0.7,
+    enableImportanceFilter: true,
+    importanceThreshold: 0.3
+};
+/**
+ * Create a budget-aware retrieval engine
+ */
+export function createBudgetRetrievalEngine(config = {}, embedder) {
+    const fullConfig = { ...DEFAULT_BUDGET_CONFIG, ...config };
+    logger.info('Creating BudgetRetrievalEngine', { ...fullConfig });
+    // Memory storage
+    const entries = new Map();
+    const entriesByTier = new Map();
+    const entriesByTag = new Map();
+    // Embedding index (if embedder provided)
+    const entryEmbeddings = new Map();
+    // Semantic cache
+    const cache = new Map();
+    let cacheStats = { hits: 0, misses: 0, hitRate: 0, avgSimilarity: 0 };
+    /**
+     * Check semantic cache for similar query
+     */
+    function checkCache(query) {
+        if (!fullConfig.enableCache)
+            return null;
+        const queryTrigrams = extractTrigrams(query.query);
+        const queryHash = hashQuery(query);
+        // Check exact match first
+        const exactEntry = cache.get(queryHash);
+        if (exactEntry) {
+            const age = Date.now() - exactEntry.timestamp;
+            if (age < fullConfig.cacheTTLMs) {
+                return exactEntry;
+            }
+        }
+        // Check similar queries
+        let bestMatch = null;
+        let bestSimilarity = 0;
+        for (const entry of cache.values()) {
+            const age = Date.now() - entry.timestamp;
+            if (age >= fullConfig.cacheTTLMs)
+                continue;
+            const similarity = trigramSimilarity(queryTrigrams, entry.trigrams);
+            if (similarity >= fullConfig.cacheSimilarityThreshold && similarity > bestSimilarity) {
+                bestSimilarity = similarity;
+                bestMatch = entry;
+            }
+        }
+        return bestMatch;
+    }
+    /**
+     * Store result in cache
+     */
+    function storeCache(query, result) {
+        if (!fullConfig.enableCache)
+            return;
+        const queryHash = hashQuery(query);
+        const trigrams = extractTrigrams(query.query);
+        cache.set(queryHash, {
+            queryHash,
+            queryText: query.query,
+            result,
+            timestamp: Date.now(),
+            trigrams
+        });
+        // Cleanup old entries
+        const now = Date.now();
+        for (const [key, entry] of cache.entries()) {
+            if (now - entry.timestamp > fullConfig.cacheTTLMs) {
+                cache.delete(key);
+            }
+        }
+    }
+    /**
+     * Calculate relevance score for an entry
+     */
+    function calculateRelevance(entry, query, anchorEntry) {
+        let relevance = 0;
+        // Base relevance from importance
+        relevance += entry.importance * 0.3;
+        // Text similarity if query provided
+        if (query.query && embedder) {
+            const queryEmb = embedder.embed(query.query);
+            const entryEmb = entryEmbeddings.get(entry.id);
+            if (queryEmb && entryEmb) {
+                const sim = cosineSimilarity(queryEmb, entryEmb);
+                relevance += sim * 0.4;
+            }
+        }
+        // Content match
+        const queryLower = query.query.toLowerCase();
+        const contentLower = entry.content.toLowerCase();
+        if (contentLower.includes(queryLower)) {
+            relevance += 0.3;
+        }
+        // Tag match
+        if (query.tags && query.tags.length > 0) {
+            const entryTags = new Set(entry.tags.map(t => t.toLowerCase()));
+            const matchCount = query.tags.filter(t => entryTags.has(t.toLowerCase())).length;
+            relevance += (matchCount / query.tags.length) * 0.2;
+        }
+        // Hop-based relevance from anchor
+        if (anchorEntry) {
+            // Entries referenced by anchor = hop 1
+            const anchor = anchorEntry;
+            const ent = entry;
+            if (anchor.references?.includes(ent.id)) {
+                return relevance + 0.5 - 0.1; // hop 1 bonus
+            }
+            // Entries referencing anchor = hop 1
+            if (ent.references?.includes(anchor.id)) {
+                return relevance + 0.5 - 0.1;
+            }
+            // Same session = hop 2
+            if (anchor.sessionId && anchor.sessionId === ent.sessionId) {
+                return relevance + 0.4 - 0.2;
+            }
+        }
+        return Math.min(1, relevance);
+    }
+    /**
+     * Perform budget-aware retrieval
+     */
+    async function retrieve(query) {
+        const startMs = Date.now();
+        const maxTokens = query.maxTokens || fullConfig.maxTokens;
+        logger.debug('Retrieving with budget', { query: query.query, maxTokens });
+        // Check cache first
+        const cached = checkCache(query);
+        if (cached) {
+            cacheStats.hits++;
+            cacheStats.avgSimilarity = (cacheStats.avgSimilarity * (cacheStats.hits - 1) + (cached.result.cacheSimilarity || 1)) / cacheStats.hits;
+            cacheStats.hitRate = cacheStats.hits / (cacheStats.hits + cacheStats.misses);
+            logger.debug('Cache hit', { similarity: cached.result.cacheSimilarity });
+            return {
+                ...cached.result,
+                cacheHit: true,
+                processingMs: Date.now() - startMs
+            };
+        }
+        cacheStats.misses++;
+        cacheStats.hitRate = cacheStats.hits / (cacheStats.hits + cacheStats.misses);
+        // Get anchor entry if specified
+        const anchorEntry = query.anchorId ? entries.get(query.anchorId) : undefined;
+        // Collect candidate entries
+        let candidateIds;
+        if (query.tier) {
+            const tierSet = entriesByTier.get(query.tier);
+            candidateIds = tierSet ? Array.from(tierSet) : [];
+        }
+        else {
+            candidateIds = Array.from(entries.keys());
+        }
+        // Filter by tags if specified
+        if (query.tags && query.tags.length > 0) {
+            candidateIds = candidateIds.filter(id => {
+                const entry = entries.get(id);
+                if (!entry)
+                    return false;
+                const entryTags = new Set(entry.tags.map(t => t.toLowerCase()));
+                return query.tags.some(t => entryTags.has(t.toLowerCase()));
+            });
+        }
+        // Score and filter candidates
+        const scoredCandidates = [];
+        for (const id of candidateIds) {
+            const entry = entries.get(id);
+            if (!entry)
+                continue;
+            // Importance filter
+            if (fullConfig.enableImportanceFilter && entry.importance < fullConfig.importanceThreshold) {
+                continue;
+            }
+            if (query.importanceThreshold && entry.importance < query.importanceThreshold) {
+                continue;
+            }
+            const relevance = calculateRelevance(entry, query, anchorEntry);
+            const tokenEstimate = estimateTokens(entry);
+            // Determine hop level
+            let hopLevel = 0;
+            if (anchorEntry) {
+                if (id === anchorEntry.id) {
+                    hopLevel = 0;
+                }
+                else if ((anchorEntry.references?.includes(id)) || (entry.references?.includes(anchorEntry.id))) {
+                    hopLevel = 1;
+                }
+                else {
+                    hopLevel = 2;
+                }
+            }
+            scoredCandidates.push({
+                entry,
+                relevance,
+                hopLevel,
+                tokenEstimate,
+                cumulativeTokens: 0 // Will be calculated after sorting
+            });
+        }
+        // Sort deterministically
+        const sorted = deterministicSort(scoredCandidates);
+        // Build budget-constrained result
+        const budget = {
+            maxTokens,
+            usedTokens: 0,
+            remainingTokens: maxTokens,
+            entriesSelected: 0
+        };
+        const selected = [];
+        for (const candidate of sorted) {
+            if (budget.remainingTokens < candidate.tokenEstimate) {
+                continue; // Can't afford this entry
+            }
+            if (candidate.hopLevel > fullConfig.maxHops) {
+                continue; // Exceeded max hops
+            }
+            candidate.cumulativeTokens = budget.usedTokens + candidate.tokenEstimate;
+            budget.usedTokens = candidate.cumulativeTokens;
+            budget.remainingTokens = maxTokens - budget.usedTokens;
+            budget.entriesSelected++;
+            selected.push(candidate);
+        }
+        const result = {
+            entries: selected,
+            tokensUsed: budget.usedTokens,
+            budgetRemaining: budget.remainingTokens,
+            cacheHit: false,
+            queryHash: hashQuery(query),
+            processingMs: Date.now() - startMs,
+            totalAvailable: candidateIds.length
+        };
+        // Store in cache
+        storeCache(query, result);
+        logger.debug('Retrieval complete', {
+            tokensUsed: result.tokensUsed,
+            entriesSelected: result.entries.length,
+            processingMs: result.processingMs,
+            cacheHit: result.cacheHit
+        });
+        return result;
+    }
+    /**
+     * Index a memory entry
+     */
+    function index(entry) {
+        entries.set(entry.id, entry);
+        // Index by tier
+        if (!entriesByTier.has(entry.tier)) {
+            entriesByTier.set(entry.tier, new Set());
+        }
+        entriesByTier.get(entry.tier).add(entry.id);
+        // Index by tags
+        for (const tag of entry.tags) {
+            if (!entriesByTag.has(tag)) {
+                entriesByTag.set(tag, new Set());
+            }
+            entriesByTag.get(tag).add(entry.id);
+        }
+        // Embed for semantic search
+        if (embedder) {
+            entryEmbeddings.set(entry.id, embedder.embed(entry.content));
+        }
+    }
+    /**
+     * Remove entry from index
+     */
+    function remove(id) {
+        const entry = entries.get(id);
+        if (!entry)
+            return;
+        entries.delete(id);
+        // Remove from tier index
+        entriesByTier.get(entry.tier)?.delete(id);
+        // Remove from tag indices
+        for (const tag of entry.tags) {
+            entriesByTag.get(tag)?.delete(id);
+        }
+        // Remove embedding
+        entryEmbeddings.delete(id);
+    }
+    /**
+     * Clear all entries
+     */
+    function clear() {
+        entries.clear();
+        entriesByTier.clear();
+        entriesByTag.clear();
+        entryEmbeddings.clear();
+        cache.clear();
+        logger.info('Budget retrieval engine cleared');
+    }
+    /**
+     * Get cache statistics
+     */
+    function getCacheStats() {
+        return { ...cacheStats };
+    }
+    /**
+     * Update configuration
+     */
+    function updateConfig(config) {
+        Object.assign(fullConfig, config);
+        logger.debug('Config updated', config);
+    }
+    return {
+        retrieve,
+        index,
+        remove,
+        clear,
+        getCacheStats,
+        updateConfig
+    };
+}
+// Export utilities for external use
+export { extractTrigrams, trigramSimilarity, estimateTokens, hashQuery };

package/dist/core/memory/index.js

@@ -12,3 +12,4 @@ export { createHybridSearchEngine, createBM25Index, bm25Score, normalizeBM25Scor
 export { createPatternRecognizer } from './pattern-recognizer.js';
 export { createMemoryObserver } from './observer.js';
 export { createBinaryVectorCompressor, quantizeToBinary, hammingDistance, hammingToSimilarity, createBinaryIndex, addToBinaryIndex, searchBinaryIndex, getCompressionStats, serializeBinaryIndex, deserializeBinaryIndex, DEFAULT_BINARY_CONFIG, EMBEDDING_DIMENSIONS } from './binary-vector.js';
+export { createBudgetRetrievalEngine, DEFAULT_BUDGET_CONFIG, extractTrigrams, trigramSimilarity, estimateTokens, hashQuery } from './budget-retrieval.js';

package/dist/version.js

@@ -1 +1 @@
-export const VERSION = '2.3.4';
+export const VERSION = '2.3.5';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mark-improving-agent",
-  "version": "2.3.4",
+  "version": "2.3.5",
   "description": "Self-evolving AI agent with permanent memory, identity continuity, and self-evolution — for AI agents that need to remember, learn, and evolve across sessions",
   "type": "module",
   "main": "./dist/index.js",