mark-improving-agent 2.3.3 → 2.3.5

package/VERSION

@@ -1 +1 @@
-2.3.3
+2.3.5

package/dist/core/memory/binary-vector.js

@@ -0,0 +1,405 @@
+/**
+ * Binary Vector Memory Compression
+ *
+ * High-performance binary vector quantization for memory embeddings.
+ * Based on QuIVer: Rethinking ANN Graph Topology via Training-Free Binary Quantization (Xiao et al., 2026)
+ *
+ * Achieves 32x compression: 1536-dim float32 (6144 bytes) → 192 bytes binary
+ * Uses Hamming distance (XOR + POPCOUNT) for fast similarity search.
+ *
+ * @module core/memory
+ * @fileoverview Binary vector quantization for memory compression
+ */
+import { createLogger } from '../../utils/logger.js';
+const logger = createLogger('[BinaryVector]');
+/**
+ * Default configuration for 1536-dim embeddings (OpenAI text-embedding-3-small)
+ */
+export const DEFAULT_BINARY_CONFIG = {
+    dimension: 1536,
+    normalized: true,
+    trackStats: true,
+};
+/**
+ * Create a binary vector index
+ *
+ * @param config - Configuration for the index
+ * @returns Empty binary index ready for vectors
+ */
+export function createBinaryIndex(config) {
+    logger.info('Creating binary vector index', { dimension: config.dimension });
+    return {
+        vectors: new Uint8Array(0),
+        indices: [],
+        metadata: [],
+        config,
+    };
+}
+/**
+ * Quantize a float32 vector to binary using sign-bit quantization
+ *
+ * @param vector - Float32 array (dimension length)
+ * @param normalized - Whether vector is L2 normalized (skip magnitude check if true)
+ * @returns Uint8Array of packed binary bits
+ *
+ * @example
+ * ```typescript
+ * const vector = new Float32Array([0.5, -0.3, 0.1, -0.8]);
+ * const binary = quantizeToBinary(vector);
+ * // Result: Uint8Array [0b10001001] (4 dimensions = 4 bits = 1 byte)
+ * ```
+ */
+export function quantizeToBinary(vector, normalized = true) {
+    const dim = vector.length;
+    const bytes = Math.ceil(dim / 8);
+    const result = new Uint8Array(bytes);
+    for (let i = 0; i < dim; i++) {
+        // Sign bit: positive = 1, negative = 0
+        const bitIndex = i % 8;
+        const byteIndex = Math.floor(i / 8);
+        if (vector[i] >= 0) {
+            result[byteIndex] |= (1 << (7 - bitIndex));
+        }
+    }
+    return result;
+}
+/**
+ * Dequantize binary vector back to approximate float32
+ *
+ * Note: This is lossy - returns center-of-mass values (±0.5 for normalized vectors)
+ *
+ * @param binary - Packed binary bits
+ * @param dimension - Original dimension
+ * @param normalized - Whether to return normalized values
+ * @returns Approximate float32 vector
+ */
+export function dequantizeFromBinary(binary, dimension, normalized = true) {
+    const result = new Float32Array(dimension);
+    for (let i = 0; i < dimension; i++) {
+        const bitIndex = i % 8;
+        const byteIndex = Math.floor(i / 8);
+        const bit = (binary[byteIndex] >> (7 - bitIndex)) & 1;
+        // For normalized vectors: 1 → +0.5, 0 → -0.5
+        // This gives approximate reconstruction
+        result[i] = normalized ? (bit ? 0.5 : -0.5) : (bit ? 0 : 0);
+    }
+    return result;
+}
+/**
+ * Calculate Hamming distance between two binary vectors
+ *
+ * @param a - First binary vector
+ * @param b - Second binary vector
+ * @returns Hamming distance (count of differing bits)
+ *
+ * @example
+ * ```typescript
+ * const dist = hammingDistance(binaryA, binaryB);
+ * // dist = 0 means identical, dist = dimension means opposite
+ * ```
+ */
+export function hammingDistance(a, b) {
+    if (a.length !== b.length) {
+        throw new Error(`Binary vectors must have same length: ${a.length} vs ${b.length}`);
+    }
+    let distance = 0;
+    for (let i = 0; i < a.length; i++) {
+        // XOR gives 1s where bits differ, POPCOUNT counts them
+        const xored = a[i] ^ b[i];
+        // POPCOUNT via built-in
+        let count = xored;
+        count = (count & 0x55) + ((count >> 1) & 0x55);
+        count = (count & 0x33) + ((count >> 2) & 0x33);
+        count = (count & 0x0F) + ((count >> 4) & 0x0F);
+        distance += count;
+    }
+    return distance;
+}
+/**
+ * Calculate similarity from Hamming distance
+ *
+ * @param hammingDist - Hamming distance
+ * @param dimension - Vector dimension
+ * @returns Similarity score 0-1 (1 = identical)
+ */
+export function hammingToSimilarity(hammingDist, dimension) {
+    return 1 - (hammingDist / dimension);
+}
+/**
+ * Add vectors to the binary index
+ *
+ * @param index - Binary index to add to
+ * @param vectors - Float32Array vectors (each dimension-long)
+ * @param ids - Unique identifiers for each vector
+ * @param metadata - Optional metadata (importance, timestamp)
+ * @returns Updated index
+ */
+export function addToBinaryIndex(index, vectors, ids, metadata) {
+    const dim = index.config.dimension;
+    if (vectors.length !== ids.length) {
+        throw new Error(`Vectors count (${vectors.length}) must match IDs count (${ids.length})`);
+    }
+    // Calculate total bytes needed
+    const bytesPerVector = Math.ceil(dim / 8);
+    const newTotalBytes = index.vectors.length + (vectors.length * bytesPerVector);
+    // Create new expanded vectors array
+    const newVectors = new Uint8Array(newTotalBytes);
+    if (index.vectors.length > 0) {
+        newVectors.set(index.vectors);
+    }
+    // Quantize and add each vector
+    let offset = index.vectors.length;
+    for (let i = 0; i < vectors.length; i++) {
+        if (vectors[i].length !== dim) {
+            throw new Error(`Vector dimension ${vectors[i].length} doesn't match index dimension ${dim}`);
+        }
+        const binary = quantizeToBinary(vectors[i], index.config.normalized);
+        newVectors.set(binary, offset);
+        offset += bytesPerVector;
+    }
+    return {
+        vectors: newVectors,
+        indices: [...index.indices, ...ids.map((_, i) => index.indices.length + i)],
+        metadata: [...index.metadata, ...metadata || ids.map((id, i) => ({ id, importance: 0.5, timestamp: Date.now() }))],
+        config: index.config,
+    };
+}
+/**
+ * Search for similar vectors in the binary index using Hamming distance
+ *
+ * @param index - Binary index to search
+ * @param queryVector - Query vector to search for
+ * @param k - Number of results to return
+ * @returns Top-k results sorted by similarity (highest first)
+ *
+ * @example
+ * ```typescript
+ * const results = searchBinaryIndex(index, queryEmbedding, 10);
+ * // Returns top 10 most similar memories
+ * ```
+ */
+export function searchBinaryIndex(index, queryVector, k) {
+    const dim = index.config.dimension;
+    const bytesPerVector = Math.ceil(dim / 8);
+    if (queryVector.length !== dim) {
+        throw new Error(`Query dimension ${queryVector.length} doesn't match index dimension ${dim}`);
+    }
+    // Quantize query
+    const queryBinary = quantizeToBinary(queryVector, index.config.normalized);
+    // Search all vectors
+    const results = [];
+    let totalHamming = 0;
+    for (let i = 0; i < index.indices.length; i++) {
+        const offset = i * bytesPerVector;
+        const vectorBinary = index.vectors.subarray(offset, offset + bytesPerVector);
+        const hDist = hammingDistance(queryBinary, vectorBinary);
+        totalHamming += hDist;
+        results.push({
+            index: i,
+            id: index.metadata[i]?.id || String(index.indices[i]),
+            hammingDistance: hDist,
+            similarity: hammingToSimilarity(hDist, dim),
+            metadata: index.metadata[i] || { importance: 0.5, timestamp: Date.now() },
+        });
+    }
+    // Sort by similarity (highest first)
+    results.sort((a, b) => b.similarity - a.similarity);
+    // Return top-k
+    const topK = results.slice(0, k);
+    if (index.config.trackStats) {
+        logger.debug(`Search completed: ${index.indices.length} vectors, avg Hamming: ${(totalHamming / index.indices.length).toFixed(2)}`);
+    }
+    return topK;
+}
+/**
+ * Get compression statistics
+ *
+ * @param index - Binary index
+ * @returns Compression statistics
+ */
+export function getCompressionStats(index) {
+    const dim = index.config.dimension;
+    const float32Bytes = dim * 4; // 4 bytes per float32
+    const binaryBytes = Math.ceil(dim / 8);
+    const vectorsCount = index.indices.length;
+    const originalBytes = vectorsCount * float32Bytes;
+    const compressedBytes = vectorsCount * binaryBytes;
+    return {
+        originalBytes,
+        compressedBytes,
+        compressionRatio: originalBytes / compressedBytes,
+        vectorsStored: vectorsCount,
+        dimension: dim,
+        avgHammingDistance: 0, // Calculated on demand
+    };
+}
+/**
+ * Save binary index to binary format
+ *
+ * @param index - Binary index to save
+ * @returns Buffer containing serialized index
+ */
+export function serializeBinaryIndex(index) {
+    const configBytes = JSON.stringify(index.config).length;
+    const metadataBytes = JSON.stringify(index.metadata).length;
+    const indicesBytes = index.indices.length * 4; // 4 bytes per number
+    const totalBytes = 4 + configBytes + 4 + metadataBytes + 4 + indicesBytes + index.vectors.length + 4 + index.indices.length * 8;
+    const buffer = new ArrayBuffer(totalBytes);
+    const view = new DataView(buffer);
+    const uint8 = new Uint8Array(buffer);
+    let offset = 0;
+    // Config
+    const configStr = JSON.stringify(index.config);
+    view.setUint32(offset, configStr.length);
+    offset += 4;
+    uint8.set(new TextEncoder().encode(configStr), offset);
+    offset += configStr.length;
+    // Metadata
+    const metadataStr = JSON.stringify(index.metadata);
+    view.setUint32(offset, metadataStr.length);
+    offset += 4;
+    uint8.set(new TextEncoder().encode(metadataStr), offset);
+    offset += metadataStr.length;
+    // Indices
+    view.setUint32(offset, index.indices.length);
+    offset += 4;
+    for (const idx of index.indices) {
+        view.setUint32(offset, idx);
+        offset += 4;
+    }
+    // Vectors
+    uint8.set(index.vectors, offset);
+    offset += index.vectors.length;
+    // Stored IDs count for compatibility
+    view.setUint32(offset, index.metadata.length);
+    return buffer;
+}
+/**
+ * Load binary index from buffer
+ *
+ * @param buffer - Serialized index buffer
+ * @returns Reconstructed BinaryIndex
+ */
+export function deserializeBinaryIndex(buffer) {
+    const view = new DataView(buffer);
+    const uint8 = new Uint8Array(buffer);
+    let offset = 0;
+    // Config
+    const configLen = view.getUint32(offset);
+    offset += 4;
+    const configStr = new TextDecoder().decode(uint8.subarray(offset, offset + configLen));
+    offset += configLen;
+    const config = JSON.parse(configStr);
+    // Metadata
+    const metadataLen = view.getUint32(offset);
+    offset += 4;
+    const metadataStr = new TextDecoder().decode(uint8.subarray(offset, offset + metadataLen));
+    offset += metadataLen;
+    const metadata = JSON.parse(metadataStr);
+    // Indices
+    const indicesLen = view.getUint32(offset);
+    offset += 4;
+    const indices = [];
+    for (let i = 0; i < indicesLen; i++) {
+        indices.push(view.getUint32(offset));
+        offset += 4;
+    }
+    // Vectors
+    const vectorsLen = buffer.byteLength - offset - 4;
+    const vectors = uint8.subarray(offset, offset + vectorsLen);
+    return {
+        vectors,
+        indices,
+        metadata,
+        config,
+    };
+}
+/**
+ * Create a binary vector compressor for memory embeddings
+ * Integrates with existing HeartFlow memory system
+ *
+ * @param config - Configuration for the compressor
+ * @returns Memory compression interface
+ */
+export function createBinaryVectorCompressor(config = {}) {
+    const fullConfig = { ...DEFAULT_BINARY_CONFIG, ...config };
+    let index = createBinaryIndex(fullConfig);
+    logger.info('Binary vector compressor initialized', { ...fullConfig });
+    return {
+        /**
+         * Compress and store an embedding
+         */
+        store(id, embedding, importance = 0.5) {
+            index = addToBinaryIndex(index, [embedding], [id], [{ id, importance, timestamp: Date.now() }]);
+            logger.debug(`Stored embedding ${id}, total: ${index.indices.length}`);
+        },
+        /**
+         * Compress and store multiple embeddings
+         */
+        storeBatch(ids, embeddings, importances) {
+            const metadata = ids.map((id, i) => ({
+                id,
+                importance: importances?.[i] ?? 0.5,
+                timestamp: Date.now(),
+            }));
+            index = addToBinaryIndex(index, embeddings, ids, metadata);
+            logger.debug(`Stored ${ids.length} embeddings, total: ${index.indices.length}`);
+        },
+        /**
+         * Search for similar embeddings
+         */
+        search(queryEmbedding, k = 10) {
+            return searchBinaryIndex(index, queryEmbedding, k);
+        },
+        /**
+         * Get compression statistics
+         */
+        getStats() {
+            const stats = getCompressionStats(index);
+            return {
+                ...stats,
+                stored: index.indices.length,
+                compressionPercent: `${stats.compressionRatio.toFixed(1)}x`,
+            };
+        },
+        /**
+         * Clear all stored vectors
+         */
+        clear() {
+            index = createBinaryIndex(fullConfig);
+            logger.info('Binary vector index cleared');
+        },
+        /**
+         * Export serialized index
+         */
+        export() {
+            return serializeBinaryIndex(index);
+        },
+        /**
+         * Import serialized index
+         */
+        import(buffer) {
+            index = deserializeBinaryIndex(buffer);
+            logger.info(`Imported binary index with ${index.indices.length} vectors`);
+        },
+    };
+}
+/**
+ * Common embedding dimensions for major embedding models
+ */
+export const EMBEDDING_DIMENSIONS = {
+    /** OpenAI text-embedding-3-small, text-embedding-3-large */
+    OPENAI_1536: 1536,
+    /** OpenAI text-embedding-ada-002 */
+    OPENAI_1536_ADA: 1536,
+    /** OpenAI text-embedding-3-large (3072 dim) */
+    OPENAI_3072: 3072,
+    /** Cohere embed-english-v3.0 */
+    COHERE_1024: 1024,
+    /** Cohere embed-multilingual-v3.0 */
+    COHERE_MULTILINGUAL: 1024,
+    /** Vertex AI textembedding-gecko */
+    VERTEX_GECKO: 768,
+    /** Default dimension */
+    DEFAULT: 1536,
+};

package/dist/core/memory/budget-retrieval.js

@@ -0,0 +1,420 @@
+/**
+ * Budget-Aware Memory Retrieval System
+ *
+ * A deterministic, token-budget-conscious memory retrieval system that:
+ * - Respects max token budgets during retrieval
+ * - Provides sub-millisecond query latency
+ * - Ensures identical queries return identical results
+ * - Supports semantic caching for similar queries
+ *
+ * Based on: Doorman11991/budget-aware-mcp (https://github.com/Doorman11991/budget-aware-mcp)
+ *
+ * Key concepts:
+ * - Hop-based graph walk: expand outward from anchor symbols hop by hop
+ * - Token budget: retrieval stops when budget is exhausted
+ * - Deterministic ordering: alphabetical within each hop level
+ * - Semantic cache: similar queries hit cache instantly
+ *
+ * @module core/memory
+ * @fileoverview Budget-aware memory retrieval with token control
+ */
+import { createLogger } from '../../utils/logger.js';
+import { cosineSimilarity } from './embedder.js';
+const logger = createLogger('[BudgetRetrieval]');
+/**
+ * Trigram-based similarity for semantic caching
+ */
+function extractTrigrams(text) {
+    const normalized = text.toLowerCase().replace(/[^a-z0-9\s]/g, ' ');
+    const words = normalized.split(/\s+/).filter(w => w.length > 0);
+    const trigrams = [];
+    for (const word of words) {
+        if (word.length >= 3) {
+            for (let i = 0; i <= word.length - 3; i++) {
+                trigrams.push(word.slice(i, i + 3));
+            }
+        }
+    }
+    return trigrams;
+}
+/**
+ * Jaccard similarity between two trigram sets
+ */
+function trigramSimilarity(a, b) {
+    if (a.length === 0 || b.length === 0)
+        return 0;
+    const setA = new Set(a);
+    const setB = new Set(b);
+    const intersection = new Set([...setA].filter(x => setB.has(x)));
+    const union = new Set([...setA, ...setB]);
+    return intersection.size / union.size;
+}
+/**
+ * Estimate tokens for a memory entry (rough approximation)
+ */
+function estimateTokens(entry) {
+    // Rough estimate: ~4 characters per token for typical English text
+    // Plus overhead for metadata
+    const contentTokens = Math.ceil(entry.content.length / 4);
+    const metadataTokens = 20; // tags, id, timestamps, etc.
+    return contentTokens + metadataTokens;
+}
+/**
+ * Create a deterministic hash from query
+ */
+function hashQuery(query) {
+    const parts = [
+        query.query.toLowerCase().trim(),
+        query.anchorId || '',
+        query.tier || '',
+        (query.maxTokens || 0).toString(),
+        (query.tags || []).sort().join(','),
+        (query.importanceThreshold || 0).toString()
+    ];
+    // Simple hash function for determinism
+    const str = parts.join('|');
+    let hash = 0;
+    for (let i = 0; i < str.length; i++) {
+        const char = str.charCodeAt(i);
+        hash = ((hash << 5) - hash) + char;
+        hash = hash & hash; // Convert to 32-bit integer
+    }
+    return Math.abs(hash).toString(36);
+}
+/**
+ * Alphabetical sort for deterministic ordering within hop levels
+ */
+function deterministicSort(entries) {
+    return entries.sort((a, b) => {
+        // First by hop level
+        if (a.hopLevel !== b.hopLevel) {
+            return a.hopLevel - b.hopLevel;
+        }
+        // Then alphabetically by content
+        return a.entry.content.localeCompare(b.entry.content);
+    });
+}
+/**
+ * Default configuration
+ */
+export const DEFAULT_BUDGET_CONFIG = {
+    maxTokens: 4000,
+    tokensPerEntry: 200,
+    maxHops: 3,
+    enableCache: true,
+    cacheTTLMs: 5 * 60 * 1000, // 5 minutes
+    cacheSimilarityThreshold: 0.7,
+    enableImportanceFilter: true,
+    importanceThreshold: 0.3
+};
+/**
+ * Create a budget-aware retrieval engine
+ */
+export function createBudgetRetrievalEngine(config = {}, embedder) {
+    const fullConfig = { ...DEFAULT_BUDGET_CONFIG, ...config };
+    logger.info('Creating BudgetRetrievalEngine', { ...fullConfig });
+    // Memory storage
+    const entries = new Map();
+    const entriesByTier = new Map();
+    const entriesByTag = new Map();
+    // Embedding index (if embedder provided)
+    const entryEmbeddings = new Map();
+    // Semantic cache
+    const cache = new Map();
+    let cacheStats = { hits: 0, misses: 0, hitRate: 0, avgSimilarity: 0 };
+    /**
+     * Check semantic cache for similar query
+     */
+    function checkCache(query) {
+        if (!fullConfig.enableCache)
+            return null;
+        const queryTrigrams = extractTrigrams(query.query);
+        const queryHash = hashQuery(query);
+        // Check exact match first
+        const exactEntry = cache.get(queryHash);
+        if (exactEntry) {
+            const age = Date.now() - exactEntry.timestamp;
+            if (age < fullConfig.cacheTTLMs) {
+                return exactEntry;
+            }
+        }
+        // Check similar queries
+        let bestMatch = null;
+        let bestSimilarity = 0;
+        for (const entry of cache.values()) {
+            const age = Date.now() - entry.timestamp;
+            if (age >= fullConfig.cacheTTLMs)
+                continue;
+            const similarity = trigramSimilarity(queryTrigrams, entry.trigrams);
+            if (similarity >= fullConfig.cacheSimilarityThreshold && similarity > bestSimilarity) {
+                bestSimilarity = similarity;
+                bestMatch = entry;
+            }
+        }
+        return bestMatch;
+    }
+    /**
+     * Store result in cache
+     */
+    function storeCache(query, result) {
+        if (!fullConfig.enableCache)
+            return;
+        const queryHash = hashQuery(query);
+        const trigrams = extractTrigrams(query.query);
+        cache.set(queryHash, {
+            queryHash,
+            queryText: query.query,
+            result,
+            timestamp: Date.now(),
+            trigrams
+        });
+        // Cleanup old entries
+        const now = Date.now();
+        for (const [key, entry] of cache.entries()) {
+            if (now - entry.timestamp > fullConfig.cacheTTLMs) {
+                cache.delete(key);
+            }
+        }
+    }
+    /**
+     * Calculate relevance score for an entry
+     */
+    function calculateRelevance(entry, query, anchorEntry) {
+        let relevance = 0;
+        // Base relevance from importance
+        relevance += entry.importance * 0.3;
+        // Text similarity if query provided
+        if (query.query && embedder) {
+            const queryEmb = embedder.embed(query.query);
+            const entryEmb = entryEmbeddings.get(entry.id);
+            if (queryEmb && entryEmb) {
+                const sim = cosineSimilarity(queryEmb, entryEmb);
+                relevance += sim * 0.4;
+            }
+        }
+        // Content match
+        const queryLower = query.query.toLowerCase();
+        const contentLower = entry.content.toLowerCase();
+        if (contentLower.includes(queryLower)) {
+            relevance += 0.3;
+        }
+        // Tag match
+        if (query.tags && query.tags.length > 0) {
+            const entryTags = new Set(entry.tags.map(t => t.toLowerCase()));
+            const matchCount = query.tags.filter(t => entryTags.has(t.toLowerCase())).length;
+            relevance += (matchCount / query.tags.length) * 0.2;
+        }
+        // Hop-based relevance from anchor
+        if (anchorEntry) {
+            // Entries referenced by anchor = hop 1
+            const anchor = anchorEntry;
+            const ent = entry;
+            if (anchor.references?.includes(ent.id)) {
+                return relevance + 0.5 - 0.1; // hop 1 bonus
+            }
+            // Entries referencing anchor = hop 1
+            if (ent.references?.includes(anchor.id)) {
+                return relevance + 0.5 - 0.1;
+            }
+            // Same session = hop 2
+            if (anchor.sessionId && anchor.sessionId === ent.sessionId) {
+                return relevance + 0.4 - 0.2;
+            }
+        }
+        return Math.min(1, relevance);
+    }
+    /**
+     * Perform budget-aware retrieval
+     */
+    async function retrieve(query) {
+        const startMs = Date.now();
+        const maxTokens = query.maxTokens || fullConfig.maxTokens;
+        logger.debug('Retrieving with budget', { query: query.query, maxTokens });
+        // Check cache first
+        const cached = checkCache(query);
+        if (cached) {
+            cacheStats.hits++;
+            cacheStats.avgSimilarity = (cacheStats.avgSimilarity * (cacheStats.hits - 1) + (cached.result.cacheSimilarity || 1)) / cacheStats.hits;
+            cacheStats.hitRate = cacheStats.hits / (cacheStats.hits + cacheStats.misses);
+            logger.debug('Cache hit', { similarity: cached.result.cacheSimilarity });
+            return {
+                ...cached.result,
+                cacheHit: true,
+                processingMs: Date.now() - startMs
+            };
+        }
+        cacheStats.misses++;
+        cacheStats.hitRate = cacheStats.hits / (cacheStats.hits + cacheStats.misses);
+        // Get anchor entry if specified
+        const anchorEntry = query.anchorId ? entries.get(query.anchorId) : undefined;
+        // Collect candidate entries
+        let candidateIds;
+        if (query.tier) {
+            const tierSet = entriesByTier.get(query.tier);
+            candidateIds = tierSet ? Array.from(tierSet) : [];
+        }
+        else {
+            candidateIds = Array.from(entries.keys());
+        }
+        // Filter by tags if specified
+        if (query.tags && query.tags.length > 0) {
+            candidateIds = candidateIds.filter(id => {
+                const entry = entries.get(id);
+                if (!entry)
+                    return false;
+                const entryTags = new Set(entry.tags.map(t => t.toLowerCase()));
+                return query.tags.some(t => entryTags.has(t.toLowerCase()));
+            });
+        }
+        // Score and filter candidates
+        const scoredCandidates = [];
+        for (const id of candidateIds) {
+            const entry = entries.get(id);
+            if (!entry)
+                continue;
+            // Importance filter
+            if (fullConfig.enableImportanceFilter && entry.importance < fullConfig.importanceThreshold) {
+                continue;
+            }
+            if (query.importanceThreshold && entry.importance < query.importanceThreshold) {
+                continue;
+            }
+            const relevance = calculateRelevance(entry, query, anchorEntry);
+            const tokenEstimate = estimateTokens(entry);
+            // Determine hop level
+            let hopLevel = 0;
+            if (anchorEntry) {
+                if (id === anchorEntry.id) {
+                    hopLevel = 0;
+                }
+                else if ((anchorEntry.references?.includes(id)) || (entry.references?.includes(anchorEntry.id))) {
+                    hopLevel = 1;
+                }
+                else {
+                    hopLevel = 2;
+                }
+            }
+            scoredCandidates.push({
+                entry,
+                relevance,
+                hopLevel,
+                tokenEstimate,
+                cumulativeTokens: 0 // Will be calculated after sorting
+            });
+        }
+        // Sort deterministically
+        const sorted = deterministicSort(scoredCandidates);
+        // Build budget-constrained result
+        const budget = {
+            maxTokens,
+            usedTokens: 0,
+            remainingTokens: maxTokens,
+            entriesSelected: 0
+        };
+        const selected = [];
+        for (const candidate of sorted) {
+            if (budget.remainingTokens < candidate.tokenEstimate) {
+                continue; // Can't afford this entry
+            }
+            if (candidate.hopLevel > fullConfig.maxHops) {
+                continue; // Exceeded max hops
+            }
+            candidate.cumulativeTokens = budget.usedTokens + candidate.tokenEstimate;
+            budget.usedTokens = candidate.cumulativeTokens;
+            budget.remainingTokens = maxTokens - budget.usedTokens;
+            budget.entriesSelected++;
+            selected.push(candidate);
+        }
+        const result = {
+            entries: selected,
+            tokensUsed: budget.usedTokens,
+            budgetRemaining: budget.remainingTokens,
+            cacheHit: false,
+            queryHash: hashQuery(query),
+            processingMs: Date.now() - startMs,
+            totalAvailable: candidateIds.length
+        };
+        // Store in cache
+        storeCache(query, result);
+        logger.debug('Retrieval complete', {
+            tokensUsed: result.tokensUsed,
+            entriesSelected: result.entries.length,
+            processingMs: result.processingMs,
+            cacheHit: result.cacheHit
+        });
+        return result;
+    }
+    /**
+     * Index a memory entry
+     */
+    function index(entry) {
+        entries.set(entry.id, entry);
+        // Index by tier
+        if (!entriesByTier.has(entry.tier)) {
+            entriesByTier.set(entry.tier, new Set());
+        }
+        entriesByTier.get(entry.tier).add(entry.id);
+        // Index by tags
+        for (const tag of entry.tags) {
+            if (!entriesByTag.has(tag)) {
+                entriesByTag.set(tag, new Set());
+            }
+            entriesByTag.get(tag).add(entry.id);
+        }
+        // Embed for semantic search
+        if (embedder) {
+            entryEmbeddings.set(entry.id, embedder.embed(entry.content));
+        }
+    }
+    /**
+     * Remove entry from index
+     */
+    function remove(id) {
+        const entry = entries.get(id);
+        if (!entry)
+            return;
+        entries.delete(id);
+        // Remove from tier index
+        entriesByTier.get(entry.tier)?.delete(id);
+        // Remove from tag indices
+        for (const tag of entry.tags) {
+            entriesByTag.get(tag)?.delete(id);
+        }
+        // Remove embedding
+        entryEmbeddings.delete(id);
+    }
+    /**
+     * Clear all entries
+     */
+    function clear() {
+        entries.clear();
+        entriesByTier.clear();
+        entriesByTag.clear();
+        entryEmbeddings.clear();
+        cache.clear();
+        logger.info('Budget retrieval engine cleared');
+    }
+    /**
+     * Get cache statistics
+     */
+    function getCacheStats() {
+        return { ...cacheStats };
+    }
+    /**
+     * Update configuration
+     */
+    function updateConfig(config) {
+        Object.assign(fullConfig, config);
+        logger.debug('Config updated', config);
+    }
+    return {
+        retrieve,
+        index,
+        remove,
+        clear,
+        getCacheStats,
+        updateConfig
+    };
+}
+// Export utilities for external use
+export { extractTrigrams, trigramSimilarity, estimateTokens, hashQuery };

package/dist/core/memory/index.js

@@ -11,3 +11,5 @@ export { createContextFragmentationEngine } from './context-fragmentation.js';
 export { createHybridSearchEngine, createBM25Index, bm25Score, normalizeBM25Scores, DEFAULT_HYBRID_CONFIG } from './hybrid-search.js';
 export { createPatternRecognizer } from './pattern-recognizer.js';
 export { createMemoryObserver } from './observer.js';
+export { createBinaryVectorCompressor, quantizeToBinary, hammingDistance, hammingToSimilarity, createBinaryIndex, addToBinaryIndex, searchBinaryIndex, getCompressionStats, serializeBinaryIndex, deserializeBinaryIndex, DEFAULT_BINARY_CONFIG, EMBEDDING_DIMENSIONS } from './binary-vector.js';
+export { createBudgetRetrievalEngine, DEFAULT_BUDGET_CONFIG, extractTrigrams, trigramSimilarity, estimateTokens, hashQuery } from './budget-retrieval.js';

package/dist/core/security/execution-governor.js

@@ -0,0 +1,436 @@
+/**
+ * Execution Governor - Cross-Context Tool Call Governance
+ *
+ * Inspired by chitin (Execution kernel for AI coding agents)
+ * Provides declarative policy enforcement for tool calls across different execution contexts.
+ *
+ * Features:
+ * - Gating tool calls against typed policies
+ * - Severity ladder with lockdown counter
+ * - Tamper-evident audit chain
+ * - Bounds enforcement on push-shaped actions
+ * - Heuristic signals for blast-radius, floundering, drift
+ *
+ * @module core/security
+ * @fileoverview Execution governance for HeartFlow
+ */
+import { createLogger } from '../../utils/logger.js';
+import * as crypto from 'crypto';
+const logger = createLogger('[ExecutionGovernor]');
+// ============================================================
+// Default Configuration
+// ============================================================
+const DEFAULT_CONFIG = {
+    enableAuditChain: true,
+    enableSeverityLockdown: true,
+    maxLockdownCount: 5,
+    severityThresholds: {
+        low: 3,
+        medium: 5,
+        high: 10,
+        critical: 20,
+    },
+    policies: [
+        {
+            id: 'default-read',
+            action: 'read',
+            allow: true,
+            severity: 'info',
+        },
+        {
+            id: 'default-write',
+            action: 'write',
+            allow: true,
+            conditions: {
+                bounds: {
+                    maxLinesChanged: 500,
+                    maxFilesAffected: 10,
+                },
+                requireConfirmation: true,
+            },
+            severity: 'medium',
+        },
+        {
+            id: 'default-delete',
+            action: 'delete',
+            allow: true,
+            conditions: {
+                bounds: {
+                    maxFilesAffected: 1,
+                },
+                requireConfirmation: true,
+            },
+            severity: 'high',
+        },
+        {
+            id: 'default-execute',
+            action: 'execute',
+            allow: false,
+            severity: 'critical',
+        },
+        {
+            id: 'default-network',
+            action: 'network',
+            allow: true,
+            conditions: {
+                maxPerHour: 50,
+            },
+            severity: 'medium',
+        },
+    ],
+    defaultPolicy: {
+        id: 'default-unknown',
+        action: 'unknown',
+        allow: false,
+        severity: 'high',
+    },
+};
+// ============================================================
+// Helper Functions
+// ============================================================
+/**
+ * Classify a tool call into a canonical action
+ */
+function classifyAction(toolName, args) {
+    const lowerTool = toolName.toLowerCase();
+    // Read actions
+    if (lowerTool.includes('read') || lowerTool.includes('get') || lowerTool.includes('fetch') ||
+        lowerTool.includes('search') || lowerTool.includes('query') || lowerTool.includes('select')) {
+        return 'read';
+    }
+    // Write actions
+    if (lowerTool.includes('write') || lowerTool.includes('create') || lowerTool.includes('add') ||
+        lowerTool.includes('insert') || lowerTool.includes('post') || lowerTool.includes('put')) {
+        return 'write';
+    }
+    // Delete actions
+    if (lowerTool.includes('delete') || lowerTool.includes('remove') || lowerTool.includes('drop') ||
+        lowerTool.includes('unlink') || lowerTool.includes('rm')) {
+        return 'delete';
+    }
+    // Execute actions
+    if (lowerTool.includes('exec') || lowerTool.includes('run') || lowerTool.includes('execute') ||
+        lowerTool.includes('spawn') || lowerTool.includes('fork') || lowerTool.includes('shell')) {
+        return 'execute';
+    }
+    // Network actions
+    if (lowerTool.includes('http') || lowerTool.includes('fetch') || lowerTool.includes('request') ||
+        lowerTool.includes('send') || lowerTool.includes('post') || lowerTool.includes('url')) {
+        return 'network';
+    }
+    // Filesystem actions
+    if (lowerTool.includes('file') || lowerTool.includes('dir') || lowerTool.includes('path') ||
+        lowerTool.includes('mkdir') || lowerTool.includes('stat') || lowerTool.includes('ls')) {
+        return 'filesystem';
+    }
+    // Process actions
+    if (lowerTool.includes('process') || lowerTool.includes('pid') || lowerTool.includes('kill') ||
+        lowerTool.includes('signal')) {
+        return 'process';
+    }
+    // Memory actions
+    if (lowerTool.includes('memory') || lowerTool.includes('store') || lowerTool.includes('recall') ||
+        lowerTool.includes('embed')) {
+        return 'memory';
+    }
+    // Agent actions
+    if (lowerTool.includes('agent') || lowerTool.includes('spawn') || lowerTool.includes('delegate') ||
+        lowerTool.includes('subagent')) {
+        return 'agent';
+    }
+    return 'unknown';
+}
+/**
+ * Calculate heuristic signals from envelope context
+ */
+function calculateHeuristics(envelope) {
+    const context = envelope.context;
+    return {
+        blastRadius: context.blastRadius || 'low',
+        floundering: context.floundering || false,
+        drift: context.drift || 0,
+        repetitionCount: 0, // Tracked in agent state
+        urgency: 0.5, // Default
+        stakes: envelope.action === 'delete' || envelope.action === 'execute' ? 0.9 : 0.3,
+    };
+}
+/**
+ * Hash data for audit chain
+ */
+function hashData(data, prevHash) {
+    const payload = JSON.stringify({ data, prevHash, timestamp: Date.now() });
+    return crypto.createHash('sha256').update(payload).digest('hex');
+}
+// ============================================================
+// Execution Governor Class
+// ============================================================
+export class ExecutionGovernor {
+    config;
+    agentStates = new Map();
+    auditChain = [];
+    decisionHistory = new Map();
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        logger.info('ExecutionGovernor initialized', {
+            policiesCount: this.config.policies.length,
+            auditChainEnabled: this.config.enableAuditChain,
+        });
+    }
+    /**
+     * Process a tool call and return governance decision
+     */
+    async gate(envelope) {
+        // Auto-classify action if not provided
+        if (envelope.action === 'unknown') {
+            envelope.action = classifyAction(envelope.toolName, envelope.args);
+        }
+        // Calculate heuristic signals
+        const signals = calculateHeuristics(envelope);
+        // Get or create agent state
+        let state = this.agentStates.get(envelope.context.sessionId);
+        if (!state) {
+            state = this.createAgentState(envelope.context.sessionId);
+            this.agentStates.set(envelope.context.sessionId, state);
+        }
+        // Check for lockdown
+        if (state.locked && this.config.enableSeverityLockdown) {
+            const decision = this.createDecision(envelope, 'deny', 'Agent is in lockdown due to severity threshold breach');
+            this.appendAudit('decision', decision);
+            return decision;
+        }
+        // Check severity thresholds
+        if (this.config.enableSeverityLockdown) {
+            const severityLevel = this.checkSeverityThreshold(state);
+            if (severityLevel) {
+                state.lockdownCount++;
+                if (state.lockdownCount >= this.config.maxLockdownCount) {
+                    state.locked = true;
+                    this.appendAudit('lockdown', { sessionId: envelope.context.sessionId, count: state.lockdownCount });
+                    logger.warn('Agent locked due to severity threshold', { sessionId: envelope.context.sessionId });
+                }
+            }
+        }
+        // Find matching policy
+        const policy = this.findMatchingPolicy(envelope.action);
+        // Evaluate bounds if present
+        let reason = 'Policy evaluation';
+        let decision = 'allow';
+        if (policy) {
+            if (!policy.allow) {
+                decision = 'deny';
+                reason = `Policy ${policy.id} explicitly denies ${envelope.action}`;
+            }
+            else if (policy.conditions?.requireConfirmation) {
+                decision = 'escalate';
+                reason = `${envelope.action} requires operator confirmation`;
+            }
+            else if (policy.conditions?.bounds) {
+                const boundCheck = this.checkBounds(envelope, policy.conditions.bounds);
+                if (!boundCheck.allowed) {
+                    decision = 'deny';
+                    reason = boundCheck.reason || 'Bounds check failed';
+                }
+            }
+            // Update severity counts
+            if (this.config.enableSeverityLockdown && decision !== 'allow') {
+                state.severityCounts[policy.severity] = (state.severityCounts[policy.severity] || 0) + 1;
+            }
+        }
+        else {
+            decision = this.config.defaultPolicy.allow ? 'allow' : 'deny';
+            reason = `No matching policy, using default: ${this.config.defaultPolicy.allow ? 'allow' : 'deny'}`;
+        }
+        // Update state
+        state.totalToolCalls++;
+        state.lastActivity = Date.now();
+        // Create decision
+        const governanceDecision = this.createDecision(envelope, decision, reason, policy ?? undefined);
+        this.decisionHistory.set(envelope.id, governanceDecision);
+        this.appendAudit('decision', governanceDecision);
+        if (decision === 'deny') {
+            logger.info('Tool call denied', { envelopeId: envelope.id, action: envelope.action, reason });
+        }
+        else if (decision === 'escalate') {
+            logger.info('Tool call escalated', { envelopeId: envelope.id, action: envelope.action, reason });
+        }
+        return governanceDecision;
+    }
+    /**
+     * Create a new tool call envelope
+     */
+    createEnvelope(toolName, args, context) {
+        return {
+            id: crypto.randomUUID(),
+            action: classifyAction(toolName, args),
+            toolName,
+            args,
+            context,
+        };
+    }
+    /**
+     * Unlock a locked agent
+     */
+    unlock(sessionId) {
+        const state = this.agentStates.get(sessionId);
+        if (!state)
+            return false;
+        state.locked = false;
+        state.lockdownCount = 0;
+        state.severityCounts = {};
+        this.appendAudit('unlock', { sessionId });
+        logger.info('Agent unlocked', { sessionId });
+        return true;
+    }
+    /**
+     * Reset agent state
+     */
+    resetState(sessionId) {
+        const state = this.agentStates.get(sessionId);
+        if (!state)
+            return false;
+        const newState = this.createAgentState(sessionId);
+        this.agentStates.set(sessionId, newState);
+        this.appendAudit('state_reset', { sessionId });
+        logger.info('Agent state reset', { sessionId });
+        return true;
+    }
+    /**
+     * Get agent state
+     */
+    getAgentState(sessionId) {
+        return this.agentStates.get(sessionId) || null;
+    }
+    /**
+     * Get audit chain
+     */
+    getAuditChain() {
+        return [...this.auditChain];
+    }
+    /**
+     * Get decision from history
+     */
+    getDecision(envelopeId) {
+        return this.decisionHistory.get(envelopeId) || null;
+    }
+    /**
+     * Verify audit chain integrity
+     */
+    verifyAuditChain() {
+        for (let i = 1; i < this.auditChain.length; i++) {
+            if (this.auditChain[i].prevHash !== this.auditChain[i - 1].hash) {
+                return { valid: false, brokenAt: i };
+            }
+        }
+        return { valid: true };
+    }
+    // Private helper methods
+    createAgentState(sessionId) {
+        return {
+            sessionId,
+            severityCounts: {},
+            lockdownCount: 0,
+            totalToolCalls: 0,
+            lastActivity: Date.now(),
+            locked: false,
+        };
+    }
+    checkSeverityThreshold(state) {
+        const counts = state.severityCounts;
+        if ((counts['critical'] || 0) >= this.config.severityThresholds.critical)
+            return 'critical';
+        if ((counts['high'] || 0) >= this.config.severityThresholds.high)
+            return 'high';
+        if ((counts['medium'] || 0) >= this.config.severityThresholds.medium)
+            return 'medium';
+        if ((counts['low'] || 0) >= this.config.severityThresholds.low)
+            return 'low';
+        return null;
+    }
+    findMatchingPolicy(action) {
+        return this.config.policies.find(p => p.action === action) || null;
+    }
+    checkBounds(envelope, bounds) {
+        if (!bounds)
+            return { allowed: true };
+        // Check filesystem bounds
+        if (bounds.allowedPaths || bounds.deniedPaths) {
+            const path = this.extractPath(envelope.args);
+            if (path) {
+                if (bounds.deniedPaths?.some(p => path.includes(p))) {
+                    return { allowed: false, reason: `Path ${path} is in denied list` };
+                }
+                if (bounds.allowedPaths && !bounds.allowedPaths.some(p => path.includes(p))) {
+                    return { allowed: false, reason: `Path ${path} is not in allowed list` };
+                }
+            }
+        }
+        // Check data size
+        if (bounds.maxDataSize) {
+            const size = this.estimateDataSize(envelope.args);
+            if (size > bounds.maxDataSize) {
+                return { allowed: false, reason: `Data size ${size} exceeds limit ${bounds.maxDataSize}` };
+            }
+        }
+        return { allowed: true };
+    }
+    extractPath(args) {
+        for (const key of ['path', 'file', 'filePath', 'target', 'destination']) {
+            if (args[key] && typeof args[key] === 'string') {
+                return args[key];
+            }
+        }
+        return null;
+    }
+    estimateDataSize(args) {
+        try {
+            return JSON.stringify(args).length;
+        }
+        catch {
+            return 0;
+        }
+    }
+    createDecision(envelope, decision, reason, policy) {
+        const decisionData = {
+            envelopeId: envelope.id,
+            action: envelope.action,
+            decision,
+            reason,
+            timestamp: Date.now(),
+        };
+        return {
+            ...decisionData,
+            matchedPolicy: policy,
+            hash: hashData(decisionData, this.auditChain.length > 0 ? this.auditChain[this.auditChain.length - 1].hash : 'genesis'),
+        };
+    }
+    appendAudit(type, payload) {
+        if (!this.config.enableAuditChain)
+            return;
+        const event = {
+            type,
+            payload,
+            timestamp: Date.now(),
+            hash: '',
+            prevHash: this.auditChain.length > 0 ? this.auditChain[this.auditChain.length - 1].hash : 'genesis',
+        };
+        event.hash = hashData({ type, payload, timestamp: event.timestamp }, event.prevHash);
+        this.auditChain.push(event);
+        // Keep chain bounded to last 10000 events
+        if (this.auditChain.length > 10000) {
+            this.auditChain = this.auditChain.slice(-5000);
+        }
+    }
+}
+// ============================================================
+// Factory Function
+// ============================================================
+/**
+ * Create an ExecutionGovernor instance with optional custom config
+ */
+export function createExecutionGovernor(config) {
+    return new ExecutionGovernor(config);
+}
+// ============================================================
+// Export default config for customization
+// ============================================================
+export { DEFAULT_CONFIG };

package/dist/core/security/index.js

@@ -1,2 +1,3 @@
 export * from './privacy.js';
 export * from './agent-shield.js';
+export { ExecutionGovernor, createExecutionGovernor, DEFAULT_CONFIG } from './execution-governor.js';

package/dist/version.js

@@ -1 +1 @@
-export const VERSION = '2.3.3';
+export const VERSION = '2.3.5';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mark-improving-agent",
-  "version": "2.3.3",
+  "version": "2.3.5",
   "description": "Self-evolving AI agent with permanent memory, identity continuity, and self-evolution — for AI agents that need to remember, learn, and evolve across sessions",
   "type": "module",
   "main": "./dist/index.js",