mark-improving-agent 2.3.2 → 2.3.4

package/VERSION

@@ -1 +1 @@
-2.3.2
+2.3.4

package/dist/core/memory/binary-vector.js

@@ -0,0 +1,405 @@
+/**
+ * Binary Vector Memory Compression
+ *
+ * High-performance binary vector quantization for memory embeddings.
+ * Based on QuIVer: Rethinking ANN Graph Topology via Training-Free Binary Quantization (Xiao et al., 2026)
+ *
+ * Achieves 32x compression: 1536-dim float32 (6144 bytes) → 192 bytes binary
+ * Uses Hamming distance (XOR + POPCOUNT) for fast similarity search.
+ *
+ * @module core/memory
+ * @fileoverview Binary vector quantization for memory compression
+ */
+import { createLogger } from '../../utils/logger.js';
+const logger = createLogger('[BinaryVector]');
+/**
+ * Default configuration for 1536-dim embeddings (OpenAI text-embedding-3-small)
+ */
+export const DEFAULT_BINARY_CONFIG = {
+    dimension: 1536,
+    normalized: true,
+    trackStats: true,
+};
+/**
+ * Create a binary vector index
+ *
+ * @param config - Configuration for the index
+ * @returns Empty binary index ready for vectors
+ */
+export function createBinaryIndex(config) {
+    logger.info('Creating binary vector index', { dimension: config.dimension });
+    return {
+        vectors: new Uint8Array(0),
+        indices: [],
+        metadata: [],
+        config,
+    };
+}
+/**
+ * Quantize a float32 vector to binary using sign-bit quantization
+ *
+ * @param vector - Float32 array (dimension length)
+ * @param normalized - Whether vector is L2 normalized (skip magnitude check if true)
+ * @returns Uint8Array of packed binary bits
+ *
+ * @example
+ * ```typescript
+ * const vector = new Float32Array([0.5, -0.3, 0.1, -0.8]);
+ * const binary = quantizeToBinary(vector);
+ * // Result: Uint8Array [0b10001001] (4 dimensions = 4 bits = 1 byte)
+ * ```
+ */
+export function quantizeToBinary(vector, normalized = true) {
+    const dim = vector.length;
+    const bytes = Math.ceil(dim / 8);
+    const result = new Uint8Array(bytes);
+    for (let i = 0; i < dim; i++) {
+        // Sign bit: positive = 1, negative = 0
+        const bitIndex = i % 8;
+        const byteIndex = Math.floor(i / 8);
+        if (vector[i] >= 0) {
+            result[byteIndex] |= (1 << (7 - bitIndex));
+        }
+    }
+    return result;
+}
+/**
+ * Dequantize binary vector back to approximate float32
+ *
+ * Note: This is lossy - returns center-of-mass values (±0.5 for normalized vectors)
+ *
+ * @param binary - Packed binary bits
+ * @param dimension - Original dimension
+ * @param normalized - Whether to return normalized values
+ * @returns Approximate float32 vector
+ */
+export function dequantizeFromBinary(binary, dimension, normalized = true) {
+    const result = new Float32Array(dimension);
+    for (let i = 0; i < dimension; i++) {
+        const bitIndex = i % 8;
+        const byteIndex = Math.floor(i / 8);
+        const bit = (binary[byteIndex] >> (7 - bitIndex)) & 1;
+        // For normalized vectors: 1 → +0.5, 0 → -0.5
+        // This gives approximate reconstruction
+        result[i] = normalized ? (bit ? 0.5 : -0.5) : (bit ? 0 : 0);
+    }
+    return result;
+}
+/**
+ * Calculate Hamming distance between two binary vectors
+ *
+ * @param a - First binary vector
+ * @param b - Second binary vector
+ * @returns Hamming distance (count of differing bits)
+ *
+ * @example
+ * ```typescript
+ * const dist = hammingDistance(binaryA, binaryB);
+ * // dist = 0 means identical, dist = dimension means opposite
+ * ```
+ */
+export function hammingDistance(a, b) {
+    if (a.length !== b.length) {
+        throw new Error(`Binary vectors must have same length: ${a.length} vs ${b.length}`);
+    }
+    let distance = 0;
+    for (let i = 0; i < a.length; i++) {
+        // XOR gives 1s where bits differ, POPCOUNT counts them
+        const xored = a[i] ^ b[i];
+        // POPCOUNT via built-in
+        let count = xored;
+        count = (count & 0x55) + ((count >> 1) & 0x55);
+        count = (count & 0x33) + ((count >> 2) & 0x33);
+        count = (count & 0x0F) + ((count >> 4) & 0x0F);
+        distance += count;
+    }
+    return distance;
+}
+/**
+ * Calculate similarity from Hamming distance
+ *
+ * @param hammingDist - Hamming distance
+ * @param dimension - Vector dimension
+ * @returns Similarity score 0-1 (1 = identical)
+ */
+export function hammingToSimilarity(hammingDist, dimension) {
+    return 1 - (hammingDist / dimension);
+}
+/**
+ * Add vectors to the binary index
+ *
+ * @param index - Binary index to add to
+ * @param vectors - Float32Array vectors (each dimension-long)
+ * @param ids - Unique identifiers for each vector
+ * @param metadata - Optional metadata (importance, timestamp)
+ * @returns Updated index
+ */
+export function addToBinaryIndex(index, vectors, ids, metadata) {
+    const dim = index.config.dimension;
+    if (vectors.length !== ids.length) {
+        throw new Error(`Vectors count (${vectors.length}) must match IDs count (${ids.length})`);
+    }
+    // Calculate total bytes needed
+    const bytesPerVector = Math.ceil(dim / 8);
+    const newTotalBytes = index.vectors.length + (vectors.length * bytesPerVector);
+    // Create new expanded vectors array
+    const newVectors = new Uint8Array(newTotalBytes);
+    if (index.vectors.length > 0) {
+        newVectors.set(index.vectors);
+    }
+    // Quantize and add each vector
+    let offset = index.vectors.length;
+    for (let i = 0; i < vectors.length; i++) {
+        if (vectors[i].length !== dim) {
+            throw new Error(`Vector dimension ${vectors[i].length} doesn't match index dimension ${dim}`);
+        }
+        const binary = quantizeToBinary(vectors[i], index.config.normalized);
+        newVectors.set(binary, offset);
+        offset += bytesPerVector;
+    }
+    return {
+        vectors: newVectors,
+        indices: [...index.indices, ...ids.map((_, i) => index.indices.length + i)],
+        metadata: [...index.metadata, ...metadata || ids.map(() => ({ id: '', importance: 0.5, timestamp: Date.now() }))],
+        config: index.config,
+    };
+}
+/**
+ * Search for similar vectors in the binary index using Hamming distance
+ *
+ * @param index - Binary index to search
+ * @param queryVector - Query vector to search for
+ * @param k - Number of results to return
+ * @returns Top-k results sorted by similarity (highest first)
+ *
+ * @example
+ * ```typescript
+ * const results = searchBinaryIndex(index, queryEmbedding, 10);
+ * // Returns top 10 most similar memories
+ * ```
+ */
+export function searchBinaryIndex(index, queryVector, k) {
+    const dim = index.config.dimension;
+    const bytesPerVector = Math.ceil(dim / 8);
+    if (queryVector.length !== dim) {
+        throw new Error(`Query dimension ${queryVector.length} doesn't match index dimension ${dim}`);
+    }
+    // Quantize query
+    const queryBinary = quantizeToBinary(queryVector, index.config.normalized);
+    // Search all vectors
+    const results = [];
+    let totalHamming = 0;
+    for (let i = 0; i < index.indices.length; i++) {
+        const offset = i * bytesPerVector;
+        const vectorBinary = index.vectors.subarray(offset, offset + bytesPerVector);
+        const hDist = hammingDistance(queryBinary, vectorBinary);
+        totalHamming += hDist;
+        results.push({
+            index: i,
+            id: index.metadata[i]?.id || String(index.indices[i]),
+            hammingDistance: hDist,
+            similarity: hammingToSimilarity(hDist, dim),
+            metadata: index.metadata[i] || { importance: 0.5, timestamp: Date.now() },
+        });
+    }
+    // Sort by similarity (highest first)
+    results.sort((a, b) => b.similarity - a.similarity);
+    // Return top-k
+    const topK = results.slice(0, k);
+    if (index.config.trackStats) {
+        logger.debug(`Search completed: ${index.indices.length} vectors, avg Hamming: ${(totalHamming / index.indices.length).toFixed(2)}`);
+    }
+    return topK;
+}
+/**
+ * Get compression statistics
+ *
+ * @param index - Binary index
+ * @returns Compression statistics
+ */
+export function getCompressionStats(index) {
+    const dim = index.config.dimension;
+    const float32Bytes = dim * 4; // 4 bytes per float32
+    const binaryBytes = Math.ceil(dim / 8);
+    const vectorsCount = index.indices.length;
+    const originalBytes = vectorsCount * float32Bytes;
+    const compressedBytes = vectorsCount * binaryBytes;
+    return {
+        originalBytes,
+        compressedBytes,
+        compressionRatio: originalBytes / compressedBytes,
+        vectorsStored: vectorsCount,
+        dimension: dim,
+        avgHammingDistance: 0, // Calculated on demand
+    };
+}
+/**
+ * Save binary index to binary format
+ *
+ * @param index - Binary index to save
+ * @returns Buffer containing serialized index
+ */
+export function serializeBinaryIndex(index) {
+    const configBytes = JSON.stringify(index.config).length;
+    const metadataBytes = JSON.stringify(index.metadata).length;
+    const indicesBytes = index.indices.length * 4; // 4 bytes per number
+    const totalBytes = 4 + configBytes + 4 + metadataBytes + 4 + indicesBytes + index.vectors.length + 4 + index.indices.length * 8;
+    const buffer = new ArrayBuffer(totalBytes);
+    const view = new DataView(buffer);
+    const uint8 = new Uint8Array(buffer);
+    let offset = 0;
+    // Config
+    const configStr = JSON.stringify(index.config);
+    view.setUint32(offset, configStr.length);
+    offset += 4;
+    uint8.set(new TextEncoder().encode(configStr), offset);
+    offset += configStr.length;
+    // Metadata
+    const metadataStr = JSON.stringify(index.metadata);
+    view.setUint32(offset, metadataStr.length);
+    offset += 4;
+    uint8.set(new TextEncoder().encode(metadataStr), offset);
+    offset += metadataStr.length;
+    // Indices
+    view.setUint32(offset, index.indices.length);
+    offset += 4;
+    for (const idx of index.indices) {
+        view.setUint32(offset, idx);
+        offset += 4;
+    }
+    // Vectors
+    uint8.set(index.vectors, offset);
+    offset += index.vectors.length;
+    // Stored IDs count for compatibility
+    view.setUint32(offset, index.metadata.length);
+    return buffer;
+}
+/**
+ * Load binary index from buffer
+ *
+ * @param buffer - Serialized index buffer
+ * @returns Reconstructed BinaryIndex
+ */
+export function deserializeBinaryIndex(buffer) {
+    const view = new DataView(buffer);
+    const uint8 = new Uint8Array(buffer);
+    let offset = 0;
+    // Config
+    const configLen = view.getUint32(offset);
+    offset += 4;
+    const configStr = new TextDecoder().decode(uint8.subarray(offset, offset + configLen));
+    offset += configLen;
+    const config = JSON.parse(configStr);
+    // Metadata
+    const metadataLen = view.getUint32(offset);
+    offset += 4;
+    const metadataStr = new TextDecoder().decode(uint8.subarray(offset, offset + metadataLen));
+    offset += metadataLen;
+    const metadata = JSON.parse(metadataStr);
+    // Indices
+    const indicesLen = view.getUint32(offset);
+    offset += 4;
+    const indices = [];
+    for (let i = 0; i < indicesLen; i++) {
+        indices.push(view.getUint32(offset));
+        offset += 4;
+    }
+    // Vectors
+    const vectorsLen = buffer.byteLength - offset - 4;
+    const vectors = uint8.subarray(offset, offset + vectorsLen);
+    return {
+        vectors,
+        indices,
+        metadata,
+        config,
+    };
+}
+/**
+ * Create a binary vector compressor for memory embeddings
+ * Integrates with existing HeartFlow memory system
+ *
+ * @param config - Configuration for the compressor
+ * @returns Memory compression interface
+ */
+export function createBinaryVectorCompressor(config = {}) {
+    const fullConfig = { ...DEFAULT_BINARY_CONFIG, ...config };
+    let index = createBinaryIndex(fullConfig);
+    logger.info('Binary vector compressor initialized', { ...fullConfig });
+    return {
+        /**
+         * Compress and store an embedding
+         */
+        store(id, embedding, importance = 0.5) {
+            index = addToBinaryIndex(index, [embedding], [id], [{ id, importance, timestamp: Date.now() }]);
+            logger.debug(`Stored embedding ${id}, total: ${index.indices.length}`);
+        },
+        /**
+         * Compress and store multiple embeddings
+         */
+        storeBatch(ids, embeddings, importances) {
+            const metadata = ids.map((id, i) => ({
+                id,
+                importance: importances?.[i] ?? 0.5,
+                timestamp: Date.now(),
+            }));
+            index = addToBinaryIndex(index, embeddings, ids, metadata);
+            logger.debug(`Stored ${ids.length} embeddings, total: ${index.indices.length}`);
+        },
+        /**
+         * Search for similar embeddings
+         */
+        search(queryEmbedding, k = 10) {
+            return searchBinaryIndex(index, queryEmbedding, k);
+        },
+        /**
+         * Get compression statistics
+         */
+        getStats() {
+            const stats = getCompressionStats(index);
+            return {
+                ...stats,
+                stored: index.indices.length,
+                compressionPercent: `${stats.compressionRatio.toFixed(1)}x`,
+            };
+        },
+        /**
+         * Clear all stored vectors
+         */
+        clear() {
+            index = createBinaryIndex(fullConfig);
+            logger.info('Binary vector index cleared');
+        },
+        /**
+         * Export serialized index
+         */
+        export() {
+            return serializeBinaryIndex(index);
+        },
+        /**
+         * Import serialized index
+         */
+        import(buffer) {
+            index = deserializeBinaryIndex(buffer);
+            logger.info(`Imported binary index with ${index.indices.length} vectors`);
+        },
+    };
+}
+/**
+ * Common embedding dimensions for major embedding models
+ */
+export const EMBEDDING_DIMENSIONS = {
+    /** OpenAI text-embedding-3-small, text-embedding-3-large */
+    OPENAI_1536: 1536,
+    /** OpenAI text-embedding-ada-002 */
+    OPENAI_1536_ADA: 1536,
+    /** OpenAI text-embedding-3-large (3072 dim) */
+    OPENAI_3072: 3072,
+    /** Cohere embed-english-v3.0 */
+    COHERE_1024: 1024,
+    /** Cohere embed-multilingual-v3.0 */
+    COHERE_MULTILINGUAL: 1024,
+    /** Vertex AI textembedding-gecko */
+    VERTEX_GECKO: 768,
+    /** Default dimension */
+    DEFAULT: 1536,
+};

package/dist/core/memory/index.js

@@ -10,3 +10,5 @@ export * from './adaptive-rag.js';
 export { createContextFragmentationEngine } from './context-fragmentation.js';
 export { createHybridSearchEngine, createBM25Index, bm25Score, normalizeBM25Scores, DEFAULT_HYBRID_CONFIG } from './hybrid-search.js';
 export { createPatternRecognizer } from './pattern-recognizer.js';
+export { createMemoryObserver } from './observer.js';
+export { createBinaryVectorCompressor, quantizeToBinary, hammingDistance, hammingToSimilarity, createBinaryIndex, addToBinaryIndex, searchBinaryIndex, getCompressionStats, serializeBinaryIndex, deserializeBinaryIndex, DEFAULT_BINARY_CONFIG, EMBEDDING_DIMENSIONS } from './binary-vector.js';

package/dist/core/memory/observer.js

@@ -0,0 +1,350 @@
+/**
+ * Memory Observer - Silent Background Memory Writer
+ *
+ * Implements the Mnemostroma-inspired Observer pattern: the AI agent NEVER writes
+ * memory directly. Instead, this Observer sidecar silently watches all I/O,
+ * extracts entities, embeds, scores, and indexes content automatically.
+ *
+ * Key principles:
+ * - AI never writes memory — Observer does it silently
+ * - Dual async pipeline: Observer (write) + Content Branch (versioned artifacts)
+ * - Memory Hulling: separates conversational noise from extractable kernels
+ * - 20ms hot buffer retrieval, ~50ms full extraction latency
+ *
+ * Based on: GG-QandV/mnemostroma (https://github.com/GG-QandV/mnemostroma)
+ *
+ * @module core/memory
+ * @fileoverview Observer pattern for silent memory writing
+ */
+import { randomUUID } from 'crypto';
+import { createLogger } from '../../utils/logger.js';
+import { createEmbedder, cosineSimilarity } from './embedder.js';
+const logger = createLogger('[MemoryObserver]');
+const DEFAULT_OBSERVER_CONFIG = {
+    importanceThreshold: 0.3,
+    hotBufferSize: 50,
+    autoExtract: true,
+    debounceMs: 100,
+    maxShellAge: 5 * 60 * 1000, // 5 minutes
+    embedder: createEmbedder({ dimensions: 128 }),
+};
+/**
+ * Noise patterns that indicate conversational shell (not worth storing)
+ */
+const SHELL_PATTERNS = [
+    /^(hi|hello|hey|what's up|howdy)\b/i,
+    /^(thanks?|thank you|thx|ty|much appreciated)\b/i,
+    /^(okay|ok|okk|kk|sure|yes|no|nah|yeah|yep|nope)\b/i,
+    /^(lol|lmao|rofl|haha|heh)\b/i,
+    /^(bye|goodbye|see you|cya|good night|night)\b/i,
+    /^(sorry|apologies|my bad|whoops)\b/i,
+    /^(please|pls|plz|could you|would you)\b/i,
+    /^(interesting|cool|nice|awesome|great|good)\b/i,
+    /^(i see|i understand|i get it|got it|understood)\b/i,
+    /^[\s.,!?;:]*$/,
+    /^(oh|ah|uh|um|hmm|well)\b/i,
+    /^(let me know|feel free|take care|talk later)\b/i,
+];
+/**
+ * Anchor patterns that indicate important kernels (decisions, deadlines, facts)
+ */
+const ANCHOR_PATTERNS = [
+    /\b(decided|decision|agreed|chosen|selected|picked|settled)\b/i,
+    /\b(must|have to|need to|required|mandatory|essential)\b/i,
+    /\b(deadline|due|by|before|after|until|timing)\b/i,
+    /\b(never|always|don't|do not|must not|forbidden)\b/i,
+    /\b(important|critical|key|vital|priority)\b/i,
+    /\b(remember|forget|keep in mind|note that)\b/i,
+    /\b(because|since|reason|why|therefore|thus)\b/i,
+    /\b(fact|true|real|actually|indeed|in fact)\b/i,
+    /\b(but|however|although|except|except for)\b/i,
+    /\b(name|date|location|price|cost|amount)\b/i,
+    /\b(user|client|customer|they|their)\b.*\b(want|need|prefer|like)\b/i,
+];
+/**
+ * Kernel type classifiers
+ */
+function classifyKernel(content, context) {
+    const lower = content.toLowerCase();
+    const fullText = [content, ...context].join(' ').toLowerCase();
+    if (/deadline|due|by |before |after |timing/i.test(lower))
+        return 'constraint';
+    if (/decided|agreed|chosen|picked|selected/i.test(lower))
+        return 'decision';
+    if (/must|have to|need to|required|mandatory/i.test(lower))
+        return 'rule';
+    if (/fact|true|actual|indeed|in fact|real/i.test(fullText))
+        return 'fact';
+    if (/remember|forget|note|keep in mind/i.test(lower))
+        return 'context';
+    if (/name|date|location|price|amount/i.test(lower))
+        return 'entity';
+    return 'context';
+}
+/**
+ * Score importance based on content features
+ */
+function scoreImportance(content, isAnchor) {
+    let score = 0.5;
+    if (isAnchor)
+        score += 0.2;
+    const words = content.split(/\s+/).length;
+    if (words >= 5 && words <= 50)
+        score += 0.15;
+    else if (words > 100)
+        score -= 0.1;
+    if (/\d+/.test(content))
+        score += 0.1;
+    if (/[A-Z][a-z]+\s+[A-Z][a-z]+/.test(content))
+        score += 0.1;
+    if (/\?$/.test(content))
+        score += 0.05;
+    if (/!$/.test(content))
+        score += 0.05;
+    return Math.max(0, Math.min(1, score));
+}
+/**
+ * Check if content is shell (noise)
+ */
+function isShell(content) {
+    const trimmed = content.trim();
+    if (trimmed.length < 3)
+        return true;
+    for (const pattern of SHELL_PATTERNS) {
+        if (pattern.test(trimmed))
+            return true;
+    }
+    return false;
+}
+/**
+ * Extract anchors from content
+ */
+function extractAnchors(content) {
+    const anchors = [];
+    for (const pattern of ANCHOR_PATTERNS) {
+        const match = content.match(pattern);
+        if (match) {
+            anchors.push(match[0]);
+        }
+    }
+    return anchors;
+}
+function buildMemoryObserver(config = {}) {
+    const cfg = { ...DEFAULT_OBSERVER_CONFIG, ...config };
+    const hotBuffer = [];
+    const anchorIndex = new Map();
+    const tagIndex = new Map();
+    let kernelsExtracted = 0;
+    let shellDiscarded = 0;
+    let lastFlush = null;
+    let sessionId = randomUUID();
+    let previousSessionId = null;
+    /**
+     * Hull content — separate kernels from shell
+     */
+    function hull(content, context = []) {
+        const kernels = [];
+        const shell = [];
+        const sentences = content.split(/[.!?]+/).map(s => s.trim()).filter(s => s.length > 2);
+        for (const sentence of sentences) {
+            if (isShell(sentence)) {
+                shell.push(sentence);
+                shellDiscarded++;
+                continue;
+            }
+            const anchors = extractAnchors(sentence);
+            const isAnchor = anchors.length > 0;
+            const importance = scoreImportance(sentence, isAnchor);
+            if (importance >= cfg.importanceThreshold) {
+                const type = classifyKernel(sentence, context);
+                const kernel = {
+                    type,
+                    content: sentence,
+                    importance,
+                    tags: [type],
+                    anchors,
+                    sourceExcerpt: sentence.slice(0, 200),
+                };
+                if (isAnchor) {
+                    kernel.tags.push('anchor');
+                }
+                kernels.push(kernel);
+                kernelsExtracted++;
+                for (const anchor of anchors) {
+                    anchorIndex.set(anchor, {
+                        content: sentence,
+                        type,
+                        timestamp: Date.now(),
+                    });
+                }
+                for (const tag of kernel.tags) {
+                    if (!tagIndex.has(tag)) {
+                        tagIndex.set(tag, []);
+                    }
+                    tagIndex.get(tag).push(kernel);
+                }
+            }
+            else {
+                shell.push(sentence);
+                shellDiscarded++;
+            }
+        }
+        return { kernels, shell };
+    }
+    function maintainHotBuffer() {
+        while (hotBuffer.length > cfg.hotBufferSize) {
+            let minIdx = 0;
+            let minImportance = Infinity;
+            for (let i = 0; i < hotBuffer.length; i++) {
+                if (hotBuffer[i].importance < minImportance) {
+                    minImportance = hotBuffer[i].importance;
+                    minIdx = i;
+                }
+            }
+            hotBuffer.splice(minIdx, 1);
+        }
+    }
+    async function semanticSearch(query, limit = 5) {
+        try {
+            const queryEmbedding = cfg.embedder.embed(query);
+            const results = [];
+            for (const kernel of hotBuffer) {
+                const kernelEmbedding = cfg.embedder.embed(kernel.content);
+                const similarity = cosineSimilarity(queryEmbedding, kernelEmbedding);
+                if (similarity > 0.3) {
+                    results.push({ content: kernel.content, score: similarity });
+                }
+            }
+            results.sort((a, b) => b.score - a.score);
+            return results.slice(0, limit);
+        }
+        catch (err) {
+            logger.warn('Semantic search failed, falling back to keyword', { error: err });
+            const queryLower = query.toLowerCase();
+            return hotBuffer
+                .filter(k => k.content.toLowerCase().includes(queryLower))
+                .slice(0, limit)
+                .map(k => ({ content: k.content, score: k.importance }));
+        }
+    }
+    return {
+        observe(content, stream = 'semantic', context) {
+            const start = Date.now();
+            const { kernels, shell } = hull(content, context);
+            hotBuffer.push(...kernels);
+            maintainHotBuffer();
+            const processingMs = Date.now() - start;
+            logger.debug(`Observed [${stream}]: ${kernels.length} kernels extracted, ${shell.length} shell discarded (${processingMs}ms)`);
+            return {
+                id: randomUUID(),
+                stream,
+                kernels,
+                shell,
+                timestamp: Date.now(),
+                processingMs,
+            };
+        },
+        async flush(tier = 'learned') {
+            if (hotBuffer.length === 0)
+                return [];
+            const entries = [];
+            for (const kernel of hotBuffer) {
+                const entry = {
+                    id: randomUUID(),
+                    tier,
+                    content: kernel.content,
+                    importance: kernel.importance,
+                    tags: kernel.tags,
+                    timestamp: Date.now(),
+                    accessCount: 0,
+                    lastAccessed: Date.now(),
+                    source: 'self',
+                };
+                entries.push(entry);
+            }
+            lastFlush = Date.now();
+            logger.info(`Flushed ${entries.length} kernels to ${tier} tier`);
+            return entries;
+        },
+        async ctx_semantic(query, limit = 5) {
+            return semanticSearch(query, limit);
+        },
+        ctx_anchors(limit = 20) {
+            const results = Array.from(anchorIndex.entries())
+                .map(([, data]) => ({
+                content: data.content,
+                type: data.type,
+                timestamp: data.timestamp,
+            }))
+                .sort((a, b) => b.timestamp - a.timestamp);
+            return results.slice(0, limit);
+        },
+        async ctx_search(tags, limit = 10) {
+            const matchingKernels = [];
+            for (const tag of tags) {
+                const kernels = tagIndex.get(tag) || [];
+                matchingKernels.push(...kernels);
+            }
+            const seen = new Set();
+            const unique = matchingKernels.filter(k => {
+                if (seen.has(k.content))
+                    return false;
+                seen.add(k.content);
+                return true;
+            });
+            return unique.slice(0, limit).map(k => ({
+                id: randomUUID(),
+                tier: 'learned',
+                content: k.content,
+                importance: k.importance,
+                tags: k.tags,
+                timestamp: Date.now(),
+                accessCount: 0,
+                lastAccessed: Date.now(),
+            }));
+        },
+        ctx_bridge(prevSessionId) {
+            const decisions = Array.from(anchorIndex.entries())
+                .filter(([, data]) => data.type === 'decision')
+                .map(([, data]) => data.content);
+            const constraints = Array.from(anchorIndex.entries())
+                .filter(([, data]) => data.type === 'constraint' || data.type === 'rule')
+                .map(([, data]) => data.content);
+            const importantFacts = Array.from(anchorIndex.entries())
+                .filter(([, data]) => data.type === 'fact')
+                .map(([, data]) => ({
+                fact: data.content,
+                certainty: 0.8,
+            }));
+            const contextSummary = hotBuffer.length > 0
+                ? hotBuffer.slice(0, 5).map(k => k.content).join(' ')
+                : '';
+            const packet = {
+                sessionId,
+                previousSessionId: prevSessionId || previousSessionId,
+                context: contextSummary,
+                decisions,
+                constraints,
+                pendingTasks: [],
+                importantFacts,
+                timestamp: Date.now(),
+                agentVersion: '2.3.3',
+            };
+            previousSessionId = sessionId;
+            sessionId = randomUUID();
+            return packet;
+        },
+        getStats() {
+            return {
+                hotBufferSize: hotBuffer.length,
+                kernelsExtracted,
+                shellDiscarded,
+                lastFlush,
+            };
+        },
+    };
+}
+export function createMemoryObserver(config) {
+    logger.info('Creating Memory Observer (Mnemostroma-inspired)');
+    return buildMemoryObserver(config);
+}

package/dist/core/security/execution-governor.js

@@ -0,0 +1,436 @@
+/**
+ * Execution Governor - Cross-Context Tool Call Governance
+ *
+ * Inspired by chitin (Execution kernel for AI coding agents)
+ * Provides declarative policy enforcement for tool calls across different execution contexts.
+ *
+ * Features:
+ * - Gating tool calls against typed policies
+ * - Severity ladder with lockdown counter
+ * - Tamper-evident audit chain
+ * - Bounds enforcement on push-shaped actions
+ * - Heuristic signals for blast-radius, floundering, drift
+ *
+ * @module core/security
+ * @fileoverview Execution governance for HeartFlow
+ */
+import { createLogger } from '../../utils/logger.js';
+import * as crypto from 'crypto';
+const logger = createLogger('[ExecutionGovernor]');
+// ============================================================
+// Default Configuration
+// ============================================================
+const DEFAULT_CONFIG = {
+    enableAuditChain: true,
+    enableSeverityLockdown: true,
+    maxLockdownCount: 5,
+    severityThresholds: {
+        low: 3,
+        medium: 5,
+        high: 10,
+        critical: 20,
+    },
+    policies: [
+        {
+            id: 'default-read',
+            action: 'read',
+            allow: true,
+            severity: 'info',
+        },
+        {
+            id: 'default-write',
+            action: 'write',
+            allow: true,
+            conditions: {
+                bounds: {
+                    maxLinesChanged: 500,
+                    maxFilesAffected: 10,
+                },
+                requireConfirmation: true,
+            },
+            severity: 'medium',
+        },
+        {
+            id: 'default-delete',
+            action: 'delete',
+            allow: true,
+            conditions: {
+                bounds: {
+                    maxFilesAffected: 1,
+                },
+                requireConfirmation: true,
+            },
+            severity: 'high',
+        },
+        {
+            id: 'default-execute',
+            action: 'execute',
+            allow: false,
+            severity: 'critical',
+        },
+        {
+            id: 'default-network',
+            action: 'network',
+            allow: true,
+            conditions: {
+                maxPerHour: 50,
+            },
+            severity: 'medium',
+        },
+    ],
+    defaultPolicy: {
+        id: 'default-unknown',
+        action: 'unknown',
+        allow: false,
+        severity: 'high',
+    },
+};
+// ============================================================
+// Helper Functions
+// ============================================================
+/**
+ * Classify a tool call into a canonical action
+ */
+function classifyAction(toolName, args) {
+    const lowerTool = toolName.toLowerCase();
+    // Read actions
+    if (lowerTool.includes('read') || lowerTool.includes('get') || lowerTool.includes('fetch') ||
+        lowerTool.includes('search') || lowerTool.includes('query') || lowerTool.includes('select')) {
+        return 'read';
+    }
+    // Write actions
+    if (lowerTool.includes('write') || lowerTool.includes('create') || lowerTool.includes('add') ||
+        lowerTool.includes('insert') || lowerTool.includes('post') || lowerTool.includes('put')) {
+        return 'write';
+    }
+    // Delete actions
+    if (lowerTool.includes('delete') || lowerTool.includes('remove') || lowerTool.includes('drop') ||
+        lowerTool.includes('unlink') || lowerTool.includes('rm')) {
+        return 'delete';
+    }
+    // Execute actions
+    if (lowerTool.includes('exec') || lowerTool.includes('run') || lowerTool.includes('execute') ||
+        lowerTool.includes('spawn') || lowerTool.includes('fork') || lowerTool.includes('shell')) {
+        return 'execute';
+    }
+    // Network actions
+    if (lowerTool.includes('http') || lowerTool.includes('fetch') || lowerTool.includes('request') ||
+        lowerTool.includes('send') || lowerTool.includes('post') || lowerTool.includes('url')) {
+        return 'network';
+    }
+    // Filesystem actions
+    if (lowerTool.includes('file') || lowerTool.includes('dir') || lowerTool.includes('path') ||
+        lowerTool.includes('mkdir') || lowerTool.includes('stat') || lowerTool.includes('ls')) {
+        return 'filesystem';
+    }
+    // Process actions
+    if (lowerTool.includes('process') || lowerTool.includes('pid') || lowerTool.includes('kill') ||
+        lowerTool.includes('signal')) {
+        return 'process';
+    }
+    // Memory actions
+    if (lowerTool.includes('memory') || lowerTool.includes('store') || lowerTool.includes('recall') ||
+        lowerTool.includes('embed')) {
+        return 'memory';
+    }
+    // Agent actions
+    if (lowerTool.includes('agent') || lowerTool.includes('spawn') || lowerTool.includes('delegate') ||
+        lowerTool.includes('subagent')) {
+        return 'agent';
+    }
+    return 'unknown';
+}
+/**
+ * Calculate heuristic signals from envelope context
+ */
+function calculateHeuristics(envelope) {
+    const context = envelope.context;
+    return {
+        blastRadius: context.blastRadius || 'low',
+        floundering: context.floundering || false,
+        drift: context.drift || 0,
+        repetitionCount: 0, // Tracked in agent state
+        urgency: 0.5, // Default
+        stakes: envelope.action === 'delete' || envelope.action === 'execute' ? 0.9 : 0.3,
+    };
+}
+/**
+ * Hash data for audit chain
+ */
+function hashData(data, prevHash) {
+    const payload = JSON.stringify({ data, prevHash, timestamp: Date.now() });
+    return crypto.createHash('sha256').update(payload).digest('hex');
+}
+// ============================================================
+// Execution Governor Class
+// ============================================================
+export class ExecutionGovernor {
+    config;
+    agentStates = new Map();
+    auditChain = [];
+    decisionHistory = new Map();
+    constructor(config = {}) {
+        this.config = { ...DEFAULT_CONFIG, ...config };
+        logger.info('ExecutionGovernor initialized', {
+            policiesCount: this.config.policies.length,
+            auditChainEnabled: this.config.enableAuditChain,
+        });
+    }
+    /**
+     * Process a tool call and return governance decision
+     */
+    async gate(envelope) {
+        // Auto-classify action if not provided
+        if (envelope.action === 'unknown') {
+            envelope.action = classifyAction(envelope.toolName, envelope.args);
+        }
+        // Calculate heuristic signals
+        const signals = calculateHeuristics(envelope);
+        // Get or create agent state
+        let state = this.agentStates.get(envelope.context.sessionId);
+        if (!state) {
+            state = this.createAgentState(envelope.context.sessionId);
+            this.agentStates.set(envelope.context.sessionId, state);
+        }
+        // Check for lockdown
+        if (state.locked && this.config.enableSeverityLockdown) {
+            const decision = this.createDecision(envelope, 'deny', 'Agent is in lockdown due to severity threshold breach');
+            this.appendAudit('decision', decision);
+            return decision;
+        }
+        // Check severity thresholds
+        if (this.config.enableSeverityLockdown) {
+            const severityLevel = this.checkSeverityThreshold(state);
+            if (severityLevel) {
+                state.lockdownCount++;
+                if (state.lockdownCount >= this.config.maxLockdownCount) {
+                    state.locked = true;
+                    this.appendAudit('lockdown', { sessionId: envelope.context.sessionId, count: state.lockdownCount });
+                    logger.warn('Agent locked due to severity threshold', { sessionId: envelope.context.sessionId });
+                }
+            }
+        }
+        // Find matching policy
+        const policy = this.findMatchingPolicy(envelope.action);
+        // Evaluate bounds if present
+        let reason = 'Policy evaluation';
+        let decision = 'allow';
+        if (policy) {
+            if (!policy.allow) {
+                decision = 'deny';
+                reason = `Policy ${policy.id} explicitly denies ${envelope.action}`;
+            }
+            else if (policy.conditions?.requireConfirmation) {
+                decision = 'escalate';
+                reason = `${envelope.action} requires operator confirmation`;
+            }
+            else if (policy.conditions?.bounds) {
+                const boundCheck = this.checkBounds(envelope, policy.conditions.bounds);
+                if (!boundCheck.allowed) {
+                    decision = 'deny';
+                    reason = boundCheck.reason || 'Bounds check failed';
+                }
+            }
+            // Update severity counts
+            if (this.config.enableSeverityLockdown && decision !== 'allow') {
+                state.severityCounts[policy.severity] = (state.severityCounts[policy.severity] || 0) + 1;
+            }
+        }
+        else {
+            decision = this.config.defaultPolicy.allow ? 'allow' : 'deny';
+            reason = `No matching policy, using default: ${this.config.defaultPolicy.allow ? 'allow' : 'deny'}`;
+        }
+        // Update state
+        state.totalToolCalls++;
+        state.lastActivity = Date.now();
+        // Create decision
+        const governanceDecision = this.createDecision(envelope, decision, reason, policy ?? undefined);
+        this.decisionHistory.set(envelope.id, governanceDecision);
+        this.appendAudit('decision', governanceDecision);
+        if (decision === 'deny') {
+            logger.info('Tool call denied', { envelopeId: envelope.id, action: envelope.action, reason });
+        }
+        else if (decision === 'escalate') {
+            logger.info('Tool call escalated', { envelopeId: envelope.id, action: envelope.action, reason });
+        }
+        return governanceDecision;
+    }
+    /**
+     * Create a new tool call envelope
+     */
+    createEnvelope(toolName, args, context) {
+        return {
+            id: crypto.randomUUID(),
+            action: classifyAction(toolName, args),
+            toolName,
+            args,
+            context,
+        };
+    }
+    /**
+     * Unlock a locked agent
+     */
+    unlock(sessionId) {
+        const state = this.agentStates.get(sessionId);
+        if (!state)
+            return false;
+        state.locked = false;
+        state.lockdownCount = 0;
+        state.severityCounts = {};
+        this.appendAudit('unlock', { sessionId });
+        logger.info('Agent unlocked', { sessionId });
+        return true;
+    }
+    /**
+     * Reset agent state
+     */
+    resetState(sessionId) {
+        const state = this.agentStates.get(sessionId);
+        if (!state)
+            return false;
+        const newState = this.createAgentState(sessionId);
+        this.agentStates.set(sessionId, newState);
+        this.appendAudit('state_reset', { sessionId });
+        logger.info('Agent state reset', { sessionId });
+        return true;
+    }
+    /**
+     * Get agent state
+     */
+    getAgentState(sessionId) {
+        return this.agentStates.get(sessionId) || null;
+    }
+    /**
+     * Get audit chain
+     */
+    getAuditChain() {
+        return [...this.auditChain];
+    }
+    /**
+     * Get decision from history
+     */
+    getDecision(envelopeId) {
+        return this.decisionHistory.get(envelopeId) || null;
+    }
+    /**
+     * Verify audit chain integrity
+     */
+    verifyAuditChain() {
+        for (let i = 1; i < this.auditChain.length; i++) {
+            if (this.auditChain[i].prevHash !== this.auditChain[i - 1].hash) {
+                return { valid: false, brokenAt: i };
+            }
+        }
+        return { valid: true };
+    }
+    // Private helper methods
+    createAgentState(sessionId) {
+        return {
+            sessionId,
+            severityCounts: {},
+            lockdownCount: 0,
+            totalToolCalls: 0,
+            lastActivity: Date.now(),
+            locked: false,
+        };
+    }
+    checkSeverityThreshold(state) {
+        const counts = state.severityCounts;
+        if ((counts['critical'] || 0) >= this.config.severityThresholds.critical)
+            return 'critical';
+        if ((counts['high'] || 0) >= this.config.severityThresholds.high)
+            return 'high';
+        if ((counts['medium'] || 0) >= this.config.severityThresholds.medium)
+            return 'medium';
+        if ((counts['low'] || 0) >= this.config.severityThresholds.low)
+            return 'low';
+        return null;
+    }
+    findMatchingPolicy(action) {
+        return this.config.policies.find(p => p.action === action) || null;
+    }
+    checkBounds(envelope, bounds) {
+        if (!bounds)
+            return { allowed: true };
+        // Check filesystem bounds
+        if (bounds.allowedPaths || bounds.deniedPaths) {
+            const path = this.extractPath(envelope.args);
+            if (path) {
+                if (bounds.deniedPaths?.some(p => path.includes(p))) {
+                    return { allowed: false, reason: `Path ${path} is in denied list` };
+                }
+                if (bounds.allowedPaths && !bounds.allowedPaths.some(p => path.includes(p))) {
+                    return { allowed: false, reason: `Path ${path} is not in allowed list` };
+                }
+            }
+        }
+        // Check data size
+        if (bounds.maxDataSize) {
+            const size = this.estimateDataSize(envelope.args);
+            if (size > bounds.maxDataSize) {
+                return { allowed: false, reason: `Data size ${size} exceeds limit ${bounds.maxDataSize}` };
+            }
+        }
+        return { allowed: true };
+    }
+    extractPath(args) {
+        for (const key of ['path', 'file', 'filePath', 'target', 'destination']) {
+            if (args[key] && typeof args[key] === 'string') {
+                return args[key];
+            }
+        }
+        return null;
+    }
+    estimateDataSize(args) {
+        try {
+            return JSON.stringify(args).length;
+        }
+        catch {
+            return 0;
+        }
+    }
+    createDecision(envelope, decision, reason, policy) {
+        const decisionData = {
+            envelopeId: envelope.id,
+            action: envelope.action,
+            decision,
+            reason,
+            timestamp: Date.now(),
+        };
+        return {
+            ...decisionData,
+            matchedPolicy: policy,
+            hash: hashData(decisionData, this.auditChain.length > 0 ? this.auditChain[this.auditChain.length - 1].hash : 'genesis'),
+        };
+    }
+    appendAudit(type, payload) {
+        if (!this.config.enableAuditChain)
+            return;
+        const event = {
+            type,
+            payload,
+            timestamp: Date.now(),
+            hash: '',
+            prevHash: this.auditChain.length > 0 ? this.auditChain[this.auditChain.length - 1].hash : 'genesis',
+        };
+        event.hash = hashData({ type, payload, timestamp: event.timestamp }, event.prevHash);
+        this.auditChain.push(event);
+        // Keep chain bounded to last 10000 events
+        if (this.auditChain.length > 10000) {
+            this.auditChain = this.auditChain.slice(-5000);
+        }
+    }
+}
+// ============================================================
+// Factory Function
+// ============================================================
+/**
+ * Create an ExecutionGovernor instance with optional custom config
+ */
+export function createExecutionGovernor(config) {
+    return new ExecutionGovernor(config);
+}
+// ============================================================
+// Export default config for customization
+// ============================================================
+export { DEFAULT_CONFIG };

package/dist/core/security/index.js

@@ -1,2 +1,3 @@
 export * from './privacy.js';
 export * from './agent-shield.js';
+export { ExecutionGovernor, createExecutionGovernor, DEFAULT_CONFIG } from './execution-governor.js';

package/dist/version.js

@@ -1 +1 @@
-export const VERSION = '2.3.2';
+export const VERSION = '2.3.4';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mark-improving-agent",
-  "version": "2.3.2",
+  "version": "2.3.4",
   "description": "Self-evolving AI agent with permanent memory, identity continuity, and self-evolution — for AI agents that need to remember, learn, and evolve across sessions",
   "type": "module",
   "main": "./dist/index.js",