monomind 1.19.0 → 2.0.0

package/packages/@monomind/cli/dist/src/mcp-tools/neural-tools.js

@@ -1,204 +1,112 @@
 /**
- * Pattern Store MCP Tools
+ * Pattern Store MCP Tools — "neural" namespace (legacy name, kept for API compat)
  *
- * Embed text as vectors, store patterns, search by cosine similarity.
- * Uses monovector ONNX embeddings when available, deterministic hash fallback otherwise.
- * Tools are registered under the "neural" namespace for backwards compatibility.
+ * These tools embed text as vectors and search by cosine similarity.
+ * No ML training, gradient descent, or neural network inference occurs.
+ * The "train" tool embeds and stores; the "predict" tool finds similar stored
+ * patterns. Embeddings come from the shared memory/embedding-operations.ts
+ * pipeline (LanceDB bridge -> ONNX -> deterministic hash fallback).
+ *
+ * All pattern storage is delegated to intelligence.ts's LocalReasoningBank
+ * (single source of truth for patterns.json). No separate models.json store.
  */
-import { getProjectCwd } from './types.js';
-import { existsSync, readFileSync, writeFileSync, renameSync, mkdirSync, statSync } from 'node:fs';
-import { join } from 'node:path';
-const MAX_NEURAL_STORE_BYTES = 50 * 1024 * 1024; // 50 MB
-const NEURAL_RESERVED_KEYS = new Set(['__proto__', 'constructor', 'prototype']);
-// Embeddings via monovector ONNX when available; deterministic hash fallback otherwise.
-let realEmbeddings = null;
-let embeddingServiceName = 'none';
-try {
-    const monovector = await import('monovector').catch(() => null);
-    if (monovector?.getOptimizedOnnxEmbedder) {
-        await monovector.initOnnxEmbedder?.();
-        const onnxEmb = monovector.getOptimizedOnnxEmbedder();
-        if (onnxEmb?.embed) {
-            const probe = await onnxEmb.embed('test');
-            const hasNonZero = ArrayBuffer.isView(probe) || Array.isArray(probe)
-                ? [...probe].some((v) => v !== 0)
-                : false;
-            if (probe && probe.length > 0 && hasNonZero) {
-                realEmbeddings = { embed: async (text) => Array.from(await onnxEmb.embed(text)) };
-                embeddingServiceName = 'monovector/onnx';
-            }
-        }
-    }
-}
-catch {
-    // No ONNX embedding provider available, will use hash fallback
-}
-// Storage paths
-const STORAGE_DIR = '.monomind';
-const NEURAL_DIR = 'neural';
-const MODELS_FILE = 'models.json';
-const PATTERNS_FILE = 'patterns.json';
-function getNeuralDir() {
-    return join(getProjectCwd(), STORAGE_DIR, NEURAL_DIR);
-}
-function getNeuralPath() {
-    return join(getNeuralDir(), MODELS_FILE);
-}
-function ensureNeuralDir() {
-    const dir = getNeuralDir();
-    if (!existsSync(dir)) {
-        mkdirSync(dir, { recursive: true });
-    }
-}
-function loadNeuralStore() {
+const RESERVED_KEYS = new Set(['__proto__', 'constructor', 'prototype']);
+const MAX_MODEL_ID_LEN = 256;
+const MAX_PATTERNS = 10000;
+const MAX_TRAINING_ENTRIES = 1000;
+const MAX_TEXT_LENGTH = 64 * 1024;
+const MAX_SEARCH_QUERY_LENGTH = 16 * 1024;
+// Embeddings: delegate to the shared embedding pipeline (LanceDB bridge -> ONNX ->
+// deterministic hash fallback) in memory/embedding-operations.ts.
+let lastEmbeddingModel = 'unknown';
+/**
+ * Generate embedding via the shared pipeline (LanceDB bridge -> ONNX -> deterministic
+ * hash), same one used by CLI `neural` commands and memory search. Falls back to a
+ * local deterministic hash only if the shared module fails to load entirely.
+ */
+async function generateEmbedding(text, dims = 384) {
+    if (!text)
+        return new Array(dims).fill(0);
     try {
-        const path = getNeuralPath();
-        if (existsSync(path)) {
-            if (statSync(path).size > MAX_NEURAL_STORE_BYTES) {
-                return { models: {}, patterns: {}, version: '3.0.0' };
-            }
-            const raw = JSON.parse(readFileSync(path, 'utf-8'));
-            // Strip proto-polluting keys from top-level containers
-            const models = {};
-            for (const [k, v] of Object.entries(raw.models ?? {})) {
-                if (!NEURAL_RESERVED_KEYS.has(k))
-                    models[k] = v;
-            }
-            const patterns = {};
-            for (const [k, v] of Object.entries(raw.patterns ?? {})) {
-                if (!NEURAL_RESERVED_KEYS.has(k))
-                    patterns[k] = v;
-            }
-            return {
-                models: models,
-                patterns: patterns,
-                version: typeof raw.version === 'string' ? raw.version : '3.0.0',
-            };
-        }
+        const { generateEmbedding: sharedGenerateEmbedding } = await import('../memory/embedding-operations.js');
+        const result = await sharedGenerateEmbedding(text);
+        lastEmbeddingModel = result.model;
+        return result.embedding;
     }
     catch {
-        // Return empty store
-    }
-    return { models: {}, patterns: {}, version: '3.0.0' };
-}
-function saveNeuralStore(store) {
-    ensureNeuralDir();
-    const dest = getNeuralPath();
-    const tmp = `${dest}.${process.pid}.${Date.now()}.tmp`;
-    writeFileSync(tmp, JSON.stringify(store, null, 2), 'utf-8');
-    renameSync(tmp, dest);
-}
-// Generate embedding - uses real ML embeddings if available, falls back to deterministic hash
-async function generateEmbedding(text, dims = 384) {
-    // If real embeddings available and text provided, use them
-    if (realEmbeddings && text) {
-        try {
-            return await realEmbeddings.embed(text);
-        }
-        catch {
-            // Fall back to hash-based
-        }
-    }
-    // Hash-based deterministic embedding (better than pure random for consistency)
-    if (text) {
-        const hash = text.split('').reduce((acc, char, i) => {
-            return acc + char.charCodeAt(0) * (i + 1);
-        }, 0);
-        // Use hash to seed a deterministic embedding
+        // memory/embedding-operations.js unavailable — deterministic hash fallback
+        lastEmbeddingModel = 'hash-fallback';
+        const hash = text.split('').reduce((acc, char, i) => acc + char.charCodeAt(0) * (i + 1), 0);
         const embedding = [];
         let seed = hash;
         for (let i = 0; i < dims; i++) {
-            // Simple LCG random with seed
             seed = (seed * 1103515245 + 12345) & 0x7fffffff;
             embedding.push((seed / 0x7fffffff) * 2 - 1);
         }
         return embedding;
     }
-    // No text provided — return zero vector (callers should always provide text)
-    return new Array(dims).fill(0);
 }
-// Cosine similarity for pattern search
-function cosineSimilarity(a, b) {
-    if (a.length !== b.length) {
-        throw new Error(`Embedding dimension mismatch: ${a.length} vs ${b.length}. Ensure all embeddings use the same provider and dims.`);
-    }
-    let dotProduct = 0;
-    let normA = 0;
-    let normB = 0;
-    for (let i = 0; i < a.length; i++) {
-        dotProduct += a[i] * b[i];
-        normA += a[i] * a[i];
-        normB += b[i] * b[i];
-    }
-    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB) || 1);
+/** Truncate and validate a modelId string */
+function sanitizeModelId(raw) {
+    if (typeof raw !== 'string')
+        return null;
+    const id = raw.length > MAX_MODEL_ID_LEN ? raw.slice(0, MAX_MODEL_ID_LEN) : raw;
+    if (RESERVED_KEYS.has(id))
+        return null;
+    return id;
 }
 export const neuralTools = [
     {
         name: 'neural_train',
-        description: 'Embed and store patterns for similarity search',
+        description: 'Embed text as vectors and store as named patterns for later similarity search (no ML training occurs)',
         category: 'neural',
         inputSchema: {
             type: 'object',
             properties: {
                 modelId: { type: 'string', description: 'Store ID (groups related patterns)' },
                 modelType: { type: 'string', enum: ['pattern', 'embedding'], description: 'Store type' },
-                epochs: { type: 'number', description: 'Ignored (kept for backwards compatibility)' },
-                learningRate: { type: 'number', description: 'Ignored (kept for backwards compatibility)' },
+                epochs: { type: 'number', description: 'Deprecated — ignored. Patterns are embedded and stored, not trained.' },
+                learningRate: { type: 'number', description: 'Deprecated — ignored. No gradient-based training occurs.' },
                 data: { type: 'object', description: 'Data to embed and store (array of strings or {text/content/label} objects)' },
             },
             required: ['modelType'],
         },
         handler: async (input) => {
-            const store = loadNeuralStore();
-            // Cap modelId to prevent DoS via oversized object keys written to the neural store JSON.
-            const MAX_MODEL_ID_LEN = 256;
             const rawModelId = input.modelId || `model-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`;
-            const modelId = typeof rawModelId === 'string' && rawModelId.length > MAX_MODEL_ID_LEN
-                ? rawModelId.slice(0, MAX_MODEL_ID_LEN)
-                : rawModelId;
-            const RESERVED_KEYS = new Set(['__proto__', 'constructor', 'prototype']);
-            if (RESERVED_KEYS.has(modelId)) {
+            const modelId = sanitizeModelId(rawModelId);
+            if (modelId === null) {
                 return { success: false, error: 'Invalid modelId' };
             }
-            const MAX_PATTERNS = 10000;
-            if (Object.keys(store.patterns ?? {}).length >= MAX_PATTERNS) {
-                return { success: false, error: `Pattern store full (max ${MAX_PATTERNS}). Run neural_compress first.` };
-            }
-            const MAX_MODELS = 100;
-            if (Object.keys(store.models ?? {}).length >= MAX_MODELS) {
-                return { success: false, error: `Model store full (max ${MAX_MODELS}). Delete old models first.` };
-            }
             const VALID_MODEL_TYPES = new Set(['pattern', 'embedding']);
             const rawModelType = input.modelType;
             if (!rawModelType || !VALID_MODEL_TYPES.has(rawModelType)) {
                 return { success: false, error: `Invalid modelType "${rawModelType}". Must be one of: pattern, embedding` };
             }
             const modelType = rawModelType;
-            const epochs = 1;
-            const model = {
-                id: modelId,
-                name: `${modelType}-store`,
-                type: modelType,
-                status: 'indexing',
-                patternsStored: 0,
-                epochs,
-                config: {},
-            };
-            store.models[modelId] = model;
-            saveNeuralStore(store);
-            // Real training: embed training data and store as searchable patterns.
-            // Cap input array length and re-check the pattern cap inside the loop —
-            // the pre-loop check at line 199 alone allowed an attacker to bypass
-            // MAX_PATTERNS by passing a 100k-entry array.
-            const MAX_TRAINING_ENTRIES = 1000;
-            const MAX_TEXT_LENGTH = 64 * 1024;
+            // Initialize intelligence system (single source of truth)
+            const { initializeIntelligence, getReasoningBank, getAllPatterns, flushPatterns } = await import('../memory/intelligence.js');
+            const init = await initializeIntelligence();
+            if (!init.success) {
+                return { success: false, error: 'Failed to initialize intelligence system' };
+            }
+            const bank = getReasoningBank();
+            if (!bank) {
+                return { success: false, error: 'ReasoningBank not available' };
+            }
+            // Check pattern capacity
+            const existingPatterns = await getAllPatterns();
+            if (existingPatterns.length >= MAX_PATTERNS) {
+                return { success: false, error: `Pattern store full (max ${MAX_PATTERNS}). Run neural_compress first.` };
+            }
             const trainingData = input.data;
             let patternsStored = 0;
             if (trainingData) {
                 const rawEntries = Array.isArray(trainingData) ? trainingData : [trainingData];
                 const entries = rawEntries.slice(0, MAX_TRAINING_ENTRIES);
                 for (let i = 0; i < entries.length; i++) {
-                    if (Object.keys(store.patterns ?? {}).length >= MAX_PATTERNS)
+                    // Re-check capacity inside loop to prevent bypass via large arrays
+                    const currentPatterns = await getAllPatterns();
+                    if (currentPatterns.length >= MAX_PATTERNS)
                         break;
                     const entry = entries[i];
                     let text = typeof entry === 'string' ? entry
@@ -212,74 +120,37 @@ export const neuralTools = [
                         text = text.slice(0, MAX_TEXT_LENGTH);
                     const embedding = await generateEmbedding(text, 384);
                     const patternId = `${modelId}-train-${i}`;
-                    store.patterns[patternId] = {
+                    bank.store({
                         id: patternId,
-                        name: typeof entry === 'object' && entry !== null && 'label' in entry
-                            ? String(entry.label) : text.slice(0, 100),
                         type: modelType,
                         embedding,
-                        metadata: { modelId, epoch: epochs, index: i, raw: entry },
-                        createdAt: new Date().toISOString(),
-                        usageCount: 0,
-                    };
-                    patternsStored++;
-                }
-            }
-            model.status = 'ready';
-            model.patternsStored = patternsStored;
-            model.trainedAt = new Date().toISOString();
-            saveNeuralStore(store);
-            // Mirror patterns to patterns.json so CLI commands (neural patterns list,
-            // neural predict) can find patterns trained via MCP and vice versa.
-            if (patternsStored > 0) {
-                try {
-                    const patternsPath = join(getNeuralDir(), PATTERNS_FILE);
-                    let existing = [];
-                    if (existsSync(patternsPath) && statSync(patternsPath).size <= MAX_NEURAL_STORE_BYTES) {
-                        const raw = readFileSync(patternsPath, 'utf-8');
-                        const parsed = JSON.parse(raw);
-                        if (Array.isArray(parsed))
-                            existing = parsed;
-                    }
-                    const existingIds = new Set(existing.map(p => p['id']));
-                    const newEntries = Object.values(store.patterns)
-                        .filter(p => p.id.startsWith(modelId) && !existingIds.has(p.id))
-                        .map(p => ({
-                        id: p.id,
-                        type: p.type,
-                        content: p.name,
+                        content: typeof entry === 'object' && entry !== null && 'label' in entry
+                            ? String(entry.label) : text.slice(0, 100),
                         confidence: 0.8,
-                        usageCount: 0,
-                        embedding: p.embedding,
-                        createdAt: p.createdAt,
-                    }));
-                    if (newEntries.length > 0) {
-                        const merged = [...existing, ...newEntries];
-                        const tmp = `${patternsPath}.${process.pid}.${Date.now()}.tmp`;
-                        writeFileSync(tmp, JSON.stringify(merged, null, 2), 'utf-8');
-                        renameSync(tmp, patternsPath);
-                    }
-                }
-                catch {
-                    // Mirror is best-effort; don't fail the train operation
+                        metadata: { modelId, index: i, raw: entry },
+                    });
+                    patternsStored++;
                 }
             }
+            // Flush immediately to persist
+            flushPatterns();
+            const totalPatterns = (await getAllPatterns()).length;
             return {
                 success: true,
-                _realEmbedding: !!realEmbeddings,
+                _realEmbedding: lastEmbeddingModel !== 'hash-fallback',
                 modelId,
                 type: modelType,
-                status: model.status,
+                status: 'ready',
                 patternsStored,
-                totalPatterns: Object.keys(store.patterns).length,
-                epochs,
-                trainedAt: model.trainedAt,
+                totalPatterns,
+                epochs: 1,
+                trainedAt: new Date().toISOString(),
             };
         },
     },
     {
         name: 'neural_predict',
-        description: 'Find similar patterns by cosine similarity',
+        description: 'Find stored patterns most similar to input text via cosine similarity (not ML prediction)',
         category: 'neural',
         inputSchema: {
             type: 'object',
@@ -291,86 +162,38 @@ export const neuralTools = [
             required: ['input'],
         },
         handler: async (input) => {
-            const store = loadNeuralStore();
-            // Cap modelId to prevent O(n) hash/compare cost on absurdly long keys.
-            const MAX_MODEL_ID_LEN_PRED = 256;
             const rawModelIdPred = input.modelId;
-            const modelId = typeof rawModelIdPred === 'string' && rawModelIdPred.length > MAX_MODEL_ID_LEN_PRED
-                ? rawModelIdPred.slice(0, MAX_MODEL_ID_LEN_PRED)
-                : rawModelIdPred;
-            const inputText = typeof input.input === 'string' ? input.input.slice(0, 16 * 1024) : '';
-            const topK = Math.max(1, Math.min(input.topK || 3, 50));
-            const RESERVED_KEYS_PRED = new Set(['__proto__', 'constructor', 'prototype']);
-            if (modelId && (RESERVED_KEYS_PRED.has(modelId) || typeof modelId !== 'string')) {
-                return { success: false, error: 'Invalid modelId' };
-            }
-            // Find model or use default
-            const model = modelId
-                ? (Object.hasOwn(store.models, modelId) ? store.models[modelId] : undefined)
-                : Object.values(store.models).find(m => m.status === 'ready');
-            if (model && model.status !== 'ready') {
-                return { success: false, error: 'Model not ready' };
+            if (rawModelIdPred) {
+                const modelId = sanitizeModelId(rawModelIdPred);
+                if (modelId === null) {
+                    return { success: false, error: 'Invalid modelId' };
+                }
             }
-            // Generate real embedding for the input
+            const inputText = typeof input.input === 'string' ? input.input.slice(0, MAX_SEARCH_QUERY_LENGTH) : '';
+            const topK = Math.max(1, Math.min(input.topK || 3, 50));
+            const { initializeIntelligence, findSimilarPatterns } = await import('../memory/intelligence.js');
+            await initializeIntelligence();
             const startTime = performance.now();
             const embedding = await generateEmbedding(inputText, 384);
+            // Use intelligence.ts findSimilarPatterns which handles embedding + search
+            const results = await findSimilarPatterns(inputText, {
+                k: topK,
+                threshold: 0.1,
+            });
             const latency = Math.round(performance.now() - startTime);
-            // Search stored patterns via cosine similarity.
-            // Merge MCP models.json patterns with CLI patterns.json.
-            const MAX_SCAN = 10000;
-            const EARLY_EXIT_THRESHOLD = 0.1;
-            const mcpPatterns = Object.values(store.patterns);
-            const cliPatternsRaw = [];
-            try {
-                const cliPath = join(getNeuralDir(), PATTERNS_FILE);
-                if (existsSync(cliPath) && statSync(cliPath).size <= MAX_NEURAL_STORE_BYTES) {
-                    const raw = JSON.parse(readFileSync(cliPath, 'utf-8'));
-                    if (Array.isArray(raw)) {
-                        const mcpIds = new Set(mcpPatterns.map(p => p.id));
-                        for (const p of raw) {
-                            if (p?.id && !mcpIds.has(p.id) && Array.isArray(p.embedding)) {
-                                cliPatternsRaw.push(p);
-                            }
-                        }
-                    }
-                }
-            }
-            catch { /* best-effort */ }
-            const cliMapped = cliPatternsRaw.map(p => ({
-                id: p.id ?? '',
-                name: p.content ?? p.name ?? p.type ?? '',
-                type: p.type ?? 'general',
-                embedding: p.embedding ?? [],
-                metadata: {},
-                createdAt: new Date().toISOString(),
-                usageCount: 0,
+            const predictions = results.map(r => ({
+                label: r.content || r.type || r.id,
+                confidence: r.similarity ?? r.confidence,
+                patternId: r.id,
             }));
-            const storedPatterns = [...mcpPatterns, ...cliMapped].slice(0, MAX_SCAN);
-            let predictions;
-            if (storedPatterns.length > 0) {
-                const scored = [];
-                for (const p of storedPatterns) {
-                    if (!Array.isArray(p.embedding) || p.embedding.length !== embedding.length)
-                        continue;
-                    const conf = Math.max(0, cosineSimilarity(embedding, p.embedding));
-                    if (conf < EARLY_EXIT_THRESHOLD)
-                        continue;
-                    scored.push({ label: p.name || p.type || p.id, confidence: conf, patternId: p.id });
-                }
-                predictions = scored.sort((a, b) => b.confidence - a.confidence).slice(0, topK);
-            }
-            else {
-                // No patterns stored — no predictions possible
-                predictions = [];
-            }
             return {
                 success: true,
-                _realEmbedding: !!realEmbeddings,
-                _hasStoredPatterns: storedPatterns.length > 0,
-                modelId: model?.id || 'default',
+                _realEmbedding: lastEmbeddingModel !== 'hash-fallback',
+                _hasStoredPatterns: results.length > 0,
+                modelId: rawModelIdPred || 'default',
                 input: inputText,
                 predictions,
-                embedding: embedding.slice(0, 8), // Preview of embedding
+                embedding: embedding.slice(0, 8),
                 embeddingDims: embedding.length,
                 latency,
             };
@@ -392,116 +215,118 @@ export const neuralTools = [
             },
         },
         handler: async (input) => {
-            const store = loadNeuralStore();
             const action = input.action || 'list';
+            const { initializeIntelligence, getAllPatterns, getPatternsByType, getReasoningBank, findSimilarPatterns, deletePattern, flushPatterns } = await import('../memory/intelligence.js');
+            await initializeIntelligence();
             if (action === 'list') {
-                const patterns = Object.values(store.patterns);
                 const typeFilter = input.type;
-                const filtered = typeFilter ? patterns.filter(p => p.type === typeFilter) : patterns;
+                const patterns = typeFilter
+                    ? await getPatternsByType(typeFilter)
+                    : await getAllPatterns();
                 return {
-                    patterns: filtered.map(p => ({
+                    patterns: patterns.map(p => ({
                         id: p.id,
-                        name: p.name,
+                        name: p.content,
                         type: p.type,
                         usageCount: p.usageCount,
-                        createdAt: p.createdAt,
+                        createdAt: typeof p.createdAt === 'number' ? new Date(p.createdAt).toISOString() : p.createdAt,
                     })),
-                    total: filtered.length,
+                    total: patterns.length,
                 };
             }
             if (action === 'get') {
                 const patternId = input.patternId;
-                if (!patternId || NEURAL_RESERVED_KEYS.has(patternId)) {
+                if (!patternId || RESERVED_KEYS.has(patternId)) {
                     return { success: false, error: 'Invalid patternId' };
                 }
-                const pattern = Object.hasOwn(store.patterns, patternId) ? store.patterns[patternId] : undefined;
+                const bank = getReasoningBank();
+                if (!bank) {
+                    return { success: false, error: 'ReasoningBank not available' };
+                }
+                const pattern = bank.get(patternId);
                 if (!pattern) {
                     return { success: false, error: 'Pattern not found' };
                 }
-                return { success: true, pattern };
+                return {
+                    success: true,
+                    pattern: {
+                        id: pattern.id,
+                        name: pattern.content,
+                        type: pattern.type,
+                        embedding: pattern.embedding,
+                        metadata: pattern.metadata ?? {},
+                        createdAt: new Date(pattern.createdAt).toISOString(),
+                        usageCount: pattern.usageCount,
+                    },
+                };
             }
             if (action === 'store') {
-                const MAX_PATTERNS = 10000;
-                if (Object.keys(store.patterns ?? {}).length >= MAX_PATTERNS) {
+                const allPatterns = await getAllPatterns();
+                if (allPatterns.length >= MAX_PATTERNS) {
                     return { success: false, error: `Pattern store full (max ${MAX_PATTERNS}). Run neural_compress first.` };
                 }
-                const patternId = `pattern-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`;
-                // Cap name length to prevent DoS in generateEmbedding (hash path is O(n))
-                const MAX_PATTERN_NAME_LENGTH = 16 * 1024; // 16 KB
+                const MAX_PATTERN_NAME_LENGTH = 16 * 1024;
                 const rawPatternName = input.name || 'Unnamed pattern';
                 const patternName = typeof rawPatternName === 'string' && rawPatternName.length > MAX_PATTERN_NAME_LENGTH
                     ? rawPatternName.slice(0, MAX_PATTERN_NAME_LENGTH)
                     : rawPatternName;
-                // Generate embedding from pattern name/content
                 const embedding = await generateEmbedding(patternName, 384);
-                const pattern = {
+                const patternId = `pattern-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`;
+                const patternType = input.type || 'general';
+                const bank = getReasoningBank();
+                if (!bank) {
+                    return { success: false, error: 'ReasoningBank not available' };
+                }
+                bank.store({
                     id: patternId,
-                    name: patternName,
-                    type: input.type || 'general',
+                    type: patternType,
                     embedding,
+                    content: patternName,
+                    confidence: 0.8,
                     metadata: input.data || {},
-                    createdAt: new Date().toISOString(),
-                    usageCount: 0,
-                };
-                store.patterns[patternId] = pattern;
-                saveNeuralStore(store);
+                });
+                flushPatterns();
                 return {
                     success: true,
-                    _realEmbedding: !!realEmbeddings,
+                    _realEmbedding: lastEmbeddingModel !== 'hash-fallback',
                     patternId,
-                    name: pattern.name,
-                    type: pattern.type,
+                    name: patternName,
+                    type: patternType,
                     embeddingDims: embedding.length,
-                    createdAt: pattern.createdAt,
+                    createdAt: new Date().toISOString(),
                 };
             }
             if (action === 'search') {
-                // Cap query length to prevent DoS in generateEmbedding (hash path is O(n))
-                const MAX_SEARCH_QUERY_LENGTH = 16 * 1024; // 16 KB — matches neural_predict cap
                 const rawQuery = input.query;
                 const query = typeof rawQuery === 'string' && rawQuery.length > MAX_SEARCH_QUERY_LENGTH
                     ? rawQuery.slice(0, MAX_SEARCH_QUERY_LENGTH)
                     : rawQuery;
-                // Generate query embedding for real similarity search
-                const queryEmbedding = await generateEmbedding(query, 384);
-                // Calculate REAL cosine similarity against stored patterns
-                const results = Object.values(store.patterns)
-                    .map(p => ({
-                    ...p,
-                    similarity: cosineSimilarity(queryEmbedding, p.embedding),
-                }))
-                    .sort((a, b) => b.similarity - a.similarity)
-                    .slice(0, 10);
-                // Increment usageCount so prune can track retrieval frequency
-                for (const r of results) {
-                    if (Object.hasOwn(store.patterns, r.id)) {
-                        store.patterns[r.id].usageCount = (store.patterns[r.id].usageCount || 0) + 1;
-                    }
-                }
-                saveNeuralStore(store);
+                const results = await findSimilarPatterns(query || '', {
+                    k: 10,
+                    threshold: 0.1,
+                });
                 return {
                     _realSimilarity: true,
-                    _realEmbedding: !!realEmbeddings,
+                    _realEmbedding: lastEmbeddingModel !== 'hash-fallback',
                     query,
                     results: results.map(r => ({
                         id: r.id,
-                        name: r.name,
+                        name: r.content,
                         type: r.type,
-                        similarity: r.similarity,
+                        similarity: r.similarity ?? r.confidence,
                     })),
                     total: results.length,
                 };
             }
             if (action === 'delete') {
                 const patternId = input.patternId;
-                if (!patternId || NEURAL_RESERVED_KEYS.has(patternId)) {
+                if (!patternId || RESERVED_KEYS.has(patternId)) {
                     return { success: false, error: 'Invalid patternId' };
                 }
-                if (!Object.hasOwn(store.patterns, patternId)) {
+                const deleted = await deletePattern(patternId);
+                if (!deleted) {
                     return { success: false, error: 'Pattern not found' };
                 }
-                delete store.patterns[patternId];
-                saveNeuralStore(store);
                 return { success: true, deleted: patternId };
             }
             return { success: false, error: 'Unknown action' };
@@ -520,97 +345,39 @@ export const neuralTools = [
             },
         },
         handler: async (input) => {
-            const store = loadNeuralStore();
             const method = input.method || 'quantize';
-            const targetReduction = input.targetSize || 0.5;
-            const patterns = Object.values(store.patterns);
+            const { initializeIntelligence, compactPatterns, getAllPatterns } = await import('../memory/intelligence.js');
+            await initializeIntelligence();
+            const patterns = await getAllPatterns();
             if (patterns.length === 0) {
                 return { success: false, error: 'No patterns to compress. Train patterns first with neural_train.' };
             }
             const beforeCount = patterns.length;
-            const beforeSize = patterns.reduce((s, p) => s + (p.embedding?.length || 0) * 4, 0); // Float32 = 4 bytes
-            if (method === 'quantize') {
-                try {
-                    const { quantizeInt8, getQuantizationStats } = await import('../memory/memory-initializer.js');
-                    let totalCompressed = 0;
-                    for (const pattern of patterns) {
-                        if (pattern.embedding && pattern.embedding.length > 0) {
-                            const stats = getQuantizationStats(pattern.embedding);
-                            const quantized = quantizeInt8(pattern.embedding);
-                            // Store quantized metadata (keep original embedding for search)
-                            pattern._quantized = {
-                                scale: quantized.scale,
-                                zeroPoint: quantized.zeroPoint,
-                                compressionRatio: stats.compressionRatio,
-                            };
-                            totalCompressed++;
-                        }
-                    }
-                    saveNeuralStore(store);
-                    return {
-                        success: true, _real: true, method,
-                        patternsCompressed: totalCompressed,
-                        note: 'Quantization metadata stored; original embeddings retained for search accuracy.',
-                    };
-                }
-                catch {
-                    return { success: false, error: 'Quantization requires memory-initializer. Run `memory init` first.' };
-                }
-            }
-            if (method === 'prune') {
-                // Prune the least-used fraction of patterns. targetReduction (0-1) is a percentile:
-                // 0.5 removes the bottom 50% by usageCount, 0.8 removes the bottom 80%, etc.
-                const counts = Object.values(store.patterns)
-                    .map(p => p.usageCount || 0)
-                    .sort((a, b) => a - b);
-                const cutoffIdx = Math.floor(targetReduction * counts.length);
-                const threshold = counts[cutoffIdx] ?? 0;
-                const toRemove = [];
-                for (const [id, pattern] of Object.entries(store.patterns)) {
-                    if ((pattern.usageCount || 0) < threshold)
-                        toRemove.push(id);
-                }
-                for (const id of toRemove)
-                    delete store.patterns[id];
-                saveNeuralStore(store);
+            if (method === 'quantize' || method === 'distill') {
+                // Both quantize and distill map to compactPatterns (dedup by cosine similarity)
+                const threshold = method === 'quantize' ? 0.99 : 0.95;
+                const result = await compactPatterns(threshold);
                 return {
                     success: true, _real: true, method,
-                    threshold,
-                    patternsRemoved: toRemove.length,
-                    patternsBefore: beforeCount,
-                    patternsAfter: Object.keys(store.patterns).length,
+                    patternsBefore: result.before,
+                    patternsAfter: result.after,
+                    patternsRemoved: result.removed,
+                    note: method === 'quantize'
+                        ? 'Near-duplicate patterns removed (cosine > 0.99).'
+                        : `Similar patterns merged (cosine > ${threshold}).`,
                 };
             }
-            if (method === 'distill') {
-                // Merge similar patterns by cosine similarity > 0.95
-                const patternList = Object.entries(store.patterns);
-                const merged = [];
-                for (let i = 0; i < patternList.length; i++) {
-                    const [idA, a] = patternList[i];
-                    if (merged.includes(idA))
-                        continue;
-                    for (let j = i + 1; j < patternList.length; j++) {
-                        const [idB, b] = patternList[j];
-                        if (!a.embedding || !b.embedding || merged.includes(idB))
-                            continue;
-                        const sim = cosineSimilarity(a.embedding, b.embedding);
-                        if (sim > 0.95) {
-                            // Merge: average embeddings, keep higher usage count
-                            for (let k = 0; k < a.embedding.length; k++) {
-                                a.embedding[k] = (a.embedding[k] + (b.embedding[k] || 0)) / 2;
-                            }
-                            a.usageCount = Math.max(a.usageCount || 0, b.usageCount || 0);
-                            delete store.patterns[idB];
-                            merged.push(idB);
-                        }
-                    }
-                }
-                saveNeuralStore(store);
+            if (method === 'prune') {
+                // Prune maps to compactPatterns with a looser threshold
+                const targetReduction = input.targetSize || 0.5;
+                // Use a threshold proportional to target reduction (lower = more aggressive)
+                const threshold = 0.7 + (1 - targetReduction) * 0.25;
+                const result = await compactPatterns(threshold);
                 return {
                     success: true, _real: true, method,
-                    patternsMerged: merged.length,
-                    patternsBefore: beforeCount,
-                    patternsAfter: Object.keys(store.patterns).length,
+                    patternsBefore: result.before,
+                    patternsAfter: result.after,
+                    patternsRemoved: result.removed,
                 };
             }
             return { success: false, error: `Unknown method: ${method}. Use quantize, prune, or distill.` };
@@ -618,7 +385,7 @@ export const neuralTools = [
     },
     {
         name: 'neural_status',
-        description: 'Get pattern store status',
+        description: 'Get pattern store status (pattern count, embedding dimensions, store health)',
         category: 'neural',
         inputSchema: {
             type: 'object',
@@ -628,38 +395,48 @@ export const neuralTools = [
             },
         },
         handler: async (input) => {
-            const store = loadNeuralStore();
+            const { initializeIntelligence, getIntelligenceStats, getAllPatterns } = await import('../memory/intelligence.js');
+            await initializeIntelligence();
+            const stats = getIntelligenceStats();
+            const patterns = await getAllPatterns();
             if (input.modelId) {
-                const MAX_MODEL_ID_LEN_STATUS = 256;
-                const rawModelIdStatus = input.modelId;
-                const modelId = typeof rawModelIdStatus === 'string' && rawModelIdStatus.length > MAX_MODEL_ID_LEN_STATUS
-                    ? rawModelIdStatus.slice(0, MAX_MODEL_ID_LEN_STATUS)
-                    : rawModelIdStatus;
-                if (NEURAL_RESERVED_KEYS.has(modelId)) {
+                const modelId = sanitizeModelId(input.modelId);
+                if (modelId === null) {
                     return { success: false, error: 'Invalid modelId' };
                 }
-                const model = Object.hasOwn(store.models, modelId) ? store.models[modelId] : undefined;
-                if (!model) {
-                    return { success: false, error: 'Model not found' };
-                }
-                return { success: true, model };
+                // Filter patterns belonging to this model (by metadata or id prefix)
+                const modelPatterns = patterns.filter(p => p.id.startsWith(modelId) ||
+                    (p.metadata?.modelId === modelId));
+                return {
+                    success: true,
+                    model: {
+                        id: modelId,
+                        patternsStored: modelPatterns.length,
+                        status: modelPatterns.length > 0 ? 'ready' : 'empty',
+                    },
+                };
+            }
+            const byType = {};
+            for (const p of patterns) {
+                byType[p.type] = (byType[p.type] || 0) + 1;
             }
-            const models = Object.values(store.models);
-            const patterns = Object.values(store.patterns);
             return {
-                embeddingProvider: realEmbeddings ? embeddingServiceName : 'hash-based (deterministic)',
+                embeddingProvider: lastEmbeddingModel === 'unknown' ? 'not yet used' : lastEmbeddingModel,
+                sonaEnabled: stats.sonaEnabled,
                 models: {
-                    total: models.length,
-                    ready: models.filter(m => m.status === 'ready').length,
-                    indexing: models.filter(m => m.status === 'indexing').length,
+                    total: 0, // models concept removed; patterns are stored directly
+                    ready: 0,
+                    indexing: 0,
                 },
                 patterns: {
                     total: patterns.length,
-                    byType: patterns.reduce((acc, p) => {
-                        acc[p.type] = (acc[p.type] || 0) + 1;
-                        return acc;
-                    }, {}),
-                    embeddingDims: patterns.length > 0 ? patterns[0].embedding.length : 384,
+                    byType,
+                    embeddingDims: patterns.length > 0 && patterns[0].embedding ? patterns[0].embedding.length : 384,
+                },
+                intelligence: {
+                    reasoningBankSize: stats.reasoningBankSize,
+                    patternsLearned: stats.patternsLearned,
+                    trajectoriesRecorded: stats.trajectoriesRecorded,
                 },
             };
         },
@@ -676,9 +453,10 @@ export const neuralTools = [
             },
         },
         handler: async (input) => {
-            const store = loadNeuralStore();
             const target = input.target || 'balanced';
-            const patterns = Object.values(store.patterns);
+            const { initializeIntelligence, compactPatterns, getAllPatterns } = await import('../memory/intelligence.js');
+            await initializeIntelligence();
+            const patterns = await getAllPatterns();
             if (patterns.length === 0) {
                 return { success: false, error: 'No patterns to optimize. Train patterns first with neural_train.' };
             }
@@ -686,81 +464,39 @@ export const neuralTools = [
             const actions = [];
             const beforeCount = patterns.length;
             const dims = patterns[0]?.embedding?.length || 0;
-            let patternsRemoved = 0;
-            let patternsQuantized = 0;
-            let duplicatesRemoved = 0;
-            // speed / balanced: deduplicate identical or near-identical patterns
-            if (target === 'speed' || target === 'balanced') {
-                const seen = new Map(); // hash -> id
-                for (const [id, p] of Object.entries(store.patterns)) {
-                    if (!p.embedding || p.embedding.length === 0)
-                        continue;
-                    // Quick hash: first 8 dims rounded
-                    const hash = p.embedding.slice(0, 8).map(v => v.toFixed(4)).join(',');
-                    if (seen.has(hash)) {
-                        // Verify with full cosine similarity
-                        const existingId = seen.get(hash);
-                        const existing = store.patterns[existingId];
-                        if (existing && cosineSimilarity(p.embedding, existing.embedding) > 0.99) {
-                            existing.usageCount = Math.max(existing.usageCount || 0, p.usageCount || 0);
-                            delete store.patterns[id];
-                            duplicatesRemoved++;
-                        }
-                    }
-                    else {
-                        seen.set(hash, id);
-                    }
-                }
-                if (duplicatesRemoved > 0)
-                    actions.push(`Removed ${duplicatesRemoved} near-duplicate patterns`);
+            // All optimization targets delegate to compactPatterns with varying thresholds
+            let threshold;
+            switch (target) {
+                case 'speed':
+                    threshold = 0.99; // only near-exact duplicates
+                    break;
+                case 'memory':
+                    threshold = 0.90; // aggressive dedup
+                    break;
+                case 'accuracy':
+                    threshold = 0.98; // conservative
+                    break;
+                case 'balanced':
+                default:
+                    threshold = 0.95;
+                    break;
             }
-            // memory / balanced: quantize large embeddings
-            if (target === 'memory' || target === 'balanced') {
-                try {
-                    const { quantizeInt8, getQuantizationStats } = await import('../memory/memory-initializer.js');
-                    for (const p of Object.values(store.patterns)) {
-                        if (p.embedding && p.embedding.length > 0 && !p._quantized) {
-                            const stats = getQuantizationStats(p.embedding);
-                            const q = quantizeInt8(p.embedding);
-                            p._quantized = { scale: q.scale, zeroPoint: q.zeroPoint, compressionRatio: stats.compressionRatio };
-                            patternsQuantized++;
-                        }
-                    }
-                    if (patternsQuantized > 0)
-                        actions.push(`Quantized ${patternsQuantized} pattern embeddings (Int8, ~3.92x)`);
-                }
-                catch {
-                    actions.push('Quantization skipped (memory-initializer not available)');
-                }
+            const result = await compactPatterns(threshold);
+            if (result.removed > 0) {
+                actions.push(`Removed ${result.removed} similar patterns (cosine > ${threshold})`);
             }
-            // accuracy / balanced: prune low-usage, zero-embedding patterns
-            if (target === 'accuracy' || target === 'balanced') {
-                for (const [id, p] of Object.entries(store.patterns)) {
-                    if (!p.embedding || p.embedding.length === 0) {
-                        delete store.patterns[id];
-                        patternsRemoved++;
-                        continue;
-                    }
-                    // Remove patterns with all-zero embeddings (no useful signal)
-                    const norm = p.embedding.reduce((s, v) => s + v * v, 0);
-                    if (norm < 1e-10) {
-                        delete store.patterns[id];
-                        patternsRemoved++;
-                    }
-                }
-                if (patternsRemoved > 0)
-                    actions.push(`Pruned ${patternsRemoved} empty/zero-signal patterns`);
+            else {
+                actions.push('No duplicate patterns found');
             }
-            saveNeuralStore(store);
             const elapsed = Math.round(performance.now() - startTime);
             return {
                 success: true, _real: true, target,
                 actions,
-                patternsBefore: beforeCount,
-                patternsAfter: Object.keys(store.patterns).length,
-                duplicatesRemoved,
-                patternsQuantized,
-                patternsRemoved,
+                patternsBefore: result.before,
+                patternsAfter: result.after,
+                duplicatesRemoved: result.removed,
+                patternsQuantized: 0,
+                patternsRemoved: result.removed,
                 embeddingDims: dims,
                 elapsedMs: elapsed,
             };