moflo 4.10.24 → 4.10.25

package/dist/src/cli/memory/quantization.js

@@ -0,0 +1,221 @@
+/**
+ * Vector quantization + flash-attention batch ops.
+ *
+ * Extracted from `memory-initializer.ts` (#1203 decomposition) — pure,
+ * dependency-free numeric helpers. INT8 quantization (4x memory reduction)
+ * and V8-optimized batch similarity / softmax / top-k used by the HNSW and
+ * search paths. No I/O, no module state.
+ *
+ * @module memory/quantization
+ */
+// ============================================================================
+// INT8 VECTOR QUANTIZATION (4x memory reduction)
+// ============================================================================
+/**
+ * Quantize a Float32 embedding to Int8 (4x memory reduction)
+ * Uses symmetric quantization with scale factor stored per-vector
+ *
+ * @param embedding - Float32 embedding array
+ * @returns Quantized Int8 array with scale factor
+ */
+export function quantizeInt8(embedding) {
+    const arr = embedding instanceof Float32Array ? embedding : new Float32Array(embedding);
+    // Find min/max for symmetric quantization
+    let min = Infinity, max = -Infinity;
+    for (let i = 0; i < arr.length; i++) {
+        if (arr[i] < min)
+            min = arr[i];
+        if (arr[i] > max)
+            max = arr[i];
+    }
+    // Symmetric quantization: scale = max(|min|, |max|) / 127
+    const absMax = Math.max(Math.abs(min), Math.abs(max));
+    const scale = absMax / 127 || 1e-10; // Avoid division by zero
+    const zeroPoint = 0; // Symmetric quantization
+    // Quantize
+    const quantized = new Int8Array(arr.length);
+    for (let i = 0; i < arr.length; i++) {
+        // Clamp to [-127, 127] to leave room for potential rounding
+        const q = Math.round(arr[i] / scale);
+        quantized[i] = Math.max(-127, Math.min(127, q));
+    }
+    return { quantized, scale, zeroPoint };
+}
+/**
+ * Dequantize Int8 back to Float32
+ *
+ * @param quantized - Int8 quantized array
+ * @param scale - Scale factor from quantization
+ * @param zeroPoint - Zero point (usually 0 for symmetric)
+ * @returns Float32Array
+ */
+export function dequantizeInt8(quantized, scale, zeroPoint = 0) {
+    const result = new Float32Array(quantized.length);
+    for (let i = 0; i < quantized.length; i++) {
+        result[i] = (quantized[i] - zeroPoint) * scale;
+    }
+    return result;
+}
+/**
+ * Compute cosine similarity between quantized vectors
+ * Faster than dequantizing first
+ */
+export function quantizedCosineSim(a, aScale, b, bScale) {
+    if (a.length !== b.length)
+        return 0;
+    let dot = 0, normA = 0, normB = 0;
+    for (let i = 0; i < a.length; i++) {
+        dot += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    // Scales cancel out in cosine similarity for normalized vectors
+    const mag = Math.sqrt(normA * normB);
+    return mag === 0 ? 0 : dot / mag;
+}
+/**
+ * Get quantization statistics for an embedding
+ */
+export function getQuantizationStats(embedding) {
+    const len = embedding.length;
+    const originalBytes = len * 4; // Float32 = 4 bytes
+    const quantizedBytes = len + 8; // Int8 = 1 byte + 8 bytes for scale/zeroPoint
+    const compressionRatio = originalBytes / quantizedBytes;
+    return { originalBytes, quantizedBytes, compressionRatio };
+}
+// ============================================================================
+// FLASH ATTENTION-STYLE BATCH OPERATIONS (V8-Optimized)
+// ============================================================================
+/**
+ * Batch cosine similarity - compute query against multiple vectors
+ * Optimized for V8 JIT with typed arrays
+ * ~50μs per 1000 vectors (384-dim)
+ */
+export function batchCosineSim(query, vectors) {
+    const n = vectors.length;
+    const scores = new Float32Array(n);
+    if (n === 0 || query.length === 0)
+        return scores;
+    // Pre-compute query norm
+    let queryNorm = 0;
+    for (let i = 0; i < query.length; i++) {
+        queryNorm += query[i] * query[i];
+    }
+    queryNorm = Math.sqrt(queryNorm);
+    if (queryNorm === 0)
+        return scores;
+    // Compute similarities
+    for (let v = 0; v < n; v++) {
+        const vec = vectors[v];
+        const len = Math.min(query.length, vec.length);
+        let dot = 0, vecNorm = 0;
+        for (let i = 0; i < len; i++) {
+            dot += query[i] * vec[i];
+            vecNorm += vec[i] * vec[i];
+        }
+        vecNorm = Math.sqrt(vecNorm);
+        scores[v] = vecNorm === 0 ? 0 : dot / (queryNorm * vecNorm);
+    }
+    return scores;
+}
+/**
+ * Softmax normalization for attention scores
+ * Numerically stable implementation
+ */
+export function softmaxAttention(scores, temperature = 1.0) {
+    const n = scores.length;
+    const result = new Float32Array(n);
+    if (n === 0)
+        return result;
+    // Find max for numerical stability
+    let max = scores[0];
+    for (let i = 1; i < n; i++) {
+        if (scores[i] > max)
+            max = scores[i];
+    }
+    // Compute exp and sum
+    let sum = 0;
+    for (let i = 0; i < n; i++) {
+        result[i] = Math.exp((scores[i] - max) / temperature);
+        sum += result[i];
+    }
+    // Normalize
+    if (sum > 0) {
+        for (let i = 0; i < n; i++) {
+            result[i] /= sum;
+        }
+    }
+    return result;
+}
+/**
+ * Top-K selection with partial sort (O(n + k log k))
+ * More efficient than full sort for small k
+ */
+export function topKIndices(scores, k) {
+    const n = scores.length;
+    if (k >= n) {
+        // Return all indices sorted by score
+        return Array.from({ length: n }, (_, i) => i)
+            .sort((a, b) => scores[b] - scores[a]);
+    }
+    // Build min-heap of size k
+    const heap = [];
+    for (let i = 0; i < n; i++) {
+        if (heap.length < k) {
+            heap.push({ idx: i, score: scores[i] });
+            // Bubble up
+            let j = heap.length - 1;
+            while (j > 0) {
+                const parent = Math.floor((j - 1) / 2);
+                if (heap[j].score < heap[parent].score) {
+                    [heap[j], heap[parent]] = [heap[parent], heap[j]];
+                    j = parent;
+                }
+                else
+                    break;
+            }
+        }
+        else if (scores[i] > heap[0].score) {
+            // Replace min and heapify down
+            heap[0] = { idx: i, score: scores[i] };
+            let j = 0;
+            while (true) {
+                const left = 2 * j + 1, right = 2 * j + 2;
+                let smallest = j;
+                if (left < k && heap[left].score < heap[smallest].score)
+                    smallest = left;
+                if (right < k && heap[right].score < heap[smallest].score)
+                    smallest = right;
+                if (smallest === j)
+                    break;
+                [heap[j], heap[smallest]] = [heap[smallest], heap[j]];
+                j = smallest;
+            }
+        }
+    }
+    // Extract and sort descending
+    return heap.sort((a, b) => b.score - a.score).map(h => h.idx);
+}
+/**
+ * Flash Attention-style search
+ * Combines batch similarity, softmax, and top-k in one pass
+ * Returns indices and attention weights
+ */
+export function flashAttentionSearch(query, vectors, options = {}) {
+    const { k = 10, temperature = 1.0, threshold = 0 } = options;
+    // Compute batch similarity
+    const scores = batchCosineSim(query, vectors);
+    // Get top-k indices
+    const indices = topKIndices(scores, k);
+    // Filter by threshold
+    const filtered = indices.filter(i => scores[i] >= threshold);
+    // Extract scores for filtered results
+    const topScores = new Float32Array(filtered.length);
+    for (let i = 0; i < filtered.length; i++) {
+        topScores[i] = scores[filtered[i]];
+    }
+    // Compute attention weights (softmax over top-k)
+    const weights = softmaxAttention(topScores, temperature);
+    return { indices: filtered, scores: topScores, weights };
+}
+//# sourceMappingURL=quantization.js.map

package/dist/src/cli/memory/schema.js

@@ -0,0 +1,382 @@
+/**
+ * Memory database schema + schema-migration helpers.
+ *
+ * Extracted from `memory-initializer.ts` (#1203 decomposition). Holds the
+ * canonical V3 schema DDL, the initial-metadata seed, the
+ * {@link MemoryInitResult} shape, and {@link ensureSchemaColumns} (the
+ * add-missing-columns migration older consumer DBs need before a read/write).
+ *
+ * @module memory/schema
+ */
+import * as fs from 'fs';
+import { errorDetail } from '../shared/utils/error-detail.js';
+import { openDaemonDatabase } from './daemon-backend.js';
+/**
+ * Enhanced schema with pattern confidence, temporal decay, versioning
+ * Vector embeddings enabled for semantic search
+ */
+export const MEMORY_SCHEMA_V3 = `
+-- MoFlo Memory Database
+-- Version: 3.0.0
+-- Features: Pattern learning, vector embeddings, temporal decay, migration tracking
+
+PRAGMA journal_mode = WAL;
+PRAGMA synchronous = NORMAL;
+PRAGMA foreign_keys = ON;
+
+-- ============================================
+-- CORE MEMORY TABLES
+-- ============================================
+
+-- Memory entries (main storage)
+CREATE TABLE IF NOT EXISTS memory_entries (
+  id TEXT PRIMARY KEY,
+  key TEXT NOT NULL,
+  namespace TEXT DEFAULT 'default',
+  content TEXT NOT NULL,
+  type TEXT DEFAULT 'semantic' CHECK(type IN ('semantic', 'episodic', 'procedural', 'working', 'pattern')),
+
+  -- Vector embedding for semantic search (stored as JSON array)
+  embedding TEXT,
+  embedding_model TEXT DEFAULT 'local',
+  embedding_dimensions INTEGER,
+
+  -- Metadata
+  tags TEXT, -- JSON array
+  metadata TEXT, -- JSON object
+  owner_id TEXT,
+
+  -- Timestamps
+  created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+  updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+  expires_at INTEGER,
+  last_accessed_at INTEGER,
+
+  -- Access tracking for hot/cold detection
+  access_count INTEGER DEFAULT 0,
+
+  -- Status
+  status TEXT DEFAULT 'active' CHECK(status IN ('active', 'archived')),
+
+  UNIQUE(namespace, key)
+);
+
+-- Indexes for memory entries
+CREATE INDEX IF NOT EXISTS idx_memory_namespace ON memory_entries(namespace);
+CREATE INDEX IF NOT EXISTS idx_memory_key ON memory_entries(key);
+CREATE INDEX IF NOT EXISTS idx_memory_type ON memory_entries(type);
+CREATE INDEX IF NOT EXISTS idx_memory_status ON memory_entries(status);
+CREATE INDEX IF NOT EXISTS idx_memory_created ON memory_entries(created_at);
+CREATE INDEX IF NOT EXISTS idx_memory_accessed ON memory_entries(last_accessed_at);
+CREATE INDEX IF NOT EXISTS idx_memory_owner ON memory_entries(owner_id);
+
+-- ============================================
+-- PATTERN LEARNING TABLES
+-- ============================================
+
+-- Learned patterns with confidence scoring and versioning
+CREATE TABLE IF NOT EXISTS patterns (
+  id TEXT PRIMARY KEY,
+
+  -- Pattern identification
+  name TEXT NOT NULL,
+  pattern_type TEXT NOT NULL CHECK(pattern_type IN (
+    'task-routing', 'error-recovery', 'optimization', 'learning',
+    'coordination', 'prediction', 'code-pattern', 'workflow'
+  )),
+
+  -- Pattern definition
+  condition TEXT NOT NULL, -- Regex or semantic match
+  action TEXT NOT NULL, -- What to do when pattern matches
+  description TEXT,
+
+  -- Confidence scoring (0.0 - 1.0)
+  confidence REAL DEFAULT 0.5,
+  success_count INTEGER DEFAULT 0,
+  failure_count INTEGER DEFAULT 0,
+
+  -- Temporal decay
+  decay_rate REAL DEFAULT 0.01, -- How fast confidence decays
+  half_life_days INTEGER DEFAULT 30, -- Days until confidence halves without use
+
+  -- Vector embedding for semantic pattern matching
+  embedding TEXT,
+  embedding_dimensions INTEGER,
+
+  -- Versioning
+  version INTEGER DEFAULT 1,
+  parent_id TEXT REFERENCES patterns(id),
+
+  -- Metadata
+  tags TEXT, -- JSON array
+  metadata TEXT, -- JSON object
+  source TEXT, -- Where the pattern was learned from
+
+  -- Timestamps
+  created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+  updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+  last_matched_at INTEGER,
+  last_success_at INTEGER,
+  last_failure_at INTEGER,
+
+  -- Status
+  status TEXT DEFAULT 'active' CHECK(status IN ('active', 'archived', 'deprecated', 'experimental'))
+);
+
+-- Indexes for patterns
+CREATE INDEX IF NOT EXISTS idx_patterns_type ON patterns(pattern_type);
+CREATE INDEX IF NOT EXISTS idx_patterns_confidence ON patterns(confidence DESC);
+CREATE INDEX IF NOT EXISTS idx_patterns_status ON patterns(status);
+CREATE INDEX IF NOT EXISTS idx_patterns_last_matched ON patterns(last_matched_at);
+
+-- Pattern evolution history (for versioning)
+CREATE TABLE IF NOT EXISTS pattern_history (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  pattern_id TEXT NOT NULL REFERENCES patterns(id),
+  version INTEGER NOT NULL,
+
+  -- Snapshot of pattern state
+  confidence REAL,
+  success_count INTEGER,
+  failure_count INTEGER,
+  condition TEXT,
+  action TEXT,
+
+  -- What changed
+  change_type TEXT CHECK(change_type IN ('created', 'updated', 'success', 'failure', 'decay', 'merged', 'split')),
+  change_reason TEXT,
+
+  created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
+);
+
+CREATE INDEX IF NOT EXISTS idx_pattern_history_pattern ON pattern_history(pattern_id);
+
+-- ============================================
+-- LEARNING & TRAJECTORY TABLES
+-- ============================================
+
+-- Learning trajectories (SONA integration)
+CREATE TABLE IF NOT EXISTS trajectories (
+  id TEXT PRIMARY KEY,
+  session_id TEXT,
+
+  -- Trajectory state
+  status TEXT DEFAULT 'active' CHECK(status IN ('active', 'completed', 'failed', 'abandoned')),
+  verdict TEXT CHECK(verdict IN ('success', 'failure', 'partial', NULL)),
+
+  -- Context
+  task TEXT,
+  context TEXT, -- JSON object
+
+  -- Metrics
+  total_steps INTEGER DEFAULT 0,
+  total_reward REAL DEFAULT 0,
+
+  -- Timestamps
+  started_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+  ended_at INTEGER,
+
+  -- Reference to extracted pattern (if any)
+  extracted_pattern_id TEXT REFERENCES patterns(id)
+);
+
+-- Trajectory steps
+CREATE TABLE IF NOT EXISTS trajectory_steps (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  trajectory_id TEXT NOT NULL REFERENCES trajectories(id),
+  step_number INTEGER NOT NULL,
+
+  -- Step data
+  action TEXT NOT NULL,
+  observation TEXT,
+  reward REAL DEFAULT 0,
+
+  -- Metadata
+  metadata TEXT, -- JSON object
+
+  created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
+);
+
+CREATE INDEX IF NOT EXISTS idx_steps_trajectory ON trajectory_steps(trajectory_id);
+
+-- ============================================
+-- MIGRATION STATE TRACKING
+-- ============================================
+
+-- Migration state (for resume capability)
+CREATE TABLE IF NOT EXISTS migration_state (
+  id TEXT PRIMARY KEY,
+  migration_type TEXT NOT NULL, -- 'v2-to-v3', 'pattern', 'memory', etc.
+
+  -- Progress tracking
+  status TEXT DEFAULT 'pending' CHECK(status IN ('pending', 'in_progress', 'completed', 'failed', 'rolled_back')),
+  total_items INTEGER DEFAULT 0,
+  processed_items INTEGER DEFAULT 0,
+  failed_items INTEGER DEFAULT 0,
+  skipped_items INTEGER DEFAULT 0,
+
+  -- Current position (for resume)
+  current_batch INTEGER DEFAULT 0,
+  last_processed_id TEXT,
+
+  -- Source/destination info
+  source_path TEXT,
+  source_type TEXT,
+  destination_path TEXT,
+
+  -- Backup info
+  backup_path TEXT,
+  backup_created_at INTEGER,
+
+  -- Error tracking
+  last_error TEXT,
+  errors TEXT, -- JSON array of errors
+
+  -- Timestamps
+  started_at INTEGER,
+  completed_at INTEGER,
+  created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+  updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
+);
+
+-- ============================================
+-- SESSION MANAGEMENT
+-- ============================================
+
+-- Sessions for context persistence
+CREATE TABLE IF NOT EXISTS sessions (
+  id TEXT PRIMARY KEY,
+
+  -- Session state
+  state TEXT NOT NULL, -- JSON object with full session state
+  status TEXT DEFAULT 'active' CHECK(status IN ('active', 'paused', 'completed', 'expired')),
+
+  -- Context
+  project_path TEXT,
+  branch TEXT,
+
+  -- Metrics
+  tasks_completed INTEGER DEFAULT 0,
+  patterns_learned INTEGER DEFAULT 0,
+
+  -- Timestamps
+  created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+  updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+  expires_at INTEGER
+);
+
+-- ============================================
+-- VECTOR INDEX METADATA (for HNSW)
+-- ============================================
+
+-- Track HNSW index state
+CREATE TABLE IF NOT EXISTS vector_indexes (
+  id TEXT PRIMARY KEY,
+  name TEXT NOT NULL UNIQUE,
+
+  -- Index configuration
+  dimensions INTEGER NOT NULL,
+  metric TEXT DEFAULT 'cosine' CHECK(metric IN ('cosine', 'euclidean', 'dot')),
+
+  -- HNSW parameters
+  hnsw_m INTEGER DEFAULT 16,
+  hnsw_ef_construction INTEGER DEFAULT 200,
+  hnsw_ef_search INTEGER DEFAULT 100,
+
+  -- Quantization
+  quantization_type TEXT CHECK(quantization_type IN ('none', 'scalar', 'product')),
+  quantization_bits INTEGER DEFAULT 8,
+
+  -- Statistics
+  total_vectors INTEGER DEFAULT 0,
+  last_rebuild_at INTEGER,
+
+  created_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000),
+  updated_at INTEGER NOT NULL DEFAULT (strftime('%s', 'now') * 1000)
+);
+
+-- ============================================
+-- SYSTEM METADATA
+-- ============================================
+
+CREATE TABLE IF NOT EXISTS metadata (
+  key TEXT PRIMARY KEY,
+  value TEXT NOT NULL,
+  updated_at INTEGER DEFAULT (strftime('%s', 'now') * 1000)
+);
+`;
+/**
+ * Initial metadata to insert after schema creation
+ */
+export function getInitialMetadata(backend) {
+    return `
+INSERT OR REPLACE INTO metadata (key, value) VALUES
+  ('schema_version', '3.0.0'),
+  ('backend', '${backend}'),
+  ('created_at', '${new Date().toISOString()}'),
+  ('sql_js', 'true'),
+  ('vector_embeddings', 'enabled'),
+  ('pattern_learning', 'enabled'),
+  ('temporal_decay', 'enabled'),
+  ('hnsw_indexing', 'enabled'),
+  ('embeddings_version', '2');
+
+-- Create default vector index configuration
+INSERT OR IGNORE INTO vector_indexes (id, name, dimensions) VALUES
+  ('default', 'default', 768),
+  ('patterns', 'patterns', 768);
+`;
+}
+/**
+ * Ensure memory_entries table has all required columns
+ * Adds missing columns for older databases (e.g., 'content' column)
+ */
+export async function ensureSchemaColumns(dbPath) {
+    const columnsAdded = [];
+    try {
+        if (!fs.existsSync(dbPath)) {
+            return { success: true, columnsAdded: [] };
+        }
+        const db = openDaemonDatabase(dbPath);
+        // Get current columns in memory_entries
+        const tableInfo = db.exec("PRAGMA table_info(memory_entries)");
+        const existingColumns = new Set(tableInfo[0]?.values?.map(row => row[1]) || []);
+        // Required columns that may be missing in older schemas
+        // Issue #977: 'type' column was missing from this list, causing store failures on older DBs
+        const requiredColumns = [
+            { name: 'content', definition: "content TEXT DEFAULT ''" },
+            { name: 'type', definition: "type TEXT DEFAULT 'semantic'" },
+            { name: 'embedding', definition: 'embedding TEXT' },
+            { name: 'embedding_model', definition: "embedding_model TEXT DEFAULT 'local'" },
+            { name: 'embedding_dimensions', definition: 'embedding_dimensions INTEGER' },
+            { name: 'tags', definition: 'tags TEXT' },
+            { name: 'metadata', definition: 'metadata TEXT' },
+            { name: 'owner_id', definition: 'owner_id TEXT' },
+            { name: 'expires_at', definition: 'expires_at INTEGER' },
+            { name: 'last_accessed_at', definition: 'last_accessed_at INTEGER' },
+            { name: 'access_count', definition: 'access_count INTEGER DEFAULT 0' },
+            { name: 'status', definition: "status TEXT DEFAULT 'active'" }
+        ];
+        for (const col of requiredColumns) {
+            if (!existingColumns.has(col.name)) {
+                try {
+                    db.run(`ALTER TABLE memory_entries ADD COLUMN ${col.definition}`);
+                    columnsAdded.push(col.name);
+                }
+                catch (e) {
+                    // Column might already exist or other error - continue
+                }
+            }
+        }
+        db.close();
+        return { success: true, columnsAdded };
+    }
+    catch (error) {
+        return {
+            success: false,
+            columnsAdded,
+            error: errorDetail(error)
+        };
+    }
+}
+//# sourceMappingURL=schema.js.map