@monoes/monomindcli 2.8.3 → 2.9.1

package/dist/src/memory/memory-crud.js

@@ -11,6 +11,8 @@ import { safeParseEmbedding } from './memory-bridge.js';
 import { ensureSchemaColumns } from './memory-migrations.js';
 import { generateEmbedding } from './embedding-operations.js';
 import { addToHNSWIndex, rebuildSearchIndex } from './hnsw-operations.js';
+import { withDbLock } from '../utils/db-mutex.js';
+import { secureDbFilePermissions } from './file-permissions.js';
 // Re-export read operations so existing callers keep working without changes.
 export { searchEntries, listEntries, getEntry } from './memory-read.js';
 /** Maximum SQLite database file size accepted before read (256 MB). */
@@ -39,163 +41,166 @@ export async function verifyMemoryInit(dbPath, options) {
     const { verbose = false } = options || {};
     const tests = [];
     try {
-        const initSqlJs = (await import('sql.js')).default;
-        const SQL = await initSqlJs();
-        const fs = await import('fs');
-        // Guard against excessively large DB files to prevent OOM.
-        const verifyStat = fs.statSync(dbPath);
-        if (verifyStat.size > MAX_DB_FILE_BYTES) {
-            return { success: false, tests: [{ name: 'Database access', passed: false, details: `File too large: ${verifyStat.size} bytes` }], summary: { passed: 0, failed: 1, total: 1 } };
-        }
-        // Load database
-        const fileBuffer = fs.readFileSync(dbPath);
-        const db = new SQL.Database(fileBuffer);
-        // Test 1: Schema verification
-        const schemaStart = Date.now();
-        const tables = db.exec("SELECT name FROM sqlite_master WHERE type='table'");
-        const tableNames = tables[0]?.values?.map(v => v[0]) || [];
-        const expectedTables = ['memory_entries', 'patterns', 'metadata', 'vector_indexes'];
-        const missingTables = expectedTables.filter(t => !tableNames.includes(t));
-        tests.push({
-            name: 'Schema verification',
-            passed: missingTables.length === 0,
-            details: missingTables.length > 0 ? `Missing: ${missingTables.join(', ')}` : `${tableNames.length} tables found`,
-            duration: Date.now() - schemaStart
-        });
-        // Test 2: Write entry
-        const writeStart = Date.now();
-        const testId = `test_${Date.now()}`;
-        const testKey = 'verification_test';
-        const testValue = 'This is a verification test entry for memory initialization';
-        try {
-            db.run(`
+        return await withDbLock(dbPath, async () => {
+            const initSqlJs = (await import('sql.js')).default;
+            const SQL = await initSqlJs();
+            const fs = await import('fs');
+            // Guard against excessively large DB files to prevent OOM.
+            const verifyStat = fs.statSync(dbPath);
+            if (verifyStat.size > MAX_DB_FILE_BYTES) {
+                return { success: false, tests: [{ name: 'Database access', passed: false, details: `File too large: ${verifyStat.size} bytes` }], summary: { passed: 0, failed: 1, total: 1 } };
+            }
+            // Load database
+            const fileBuffer = fs.readFileSync(dbPath);
+            const db = new SQL.Database(fileBuffer);
+            // Test 1: Schema verification
+            const schemaStart = Date.now();
+            const tables = db.exec("SELECT name FROM sqlite_master WHERE type='table'");
+            const tableNames = tables[0]?.values?.map(v => v[0]) || [];
+            const expectedTables = ['memory_entries', 'patterns', 'metadata', 'vector_indexes'];
+            const missingTables = expectedTables.filter(t => !tableNames.includes(t));
+            tests.push({
+                name: 'Schema verification',
+                passed: missingTables.length === 0,
+                details: missingTables.length > 0 ? `Missing: ${missingTables.join(', ')}` : `${tableNames.length} tables found`,
+                duration: Date.now() - schemaStart
+            });
+            // Test 2: Write entry
+            const writeStart = Date.now();
+            const testId = `test_${Date.now()}`;
+            const testKey = 'verification_test';
+            const testValue = 'This is a verification test entry for memory initialization';
+            try {
+                db.run(`
         INSERT INTO memory_entries (id, key, namespace, content, type, created_at, updated_at)
         VALUES (?, ?, 'test', ?, 'semantic', ?, ?)
       `, [testId, testKey, testValue, Date.now(), Date.now()]);
-            tests.push({
-                name: 'Write entry',
-                passed: true,
-                details: 'Entry written successfully',
-                duration: Date.now() - writeStart
-            });
-        }
-        catch (e) {
-            tests.push({
-                name: 'Write entry',
-                passed: false,
-                details: e instanceof Error ? e.message : 'Write failed',
-                duration: Date.now() - writeStart
-            });
-        }
-        // Test 3: Read entry
-        const readStart = Date.now();
-        try {
-            const result = db.exec(`SELECT content FROM memory_entries WHERE id = ?`, [testId]);
-            const content = result[0]?.values[0]?.[0];
-            tests.push({
-                name: 'Read entry',
-                passed: content === testValue,
-                details: content === testValue ? 'Content matches' : 'Content mismatch',
-                duration: Date.now() - readStart
-            });
-        }
-        catch (e) {
-            tests.push({
-                name: 'Read entry',
-                passed: false,
-                details: e instanceof Error ? e.message : 'Read failed',
-                duration: Date.now() - readStart
-            });
-        }
-        // Test 4: Write with embedding
-        const embeddingStart = Date.now();
-        try {
-            const { embedding, dimensions, model } = await generateEmbedding(testValue);
-            const embeddingJson = JSON.stringify(embedding);
-            db.run(`
+                tests.push({
+                    name: 'Write entry',
+                    passed: true,
+                    details: 'Entry written successfully',
+                    duration: Date.now() - writeStart
+                });
+            }
+            catch (e) {
+                tests.push({
+                    name: 'Write entry',
+                    passed: false,
+                    details: e instanceof Error ? e.message : 'Write failed',
+                    duration: Date.now() - writeStart
+                });
+            }
+            // Test 3: Read entry
+            const readStart = Date.now();
+            try {
+                const result = db.exec(`SELECT content FROM memory_entries WHERE id = ?`, [testId]);
+                const content = result[0]?.values[0]?.[0];
+                tests.push({
+                    name: 'Read entry',
+                    passed: content === testValue,
+                    details: content === testValue ? 'Content matches' : 'Content mismatch',
+                    duration: Date.now() - readStart
+                });
+            }
+            catch (e) {
+                tests.push({
+                    name: 'Read entry',
+                    passed: false,
+                    details: e instanceof Error ? e.message : 'Read failed',
+                    duration: Date.now() - readStart
+                });
+            }
+            // Test 4: Write with embedding
+            const embeddingStart = Date.now();
+            try {
+                const { embedding, dimensions, model } = await generateEmbedding(testValue);
+                const embeddingJson = JSON.stringify(embedding);
+                db.run(`
         UPDATE memory_entries
         SET embedding = ?, embedding_dimensions = ?, embedding_model = ?
         WHERE id = ?
       `, [embeddingJson, dimensions, model, testId]);
-            tests.push({
-                name: 'Generate embedding',
-                passed: true,
-                details: `${dimensions}-dim vector (${model})`,
-                duration: Date.now() - embeddingStart
-            });
-        }
-        catch (e) {
-            tests.push({
-                name: 'Generate embedding',
-                passed: false,
-                details: e instanceof Error ? e.message : 'Embedding failed',
-                duration: Date.now() - embeddingStart
-            });
-        }
-        // Test 5: Pattern storage
-        const patternStart = Date.now();
-        try {
-            const patternId = `pattern_${Date.now()}`;
-            db.run(`
+                tests.push({
+                    name: 'Generate embedding',
+                    passed: true,
+                    details: `${dimensions}-dim vector (${model})`,
+                    duration: Date.now() - embeddingStart
+                });
+            }
+            catch (e) {
+                tests.push({
+                    name: 'Generate embedding',
+                    passed: false,
+                    details: e instanceof Error ? e.message : 'Embedding failed',
+                    duration: Date.now() - embeddingStart
+                });
+            }
+            // Test 5: Pattern storage
+            const patternStart = Date.now();
+            try {
+                const patternId = `pattern_${Date.now()}`;
+                db.run(`
         INSERT INTO patterns (id, name, pattern_type, condition, action, confidence, created_at, updated_at)
         VALUES (?, 'test-pattern', 'task-routing', 'test condition', 'test action', 0.5, ?, ?)
       `, [patternId, Date.now(), Date.now()]);
-            tests.push({
-                name: 'Pattern storage',
-                passed: true,
-                details: 'Pattern stored with confidence scoring',
-                duration: Date.now() - patternStart
-            });
-            // Cleanup test pattern
-            db.run(`DELETE FROM patterns WHERE id = ?`, [patternId]);
-        }
-        catch (e) {
-            tests.push({
-                name: 'Pattern storage',
-                passed: false,
-                details: e instanceof Error ? e.message : 'Pattern storage failed',
-                duration: Date.now() - patternStart
-            });
-        }
-        // Test 6: Vector index configuration
-        const indexStart = Date.now();
-        try {
-            const indexResult = db.exec(`SELECT name, dimensions, hnsw_m, hnsw_ef_construction FROM vector_indexes`);
-            const indexes = indexResult[0]?.values || [];
-            tests.push({
-                name: 'Vector index config',
-                passed: indexes.length > 0,
-                details: `${indexes.length} indexes configured (HNSW M=16, ef=200)`,
-                duration: Date.now() - indexStart
-            });
-        }
-        catch (e) {
-            tests.push({
-                name: 'Vector index config',
-                passed: false,
-                details: e instanceof Error ? e.message : 'Index check failed',
-                duration: Date.now() - indexStart
-            });
-        }
-        // Cleanup test entry
-        db.run(`DELETE FROM memory_entries WHERE id = ?`, [testId]);
-        // Save changes atomically
-        const data = db.export();
-        const dbTmpHealth = dbPath + '.tmp';
-        fs.writeFileSync(dbTmpHealth, Buffer.from(data));
-        fs.renameSync(dbTmpHealth, dbPath);
-        db.close();
-        const passed = tests.filter(t => t.passed).length;
-        const failed = tests.filter(t => !t.passed).length;
-        return {
-            success: failed === 0,
-            tests,
-            summary: {
-                passed,
-                failed,
-                total: tests.length
+                tests.push({
+                    name: 'Pattern storage',
+                    passed: true,
+                    details: 'Pattern stored with confidence scoring',
+                    duration: Date.now() - patternStart
+                });
+                // Cleanup test pattern
+                db.run(`DELETE FROM patterns WHERE id = ?`, [patternId]);
             }
-        };
+            catch (e) {
+                tests.push({
+                    name: 'Pattern storage',
+                    passed: false,
+                    details: e instanceof Error ? e.message : 'Pattern storage failed',
+                    duration: Date.now() - patternStart
+                });
+            }
+            // Test 6: Vector index configuration
+            const indexStart = Date.now();
+            try {
+                const indexResult = db.exec(`SELECT name, dimensions, hnsw_m, hnsw_ef_construction FROM vector_indexes`);
+                const indexes = indexResult[0]?.values || [];
+                tests.push({
+                    name: 'Vector index config',
+                    passed: indexes.length > 0,
+                    details: `${indexes.length} indexes configured (HNSW M=16, ef=200)`,
+                    duration: Date.now() - indexStart
+                });
+            }
+            catch (e) {
+                tests.push({
+                    name: 'Vector index config',
+                    passed: false,
+                    details: e instanceof Error ? e.message : 'Index check failed',
+                    duration: Date.now() - indexStart
+                });
+            }
+            // Cleanup test entry
+            db.run(`DELETE FROM memory_entries WHERE id = ?`, [testId]);
+            // Save changes atomically
+            const data = db.export();
+            const dbTmpHealth = dbPath + '.tmp';
+            fs.writeFileSync(dbTmpHealth, Buffer.from(data));
+            fs.renameSync(dbTmpHealth, dbPath);
+            secureDbFilePermissions(dbPath);
+            db.close();
+            const passed = tests.filter(t => t.passed).length;
+            const failed = tests.filter(t => !t.passed).length;
+            return {
+                success: failed === 0,
+                tests,
+                summary: {
+                    passed,
+                    failed,
+                    total: tests.length
+                }
+            };
+        });
     }
     catch (error) {
         return {
@@ -230,97 +235,112 @@ export async function storeEntry(options) {
             return { success: false, id: '', error: 'Database not initialized. Run: monomind memory init' };
         }
         await ensureSchemaColumns(dbPath);
-        const initSqlJs = (await import('sql.js')).default;
-        const SQL = await initSqlJs();
-        const storeStat = fs.statSync(dbPath);
-        if (storeStat.size > MAX_DB_FILE_BYTES) {
-            return { success: false, id: '', error: `Database file too large: ${storeStat.size} bytes` };
-        }
-        const fileBuffer = fs.readFileSync(dbPath);
-        const db = new SQL.Database(fileBuffer);
-        // memory_entries has no UNIQUE constraint on (key, namespace) — only
-        // `id TEXT PRIMARY KEY`. `INSERT OR REPLACE` only replaces a row when
-        // its PRIMARY KEY collides, so generating a fresh id on every call (as
-        // this used to do unconditionally) meant upsert never actually matched
-        // the existing row for a given key+namespace — it silently inserted a
-        // duplicate instead of replacing it. When upserting, look up the
-        // existing row's real id first and reuse it so the replace actually
-        // collides; only mint a new id when no existing row is found.
-        let id = `entry_${Date.now()}_${Math.random().toString(36).substring(7)}`;
-        let existingCreatedAt = null;
-        if (upsert) {
-            // Only match active rows — a soft-deleted row's key+namespace should
-            // not be resurrected by an unrelated upsert; that would both restore
-            // history a delete intentionally cleared and mask the deletion.
-            const existingIdResult = db.exec("SELECT id, created_at FROM memory_entries WHERE key = ? AND namespace = ? AND status = 'active' LIMIT 1", [key, namespace]);
-            const existingRow = existingIdResult[0]?.values?.[0];
-            const existingId = existingRow?.[0];
-            if (typeof existingId === 'string') {
-                id = existingId;
-                const createdAt = existingRow?.[1];
-                if (typeof createdAt === 'number')
-                    existingCreatedAt = createdAt;
+        return await withDbLock(dbPath, async () => {
+            const initSqlJs = (await import('sql.js')).default;
+            const SQL = await initSqlJs();
+            const storeStat = fs.statSync(dbPath);
+            if (storeStat.size > MAX_DB_FILE_BYTES) {
+                return { success: false, id: '', error: `Database file too large: ${storeStat.size} bytes` };
             }
-        }
-        const now = Date.now();
-        const createdAt = existingCreatedAt ?? now;
-        let embeddingJson = null;
-        let embeddingDimensions = null;
-        let embeddingModel = null;
-        if (generateEmbeddingFlag && value.length > 0) {
-            const embResult = await generateEmbedding(value);
-            embeddingJson = JSON.stringify(embResult.embedding);
-            embeddingDimensions = embResult.dimensions;
-            embeddingModel = embResult.model;
-        }
-        const insertSql = upsert
-            ? `INSERT OR REPLACE INTO memory_entries (
-          id, key, namespace, content, type,
-          embedding, embedding_dimensions, embedding_model,
-          tags, metadata, created_at, updated_at, expires_at, status
-        ) VALUES (?, ?, ?, ?, 'semantic', ?, ?, ?, ?, ?, ?, ?, ?, 'active')`
-            : `INSERT INTO memory_entries (
+            const fileBuffer = fs.readFileSync(dbPath);
+            const db = new SQL.Database(fileBuffer);
+            let id = `entry_${Date.now()}_${Math.random().toString(36).substring(7)}`;
+            let existingCreatedAt = null;
+            let matchedExisting = false;
+            if (upsert) {
+                const existingIdResult = db.exec("SELECT id, created_at FROM memory_entries WHERE key = ? AND namespace = ? AND status = 'active' LIMIT 1", [key, namespace]);
+                const existingRow = existingIdResult[0]?.values?.[0];
+                const existingId = existingRow?.[0];
+                if (typeof existingId === 'string') {
+                    id = existingId;
+                    matchedExisting = true;
+                    const createdAt = existingRow?.[1];
+                    if (typeof createdAt === 'number')
+                        existingCreatedAt = createdAt;
+                }
+            }
+            const now = Date.now();
+            const createdAt = existingCreatedAt ?? now;
+            let embeddingJson = null;
+            let embeddingDimensions = null;
+            let embeddingModel = null;
+            if (generateEmbeddingFlag && value.length > 0) {
+                const embResult = await generateEmbedding(value);
+                embeddingJson = JSON.stringify(embResult.embedding);
+                embeddingDimensions = embResult.dimensions;
+                embeddingModel = embResult.model;
+            }
+            const isUpdate = upsert && matchedExisting;
+            if (isUpdate) {
+                // #88: upsert against an existing row must UPDATE in place. The old
+                // INSERT OR REPLACE omitted access_count, confidence, importance_score,
+                // last_accessed_at, owner_id, agent_id, session_id and hardcoded
+                // metadata '{}' — every update silently wiped all learned stats.
+                db.run(`UPDATE memory_entries SET
+           content = ?,
+           embedding = ?,
+           embedding_dimensions = ?,
+           embedding_model = ?,
+           tags = ?,
+           updated_at = ?,
+           expires_at = ?
+         WHERE id = ?`, [
+                    value,
+                    embeddingJson,
+                    embeddingDimensions,
+                    embeddingModel,
+                    tags.length > 0 ? JSON.stringify(tags) : null,
+                    now,
+                    ttl ? now + (ttl * 1000) : null,
+                    id
+                ]);
+            }
+            else {
+                const insertSql = `INSERT INTO memory_entries (
           id, key, namespace, content, type,
           embedding, embedding_dimensions, embedding_model,
           tags, metadata, created_at, updated_at, expires_at, status
         ) VALUES (?, ?, ?, ?, 'semantic', ?, ?, ?, ?, ?, ?, ?, ?, 'active')`;
-        db.run(insertSql, [
-            id,
-            key,
-            namespace,
-            value,
-            embeddingJson,
-            embeddingDimensions,
-            embeddingModel,
-            tags.length > 0 ? JSON.stringify(tags) : null,
-            '{}',
-            createdAt,
-            now,
-            ttl ? now + (ttl * 1000) : null
-        ]);
-        // Save atomically
-        const data = db.export();
-        const dbTmpStore = dbPath + '.tmp';
-        fs.writeFileSync(dbTmpStore, Buffer.from(data));
-        fs.renameSync(dbTmpStore, dbPath);
-        db.close();
-        // Add to HNSW index for faster future searches
-        if (embeddingJson) {
-            const embResult = safeParseEmbedding(embeddingJson);
-            if (embResult) {
-                await addToHNSWIndex(id, embResult, {
+                db.run(insertSql, [
                     id,
                     key,
                     namespace,
-                    content: value
-                });
+                    value,
+                    embeddingJson,
+                    embeddingDimensions,
+                    embeddingModel,
+                    tags.length > 0 ? JSON.stringify(tags) : null,
+                    '{}',
+                    createdAt,
+                    now,
+                    ttl ? now + (ttl * 1000) : null
+                ]);
             }
-        }
-        return {
-            success: true,
-            id,
-            embedding: embeddingJson ? { dimensions: embeddingDimensions, model: embeddingModel } : undefined
-        };
+            // Save atomically
+            const data = db.export();
+            const dbTmpStore = dbPath + '.tmp';
+            fs.writeFileSync(dbTmpStore, Buffer.from(data));
+            fs.renameSync(dbTmpStore, dbPath);
+            secureDbFilePermissions(dbPath);
+            db.close();
+            // Add to HNSW index for faster future searches
+            if (embeddingJson) {
+                const embResult = safeParseEmbedding(embeddingJson);
+                if (embResult) {
+                    await addToHNSWIndex(id, embResult, {
+                        id,
+                        key,
+                        namespace,
+                        content: value
+                    });
+                }
+            }
+            return {
+                success: true,
+                id,
+                embedding: embeddingJson ? { dimensions: embeddingDimensions, model: embeddingModel } : undefined
+            };
+        });
     }
     catch (error) {
         return {
@@ -378,42 +398,43 @@ export async function deleteEntry(options) {
             };
         }
         await ensureSchemaColumns(dbPath);
-        const initSqlJs = (await import('sql.js')).default;
-        const SQL = await initSqlJs();
-        const deleteStat = fs.statSync(dbPath);
-        if (deleteStat.size > MAX_DB_FILE_BYTES) {
-            return { success: false, deleted: false, key, namespace, remainingEntries: 0, error: `Database file too large: ${deleteStat.size} bytes` };
-        }
-        const fileBuffer = fs.readFileSync(dbPath);
-        const db = new SQL.Database(fileBuffer);
-        const checkStmt = db.prepare(`
+        return await withDbLock(dbPath, async () => {
+            const initSqlJs = (await import('sql.js')).default;
+            const SQL = await initSqlJs();
+            const deleteStat = fs.statSync(dbPath);
+            if (deleteStat.size > MAX_DB_FILE_BYTES) {
+                return { success: false, deleted: false, key, namespace, remainingEntries: 0, error: `Database file too large: ${deleteStat.size} bytes` };
+            }
+            const fileBuffer = fs.readFileSync(dbPath);
+            const db = new SQL.Database(fileBuffer);
+            const checkStmt = db.prepare(`
       SELECT id FROM memory_entries
       WHERE status = 'active'
         AND key = ?
         AND namespace = ?
       LIMIT 1
     `);
-        checkStmt.bind([key, namespace]);
-        const checkRows = [];
-        while (checkStmt.step()) {
-            checkRows.push(checkStmt.get());
-        }
-        checkStmt.free();
-        const checkResult = checkRows.length > 0 ? [{ values: checkRows }] : [];
-        if (!checkResult[0]?.values?.[0]) {
-            const countResult = db.exec(`SELECT COUNT(*) FROM memory_entries WHERE status = 'active'`);
-            const remainingEntries = countResult[0]?.values?.[0]?.[0] || 0;
-            db.close();
-            return {
-                success: true,
-                deleted: false,
-                key,
-                namespace,
-                remainingEntries,
-                error: `Key '${key}' not found in namespace '${namespace}'`
-            };
-        }
-        db.run(`
+            checkStmt.bind([key, namespace]);
+            const checkRows = [];
+            while (checkStmt.step()) {
+                checkRows.push(checkStmt.get());
+            }
+            checkStmt.free();
+            const checkResult = checkRows.length > 0 ? [{ values: checkRows }] : [];
+            if (!checkResult[0]?.values?.[0]) {
+                const countResult = db.exec(`SELECT COUNT(*) FROM memory_entries WHERE status = 'active'`);
+                const remainingEntries = countResult[0]?.values?.[0]?.[0] || 0;
+                db.close();
+                return {
+                    success: true,
+                    deleted: false,
+                    key,
+                    namespace,
+                    remainingEntries,
+                    error: `Key '${key}' not found in namespace '${namespace}'`
+                };
+            }
+            db.run(`
       UPDATE memory_entries
       SET status = 'deleted',
           embedding = NULL,
@@ -422,23 +443,25 @@ export async function deleteEntry(options) {
         AND namespace = ?
         AND status = 'active'
     `, [key, namespace]);
-        const countResult = db.exec(`SELECT COUNT(*) FROM memory_entries WHERE status = 'active'`);
-        const remainingEntries = countResult[0]?.values?.[0]?.[0] || 0;
-        // Save updated database atomically
-        const data = db.export();
-        const dbTmpDelete = dbPath + '.tmp';
-        fs.writeFileSync(dbTmpDelete, Buffer.from(data));
-        fs.renameSync(dbTmpDelete, dbPath);
-        db.close();
-        // Invalidate the HNSW index so it rebuilds from DB on next search.
-        rebuildSearchIndex();
-        return {
-            success: true,
-            deleted: true,
-            key,
-            namespace,
-            remainingEntries
-        };
+            const countResult = db.exec(`SELECT COUNT(*) FROM memory_entries WHERE status = 'active'`);
+            const remainingEntries = countResult[0]?.values?.[0]?.[0] || 0;
+            // Save updated database atomically
+            const data = db.export();
+            const dbTmpDelete = dbPath + '.tmp';
+            fs.writeFileSync(dbTmpDelete, Buffer.from(data));
+            fs.renameSync(dbTmpDelete, dbPath);
+            secureDbFilePermissions(dbPath);
+            db.close();
+            // Invalidate the HNSW index so it rebuilds from DB on next search.
+            rebuildSearchIndex();
+            return {
+                success: true,
+                deleted: true,
+                key,
+                namespace,
+                remainingEntries
+            };
+        });
     }
     catch (error) {
         return {