om-memory-system 3.0.0

package/dist/services/turso/legacy-migrator.js

@@ -0,0 +1,540 @@
+import { existsSync, mkdirSync, readdirSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from "node:fs";
+import { basename, join } from "node:path";
+import { CONFIG } from "../../config.js";
+import { log } from "../logger.js";
+import { tursoConnectionManager } from "./connection-manager.js";
+import { tursoShardManager } from "./shard-manager.js";
+import { withSqliteFileLockRetry } from "./sqlite-handle-release.js";
+import { tursoVectorSearch } from "./vector-search.js";
+import { blobToFloat32Array } from "./vector-utils.js";
+const MIGRATION_MARKER = ".turso-migrated";
+const MIGRATION_LOCK = ".turso-migrate.lock";
+const SIDECAR_SUFFIX = ".turso-migrate.json";
+const BACKUP_SUFFIX = ".legacy.bak";
+const REEMBED_SWAP_SUFFIX = ".reembed-swap.json";
+function sidecarPath(dbPath) {
+    return `${dbPath}${SIDECAR_SUFFIX}`;
+}
+function backupPath(dbPath) {
+    return `${dbPath}${BACKUP_SUFFIX}`;
+}
+function readSidecar(dbPath) {
+    const path = sidecarPath(dbPath);
+    if (!existsSync(path))
+        return null;
+    try {
+        return JSON.parse(readFileSync(path, "utf-8"));
+    }
+    catch {
+        return null;
+    }
+}
+function writeSidecar(dbPath, sidecar) {
+    writeFileSync(sidecarPath(dbPath), JSON.stringify(sidecar, null, 2), "utf-8");
+}
+function readMarker(storagePath) {
+    const markerPath = join(storagePath, MIGRATION_MARKER);
+    if (!existsSync(markerPath))
+        return null;
+    try {
+        const raw = readFileSync(markerPath, "utf-8");
+        const parsed = JSON.parse(raw);
+        if (parsed.completedAt && Array.isArray(parsed.shards)) {
+            return parsed;
+        }
+        return null;
+    }
+    catch {
+        return null;
+    }
+}
+function writeMarker(storagePath, shards) {
+    const marker = {
+        completedAt: new Date().toISOString(),
+        shards,
+    };
+    writeFileSync(join(storagePath, MIGRATION_MARKER), JSON.stringify(marker, null, 2), "utf-8");
+}
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function acquireMigrationLock(storagePath) {
+    const lockPath = join(storagePath, MIGRATION_LOCK);
+    if (existsSync(lockPath)) {
+        try {
+            const lock = JSON.parse(readFileSync(lockPath, "utf-8"));
+            if (lock.pid && isProcessAlive(lock.pid)) {
+                return false;
+            }
+            unlinkSync(lockPath);
+        }
+        catch {
+            try {
+                unlinkSync(lockPath);
+            }
+            catch {
+                // stale or corrupt lock — try atomic create below
+            }
+        }
+    }
+    const lock = { pid: process.pid, timestamp: new Date().toISOString() };
+    try {
+        writeFileSync(lockPath, JSON.stringify(lock), { flag: "wx" });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+async function acquireMigrationLockWithRetry(storagePath, maxAttempts = 3) {
+    for (let attempt = 0; attempt < maxAttempts; attempt++) {
+        if (acquireMigrationLock(storagePath)) {
+            return;
+        }
+        await sleep(200 + attempt * 150);
+    }
+    throw new Error("Turso legacy migration locked by another process");
+}
+function releaseMigrationLock(storagePath) {
+    const lockPath = join(storagePath, MIGRATION_LOCK);
+    if (!existsSync(lockPath))
+        return;
+    try {
+        const lock = JSON.parse(readFileSync(lockPath, "utf-8"));
+        if (lock.pid === process.pid) {
+            unlinkSync(lockPath);
+        }
+    }
+    catch {
+        unlinkSync(lockPath);
+    }
+}
+async function countMemories(db) {
+    const row = await db.get(`SELECT COUNT(*) as count FROM memories`);
+    return Number(row?.count ?? 0);
+}
+async function hasMemoriesTable(db) {
+    const row = await db.get(`SELECT name FROM sqlite_master WHERE type='table' AND name='memories'`);
+    return Boolean(row);
+}
+async function isTursoVectorShardReady(db) {
+    const indexRow = await db.get(`SELECT name FROM sqlite_master WHERE type='index' AND name='memories_vec_idx'`);
+    if (!indexRow) {
+        return false;
+    }
+    const metaRow = await db.get(`SELECT value FROM shard_metadata WHERE key = 'embedding_dimensions'`);
+    if (!metaRow?.value) {
+        return false;
+    }
+    const storedDimensions = Number(metaRow.value);
+    if (!Number.isInteger(storedDimensions) || storedDimensions <= 0) {
+        return false;
+    }
+    const count = await countMemories(db);
+    if (count === 0) {
+        return true;
+    }
+    try {
+        const probe = await db.get(`SELECT vector_extract(vector) AS extracted FROM memories LIMIT 1`);
+        if (probe?.extracted == null)
+            return false;
+        const extracted = JSON.parse(String(probe.extracted));
+        return Array.isArray(extracted) && extracted.length === storedDimensions;
+    }
+    catch {
+        return false;
+    }
+}
+async function isShardMigrationComplete(dbPath) {
+    const sidecar = readSidecar(dbPath);
+    if (!sidecar || sidecar.status !== "complete") {
+        return false;
+    }
+    if (!existsSync(dbPath)) {
+        return false;
+    }
+    try {
+        const db = await tursoConnectionManager.getConnection(dbPath);
+        if (!(await hasMemoriesTable(db))) {
+            return sidecar.expectedCount === 0;
+        }
+        const count = await countMemories(db);
+        if (count !== sidecar.expectedCount) {
+            return false;
+        }
+        // Sidecar alone is not enough — require native vector index + readable vectors.
+        return isTursoVectorShardReady(db);
+    }
+    catch {
+        return false;
+    }
+}
+async function restoreFromBackup(dbPath) {
+    const backup = backupPath(dbPath);
+    if (!existsSync(backup))
+        return;
+    await tursoConnectionManager.closeConnection(dbPath);
+    if (existsSync(dbPath)) {
+        await withSqliteFileLockRetry(() => unlinkSync(dbPath));
+    }
+    await withSqliteFileLockRetry(() => renameSync(backup, dbPath));
+    const sidecarPathFile = sidecarPath(dbPath);
+    if (existsSync(sidecarPathFile)) {
+        unlinkSync(sidecarPathFile);
+    }
+    log("Legacy migration restored shard from backup", { dbPath, backup });
+}
+async function shouldRestoreFromBackup(dbPath) {
+    const backup = backupPath(dbPath);
+    if (!existsSync(backup))
+        return false;
+    if (!existsSync(dbPath)) {
+        return true;
+    }
+    try {
+        const db = await tursoConnectionManager.getConnection(dbPath);
+        const sidecar = readSidecar(dbPath);
+        // A healthy Turso vector shard must not be overwritten by .legacy.bak,
+        // except when a pending rewrite clearly did not finish importing all rows.
+        if ((await hasMemoriesTable(db)) && (await isTursoVectorShardReady(db))) {
+            if (sidecar?.status === "pending") {
+                const count = await countMemories(db);
+                if (count < sidecar.expectedCount) {
+                    return true;
+                }
+            }
+            return false;
+        }
+        if (sidecar?.status === "complete") {
+            return !(await isShardMigrationComplete(dbPath));
+        }
+        if (!(await hasMemoriesTable(db))) {
+            return true;
+        }
+        const count = await countMemories(db);
+        const expected = sidecar?.expectedCount ?? null;
+        if (expected != null && count < expected) {
+            return true;
+        }
+        if (sidecar?.status === "pending") {
+            return true;
+        }
+    }
+    catch {
+        return true;
+    }
+    return false;
+}
+function rowToRecord(row) {
+    const vector = blobToFloat32Array(row.vector);
+    if (!vector)
+        return null;
+    const tagsVector = blobToFloat32Array(row.tags_vector) ?? undefined;
+    return {
+        id: String(row.id),
+        content: String(row.content),
+        vector,
+        tagsVector,
+        containerTag: String(row.container_tag),
+        tags: row.tags ? String(row.tags) : undefined,
+        type: row.type ? String(row.type) : undefined,
+        createdAt: Number(row.created_at),
+        updatedAt: Number(row.updated_at),
+        metadata: row.metadata ? String(row.metadata) : undefined,
+        displayName: row.display_name ? String(row.display_name) : undefined,
+        userName: row.user_name ? String(row.user_name) : undefined,
+        userEmail: row.user_email ? String(row.user_email) : undefined,
+        projectPath: row.project_path ? String(row.project_path) : undefined,
+        projectName: row.project_name ? String(row.project_name) : undefined,
+        gitRepoUrl: row.git_repo_url ? String(row.git_repo_url) : undefined,
+    };
+}
+async function migrateMemoryShard(dbPath) {
+    if (await isShardMigrationComplete(dbPath)) {
+        const sidecar = readSidecar(dbPath);
+        return sidecar;
+    }
+    if (await shouldRestoreFromBackup(dbPath)) {
+        await restoreFromBackup(dbPath);
+    }
+    if (await isShardMigrationComplete(dbPath)) {
+        return readSidecar(dbPath);
+    }
+    const db = await tursoConnectionManager.getConnection(dbPath);
+    const hasTable = await hasMemoriesTable(db);
+    if (!hasTable) {
+        await tursoShardManager.initShardDb(db);
+        const sidecar = {
+            sourceCount: 0,
+            expectedCount: 0,
+            importedCount: 0,
+            skippedCount: 0,
+            status: "complete",
+        };
+        writeSidecar(dbPath, sidecar);
+        return sidecar;
+    }
+    if (await isTursoVectorShardReady(db)) {
+        const count = await countMemories(db);
+        const sidecar = {
+            sourceCount: count,
+            expectedCount: count,
+            importedCount: count,
+            skippedCount: 0,
+            status: "complete",
+        };
+        writeSidecar(dbPath, sidecar);
+        return sidecar;
+    }
+    const rows = await db.all(`SELECT * FROM memories`);
+    const records = [];
+    const pinnedIds = [];
+    const skippedIds = [];
+    for (const row of rows) {
+        const record = rowToRecord(row);
+        if (!record) {
+            skippedIds.push(String(row.id));
+            log("Legacy migration skipped memory without vector", { memoryId: row.id, dbPath });
+            continue;
+        }
+        records.push(record);
+        if (Number(row.is_pinned) === 1) {
+            pinnedIds.push(record.id);
+        }
+    }
+    const sourceCount = rows.length;
+    const skippedCount = skippedIds.length;
+    if (skippedCount > 0) {
+        throw new Error(`Legacy migration aborted for ${dbPath}: ${skippedCount} of ${sourceCount} memories have unreadable vectors (${skippedIds.join(", ")})`);
+    }
+    const sourceDimensions = records[0]?.vector.length ?? CONFIG.embeddingDimensions;
+    if (!Number.isInteger(sourceDimensions) || sourceDimensions <= 0 || sourceDimensions > 65536) {
+        throw new Error(`Legacy migration aborted for ${dbPath}: invalid source vector dimensions ${sourceDimensions}`);
+    }
+    const dimensionMismatches = records.filter((record) => record.vector.length !== sourceDimensions ||
+        (record.tagsVector != null && record.tagsVector.length !== sourceDimensions));
+    if (dimensionMismatches.length > 0) {
+        const sample = dimensionMismatches
+            .slice(0, 5)
+            .map((record) => `${record.id}(vector=${record.vector.length}` +
+            (record.tagsVector ? `,tags=${record.tagsVector.length}` : "") +
+            ")")
+            .join(", ");
+        throw new Error(`Legacy migration aborted for ${dbPath}: ${dimensionMismatches.length} memories have inconsistent vector dimensions; expected ${sourceDimensions} (${sample})`);
+    }
+    const expectedCount = records.length;
+    writeSidecar(dbPath, {
+        sourceCount,
+        expectedCount,
+        importedCount: 0,
+        skippedCount,
+        status: "pending",
+    });
+    const backup = backupPath(dbPath);
+    await tursoConnectionManager.closeConnection(dbPath);
+    if (existsSync(dbPath)) {
+        await withSqliteFileLockRetry(() => renameSync(dbPath, backup));
+    }
+    const freshDb = await tursoConnectionManager.getConnection(dbPath);
+    const sourceModel = sourceDimensions === CONFIG.embeddingDimensions ? CONFIG.embeddingModel : "legacy-unknown";
+    await tursoShardManager.initShardDb(freshDb, sourceDimensions, sourceModel);
+    const BATCH_SIZE = 50;
+    await freshDb.transaction("write", async (tx) => {
+        for (let offset = 0; offset < records.length; offset += BATCH_SIZE) {
+            const batch = records.slice(offset, offset + BATCH_SIZE);
+            for (const record of batch) {
+                await tursoVectorSearch.insertVectorInTransaction(tx, record);
+            }
+        }
+        for (const memoryId of pinnedIds) {
+            await tx.execute({
+                sql: `UPDATE memories SET is_pinned = 1 WHERE id = ?`,
+                args: [memoryId],
+            });
+        }
+    });
+    const importedCount = await countMemories(freshDb);
+    if (importedCount !== expectedCount) {
+        throw new Error(`Legacy migration count mismatch for ${dbPath}: expected ${expectedCount}, got ${importedCount}`);
+    }
+    const completedSidecar = {
+        sourceCount,
+        expectedCount,
+        importedCount,
+        skippedCount,
+        status: "complete",
+    };
+    writeSidecar(dbPath, completedSidecar);
+    log("Legacy memory shard migrated to Turso vectors", {
+        dbPath,
+        memories: expectedCount,
+        backupPath: backup,
+    });
+    return completedSidecar;
+}
+async function migrateDirectory(dirName) {
+    const dir = join(CONFIG.storagePath, dirName);
+    const results = [];
+    if (!existsSync(dir))
+        return results;
+    for (const file of readdirSync(dir)) {
+        if (!file.endsWith(".db") || file.includes(BACKUP_SUFFIX))
+            continue;
+        const dbPath = join(dir, file);
+        const sidecar = await migrateMemoryShard(dbPath);
+        if (sidecar) {
+            results.push({ path: dbPath, sidecar });
+        }
+    }
+    return results;
+}
+function listAllShardDbPaths() {
+    const paths = [];
+    for (const dirName of ["users", "projects"]) {
+        const dir = join(CONFIG.storagePath, dirName);
+        if (!existsSync(dir))
+            continue;
+        for (const file of readdirSync(dir)) {
+            if (!file.endsWith(".db") || file.includes(BACKUP_SUFFIX))
+                continue;
+            paths.push(join(dir, file));
+        }
+    }
+    return paths;
+}
+async function allShardsComplete() {
+    const paths = listAllShardDbPaths();
+    if (paths.length === 0)
+        return true;
+    for (const dbPath of paths) {
+        if (!(await isShardMigrationComplete(dbPath))) {
+            return false;
+        }
+    }
+    return true;
+}
+function recoverInterruptedReembedSwaps() {
+    for (const dirName of ["users", "projects"]) {
+        const dir = join(CONFIG.storagePath, dirName);
+        if (!existsSync(dir))
+            continue;
+        for (const file of readdirSync(dir)) {
+            if (!file.endsWith(REEMBED_SWAP_SUFFIX))
+                continue;
+            const statePath = join(dir, file);
+            const expectedDbPath = statePath.slice(0, -REEMBED_SWAP_SUFFIX.length);
+            try {
+                const state = JSON.parse(readFileSync(statePath, "utf-8"));
+                const stagedPath = state.stagedPath;
+                const backupPath = state.backupPath;
+                const valid = state.dbPath === expectedDbPath &&
+                    typeof stagedPath === "string" &&
+                    stagedPath.startsWith(`${expectedDbPath}.reembed-`) &&
+                    stagedPath.endsWith(".tmp") &&
+                    typeof backupPath === "string" &&
+                    backupPath.startsWith(`${expectedDbPath}.pre-reembed-`) &&
+                    backupPath.endsWith(".bak");
+                if (!valid) {
+                    throw new Error("invalid re-embed swap state paths");
+                }
+                if (existsSync(expectedDbPath)) {
+                    if (existsSync(stagedPath))
+                        unlinkSync(stagedPath);
+                }
+                else if (existsSync(stagedPath)) {
+                    renameSync(stagedPath, expectedDbPath);
+                }
+                else if (existsSync(backupPath)) {
+                    renameSync(backupPath, expectedDbPath);
+                }
+                else {
+                    throw new Error("neither staged replacement nor source backup exists");
+                }
+                unlinkSync(statePath);
+                log("Recovered interrupted re-embed shard swap", { dbPath: expectedDbPath });
+            }
+            catch (error) {
+                throw new Error(`Failed to recover re-embed swap ${statePath}: ${String(error)}`, {
+                    cause: error,
+                });
+            }
+        }
+    }
+}
+async function reconcileShardRegistry() {
+    const parsedShards = [];
+    for (const path of listAllShardDbPaths()) {
+        const match = /^(user|project)_([a-f0-9]{16})_shard_(\d+)\.db$/.exec(basename(path));
+        if (!match) {
+            log("Legacy migration left unrecognized shard filename unregistered", { path });
+            continue;
+        }
+        const db = await tursoConnectionManager.getConnection(path);
+        parsedShards.push({
+            path,
+            scope: match[1],
+            scopeHash: match[2],
+            shardIndex: Number(match[3]),
+            vectorCount: await countMemories(db),
+        });
+    }
+    const highestIndexByScope = new Map();
+    for (const shard of parsedShards) {
+        const key = `${shard.scope}:${shard.scopeHash}`;
+        highestIndexByScope.set(key, Math.max(highestIndexByScope.get(key) ?? -1, shard.shardIndex));
+    }
+    for (const shard of parsedShards) {
+        const key = `${shard.scope}:${shard.scopeHash}`;
+        await tursoShardManager.registerExistingShard(shard.scope, shard.scopeHash, shard.shardIndex, shard.path, shard.vectorCount, shard.shardIndex === highestIndexByScope.get(key));
+    }
+}
+export async function runLegacyTursoMigration() {
+    if (!existsSync(CONFIG.storagePath)) {
+        mkdirSync(CONFIG.storagePath, { recursive: true });
+    }
+    recoverInterruptedReembedSwaps();
+    const marker = readMarker(CONFIG.storagePath);
+    if (marker && (await allShardsComplete())) {
+        await reconcileShardRegistry();
+        return;
+    }
+    await acquireMigrationLockWithRetry(CONFIG.storagePath);
+    try {
+        const dbFiles = ["metadata.db", "user-prompts.db", "user-profiles.db", "ai-sessions.db"];
+        for (const file of dbFiles) {
+            const path = join(CONFIG.storagePath, file);
+            if (!existsSync(path))
+                continue;
+            await tursoConnectionManager.getConnection(path);
+        }
+        const migratedShards = [];
+        for (const dirName of ["users", "projects"]) {
+            const dirResults = await migrateDirectory(dirName);
+            for (const { path, sidecar } of dirResults) {
+                migratedShards.push({
+                    path,
+                    expectedCount: sidecar.expectedCount,
+                    importedCount: sidecar.importedCount,
+                });
+            }
+        }
+        if (!(await allShardsComplete())) {
+            throw new Error("Turso legacy migration incomplete: not all shards verified");
+        }
+        await reconcileShardRegistry();
+        writeMarker(CONFIG.storagePath, migratedShards);
+        log("Turso legacy migration complete", { storagePath: CONFIG.storagePath });
+    }
+    finally {
+        releaseMigrationLock(CONFIG.storagePath);
+    }
+}

package/dist/services/turso/lifecycle.d.ts

@@ -0,0 +1 @@
+export declare function closeTursoAndInvalidateCaches(): Promise<void>;

package/dist/services/turso/lifecycle.js

@@ -0,0 +1,14 @@
+import { tursoConnectionManager } from "./connection-manager.js";
+import { tursoShardManager } from "./shard-manager.js";
+import { resetTursoReady } from "./ready.js";
+import { userPromptManager } from "../user-prompt/user-prompt-manager.js";
+import { userProfileManager } from "../user-profile/user-profile-manager.js";
+import { aiSessionManager } from "../ai/session/ai-session-manager.js";
+export async function closeTursoAndInvalidateCaches() {
+    await tursoConnectionManager.closeAll();
+    tursoShardManager.reset();
+    resetTursoReady();
+    userPromptManager.reset();
+    userProfileManager.reset();
+    aiSessionManager.reset();
+}

package/dist/services/turso/operation-lock.d.ts

@@ -0,0 +1,2 @@
+export declare function assertNoTursoMigrationInProgress(): void;
+export declare function acquireTursoOperationLock(operation: string): () => void;

package/dist/services/turso/operation-lock.js

@@ -0,0 +1,61 @@
+import { existsSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import { CONFIG } from "../../config.js";
+const OPERATION_LOCK = ".turso-operation.lock";
+const LEGACY_MIGRATION_LOCK = ".turso-migrate.lock";
+function isProcessAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function readLiveLock(path) {
+    if (!existsSync(path))
+        return null;
+    try {
+        const state = JSON.parse(readFileSync(path, "utf-8"));
+        if (Number.isInteger(state.pid) && state.pid > 0 && isProcessAlive(state.pid)) {
+            return state;
+        }
+    }
+    catch {
+        // Corrupt locks are stale and removed below.
+    }
+    try {
+        unlinkSync(path);
+    }
+    catch {
+        // Already removed (race) or held open (Windows) — either way the lock is
+        // gone or unowned; do not let cleanup failure block writes.
+    }
+    return null;
+}
+export function assertNoTursoMigrationInProgress() {
+    for (const file of [OPERATION_LOCK, LEGACY_MIGRATION_LOCK]) {
+        const path = join(CONFIG.storagePath, file);
+        const state = readLiveLock(path);
+        if (state) {
+            throw new Error(`Database migration is in progress${state.operation ? ` (${state.operation})` : ""} ` +
+                `in process ${state.pid}; writes are temporarily blocked`);
+        }
+    }
+}
+export function acquireTursoOperationLock(operation) {
+    assertNoTursoMigrationInProgress();
+    const path = join(CONFIG.storagePath, OPERATION_LOCK);
+    const state = {
+        pid: process.pid,
+        timestamp: new Date().toISOString(),
+        operation,
+    };
+    writeFileSync(path, JSON.stringify(state), { flag: "wx" });
+    return () => {
+        const current = readLiveLock(path);
+        if (current?.pid === process.pid && existsSync(path)) {
+            unlinkSync(path);
+        }
+    };
+}

package/dist/services/turso/ready.d.ts

@@ -0,0 +1,2 @@
+export declare function ensureTursoReady(): Promise<void>;
+export declare function resetTursoReady(): void;

package/dist/services/turso/ready.js

@@ -0,0 +1,36 @@
+import { runLegacyTursoMigration } from "./legacy-migrator.js";
+import { runStartupTagPrefixGate } from "../tag-prefix-migration.js";
+import { tursoShardManager } from "./shard-manager.js";
+import { log } from "../logger.js";
+let initPromise = null;
+let isReady = false;
+export async function ensureTursoReady() {
+    if (isReady)
+        return;
+    if (initPromise)
+        return initPromise;
+    initPromise = (async () => {
+        try {
+            await runLegacyTursoMigration();
+            // Gate every read/write on the one-time container tag prefix migration:
+            // either the store is fully on omms_, or initialisation fails loudly
+            // (no silent dual-prefix fallback). Skipped under
+            // OMMS_SKIP_TAG_PREFIX_MIGRATION=1 (tests).
+            await runStartupTagPrefixGate();
+            const { shardPathMigrationService } = await import("../shard-path-migration-service.js");
+            await shardPathMigrationService.recoverInterruptedSwap();
+            await tursoShardManager.getAllShards("user", "");
+            isReady = true;
+        }
+        catch (error) {
+            initPromise = null;
+            log("Turso ready gate failed", { error: String(error) });
+            throw error;
+        }
+    })();
+    return initPromise;
+}
+export function resetTursoReady() {
+    isReady = false;
+    initPromise = null;
+}