@lotargo/memory_plugin 1.3.2 → 1.4.5

package/mcp-server/cli.js

@@ -1,4 +1,4 @@
-#!/usr/bin/env node
+#!/usr/bin/env node
 import readline from "readline";
 import { join } from "node:path";
 import { getConfig, updateConfig, resetConfig } from "./config/config_manager.js";
@@ -91,9 +91,11 @@ async function getQuickStats() {
   let docCount = 0;
   let chunkCount = 0;
   try {
-    const db = getDatabase();
-    docCount = db.prepare("SELECT COUNT(*) as cnt FROM documents").get().cnt;
-    chunkCount = db.prepare("SELECT COUNT(*) as cnt FROM micro_chunks").get().cnt;
+    const db = await getDatabase();
+    const docRow = await db.prepare("SELECT COUNT(*) as cnt FROM documents").get();
+    docCount = docRow ? docRow.cnt : 0;
+    const chunkRow = await db.prepare("SELECT COUNT(*) as cnt FROM micro_chunks").get();
+    chunkCount = chunkRow ? chunkRow.cnt : 0;
   } catch (e) {}
 
   let factCount = 0;
@@ -658,6 +660,31 @@ export async function runCli() {
     return;
   }
 
+  if (cliArgs.includes("login")) {
+    console.log("\n  [CLOUD] Starting Turso cloud authorization...");
+    const { loginToCloud } = await import("./admin/auth.js");
+    try {
+      const secrets = await loginToCloud();
+      console.log(`\n  \x1b[32m[OK] Successfully signed in to the cloud! Connected to endpoint: ${secrets.dbUrl}\x1b[0m\n`);
+    } catch (e) {
+      console.error(`\n  \x1b[31m[ERROR] Authorization failed: ${e.message}\x1b[0m\n`);
+      process.exit(1);
+    }
+    return;
+  }
+
+  if (cliArgs.includes("logout")) {
+    console.log("\n  [CLOUD] Signing out of the cloud...");
+    const { logoutFromCloud } = await import("./admin/auth.js");
+    const deleted = logoutFromCloud();
+    if (deleted) {
+      console.log("  \x1b[32m[OK] You have been signed out. Encrypted secrets removed. Mode reverted to only-local.\x1b[0m\n");
+    } else {
+      console.log("  [*] Mode reverted to only-local. No session tokens were found.\x1b[0m\n");
+    }
+    return;
+  }
+
   let running = true;
   let selectedIndex = 0;
 
@@ -760,6 +787,27 @@ export async function runCli() {
           },
         ],
       },
+      {
+        title: "Cloud Synchronization & Turso",
+        items: [
+          {
+            label: "[CLOUD] Login to Turso Cloud",
+            value: "cloud_login",
+            info: "Perform secure OAuth/Device login flow with loopback listener and local AES-256 key encryption",
+          },
+          {
+            label: "[CLOUD] Logout",
+            value: "cloud_logout",
+            info: "Sign out, purge encrypted secrets, and revert mode to only-local",
+          },
+          {
+            label: "Operational Mode",
+            badge: config.mode.toUpperCase(),
+            value: "cloud_mode",
+            info: "Choose Operational Mode: only-local | only-cloud | hybrid-sync",
+          },
+        ],
+      },
       {
         title: "Global Prompt & Integration Management",
         items: [
@@ -1103,10 +1151,11 @@ export async function runCli() {
                 await writeMemory(key, updated);
                 let links = 0;
                 try {
-                  const db = getDatabase();
-                  links = db
+                  const db = await getDatabase();
+                  const runRes = await db
                     .prepare("UPDATE knowledge_links SET fact_text = ? WHERE fact_key = ? AND fact_text = ?")
-                    .run(newText, key, factText(selectedEntry)).changes;
+                    .run(newText, key, factText(selectedEntry));
+                  links = runRes ? runRes.changes : 0;
                 } catch (e) {}
                 console.clear();
                 console.log(`\n  [OK] Fact updated successfully${links ? `, ${links} doc link(s) updated` : ""}.\n`);
@@ -1234,8 +1283,8 @@ export async function runCli() {
       case "rag_docs": {
         let docRunning = true;
         while (docRunning) {
-          const db = getDatabase();
-          const docs = db.prepare("SELECT id, title, path, created_at FROM documents ORDER BY created_at DESC").all();
+          const db = await getDatabase();
+          const docs = await db.prepare("SELECT id, title, path, created_at FROM documents ORDER BY created_at DESC").all();
 
           if (!docs || docs.length === 0) {
             console.clear();
@@ -1295,9 +1344,11 @@ export async function runCli() {
           });
 
           if (actionRes.action === "select" && actionRes.value === "info") {
-            const secCount = db.prepare("SELECT COUNT(*) as cnt FROM sections WHERE doc_id = ?").get(targetDoc.id).cnt;
-            const chunkCount = db.prepare("SELECT COUNT(*) as cnt FROM micro_chunks WHERE doc_id = ?").get(targetDoc.id).cnt;
-            const sampleSections = db.prepare("SELECT heading FROM sections WHERE doc_id = ? LIMIT 5").all(targetDoc.id);
+            const secCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM sections WHERE doc_id = ?").get(targetDoc.id);
+            const secCount = secCountRow ? secCountRow.cnt : 0;
+            const chunkCountRow = await db.prepare("SELECT COUNT(*) as cnt FROM micro_chunks WHERE doc_id = ?").get(targetDoc.id);
+            const chunkCount = chunkCountRow ? chunkCountRow.cnt : 0;
+            const sampleSections = await db.prepare("SELECT heading FROM sections WHERE doc_id = ? LIMIT 5").all(targetDoc.id);
 
             console.clear();
             const line = "─".repeat(PANEL_WIDTH - 2);
@@ -1776,6 +1827,51 @@ export async function runCli() {
         }
         break;
       }
+      case "cloud_login": {
+        console.clear();
+        console.log("\n  [CLOUD] Starting Turso cloud authorization...");
+        const { loginToCloud } = await import("./admin/auth.js");
+        try {
+          const secrets = await loginToCloud();
+          console.log(`\n  \x1b[32m[OK] Successfully signed in to the cloud! Connected to endpoint: ${secrets.dbUrl}\x1b[0m\n`);
+        } catch (e) {
+          console.error(`\n  \x1b[31m[ERROR] Authorization failed: ${e.message}\x1b[0m\n`);
+        }
+        await waitForEnter();
+        break;
+      }
+      case "cloud_logout": {
+        console.clear();
+        console.log("\n  [CLOUD] Signing out of the cloud...");
+        const { logoutFromCloud } = await import("./admin/auth.js");
+        const deleted = logoutFromCloud();
+        if (deleted) {
+          console.log("  \x1b[32m[OK] You have been signed out. Encrypted secrets removed. Mode reverted to only-local.\x1b[0m\n");
+        } else {
+          console.log("  [*] Mode reverted to only-local. No session tokens were found.\x1b[0m\n");
+        }
+        await waitForEnter();
+        break;
+      }
+      case "cloud_mode": {
+        const modeItems = [
+          { label: "only-local (Local only)", value: "only-local", info: "Fully private, offline-first mode (everything stored on disk)" },
+          { label: "only-cloud (Cloud only)", value: "only-cloud", info: "Fully serverless cloud mode with no local caching" },
+          { label: "hybrid-sync (Local with background sync)", value: "hybrid-sync", info: "Instant local operations with a background sync daemon" },
+        ];
+        const initialIdx = Math.max(0, modeItems.findIndex((i) => i.value === config.mode));
+        const subRes = await selectSimpleMenu({
+          title: "CHOOSE OPERATIONAL MODE",
+          subtitle: "Configure database storage and cloud sync behavior",
+          items: modeItems,
+          initialIndex: initialIdx,
+        });
+
+        if (subRes.action === "select") {
+          updateConfig({ mode: subRes.value });
+        }
+        break;
+      }
       case "enable_prompt": {
         const { enableGlobalPrompt } = await import("./prompt_manager.js");
         const results = await enableGlobalPrompt();

package/mcp-server/config/auth_store.js

@@ -0,0 +1,137 @@
+import fs from "node:fs";
+import path from "node:path";
+import crypto from "node:crypto";
+import os from "node:os";
+import { execSync } from "node:child_process";
+import { MEMORY_DIR, ensureDirSync } from "../memory.js";
+
+const SECRETS_FILE = path.join(MEMORY_DIR, "auth_secrets.enc");
+
+// Stable per-machine identifier. Must NOT rely on volatile values (e.g.
+// os.networkInterfaces() — VPN adapters, hotspot IPs and IPv6 privacy
+// addresses rotate constantly and would silently change the AES key).
+function getMachineId() {
+  try {
+    if (process.platform === "win32") {
+      const out = execSync("reg query HKLM\\SOFTWARE\\Microsoft\\Cryptography /v MachineGuid", {
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"],
+      });
+      const m = out.match(/MachineGuid\s+REG_SZ\s+([0-9a-fA-F-]{36})/i);
+      if (m) return m[1].toLowerCase();
+    } else if (process.platform === "linux") {
+      for (const p of ["/etc/machine-id", "/var/lib/dbus/machine-id"]) {
+        try {
+          const v = fs.readFileSync(p, "utf8").trim();
+          if (v) return v;
+        } catch {}
+      }
+    } else if (process.platform === "darwin") {
+      const out = execSync("ioreg -rd1 -c IOPlatformExpertDevice", {
+        encoding: "utf8",
+        stdio: ["ignore", "pipe", "ignore"],
+      });
+      const m = out.match(/"IOPlatformUUID"\s*=\s*"([^"]+)"/);
+      if (m) return m[1];
+    }
+  } catch {}
+  return null;
+}
+
+// Generate a deterministic hardware + system fingerprint (stable across reboots,
+// network changes and user sessions on the same machine).
+function getSystemFingerprint() {
+  const parts = [
+    getMachineId() || "no-machine-id",
+    os.hostname() || "localhost",
+    os.userInfo()?.username || "default_user",
+    os.platform() || "unknown",
+    os.arch() || "unknown",
+  ];
+  return parts.join("|");
+}
+
+// Derive a 256-bit (32 bytes) key using PBKDF2 with salt derived from fingerprint
+function deriveEncryptionKey() {
+  const fingerprint = getSystemFingerprint();
+  const salt = crypto.createHash("sha256").update(fingerprint).digest();
+  // PBKDF2 with 10,000 iterations to derive a secure 32-byte key
+  return crypto.pbkdf2Sync(fingerprint, salt, 10000, 32, "sha256");
+}
+
+// Encrypt data using AES-256-GCM
+export function encryptData(plainText) {
+  const key = deriveEncryptionKey();
+  const iv = crypto.randomBytes(12); // 96-bit IV is standard for GCM
+  const cipher = crypto.createCipheriv("aes-256-gcm", key, iv);
+
+  let encrypted = cipher.update(plainText, "utf8", "hex");
+  encrypted += cipher.final("hex");
+
+  const authTag = cipher.getAuthTag().toString("hex");
+
+  // Format as: iv_hex:auth_tag_hex:encrypted_hex
+  return `${iv.toString("hex")}:${authTag}:${encrypted}`;
+}
+
+// Decrypt data using AES-256-GCM
+export function decryptData(encryptedStr) {
+  const parts = encryptedStr.split(":");
+  if (parts.length !== 3) {
+    throw new Error("Invalid encrypted format. Expected iv:tag:ciphertext");
+  }
+
+  const [ivHex, authTagHex, encryptedHex] = parts;
+  const key = deriveEncryptionKey();
+  const iv = Buffer.from(ivHex, "hex");
+  const authTag = Buffer.from(authTagHex, "hex");
+
+  const decipher = crypto.createDecipheriv("aes-256-gcm", key, iv);
+  decipher.setAuthTag(authTag);
+
+  let decrypted = decipher.update(encryptedHex, "hex", "utf8");
+  decrypted += decipher.final("utf8");
+
+  return decrypted;
+}
+
+// Save secrets securely
+export function saveSecrets(secrets) {
+  ensureDirSync();
+  const plainText = JSON.stringify(secrets);
+  const encrypted = encryptData(plainText);
+  fs.writeFileSync(SECRETS_FILE, encrypted, "utf-8");
+}
+
+// Load secrets securely
+export function loadSecrets() {
+  if (!fs.existsSync(SECRETS_FILE)) {
+    return null;
+  }
+  try {
+    const encrypted = fs.readFileSync(SECRETS_FILE, "utf-8").trim();
+    if (!encrypted) return null;
+    const decrypted = decryptData(encrypted);
+    return JSON.parse(decrypted);
+  } catch (err) {
+    console.error(
+      "Failed to decrypt or load cloud secrets:",
+      err.message,
+      "— the file was encrypted with a different machine key. Re-run login to recreate it."
+    );
+    return null;
+  }
+}
+
+// Delete secrets from disk
+export function deleteSecrets() {
+  if (fs.existsSync(SECRETS_FILE)) {
+    try {
+      fs.unlinkSync(SECRETS_FILE);
+      return true;
+    } catch (err) {
+      console.error("Failed to delete secrets file:", err.message);
+    }
+  }
+  return false;
+}

package/mcp-server/config/config_manager.js

@@ -14,6 +14,11 @@ export const DEFAULT_CONFIG = {
   gpuAttentionBudget: 2000000, // GPU micro-batch attention budget [1M - 16M] (default 2.0M ~1.5GB VRAM)
   onnxThreads: 0,          // ONNX WASM threads: 0 = auto-detect CPU cores, or 1-16
   executionDevice: "cpu",  // "cpu" | "webgpu"
+  mode: "only-local",     // "only-local" | "only-cloud" | "hybrid-sync"
+  tursoUrl: "",           // Connection endpoint URL for Turso DB
+  failoverUrl: "",        // Failover connection endpoint URL (Fly.io + LiteFS)
+  authorized: false,      // True once the user completed cloud login (token stored encrypted)
+  username: "",           // Account username from the Turso OAuth profile
 };
 
 let cachedConfig = null;

package/mcp-server/db/database.js

@@ -1,41 +1,225 @@
 import { DatabaseSync } from "node:sqlite";
 import { join } from "path";
 import { existsSync, mkdirSync } from "fs";
-import { MEMORY_DIR, ensureDir } from "../memory.js";
+import { MEMORY_DIR } from "../memory.js";
 import { runMigrations } from "./migrations.js";
+import { getConfig } from "../config/config_manager.js";
+import { loadSecrets } from "../config/auth_store.js";
+import { createClient } from "@libsql/client";
 
 let dbInstance = null;
+let dbInitPromise = null;
 
 export const STORAGE_DIR = join(MEMORY_DIR, "storage");
 export const BLOBS_DIR = join(STORAGE_DIR, "blobs");
 export const MODELS_DIR = join(STORAGE_DIR, "models");
 export const DB_PATH = join(STORAGE_DIR, "memory.sqlite");
 
-export function getDatabase(customPath = null) {
-  if (dbInstance && !customPath) {
-    return dbInstance;
+class DatabaseWrapper {
+  constructor(localDb, cloudClient, mode, failoverClient = null) {
+    this.localDb = localDb;
+    this.cloudClient = cloudClient;
+    this.mode = mode;
+    this.failoverClient = failoverClient;
+    this.usingFailover = false;
+    this.consecutiveFailures = 0;
   }
 
-  const dbPath = customPath || DB_PATH;
-  const parentDir = join(dbPath, "..");
-  if (!existsSync(parentDir)) {
-    mkdirSync(parentDir, { recursive: true });
+  async runWithRetry(fn) {
+    let attempts = 0;
+    const maxAttempts = 3;
+    const timeoutMs = 10000;
+
+    while (attempts < maxAttempts) {
+      attempts++;
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+      try {
+        const client = (this.usingFailover && this.failoverClient) ? this.failoverClient : this.cloudClient;
+        const result = await Promise.race([
+          fn(client),
+          new Promise((_, reject) => {
+            controller.signal.addEventListener("abort", () => {
+              reject(new Error("Database operation timed out after 10 seconds"));
+            });
+          })
+        ]);
+        clearTimeout(timeoutId);
+        // Successful operation, reset consecutive failures
+        this.consecutiveFailures = 0;
+        return result;
+      } catch (err) {
+        clearTimeout(timeoutId);
+        if (attempts >= maxAttempts) {
+          this.consecutiveFailures++;
+          if (this.consecutiveFailures >= 3 && this.failoverClient && !this.usingFailover) {
+            console.warn("[WARN] Turso is temporarily unreachable. Switching to LiteFS failover replica...");
+            this.usingFailover = true;
+            // Retry the operation on the failover client
+            return this.runWithRetry(fn);
+          }
+          throw err;
+        }
+        // Small delay before retrying (exponential backoff / fixed delay)
+        await new Promise((resolve) => setTimeout(resolve, 200));
+      }
+    }
+  }
+
+  async exec(sql) {
+    if (this.mode === "only-cloud" && (this.cloudClient || this.failoverClient)) {
+      const trimmed = sql.trim().replace(/;$/, "").toUpperCase();
+      if (trimmed === "BEGIN" || trimmed === "BEGIN IMMEDIATE" || trimmed === "COMMIT" || trimmed === "ROLLBACK") {
+        return;
+      }
+      return await this.runWithRetry(async (client) => {
+        return await client.executeMultiple(sql);
+      });
+    } else {
+      return this.localDb.exec(sql);
+    }
+  }
+
+  prepare(sql) {
+    const self = this;
+    return {
+      async run(...args) {
+        if (self.mode === "only-cloud" && (self.cloudClient || self.failoverClient)) {
+          const res = await self.runWithRetry(async (client) => {
+            return await client.execute({ sql, args });
+          });
+          return {
+            changes: res.rowsAffected || 0,
+            lastInsertRowid: res.lastInsertRowid !== undefined ? Number(res.lastInsertRowid) : undefined,
+          };
+        } else {
+          return self.localDb.prepare(sql).run(...args);
+        }
+      },
+      async get(...args) {
+        if (self.mode === "only-cloud" && (self.cloudClient || self.failoverClient)) {
+          const res = await self.runWithRetry(async (client) => {
+            return await client.execute({ sql, args });
+          });
+          return res.rows[0];
+        } else {
+          return self.localDb.prepare(sql).get(...args);
+        }
+      },
+      async all(...args) {
+        if (self.mode === "only-cloud" && (self.cloudClient || self.failoverClient)) {
+          const res = await self.runWithRetry(async (client) => {
+            return await client.execute({ sql, args });
+          });
+          return res.rows;
+        } else {
+          return self.localDb.prepare(sql).all(...args);
+        }
+      },
+    };
+  }
+
+  close() {
+    if (this.localDb) {
+      try {
+        this.localDb.close();
+      } catch (e) {}
+      this.localDb = null;
+    }
+    if (this.cloudClient) {
+      try {
+        this.cloudClient.close();
+      } catch (e) {}
+      this.cloudClient = null;
+    }
+    if (this.failoverClient) {
+      try {
+        this.failoverClient.close();
+      } catch (e) {}
+      this.failoverClient = null;
+    }
+  }
+}
+
+async function openDatabase(customPath, mode) {
+  const config = getConfig();
+  let localDb = null;
+  if (mode !== "only-cloud") {
+    const dbPath = customPath || DB_PATH;
+    const parentDir = join(dbPath, "..");
+    if (!existsSync(parentDir)) {
+      mkdirSync(parentDir, { recursive: true });
+    }
+    localDb = new DatabaseSync(dbPath);
+    localDb.exec("PRAGMA foreign_keys = ON;");
+    localDb.exec("PRAGMA journal_mode = WAL;");
+  }
+
+  let cloudClient = null;
+  let failoverClient = null;
+  if (mode === "only-cloud" || mode === "hybrid-sync") {
+    const secrets = loadSecrets();
+    const tursoUrl = customPath && customPath.startsWith("libsql:") ? customPath : (secrets?.dbUrl || config.tursoUrl);
+    const failoverUrl = config.failoverUrl || "";
+    const token = secrets?.token;
+
+    if (tursoUrl) {
+      cloudClient = createClient({
+        url: tursoUrl,
+        authToken: token || undefined,
+      });
+      if (failoverUrl) {
+        failoverClient = createClient({
+          url: failoverUrl,
+          authToken: token || undefined,
+        });
+      }
+      // In hybrid-sync mode, ensure remote schema is also fully migrated and up to date
+      if (mode === "hybrid-sync") {
+        const cloudDbWrapper = new DatabaseWrapper(null, cloudClient, "only-cloud", failoverClient);
+        await runMigrations(cloudDbWrapper);
+      }
+    } else if (mode === "only-cloud") {
+      throw new Error("Turso URL is required for only-cloud mode. Please login first.");
+    }
+  }
+
+  const wrappedDb = new DatabaseWrapper(localDb, cloudClient, mode, failoverClient);
+
+  // Initialize/run migrations
+  await runMigrations(wrappedDb);
+
+  if (!customPath) {
+    dbInstance = wrappedDb;
   }
 
-  const db = new DatabaseSync(dbPath);
-  db.exec("PRAGMA foreign_keys = ON;");
-  db.exec("PRAGMA journal_mode = WAL;");
+  return wrappedDb;
+}
 
-  runMigrations(db);
+export async function getDatabase(customPath = null, forceMode = null) {
+  const config = getConfig();
+  const mode = forceMode || config.mode || "only-local";
 
   if (!customPath) {
-    dbInstance = db;
+    if (dbInstance && dbInstance.mode === mode) {
+      return dbInstance;
+    }
+    // Deduplicate concurrent default-DB initialization so migrations never run
+    // on multiple connections at once (avoids "database is locked" crashes).
+    if (!dbInitPromise) {
+      dbInitPromise = openDatabase(null, mode).finally(() => {
+        dbInitPromise = null;
+      });
+    }
+    return await dbInitPromise;
   }
 
-  return db;
+  return openDatabase(customPath, mode);
 }
 
 export function closeDatabase() {
+  dbInitPromise = null;
   if (dbInstance) {
     dbInstance.close();
     dbInstance = null;