@lotargo/memory_plugin 1.3.2 → 1.4.0

package/mcp-server/db/database.js

@@ -1,8 +1,11 @@
 import { DatabaseSync } from "node:sqlite";
 import { join } from "path";
 import { existsSync, mkdirSync } from "fs";
-import { MEMORY_DIR, ensureDir } from "../memory.js";
+import { MEMORY_DIR } from "../memory.js";
 import { runMigrations } from "./migrations.js";
+import { getConfig } from "../config/config_manager.js";
+import { loadSecrets } from "../config/auth_store.js";
+import { createClient } from "@libsql/client";
 
 let dbInstance = null;
 
@@ -11,28 +14,192 @@ export const BLOBS_DIR = join(STORAGE_DIR, "blobs");
 export const MODELS_DIR = join(STORAGE_DIR, "models");
 export const DB_PATH = join(STORAGE_DIR, "memory.sqlite");
 
-export function getDatabase(customPath = null) {
-  if (dbInstance && !customPath) {
+class DatabaseWrapper {
+  constructor(localDb, cloudClient, mode, failoverClient = null) {
+    this.localDb = localDb;
+    this.cloudClient = cloudClient;
+    this.mode = mode;
+    this.failoverClient = failoverClient;
+    this.usingFailover = false;
+    this.consecutiveFailures = 0;
+  }
+
+  async runWithRetry(fn) {
+    let attempts = 0;
+    const maxAttempts = 3;
+    const timeoutMs = 10000;
+
+    while (attempts < maxAttempts) {
+      attempts++;
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+
+      try {
+        const client = (this.usingFailover && this.failoverClient) ? this.failoverClient : this.cloudClient;
+        const result = await Promise.race([
+          fn(client),
+          new Promise((_, reject) => {
+            controller.signal.addEventListener("abort", () => {
+              reject(new Error("Database operation timed out after 10 seconds"));
+            });
+          })
+        ]);
+        clearTimeout(timeoutId);
+        // Successful operation, reset consecutive failures
+        this.consecutiveFailures = 0;
+        return result;
+      } catch (err) {
+        clearTimeout(timeoutId);
+        if (attempts >= maxAttempts) {
+          this.consecutiveFailures++;
+          if (this.consecutiveFailures >= 3 && this.failoverClient && !this.usingFailover) {
+            console.warn("[WARN] Turso is temporarily unreachable. Switching to LiteFS failover replica...");
+            this.usingFailover = true;
+            // Retry the operation on the failover client
+            return this.runWithRetry(fn);
+          }
+          throw err;
+        }
+        // Small delay before retrying (exponential backoff / fixed delay)
+        await new Promise((resolve) => setTimeout(resolve, 200));
+      }
+    }
+  }
+
+  async exec(sql) {
+    if (this.mode === "only-cloud" && (this.cloudClient || this.failoverClient)) {
+      const trimmed = sql.trim().replace(/;$/, "").toUpperCase();
+      if (trimmed === "BEGIN" || trimmed === "BEGIN IMMEDIATE" || trimmed === "COMMIT" || trimmed === "ROLLBACK") {
+        return;
+      }
+      return await this.runWithRetry(async (client) => {
+        return await client.executeMultiple(sql);
+      });
+    } else {
+      return this.localDb.exec(sql);
+    }
+  }
+
+  prepare(sql) {
+    const self = this;
+    return {
+      async run(...args) {
+        if (self.mode === "only-cloud" && (self.cloudClient || self.failoverClient)) {
+          const res = await self.runWithRetry(async (client) => {
+            return await client.execute({ sql, args });
+          });
+          return {
+            changes: res.rowsAffected || 0,
+            lastInsertRowid: res.lastInsertRowid !== undefined ? Number(res.lastInsertRowid) : undefined,
+          };
+        } else {
+          return self.localDb.prepare(sql).run(...args);
+        }
+      },
+      async get(...args) {
+        if (self.mode === "only-cloud" && (self.cloudClient || self.failoverClient)) {
+          const res = await self.runWithRetry(async (client) => {
+            return await client.execute({ sql, args });
+          });
+          return res.rows[0];
+        } else {
+          return self.localDb.prepare(sql).get(...args);
+        }
+      },
+      async all(...args) {
+        if (self.mode === "only-cloud" && (self.cloudClient || self.failoverClient)) {
+          const res = await self.runWithRetry(async (client) => {
+            return await client.execute({ sql, args });
+          });
+          return res.rows;
+        } else {
+          return self.localDb.prepare(sql).all(...args);
+        }
+      },
+    };
+  }
+
+  close() {
+    if (this.localDb) {
+      try {
+        this.localDb.close();
+      } catch (e) {}
+      this.localDb = null;
+    }
+    if (this.cloudClient) {
+      try {
+        this.cloudClient.close();
+      } catch (e) {}
+      this.cloudClient = null;
+    }
+    if (this.failoverClient) {
+      try {
+        this.failoverClient.close();
+      } catch (e) {}
+      this.failoverClient = null;
+    }
+  }
+}
+
+export async function getDatabase(customPath = null, forceMode = null) {
+  const config = getConfig();
+  const mode = forceMode || config.mode || "only-local";
+
+  if (dbInstance && !customPath && dbInstance.mode === mode) {
     return dbInstance;
   }
 
-  const dbPath = customPath || DB_PATH;
-  const parentDir = join(dbPath, "..");
-  if (!existsSync(parentDir)) {
-    mkdirSync(parentDir, { recursive: true });
+  let localDb = null;
+  if (mode !== "only-cloud") {
+    const dbPath = customPath || DB_PATH;
+    const parentDir = join(dbPath, "..");
+    if (!existsSync(parentDir)) {
+      mkdirSync(parentDir, { recursive: true });
+    }
+    localDb = new DatabaseSync(dbPath);
+    localDb.exec("PRAGMA foreign_keys = ON;");
+    localDb.exec("PRAGMA journal_mode = WAL;");
+  }
+
+  let cloudClient = null;
+  let failoverClient = null;
+  if (mode === "only-cloud" || mode === "hybrid-sync") {
+    const secrets = loadSecrets();
+    const tursoUrl = customPath && customPath.startsWith("libsql:") ? customPath : (secrets?.dbUrl || config.tursoUrl);
+    const failoverUrl = config.failoverUrl || "";
+    const token = secrets?.token;
+
+    if (tursoUrl) {
+      cloudClient = createClient({
+        url: tursoUrl,
+        authToken: token || undefined,
+      });
+      if (failoverUrl) {
+        failoverClient = createClient({
+          url: failoverUrl,
+          authToken: token || undefined,
+        });
+      }
+      // In hybrid-sync mode, ensure remote schema is also fully migrated and up to date
+      if (mode === "hybrid-sync") {
+        const cloudDbWrapper = new DatabaseWrapper(null, cloudClient, "only-cloud", failoverClient);
+        await runMigrations(cloudDbWrapper);
+      }
+    } else if (mode === "only-cloud") {
+      throw new Error("Turso URL is required for only-cloud mode. Please login first.");
+    }
   }
 
-  const db = new DatabaseSync(dbPath);
-  db.exec("PRAGMA foreign_keys = ON;");
-  db.exec("PRAGMA journal_mode = WAL;");
+  const wrappedDb = new DatabaseWrapper(localDb, cloudClient, mode, failoverClient);
 
-  runMigrations(db);
+  // Initialize/run migrations
+  await runMigrations(wrappedDb);
 
   if (!customPath) {
-    dbInstance = db;
+    dbInstance = wrappedDb;
   }
 
-  return db;
+  return wrappedDb;
 }
 
 export function closeDatabase() {

package/mcp-server/db/migrations.js

@@ -2,9 +2,9 @@ const MIGRATIONS = [
   {
     version: 1,
     name: "001_initial_rag_schema",
-    up: (db) => {
+    up: async (db) => {
       // 1. Documents Table
-      db.exec(`
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS documents (
             id TEXT PRIMARY KEY,
             path TEXT UNIQUE NOT NULL,
@@ -19,7 +19,7 @@ const MIGRATIONS = [
       `);
 
       // 2. Sections Table
-      db.exec(`
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS sections (
             id TEXT PRIMARY KEY,
             doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
@@ -31,7 +31,7 @@ const MIGRATIONS = [
       `);
 
       // 3. Micro-Chunks Table
-      db.exec(`
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS micro_chunks (
             id TEXT PRIMARY KEY,
             section_id TEXT NOT NULL REFERENCES sections(id) ON DELETE CASCADE,
@@ -43,7 +43,7 @@ const MIGRATIONS = [
       `);
 
       // 4. Full-Text Search (BM25 Index via SQLite FTS5)
-      db.exec(`
+      await db.exec(`
         CREATE VIRTUAL TABLE IF NOT EXISTS micro_chunks_fts USING fts5(
             id UNINDEXED,
             content,
@@ -52,7 +52,7 @@ const MIGRATIONS = [
       `);
 
       // 5. GraphRAG Lite Edges Table
-      db.exec(`
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS graph_edges (
             source_id TEXT NOT NULL,
             target_id TEXT NOT NULL,
@@ -65,15 +65,15 @@ const MIGRATIONS = [
   {
     version: 2,
     name: "002_agent_knowledge_graph",
-    up: (db) => {
+    up: async (db) => {
       try {
-        db.exec(`ALTER TABLE graph_edges ADD COLUMN metadata_json TEXT;`);
+        await db.exec(`ALTER TABLE graph_edges ADD COLUMN metadata_json TEXT;`);
       } catch (e) {}
       try {
-        db.exec(`ALTER TABLE graph_edges ADD COLUMN created_at INTEGER;`);
+        await db.exec(`ALTER TABLE graph_edges ADD COLUMN created_at INTEGER;`);
       } catch (e) {}
 
-      db.exec(`
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS knowledge_links (
             id TEXT PRIMARY KEY,
             fact_key TEXT NOT NULL,
@@ -92,8 +92,8 @@ const MIGRATIONS = [
   {
     version: 3,
     name: "003_medium_chunks_hierarchy",
-    up: (db) => {
-      db.exec(`
+    up: async (db) => {
+      await db.exec(`
         CREATE TABLE IF NOT EXISTS medium_chunks (
             id TEXT PRIMARY KEY,
             section_id TEXT NOT NULL REFERENCES sections(id) ON DELETE CASCADE,
@@ -106,43 +106,72 @@ const MIGRATIONS = [
       `);
 
       try {
-        db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`);
+        await db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`);
       } catch (e) {}
     },
   },
 ];
 
-export function runMigrations(db) {
-  const versionRow = db.prepare("PRAGMA user_version;").get();
-  const currentVersion = versionRow ? versionRow.user_version : 0;
+export async function runMigrations(db) {
+  let currentVersion = 0;
+  try {
+    const row = await db.prepare("SELECT MAX(version) as v FROM schema_migrations;").get();
+    currentVersion = row ? row.v || 0 : 0;
+  } catch (e) {
+    try {
+      const versionRow = await db.prepare("PRAGMA user_version;").get();
+      currentVersion = versionRow ? (versionRow.user_version || 0) : 0;
+    } catch (e2) {}
+  }
+
+  // Ensure schema_migrations table exists for future
+  try {
+    await db.exec(`CREATE TABLE IF NOT EXISTS schema_migrations (version INTEGER PRIMARY KEY);`);
+  } catch (e) {}
+
+  // Create notebooks table if not exists
+  try {
+    await db.exec(`
+      CREATE TABLE IF NOT EXISTS notebooks (
+          key TEXT PRIMARY KEY,
+          content TEXT NOT NULL,
+          updated_at INTEGER NOT NULL
+      );
+    `);
+  } catch (e) {}
 
   for (const migration of MIGRATIONS) {
     if (migration.version > currentVersion) {
-      db.exec("BEGIN IMMEDIATE;");
+      await db.exec("BEGIN;");
       try {
-        migration.up(db);
-        db.exec("COMMIT;");
-        db.exec(`PRAGMA user_version = ${migration.version};`);
+        await migration.up(db);
+        await db.prepare("INSERT INTO schema_migrations (version) VALUES (?);").run(migration.version);
+        await db.exec("COMMIT;");
+        try {
+          await db.exec(`PRAGMA user_version = ${migration.version};`);
+        } catch (e) {}
       } catch (err) {
-        db.exec("ROLLBACK;");
+        try {
+          await db.exec("ROLLBACK;");
+        } catch (e) {}
         throw new Error(`Migration ${migration.name} failed: ${err.message}`);
       }
     }
   }
 
   // Defensive table & column check for medium_chunks hierarchy
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS medium_chunks (
-        id TEXT PRIMARY KEY,
-        section_id TEXT NOT NULL REFERENCES sections(id) ON DELETE CASCADE,
-        doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
-        content TEXT NOT NULL,
-        block_type TEXT NOT NULL,
-        token_count INTEGER NOT NULL,
-        created_at INTEGER
-    );
-  `);
   try {
-    db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`);
+    await db.exec(`
+      CREATE TABLE IF NOT EXISTS medium_chunks (
+          id TEXT PRIMARY KEY,
+          section_id TEXT NOT NULL REFERENCES sections(id) ON DELETE CASCADE,
+          doc_id TEXT NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+          content TEXT NOT NULL,
+          block_type TEXT NOT NULL,
+          token_count INTEGER NOT NULL,
+          created_at INTEGER
+      );
+    `);
+    await db.exec(`ALTER TABLE micro_chunks ADD COLUMN medium_id TEXT REFERENCES medium_chunks(id) ON DELETE CASCADE;`);
   } catch (e) {}
 }

package/mcp-server/db/sync_queue.js

@@ -0,0 +1,211 @@
+let isSyncing = false;
+
+async function processSyncTask(db, task) {
+  if (task.action === "write_memory") {
+    await db.cloudClient.execute({
+      sql: `
+        INSERT INTO notebooks (key, content, updated_at)
+        VALUES (?, ?, ?)
+        ON CONFLICT(key) DO UPDATE SET content = excluded.content, updated_at = excluded.updated_at;
+      `,
+      args: [task.key_or_id, task.payload, task.created_at],
+    });
+    return;
+  }
+
+  if (task.action === "delete_document") {
+    const docId = task.key_or_id;
+    const docRow = await db.cloudClient.execute({
+      sql: "SELECT id FROM documents WHERE id = ? OR path = ?;",
+      args: [docId, docId],
+    });
+    if (docRow.rows.length > 0) {
+      const realDocId = docRow.rows[0].id;
+      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks_fts WHERE id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM medium_chunks WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM sections WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?;", args: [realDocId, realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM knowledge_links WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM documents WHERE id = ?;", args: [realDocId] });
+    }
+    return;
+  }
+
+  if (task.action === "ingest_document") {
+    const data = JSON.parse(task.payload);
+    const doc = data.document;
+
+    // 1. Delete existing doc from cloud if any
+    const existingDocRow = await db.cloudClient.execute({
+      sql: "SELECT id FROM documents WHERE id = ? OR path = ?;",
+      args: [doc.id, doc.path],
+    });
+    if (existingDocRow.rows.length > 0) {
+      const realDocId = existingDocRow.rows[0].id;
+      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks_fts WHERE id IN (SELECT id FROM micro_chunks WHERE doc_id = ?);", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM micro_chunks WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM medium_chunks WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM sections WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM graph_edges WHERE source_id = ? OR target_id = ?;", args: [realDocId, realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM knowledge_links WHERE doc_id = ?;", args: [realDocId] });
+      await db.cloudClient.execute({ sql: "DELETE FROM documents WHERE id = ?;", args: [realDocId] });
+    }
+
+    // 2. Insert document
+    await db.cloudClient.execute({
+      sql: `
+        INSERT INTO documents (id, path, blob_hash, title, checksum, toc_json, metadata_json, created_at, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?);
+      `,
+      args: [
+        doc.id,
+        doc.path,
+        doc.blob_hash,
+        doc.title,
+        doc.checksum,
+        doc.toc_json ? (typeof doc.toc_json === "string" ? doc.toc_json : JSON.stringify(doc.toc_json)) : null,
+        doc.metadata_json ? (typeof doc.metadata_json === "string" ? doc.metadata_json : JSON.stringify(doc.metadata_json)) : null,
+        doc.created_at,
+        doc.updated_at,
+      ],
+    });
+
+    // 3. Insert sections
+    if (Array.isArray(data.sections)) {
+      for (const s of data.sections) {
+        await db.cloudClient.execute({
+          sql: "INSERT INTO sections (id, doc_id, heading, breadcrumbs, content, token_count) VALUES (?, ?, ?, ?, ?, ?);",
+          args: [s.id, doc.id, s.heading, s.breadcrumbs, s.content, s.token_count],
+        });
+      }
+    }
+
+    // 4. Insert medium_chunks
+    if (Array.isArray(data.medium_chunks)) {
+      for (const m of data.medium_chunks) {
+        await db.cloudClient.execute({
+          sql: "INSERT INTO medium_chunks (id, section_id, doc_id, content, block_type, token_count, created_at) VALUES (?, ?, ?, ?, ?, ?, ?);",
+          args: [m.id, m.section_id, doc.id, m.content, m.block_type, m.token_count, m.created_at || Date.now()],
+        });
+      }
+    }
+
+    // 5. Insert micro_chunks & FTS
+    if (Array.isArray(data.micro_chunks)) {
+      for (const mc of data.micro_chunks) {
+        let vecBuf = Buffer.alloc(0);
+        if (mc.vector) {
+          if (Buffer.isBuffer(mc.vector)) {
+            vecBuf = mc.vector;
+          } else if (typeof mc.vector === "string") {
+            vecBuf = Buffer.from(mc.vector, "base64");
+          } else if (mc.vector.type === "Buffer" && Array.isArray(mc.vector.data)) {
+            vecBuf = Buffer.from(mc.vector.data);
+          } else if (Array.isArray(mc.vector)) {
+            vecBuf = Buffer.from(mc.vector);
+          }
+        }
+        await db.cloudClient.execute({
+          sql: "INSERT INTO micro_chunks (id, section_id, doc_id, content, vector, token_count, medium_id) VALUES (?, ?, ?, ?, ?, ?, ?);",
+          args: [mc.id, mc.section_id, doc.id, mc.content, vecBuf, mc.token_count, mc.medium_id || null],
+        });
+
+        // Insert into remote FTS
+        try {
+          await db.cloudClient.execute({
+            sql: "INSERT INTO micro_chunks_fts (id, content, breadcrumbs) VALUES (?, ?, ?);",
+            args: [mc.id, mc.content, mc.breadcrumbs || ""],
+          });
+        } catch (ftsErr) {
+          console.warn("FTS insertion failed on cloud:", ftsErr.message);
+        }
+      }
+    }
+
+    // 6. Insert graph_edges
+    if (Array.isArray(data.graph_edges)) {
+      for (const e of data.graph_edges) {
+        await db.cloudClient.execute({
+          sql: "INSERT OR IGNORE INTO graph_edges (source_id, target_id, relation_type, metadata_json, created_at) VALUES (?, ?, ?, ?, ?);",
+          args: [e.source_id, e.target_id, e.relation_type, e.metadata_json ? (typeof e.metadata_json === "string" ? e.metadata_json : JSON.stringify(e.metadata_json)) : null, e.created_at || Date.now()],
+        });
+      }
+    }
+  }
+}
+
+export async function enqueueSyncTask(action, keyOrId, payload = null) {
+  const { getDatabase } = await import("./database.js");
+  const db = await getDatabase();
+  if (db.mode === "only-cloud") return; // No need to queue in only-cloud mode
+
+  // Ensure queue table exists
+  await db.exec(`
+    CREATE TABLE IF NOT EXISTS sync_queue (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        action TEXT NOT NULL,
+        key_or_id TEXT NOT NULL,
+        payload TEXT,
+        created_at INTEGER NOT NULL
+    );
+  `);
+
+  await db.prepare(`
+    INSERT INTO sync_queue (action, key_or_id, payload, created_at)
+    VALUES (?, ?, ?, ?);
+  `).run(action, keyOrId, payload ? (typeof payload === "string" ? payload : JSON.stringify(payload)) : null, Date.now());
+
+  // Trigger background sync worker asynchronously
+  triggerBackgroundSync().catch((err) => {
+    console.error("Background sync trigger error:", err.message);
+  });
+}
+
+export async function triggerBackgroundSync() {
+  if (isSyncing) return;
+  isSyncing = true;
+
+  try {
+    const { getDatabase } = await import("./database.js");
+    const db = await getDatabase();
+    if (db.mode !== "hybrid-sync" || !db.cloudClient) {
+      isSyncing = false;
+      return;
+    }
+
+    // Ensure table exists
+    await db.exec(`
+      CREATE TABLE IF NOT EXISTS sync_queue (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          action TEXT NOT NULL,
+          key_or_id TEXT NOT NULL,
+          payload TEXT,
+          created_at INTEGER NOT NULL
+      );
+    `);
+
+    // Fetch tasks ordered by id
+    const tasks = await db.prepare("SELECT * FROM sync_queue ORDER BY id ASC LIMIT 50;").all();
+    if (tasks.length === 0) {
+      isSyncing = false;
+      return;
+    }
+
+    for (const task of tasks) {
+      try {
+        await processSyncTask(db, task);
+        // On success, delete from queue
+        await db.prepare("DELETE FROM sync_queue WHERE id = ?;").run(task.id);
+      } catch (err) {
+        console.error(`Failed to process sync task ${task.id} (${task.action}):`, err.message, err.stack);
+        // Stop processing this batch to preserve order on error, retry next time
+        break;
+      }
+    }
+  } catch (err) {
+    console.error("Error during background sync execution:", err.message);
+  } finally {
+    isSyncing = false;
+  }
+}

package/mcp-server/graph/graph_extractor.js

@@ -1,20 +1,43 @@
+const ignoredKeywords = new Set([
+  "const", "let", "var", "function", "class", "import", "export", "from", "return", "if", "for", "while", "switch", "case", "default",
+  "def", "self", "lambda", "pass", "yield", "async", "await", "with", "except", "try", "catch", "finally",
+  "func", "type", "struct", "interface", "chan", "map", "go", "defer", "package", "range",
+  "fn", "enum", "trait", "impl", "pub", "mut", "ref", "self", "Self", "match", "use", "mod", "crate",
+  "namespace", "template", "typename", "public", "private", "protected", "virtual", "override", "using", "inline", "static", "constexpr", "extern", "explicit", "friend", "operator", "throw",
+  "record", "synchronized", "final", "void", "throws", "new", "this", "super", "fun", "val", "null", "true", "false",
+  "internal", "readonly", "base", "get", "set", "echo", "exit", "die", "require", "include",
+  "module", "end", "extend", "attr_accessor", "attr_reader", "attr_writer", "nil", "puts", "raise"
+]);
+
 export function extractSymbolsFromContent(content) {
+  if (!content) return [];
   const symbols = new Set();
 
-  const jsTsRegex = /(?:function|class|interface|type|enum|const|let|var)\s+([a-zA-Z0-9_$]+)/g;
-  let match;
-  while ((match = jsTsRegex.exec(content)) !== null) {
-    const symbol = match[1];
-    if (symbol.length > 2 && !["const", "let", "var", "function", "class", "import", "export", "from", "return", "if", "for", "while"].includes(symbol)) {
-      symbols.add(symbol);
-    }
-  }
+  const patterns = [
+    // JS/TS
+    /(?:function|class|interface|type|enum|const|let|var)\s+([a-zA-Z0-9_$]+)/g,
+    // Python / PHP / Ruby
+    /(?:def|class|function|module|trait)\s+([a-zA-Z0-9_!?=]+)/g,
+    // Go / Rust / C++ / Java / Kotlin / C# / PHP (Types)
+    /\b(?:class|struct|interface|record|enum|trait|type|namespace)\s+([a-zA-Z0-9_]+)/g,
+    // Go (Functions)
+    /\bfunc\s+(?:\([^)]+\)\s+)?([a-zA-Z0-9_]+)\s*\(/g,
+    // Rust (Functions)
+    /\bfn\s+([a-zA-Z0-9_]+)/g,
+    // Kotlin (Functions)
+    /\bfun\s+([a-zA-Z0-9_]+)/g,
+    // Java / C# / C++ (Methods/Functions)
+    /\b(?:public|protected|private|static|synchronized|final|async|virtual|override|readonly)*\s*[\w<>\[\]]+\s+([a-zA-Z0-9_]+)\s*\([^)]*\)\s*(?:const|override|noexcept|throws\s+[\w,\s]+|\s)*\s*[{;]/g
+  ];
 
-  const pyRegex = /(?:def|class)\s+([a-zA-Z0-9_]+)/g;
-  while ((match = pyRegex.exec(content)) !== null) {
-    const symbol = match[1];
-    if (symbol.length > 2 && !["def", "class", "self", "return", "import", "from"].includes(symbol)) {
-      symbols.add(symbol);
+  for (const pattern of patterns) {
+    let match;
+    pattern.lastIndex = 0;
+    while ((match = pattern.exec(content)) !== null) {
+      const symbol = match[1];
+      if (symbol && symbol.length > 2 && !ignoredKeywords.has(symbol)) {
+        symbols.add(symbol);
+      }
     }
   }
 
@@ -52,21 +75,21 @@ export function buildGraphEdges(docId, hierarchy) {
   return edges;
 }
 
-export function saveGraphEdges(db, edges) {
+export async function saveGraphEdges(db, edges) {
   const stmt = db.prepare(`
     INSERT OR IGNORE INTO graph_edges (source_id, target_id, relation_type)
     VALUES (?, ?, ?);
   `);
   for (const edge of edges) {
-    stmt.run(edge.source_id, edge.target_id, edge.relation_type);
+    await stmt.run(edge.source_id, edge.target_id, edge.relation_type);
   }
 }
 
-export function getRelatedSymbols(db, sectionId) {
+export async function getRelatedSymbols(db, sectionId) {
   const stmt = db.prepare(`
     SELECT target_id, relation_type FROM graph_edges
     WHERE source_id = ? AND relation_type = 'DEFINES_SYMBOL';
   `);
-  const rows = stmt.all(sectionId);
+  const rows = await stmt.all(sectionId);
   return rows.map((r) => r.target_id.replace("symbol:", ""));
 }