@melandlabs/memory-store 0.2.0 → 0.2.2

package/dist/server/cli-http.js

@@ -1,12 +1,22 @@
 #!/usr/bin/env node
-import { serve } from '@hono/node-server';
-import { Hono } from 'hono';
 import { buildMemoryRecordEmbeddingDocument } from '@melandlabs/ai/memory';
 import { ChromaVectorStore } from '@melandlabs/rag';
 import { mkdirSync } from 'fs';
 import { homedir } from 'os';
-import { join, dirname } from 'path';
+import { dirname, join } from 'path';
 import { SQLiteRawMessageManager } from '@melandlabs/sqlite';
+import { serve } from '@hono/node-server';
+import { Hono } from 'hono';
+
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
 
 // src/contracts.ts
 function normalizeTimestampToMs(value) {
@@ -30,14 +40,10 @@ function rawMessageToMemoryRecord(message) {
     embeddingContentHash: message.embeddingContentHash
   };
 }
-
-// src/storage/chroma-memory-index.ts
-var DEFAULT_RAW_MESSAGES_COLLECTION = "opencontext_raw_messages";
-var RAW_VECTOR_BACKEND_ENV_KEYS = [
-  "RAW_MESSAGE_VECTOR_STORE_BACKEND",
-  "MEMORY_VECTOR_STORE_BACKEND",
-  "VECTOR_STORE_BACKEND"
-];
+var init_contracts = __esm({
+  "src/contracts.ts"() {
+  }
+});
 function getBackend(keys) {
   for (const key of keys) {
     const value = process.env[key]?.trim().toLowerCase();
@@ -140,21 +146,51 @@ async function searchRawMessagesWithChroma(input) {
     metadata: result.metadata ?? {}
   })).slice(0, input.limit);
 }
+var DEFAULT_RAW_MESSAGES_COLLECTION, RAW_VECTOR_BACKEND_ENV_KEYS;
+var init_chroma_memory_index = __esm({
+  "src/storage/chroma-memory-index.ts"() {
+    init_contracts();
+    DEFAULT_RAW_MESSAGES_COLLECTION = "opencontext_raw_messages";
+    RAW_VECTOR_BACKEND_ENV_KEYS = [
+      "RAW_MESSAGE_VECTOR_STORE_BACKEND",
+      "MEMORY_VECTOR_STORE_BACKEND",
+      "VECTOR_STORE_BACKEND"
+    ];
+  }
+});
 
 // src/storage/postgres-raw-message-factory.ts
-var factory = null;
 async function resolvePostgresFactory(env) {
-  return null;
+  if (!factory) return null;
+  return await factory(env);
 }
 function hasPostgresFactory() {
   return factory !== null;
 }
-var manager = null;
-var DEFAULT_SQLITE_PATH = join(homedir(), ".opencontext", "memory", "store.db");
+var factory;
+var init_postgres_raw_message_factory = __esm({
+  "src/storage/postgres-raw-message-factory.ts"() {
+    factory = null;
+  }
+});
+
+// src/storage/sqlite-raw-message-store.ts
+var sqlite_raw_message_store_exports = {};
+__export(sqlite_raw_message_store_exports, {
+  __resetSQLiteRawMessageManagerForTests: () => __resetSQLiteRawMessageManagerForTests,
+  closeSQLiteRawMessageManager: () => closeSQLiteRawMessageManager,
+  getSQLiteRawMessageManager: () => getSQLiteRawMessageManager,
+  isSQLiteRawMessageStorageAvailable: () => isSQLiteRawMessageStorageAvailable,
+  lexicalSearchRawMessages: () => lexicalSearchRawMessages,
+  resolveSQLiteRawMessageDbPath: () => resolveSQLiteRawMessageDbPath
+});
 function resolveSQLiteRawMessageDbPath() {
   const fromEnv = process.env.MEMORY_STORE_DB_PATH?.trim();
   return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_SQLITE_PATH;
 }
+function isSQLiteRawMessageStorageAvailable(_env) {
+  return true;
+}
 async function getSQLiteRawMessageManager(_env) {
   if (!manager) {
     const dbPath = resolveSQLiteRawMessageDbPath();
@@ -174,8 +210,35 @@ async function closeSQLiteRawMessageManager() {
   await manager.close();
   manager = null;
 }
+function __resetSQLiteRawMessageManagerForTests() {
+  manager = null;
+}
+async function lexicalSearchRawMessages(input) {
+  const mgr = await getSQLiteRawMessageManager();
+  if (typeof mgr.lexicalSearchMessages !== "function") {
+    return [];
+  }
+  return mgr.lexicalSearchMessages(input);
+}
+var manager, DEFAULT_SQLITE_PATH;
+var init_sqlite_raw_message_store = __esm({
+  "src/storage/sqlite-raw-message-store.ts"() {
+    init_chroma_memory_index();
+    manager = null;
+    DEFAULT_SQLITE_PATH = join(homedir(), ".opencontext", "memory", "store.db");
+  }
+});
 
 // src/storage/raw-message-store.ts
+var raw_message_store_exports = {};
+__export(raw_message_store_exports, {
+  closeRawMessageStore: () => closeRawMessageStore,
+  configureRawMessageStore: () => configureRawMessageStore,
+  createRawMessageStore: () => createRawMessageStore,
+  getRawMessageManager: () => getRawMessageManager,
+  getRawMessageStorageBackend: () => getRawMessageStorageBackend,
+  isRawMessageStorageAvailable: () => isRawMessageStorageAvailable
+});
 function resolveBackend(env) {
   const explicit = process.env.OPENCONTEXT_MEMORY_STORE_BACKEND?.trim().toLowerCase();
   if (explicit === "postgres") return "postgres";
@@ -185,7 +248,7 @@ function resolveEnv(_env) {
   return {};
 }
 function createRawMessageStore(options = {}) {
-  resolveEnv(options.env);
+  const env = resolveEnv(options.env);
   return {
     getBackend() {
       return resolveBackend();
@@ -195,7 +258,7 @@ function createRawMessageStore(options = {}) {
     },
     async getManager() {
       if (resolveBackend() === "postgres") {
-        const pg = await resolvePostgresFactory();
+        const pg = await resolvePostgresFactory(env);
         if (!pg) {
           throw new Error(
             "OPENCONTEXT_MEMORY_STORE_BACKEND=postgres but no Postgres factory is registered. Call registerPostgresFactory() at startup, or unset the env var to use the default SQLite backend."
@@ -210,14 +273,47 @@ function createRawMessageStore(options = {}) {
     }
   };
 }
-var moduleStore = null;
-var moduleConfig = null;
+function configureRawMessageStore(config) {
+  moduleConfig = config;
+  moduleStore = createRawMessageStore({ env: config.env });
+  return moduleStore;
+}
+function getRawMessageStorageBackend() {
+  if (!moduleStore) {
+    moduleStore = createRawMessageStore({ env: moduleConfig?.env });
+  }
+  return moduleStore.getBackend();
+}
 function isRawMessageStorageAvailable() {
   if (!moduleStore) {
     moduleStore = createRawMessageStore({ env: moduleConfig?.env });
   }
   return moduleStore.isAvailable();
 }
+async function getRawMessageManager() {
+  if (!moduleStore) {
+    moduleStore = createRawMessageStore({ env: moduleConfig?.env });
+  }
+  return moduleStore.getManager();
+}
+async function closeRawMessageStore() {
+  if (!moduleStore) return;
+  await moduleStore.close();
+  moduleStore = null;
+}
+var moduleStore, moduleConfig;
+var init_raw_message_store = __esm({
+  "src/storage/raw-message-store.ts"() {
+    init_postgres_raw_message_factory();
+    init_sqlite_raw_message_store();
+    moduleStore = null;
+    moduleConfig = null;
+  }
+});
+
+// src/search/unified-search.ts
+init_chroma_memory_index();
+init_raw_message_store();
 
 // src/search/utilities.ts
 var DEFAULT_LIMIT = 10;
@@ -361,7 +457,7 @@ function createUnifiedSearch(deps = {}) {
           warnings.push({
             source: "memory",
             code: "memory_search_failed",
-            message: error.message ?? "memory_search_failed"
+            message: `Memory search failed: ${error.message ?? "Unknown error"}. Using keyword search fallback.`
           });
         }
       } else {
@@ -401,13 +497,7 @@ function createUnifiedSearch(deps = {}) {
           message: error.message ?? "insights_search_failed"
         });
       }
-    } else if (sources.includes("insights")) {
-      warnings.push({
-        source: "insights",
-        code: "insights_search_not_configured",
-        message: "Insights search is not configured for this host."
-      });
-    }
+    } else if (sources.includes("insights")) ;
     const knowledgeHits = [];
     if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
       try {
@@ -432,13 +522,7 @@ function createUnifiedSearch(deps = {}) {
           message: error.message ?? "knowledge_search_failed"
         });
       }
-    } else if (sources.includes("knowledge")) {
-      warnings.push({
-        source: "knowledge",
-        code: "knowledge_search_not_configured",
-        message: "Knowledge search is not configured for this host."
-      });
-    }
+    } else if (sources.includes("knowledge")) ;
     const merged = mergeAcrossSources({
       memorySubs,
       insightHits,
@@ -456,7 +540,33 @@ function createUnifiedSearch(deps = {}) {
   }
   async function runMemorySource(input, limit, threshold, warnings) {
     if (typeof deps.embedQuery !== "function") {
-      throw new Error("embedQuery is not configured");
+      warnings.push({
+        source: "memory",
+        code: "memory_lexical_search_fallback",
+        message: "Semantic search not configured, using keyword search as fallback"
+      });
+      const { lexicalSearchRawMessages: lexicalSearchRawMessages2 } = await Promise.resolve().then(() => (init_sqlite_raw_message_store(), sqlite_raw_message_store_exports));
+      const keywords = deriveLexicalKeywords(input.query);
+      let lexical2 = [];
+      if (keywords.length > 0) {
+        try {
+          const filters2 = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
+          const results = await Promise.all(
+            filters2.map(
+              (botId) => lexicalSearchRawMessages2({
+                userId: input.userId,
+                keywords,
+                limit: Math.ceil(limit / filters2.length),
+                botId
+              })
+            )
+          );
+          lexical2 = results.flat().filter(Boolean).map(toMemoryResult);
+        } catch (error) {
+          logger.warn?.("[memory-store] Default lexical search failed:", error);
+        }
+      }
+      return { semantic: [], lexical: lexical2 };
     }
     const queryEmbedding = await deps.embedQuery({
       userId: input.userId,
@@ -503,6 +613,28 @@ function createUnifiedSearch(deps = {}) {
         )
       )).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
     }
+    if (semantic.length === 0) {
+      try {
+        const { getRawMessageManager: getRawMessageManager2 } = await Promise.resolve().then(() => (init_raw_message_store(), raw_message_store_exports));
+        const manager2 = await getRawMessageManager2();
+        if (typeof manager2.searchMessagesSemantically === "function") {
+          const results = await manager2.searchMessagesSemantically({
+            userId: input.userId,
+            queryEmbedding,
+            limit
+          });
+          semantic = results.map((r) => ({
+            type: "memory",
+            id: r.id,
+            content: r.content,
+            similarity: r.similarity,
+            metadata: r.metadata ?? {}
+          }));
+        }
+      } catch (error) {
+        logger.warn?.("[memory-store] SQLite semantic search failed:", error);
+      }
+    }
     let lexical = [];
     if (typeof deps.searchRawMessagesLexical === "function") {
       const keywords = deriveLexicalKeywords(input.query);
@@ -585,6 +717,8 @@ function mergeAcrossSources(input) {
 }
 
 // src/http.ts
+init_chroma_memory_index();
+init_raw_message_store();
 async function embedMissingMessages(messages, deps) {
   if (typeof deps.embedQuery !== "function") {
     return messages;
@@ -619,7 +753,23 @@ async function startHttpServer(options = {}) {
   const rawStore = createRawMessageStore({
     env: options.env
   });
-  const search = createUnifiedSearch(options.unified);
+  const manager2 = await rawStore.getManager();
+  const search = createUnifiedSearch({
+    ...options.unified,
+    searchRawMessagesLexical: async (input) => {
+      if (typeof manager2.lexicalSearchMessages === "function") {
+        const results = await manager2.lexicalSearchMessages(input);
+        return results.filter(Boolean).map((r) => ({
+          type: "memory",
+          id: r.id,
+          content: r.content,
+          similarity: r.similarity,
+          metadata: r.metadata ?? {}
+        }));
+      }
+      return [];
+    }
+  });
   const app = new Hono();
   app.get("/health", (c) => c.json({ ok: true, store: "memory", ts: Date.now() }));
   app.post("/v1/search", async (c) => {
@@ -647,20 +797,20 @@ async function startHttpServer(options = {}) {
     if (!Array.isArray(body.messages) || !body.userId) {
       return c.json({ error: "userId and messages[] required" }, 400);
     }
-    const manager2 = await rawStore.getManager();
+    const manager3 = await rawStore.getManager();
     const incoming = body.messages;
     const messages = body.embedOnInsert === true ? await embedMissingMessages(
       incoming.map((m) => ({ ...m, userId: m.userId ?? body.userId })),
       options.unified ?? {}
     ) : incoming.map((m) => ({ ...m, userId: m.userId ?? body.userId }));
     let result;
-    if (typeof manager2.upsertRawMessages === "function") {
-      result = await manager2.upsertRawMessages({
+    if (typeof manager3.upsertRawMessages === "function") {
+      result = await manager3.upsertRawMessages({
         userId: body.userId,
         messages
       });
-    } else if (typeof manager2.storeMessages === "function") {
-      const ids = await manager2.storeMessages(messages);
+    } else if (typeof manager3.storeMessages === "function") {
+      const ids = await manager3.storeMessages(messages);
       result = { inserted: ids.length, ids };
     } else {
       return c.json(
@@ -675,14 +825,33 @@ async function startHttpServer(options = {}) {
     } catch (error) {
       console.warn("[memory-store/http] chroma upsert failed:", error);
     }
+    try {
+      const messagesWithEmbeddings = messages.filter(
+        (m) => Array.isArray(m.embedding) && m.embedding.length > 0
+      );
+      if (messagesWithEmbeddings.length > 0) {
+        for (const message of messagesWithEmbeddings) {
+          if (typeof manager3.upsertVectorForMessage === "function") {
+            manager3.upsertVectorForMessage(message.messageId, message.embedding);
+          }
+        }
+        console.log(
+          "[memory-store/http] Updated sqlite-vec vector table for",
+          messagesWithEmbeddings.length,
+          "message(s)"
+        );
+      }
+    } catch (error) {
+      console.warn("[memory-store/http] sqlite-vec vector update failed:", error);
+    }
     return c.json({ ok: true, count: messages.length, result });
   });
   app.get("/v1/raw-messages/:id", async (c) => {
     const id = c.req.param("id");
     const userId = c.req.query("userId");
     if (!userId) return c.json({ error: "userId query param required" }, 400);
-    const manager2 = await rawStore.getManager();
-    const row = await manager2.getMessageById?.(id);
+    const manager3 = await rawStore.getManager();
+    const row = await manager3.getMessageById?.(id);
     if (!row) return c.json({ error: "not found" }, 404);
     return c.json({ message: row });
   });
@@ -697,7 +866,11 @@ async function startHttpServer(options = {}) {
   };
 }
 
+// src/server/cli-shared.ts
+init_raw_message_store();
+
 // src/storage/sqlite-vector-index.ts
+init_sqlite_raw_message_store();
 function getMemoryVectorStoreBackend() {
   const backend = (process.env.INSIGHT_VECTOR_STORE_BACKEND || process.env.MEMORY_VECTOR_STORE_BACKEND || process.env.VECTOR_STORE_BACKEND || "sqlite-vec").toLowerCase();
   return backend === "chroma" ? "chroma" : "sqlite-vec";
@@ -811,11 +984,26 @@ async function buildUnified(args) {
   const wantsAiRag = args.embeddingProvider === "local" || args.memoryBackend === "chroma" || args.insightsBackend === "chroma" || args.knowledgeBackend === "chroma";
   const aiRag = wantsAiRag ? await loadAiRag() : null;
   if (args.embeddingProvider === "local") {
-    const provider = new aiRag.LocalTransformersEmbeddingProvider({ modelName: args.embeddingModel });
-    unified.embedQuery = async ({ query }) => provider.embedQuery(query);
-    log(
-      `embedQuery wired via LocalTransformersEmbeddingProvider (model=${args.embeddingModel ?? "Xenova/all-MiniLM-L6-v2"})`
-    );
+    try {
+      if (!aiRag) {
+        throw new Error("ai-rag modules not loaded");
+      }
+      const provider = new aiRag.LocalTransformersEmbeddingProvider({ modelName: args.embeddingModel });
+      unified.embedQuery = async ({ query }) => provider.embedQuery(query);
+      log(
+        `embedQuery wired via LocalTransformersEmbeddingProvider (model=${args.embeddingModel ?? "Xenova/all-MiniLM-L6-v2"})`
+      );
+    } catch (error) {
+      log(
+        `Warning: Failed to initialize LocalTransformersEmbeddingProvider: ${error.message}`
+      );
+      log(
+        "Semantic search will be disabled. The server will continue with keyword-only search."
+      );
+      log(
+        "To fix: Ensure the model is downloaded or check your network connection to huggingface.co"
+      );
+    }
   } else if (args.embeddingProvider === "openrouter") {
     const apiKey = process.env.OPENROUTER_API_KEY;
     if (!apiKey) {