@melandlabs/memory-store 0.1.6 → 0.2.1

package/dist/search/unified-search.js

@@ -2,16 +2,24 @@ import '@melandlabs/ai/memory';
 import { ChromaVectorStore } from '@melandlabs/rag';
 import { mkdirSync } from 'fs';
 import { homedir } from 'os';
-import { join, dirname } from 'path';
+import { dirname, join } from 'path';
 import { SQLiteRawMessageManager } from '@melandlabs/sqlite';
 
-// src/storage/chroma-memory-index.ts
-var DEFAULT_RAW_MESSAGES_COLLECTION = "opencontext_raw_messages";
-var RAW_VECTOR_BACKEND_ENV_KEYS = [
-  "RAW_MESSAGE_VECTOR_STORE_BACKEND",
-  "MEMORY_VECTOR_STORE_BACKEND",
-  "VECTOR_STORE_BACKEND"
-];
+var __defProp = Object.defineProperty;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __esm = (fn, res) => function __init() {
+  return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
+};
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+
+// src/contracts.ts
+var init_contracts = __esm({
+  "src/contracts.ts"() {
+  }
+});
 function getBackend(keys) {
   for (const key of keys) {
     const value = process.env[key]?.trim().toLowerCase();
@@ -62,21 +70,51 @@ async function searchRawMessagesWithChroma(input) {
     metadata: result.metadata ?? {}
   })).slice(0, input.limit);
 }
+var DEFAULT_RAW_MESSAGES_COLLECTION, RAW_VECTOR_BACKEND_ENV_KEYS;
+var init_chroma_memory_index = __esm({
+  "src/storage/chroma-memory-index.ts"() {
+    init_contracts();
+    DEFAULT_RAW_MESSAGES_COLLECTION = "opencontext_raw_messages";
+    RAW_VECTOR_BACKEND_ENV_KEYS = [
+      "RAW_MESSAGE_VECTOR_STORE_BACKEND",
+      "MEMORY_VECTOR_STORE_BACKEND",
+      "VECTOR_STORE_BACKEND"
+    ];
+  }
+});
 
 // src/storage/postgres-raw-message-factory.ts
-var factory = null;
 async function resolvePostgresFactory(env) {
-  return null;
+  if (!factory) return null;
+  return await factory(env);
 }
 function hasPostgresFactory() {
   return factory !== null;
 }
-var manager = null;
-var DEFAULT_SQLITE_PATH = join(homedir(), ".opencontext", "memory", "store.db");
+var factory;
+var init_postgres_raw_message_factory = __esm({
+  "src/storage/postgres-raw-message-factory.ts"() {
+    factory = null;
+  }
+});
+
+// src/storage/sqlite-raw-message-store.ts
+var sqlite_raw_message_store_exports = {};
+__export(sqlite_raw_message_store_exports, {
+  __resetSQLiteRawMessageManagerForTests: () => __resetSQLiteRawMessageManagerForTests,
+  closeSQLiteRawMessageManager: () => closeSQLiteRawMessageManager,
+  getSQLiteRawMessageManager: () => getSQLiteRawMessageManager,
+  isSQLiteRawMessageStorageAvailable: () => isSQLiteRawMessageStorageAvailable,
+  lexicalSearchRawMessages: () => lexicalSearchRawMessages,
+  resolveSQLiteRawMessageDbPath: () => resolveSQLiteRawMessageDbPath
+});
 function resolveSQLiteRawMessageDbPath() {
   const fromEnv = process.env.MEMORY_STORE_DB_PATH?.trim();
   return fromEnv && fromEnv.length > 0 ? fromEnv : DEFAULT_SQLITE_PATH;
 }
+function isSQLiteRawMessageStorageAvailable(_env) {
+  return true;
+}
 async function getSQLiteRawMessageManager(_env) {
   if (!manager) {
     const dbPath = resolveSQLiteRawMessageDbPath();
@@ -96,8 +134,35 @@ async function closeSQLiteRawMessageManager() {
   await manager.close();
   manager = null;
 }
+function __resetSQLiteRawMessageManagerForTests() {
+  manager = null;
+}
+async function lexicalSearchRawMessages(input) {
+  const mgr = await getSQLiteRawMessageManager();
+  if (typeof mgr.lexicalSearchMessages !== "function") {
+    return [];
+  }
+  return mgr.lexicalSearchMessages(input);
+}
+var manager, DEFAULT_SQLITE_PATH;
+var init_sqlite_raw_message_store = __esm({
+  "src/storage/sqlite-raw-message-store.ts"() {
+    init_chroma_memory_index();
+    manager = null;
+    DEFAULT_SQLITE_PATH = join(homedir(), ".opencontext", "memory", "store.db");
+  }
+});
 
 // src/storage/raw-message-store.ts
+var raw_message_store_exports = {};
+__export(raw_message_store_exports, {
+  closeRawMessageStore: () => closeRawMessageStore,
+  configureRawMessageStore: () => configureRawMessageStore,
+  createRawMessageStore: () => createRawMessageStore,
+  getRawMessageManager: () => getRawMessageManager,
+  getRawMessageStorageBackend: () => getRawMessageStorageBackend,
+  isRawMessageStorageAvailable: () => isRawMessageStorageAvailable
+});
 function resolveBackend(env) {
   const explicit = process.env.OPENCONTEXT_MEMORY_STORE_BACKEND?.trim().toLowerCase();
   if (explicit === "postgres") return "postgres";
@@ -107,7 +172,7 @@ function resolveEnv(_env) {
   return {};
 }
 function createRawMessageStore(options = {}) {
-  resolveEnv(options.env);
+  const env = resolveEnv(options.env);
   return {
     getBackend() {
       return resolveBackend();
@@ -117,7 +182,7 @@ function createRawMessageStore(options = {}) {
     },
     async getManager() {
       if (resolveBackend() === "postgres") {
-        const pg = await resolvePostgresFactory();
+        const pg = await resolvePostgresFactory(env);
         if (!pg) {
           throw new Error(
             "OPENCONTEXT_MEMORY_STORE_BACKEND=postgres but no Postgres factory is registered. Call registerPostgresFactory() at startup, or unset the env var to use the default SQLite backend."
@@ -132,14 +197,47 @@ function createRawMessageStore(options = {}) {
     }
   };
 }
-var moduleStore = null;
-var moduleConfig = null;
+function configureRawMessageStore(config) {
+  moduleConfig = config;
+  moduleStore = createRawMessageStore({ env: config.env });
+  return moduleStore;
+}
+function getRawMessageStorageBackend() {
+  if (!moduleStore) {
+    moduleStore = createRawMessageStore({ env: moduleConfig?.env });
+  }
+  return moduleStore.getBackend();
+}
 function isRawMessageStorageAvailable() {
   if (!moduleStore) {
     moduleStore = createRawMessageStore({ env: moduleConfig?.env });
   }
   return moduleStore.isAvailable();
 }
+async function getRawMessageManager() {
+  if (!moduleStore) {
+    moduleStore = createRawMessageStore({ env: moduleConfig?.env });
+  }
+  return moduleStore.getManager();
+}
+async function closeRawMessageStore() {
+  if (!moduleStore) return;
+  await moduleStore.close();
+  moduleStore = null;
+}
+var moduleStore, moduleConfig;
+var init_raw_message_store = __esm({
+  "src/storage/raw-message-store.ts"() {
+    init_postgres_raw_message_factory();
+    init_sqlite_raw_message_store();
+    moduleStore = null;
+    moduleConfig = null;
+  }
+});
+
+// src/search/unified-search.ts
+init_chroma_memory_index();
+init_raw_message_store();
 
 // src/search/utilities.ts
 var DEFAULT_LIMIT = 10;
@@ -169,7 +267,14 @@ function clampUnifiedMemorySearchThreshold(threshold) {
   }
   return Math.min(1, Math.max(-1, parsed));
 }
-function mergeUnifiedMemorySearchResults(results, limit) {
+function mergeUnifiedMemorySearchResults(results, limit, options = {}) {
+  const strategy = normalizeUnifiedMemoryMergeStrategy(options.strategy);
+  if (strategy === "rrf") {
+    const lists = options.rankedLists ?? [
+      { name: "memory-semantic", hits: results }
+    ];
+    return mergeUnifiedMemorySearchResultsRrf(lists, limit);
+  }
   return [...results].sort((a, b) => {
     const scoreDelta = b.similarity - a.similarity;
     if (scoreDelta !== 0) {
@@ -178,6 +283,42 @@ function mergeUnifiedMemorySearchResults(results, limit) {
     return a.type.localeCompare(b.type) || a.id.localeCompare(b.id);
   }).slice(0, limit);
 }
+function normalizeUnifiedMemoryMergeStrategy(value) {
+  return value === "rrf" ? "rrf" : "similarity";
+}
+var DEFAULT_RRF_K = 60;
+function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K) {
+  if (!Array.isArray(lists) || lists.length === 0) {
+    return [];
+  }
+  const safeK = Number.isFinite(k) && k > 0 ? k : DEFAULT_RRF_K;
+  const scores = /* @__PURE__ */ new Map();
+  const order = [];
+  for (const list of lists) {
+    for (let index = 0; index < list.hits.length; index += 1) {
+      const hit = list.hits[index];
+      const key = `${hit.type}::${hit.id}`;
+      const contribution = 1 / (safeK + index + 1);
+      const existing = scores.get(key);
+      if (existing) {
+        existing.rrf += contribution;
+      } else {
+        scores.set(key, { hit, rrf: contribution });
+        order.push(key);
+      }
+    }
+  }
+  const ranked = order.map((key) => scores.get(key)).sort((a, b) => {
+    if (b.rrf !== a.rrf) {
+      return b.rrf - a.rrf;
+    }
+    return a.hit.type.localeCompare(b.hit.type) || a.hit.id.localeCompare(b.hit.id);
+  }).slice(0, limit);
+  return ranked.map(({ hit, rrf }) => ({
+    ...hit,
+    metadata: { ...hit.metadata, rrfScore: rrf }
+  }));
+}
 function toKnowledgeResult(result) {
   return {
     type: "knowledge",
@@ -209,6 +350,9 @@ function isRawMemorySemanticResult(result) {
 }
 
 // src/search/unified-search.ts
+function deriveLexicalKeywords(query) {
+  return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
+}
 function createUnifiedSearch(deps = {}) {
   const logger = deps.logger ?? console;
   async function searchUnifiedMemory(input) {
@@ -216,8 +360,8 @@ function createUnifiedSearch(deps = {}) {
     const sources = normalizeUnifiedMemorySearchSources(input.sources);
     const limit = clampUnifiedMemorySearchLimit(input.limit);
     const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
+    const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy);
     const warnings = [];
-    const results = [];
     if (!query) {
       return {
         query,
@@ -227,17 +371,17 @@ function createUnifiedSearch(deps = {}) {
         warnings
       };
     }
+    let memorySubs;
     if (sources.includes("memory")) {
       if (isRawMessageStorageAvailable()) {
         try {
-          const memoryHits = await runMemorySource(input, limit, threshold);
-          results.push(...memoryHits);
+          memorySubs = await runMemorySource(input, limit, threshold, warnings);
         } catch (error) {
           logger.warn?.("[memory-store] memory source failed:", error);
           warnings.push({
             source: "memory",
             code: "memory_search_failed",
-            message: error.message ?? "memory_search_failed"
+            message: `Memory search failed: ${error.message ?? "Unknown error"}. Using keyword search fallback.`
           });
         }
       } else {
@@ -248,9 +392,10 @@ function createUnifiedSearch(deps = {}) {
         });
       }
     }
+    const insightHits = [];
     if (sources.includes("insights") && typeof deps.searchInsights === "function") {
       try {
-        const insightHits = await deps.searchInsights({
+        const hits = await deps.searchInsights({
           userId: input.userId,
           query: input.query,
           limit,
@@ -259,8 +404,8 @@ function createUnifiedSearch(deps = {}) {
           includeArchived: input.includeArchivedInsights,
           authToken: input.authToken
         });
-        for (const hit of insightHits) {
-          results.push({
+        for (const hit of hits) {
+          insightHits.push({
             type: "insight",
             id: hit.id,
             content: hit.content,
@@ -276,16 +421,11 @@ function createUnifiedSearch(deps = {}) {
           message: error.message ?? "insights_search_failed"
         });
       }
-    } else if (sources.includes("insights")) {
-      warnings.push({
-        source: "insights",
-        code: "insights_search_not_configured",
-        message: "Insights search is not configured for this host."
-      });
-    }
+    } else if (sources.includes("insights")) ;
+    const knowledgeHits = [];
     if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
       try {
-        const knowledgeHits = await deps.searchKnowledge({
+        const hits = await deps.searchKnowledge({
           userId: input.userId,
           query: input.query,
           options: {
@@ -295,8 +435,8 @@ function createUnifiedSearch(deps = {}) {
           },
           authToken: input.authToken
         });
-        for (const hit of knowledgeHits) {
-          results.push(toKnowledgeResult(hit));
+        for (const hit of hits) {
+          knowledgeHits.push(toKnowledgeResult(hit));
         }
       } catch (error) {
         logger.warn?.("[memory-store] knowledge source failed:", error);
@@ -306,14 +446,14 @@ function createUnifiedSearch(deps = {}) {
           message: error.message ?? "knowledge_search_failed"
         });
       }
-    } else if (sources.includes("knowledge")) {
-      warnings.push({
-        source: "knowledge",
-        code: "knowledge_search_not_configured",
-        message: "Knowledge search is not configured for this host."
-      });
-    }
-    const merged = mergeUnifiedMemorySearchResults(results, limit);
+    } else if (sources.includes("knowledge")) ;
+    const merged = mergeAcrossSources({
+      memorySubs,
+      insightHits,
+      knowledgeHits,
+      limit,
+      strategy: mergeStrategy
+    });
     return {
       query: input.query,
       sources,
@@ -322,9 +462,35 @@ function createUnifiedSearch(deps = {}) {
       warnings
     };
   }
-  async function runMemorySource(input, limit, threshold) {
+  async function runMemorySource(input, limit, threshold, warnings) {
     if (typeof deps.embedQuery !== "function") {
-      throw new Error("embedQuery is not configured");
+      warnings.push({
+        source: "memory",
+        code: "memory_lexical_search_fallback",
+        message: "Semantic search not configured, using keyword search as fallback"
+      });
+      const { lexicalSearchRawMessages: lexicalSearchRawMessages2 } = await Promise.resolve().then(() => (init_sqlite_raw_message_store(), sqlite_raw_message_store_exports));
+      const keywords = deriveLexicalKeywords(input.query);
+      let lexical2 = [];
+      if (keywords.length > 0) {
+        try {
+          const filters2 = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
+          const results = await Promise.all(
+            filters2.map(
+              (botId) => lexicalSearchRawMessages2({
+                userId: input.userId,
+                keywords,
+                limit: Math.ceil(limit / filters2.length),
+                botId
+              })
+            )
+          );
+          lexical2 = results.flat().filter(Boolean).map(toMemoryResult);
+        } catch (error) {
+          logger.warn?.("[memory-store] Default lexical search failed:", error);
+        }
+      }
+      return { semantic: [], lexical: lexical2 };
     }
     const queryEmbedding = await deps.embedQuery({
       userId: input.userId,
@@ -332,9 +498,10 @@ function createUnifiedSearch(deps = {}) {
       authToken: input.authToken
     });
     const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
+    let semantic = [];
     if (isRawMessageChromaEnabled()) {
       try {
-        const chromaHits = (await Promise.all(
+        semantic = (await Promise.all(
           filters.map((filter) => {
             const botId = "botId" in filter ? filter.botId : void 0;
             return searchRawMessagesWithChroma({
@@ -349,18 +516,16 @@ function createUnifiedSearch(deps = {}) {
         logger.log?.("[memory-store] Raw message semantic search completed", {
           backend: "chroma",
           dimensions: queryEmbedding.length,
-          count: chromaHits.length
+          count: semantic.length
         });
-        return chromaHits;
       } catch (error) {
         logger.warn?.(
           "[memory-store] Chroma raw message search failed; falling back to database search:",
           error
         );
       }
-    }
-    if (typeof deps.searchRawMessagesAnn === "function") {
-      const rows = (await Promise.all(
+    } else if (typeof deps.searchRawMessagesAnn === "function") {
+      semantic = (await Promise.all(
         filters.map(
           (filter) => deps.searchRawMessagesAnn({
             userId: input.userId,
@@ -371,18 +536,108 @@ function createUnifiedSearch(deps = {}) {
           })
         )
       )).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
-      return rows;
     }
-    return [];
+    if (semantic.length === 0) {
+      try {
+        const { getRawMessageManager: getRawMessageManager2 } = await Promise.resolve().then(() => (init_raw_message_store(), raw_message_store_exports));
+        const manager2 = await getRawMessageManager2();
+        if (typeof manager2.searchMessagesSemantically === "function") {
+          const results = await manager2.searchMessagesSemantically({
+            userId: input.userId,
+            queryEmbedding,
+            limit
+          });
+          semantic = results.map((r) => ({
+            type: "memory",
+            id: r.id,
+            content: r.content,
+            similarity: r.similarity,
+            metadata: r.metadata ?? {}
+          }));
+        }
+      } catch (error) {
+        logger.warn?.("[memory-store] SQLite semantic search failed:", error);
+      }
+    }
+    let lexical = [];
+    if (typeof deps.searchRawMessagesLexical === "function") {
+      const keywords = deriveLexicalKeywords(input.query);
+      if (keywords.length > 0) {
+        try {
+          const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
+          lexical = (await Promise.all(
+            lexFilters.map(
+              (botId) => deps.searchRawMessagesLexical({
+                userId: input.userId,
+                keywords,
+                limit,
+                botId
+              })
+            )
+          )).flat().filter(isRawMemorySemanticResult).map((hit) => ({
+            ...toMemoryResult(hit),
+            metadata: { ...hit.metadata, scoring: "bm25" }
+          }));
+        } catch (error) {
+          logger.warn?.("[memory-store] lexical memory search failed:", error);
+          warnings.push({
+            source: "memory",
+            code: "memory_lexical_search_failed",
+            message: error.message ?? "memory_lexical_search_failed"
+          });
+        }
+      }
+    } else if (input.mergeStrategy === "rrf") {
+      warnings.push({
+        source: "memory",
+        code: "memory_lexical_search_not_configured",
+        message: "RRF merge requested but lexical search is not configured; falling back to semantic-only RRF."
+      });
+    }
+    return { semantic, lexical };
   }
   async function searchRawMemorySemantically(input) {
     const limit = clampUnifiedMemorySearchLimit(input.limit);
     const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
-    return runMemorySource(input, limit, threshold);
+    const sub = await runMemorySource(input, limit, threshold, []);
+    return mergeAcrossSources({
+      memorySubs: sub,
+      insightHits: [],
+      knowledgeHits: [],
+      limit,
+      strategy: normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy)
+    });
   }
   return { searchUnifiedMemory, searchRawMemorySemantically };
 }
+function mergeAcrossSources(input) {
+  const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
+  const all = [
+    ...memorySubs.semantic,
+    ...memorySubs.lexical,
+    ...input.insightHits,
+    ...input.knowledgeHits
+  ];
+  if (input.strategy !== "rrf") {
+    return mergeUnifiedMemorySearchResults(all, input.limit);
+  }
+  const lists = [];
+  if (memorySubs.semantic.length > 0) {
+    lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
+  }
+  if (memorySubs.lexical.length > 0) {
+    lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
+  }
+  if (input.insightHits.length > 0) {
+    lists.push({ name: "insights", hits: input.insightHits });
+  }
+  if (input.knowledgeHits.length > 0) {
+    lists.push({ name: "knowledge", hits: input.knowledgeHits });
+  }
+  if (lists.length <= 1) {
+    return mergeUnifiedMemorySearchResults(all, input.limit);
+  }
+  return mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
+}
 
-export { clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult };
-//# sourceMappingURL=unified-search.js.map
-//# sourceMappingURL=unified-search.js.map
+export { clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, mergeUnifiedMemorySearchResultsRrf, normalizeUnifiedMemoryMergeStrategy, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult };