@melandlabs/memory-store 0.1.2 → 0.1.5

package/dist/server/cli-mcp.js

@@ -465,6 +465,34 @@ function createUnifiedSearch(deps = {}) {
 // src/mcp.ts
 var DEFAULT_NAME = "@melandlabs/memory-store";
 var DEFAULT_VERSION = "0.1.0";
+async function embedMissingMessages(messages, deps) {
+  if (typeof deps.embedQuery !== "function") {
+    return messages;
+  }
+  const out = [];
+  for (const message of messages) {
+    if (Array.isArray(message.embedding) && message.embedding.length > 0) {
+      out.push(message);
+      continue;
+    }
+    if (typeof message.content !== "string" || message.content.length === 0) {
+      out.push(message);
+      continue;
+    }
+    const vector = await deps.embedQuery({
+      userId: message.userId,
+      query: message.content
+    });
+    out.push({
+      ...message,
+      embedding: vector,
+      embeddingModel: message.embeddingModel ?? "server",
+      embeddingDimensions: vector.length,
+      embeddingUpdatedAt: Date.now()
+    });
+  }
+  return out;
+}
 async function startMcpServer(options = {}) {
   const server = new McpServer({
     name: options.name ?? DEFAULT_NAME,
@@ -490,6 +518,9 @@ async function startMcpServer(options = {}) {
   };
   const writeSchema = {
     userId: z.string(),
+    embedOnInsert: z.boolean().optional().describe(
+      "When true AND the host wired `unified.embedQuery`, fill in any missing `embedding` server-side via that embedder. Required when the active memory backend is sqlite-vec and the caller doesn't pre-embed."
+    ),
     message: z.object({
       id: z.string().optional(),
       messageId: z.string().optional(),
@@ -500,6 +531,9 @@ async function startMcpServer(options = {}) {
       channel: z.string().optional(),
       person: z.string().optional(),
       timestamp: z.number().optional(),
+      embedding: z.array(z.number()).optional(),
+      embeddingModel: z.string().optional(),
+      embeddingDimensions: z.number().optional(),
       metadata: z.record(z.string(), z.unknown()).optional()
     }).passthrough()
   };
@@ -532,17 +566,32 @@ async function startMcpServer(options = {}) {
     async (args) => {
       const a = args;
       const manager2 = await rawStore.getManager();
-      const result = await manager2.upsertRawMessages?.({
-        userId: a.userId,
-        messages: [a.message]
-      });
+      const incoming = [a.message].map((m) => ({ ...m, userId: m.userId ?? a.userId }));
+      const messages = a.embedOnInsert === true ? await embedMissingMessages(incoming, options.unified ?? {}) : incoming;
+      let result;
+      if (typeof manager2.upsertRawMessages === "function") {
+        result = await manager2.upsertRawMessages({
+          userId: a.userId,
+          messages
+        });
+      } else if (typeof manager2.storeMessages === "function") {
+        const ids = await manager2.storeMessages(messages);
+        result = { inserted: ids.length, ids };
+      } else {
+        throw new Error("active raw-message manager exposes neither upsertRawMessages nor storeMessages");
+      }
       try {
-        await upsertRawMessagesToChroma([a.message]);
+        await upsertRawMessagesToChroma(messages);
       } catch (error) {
         console.warn("[memory-store/mcp] chroma upsert failed:", error);
       }
       return {
-        content: [{ type: "text", text: JSON.stringify({ ok: true, result }) }]
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify({ ok: true, count: messages.length, result })
+          }
+        ]
       };
     }
   );
@@ -572,9 +621,348 @@ async function startMcpServer(options = {}) {
   return server;
 }
 
+// src/storage/sqlite-vector-index.ts
+function getMemoryVectorStoreBackend() {
+  const backend = (process.env.INSIGHT_VECTOR_STORE_BACKEND || process.env.MEMORY_VECTOR_STORE_BACKEND || process.env.VECTOR_STORE_BACKEND || "sqlite-vec").toLowerCase();
+  return backend === "chroma" ? "chroma" : "sqlite-vec";
+}
+function isInsightSQLiteVecEnabled(_env) {
+  return getMemoryVectorStoreBackend() === "sqlite-vec";
+}
+async function getInsightSQLiteVecStore(_env) {
+  const dbPath = resolveSQLiteRawMessageDbPath();
+  const { getSQLiteVecStore } = await import('@melandlabs/rag/sqlite-vec-store');
+  return await getSQLiteVecStore(dbPath, void 0, {
+    collectionName: process.env.SQLITE_VEC_INSIGHTS_COLLECTION || "opencontext_insights"
+  });
+}
+async function searchInsightsWithSQLiteVec(input, env) {
+  const store = await getInsightSQLiteVecStore();
+  const overfetchLimit = Math.max(input.limit * 8, input.limit);
+  const results = await store.similaritySearchWithOptions(input.queryEmbedding, {
+    limit: overfetchLimit,
+    filter: { userId: input.userId }
+  });
+  return results.filter((result) => result.score >= input.threshold).filter((result) => {
+    const metadata = result.metadata ?? {};
+    if (input.botIds?.length && !input.botIds.includes(String(metadata.botId ?? ""))) {
+      return false;
+    }
+    return input.includeArchived || metadata.archived !== true;
+  }).slice(0, input.limit);
+}
+
+// src/server/cli-shared.ts
+async function loadAiRag() {
+  try {
+    const [local, chroma] = await Promise.all([
+      import('@melandlabs/ai-rag/local-transformers-embedding-provider'),
+      import('@melandlabs/ai-rag/chroma-store')
+    ]);
+    return {
+      LocalTransformersEmbeddingProvider: local.LocalTransformersEmbeddingProvider,
+      ChromaVectorStore: chroma.ChromaVectorStore
+    };
+  } catch (error) {
+    throw new Error(
+      `--embedding-provider local / --*-backend=chroma require @melandlabs/ai-rag to be installed. Run \`pnpm add @melandlabs/ai-rag\` and try again. (${error.message})`
+    );
+  }
+}
+function parseUnifiedArgs(argv) {
+  const env = process.env;
+  const args = {
+    embeddingProvider: env.EMBEDDING_PROVIDER ?? "none",
+    embeddingModel: env.EMBEDDING_MODEL,
+    chromaUrl: env.CHROMA_URL,
+    memoryBackend: env.MEMORY_BACKEND ?? "none",
+    insightsBackend: env.INSIGHTS_BACKEND ?? "none",
+    insightsCollection: env.INSIGHTS_COLLECTION ?? "opencontext_insights",
+    knowledgeBackend: env.KNOWLEDGE_BACKEND ?? "none",
+    knowledgeCollection: env.KNOWLEDGE_COLLECTION ?? "opencontext_knowledge"
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    const next = argv[i + 1];
+    const takeValue = () => {
+      if (next === void 0) throw new Error(`[memory-store/http] ${arg} requires a value`);
+      i += 1;
+      return next;
+    };
+    switch (arg) {
+      case "--embedding-provider":
+        args.embeddingProvider = takeValue();
+        break;
+      case "--embedding-model":
+        args.embeddingModel = takeValue();
+        break;
+      case "--chroma-url":
+        args.chromaUrl = takeValue();
+        break;
+      case "--memory-backend":
+        args.memoryBackend = takeValue();
+        break;
+      case "--insights-backend":
+        args.insightsBackend = takeValue();
+        break;
+      case "--insights-collection":
+        args.insightsCollection = takeValue();
+        break;
+      case "--knowledge-backend":
+        args.knowledgeBackend = takeValue();
+        break;
+      case "--knowledge-collection":
+        args.knowledgeCollection = takeValue();
+        break;
+    }
+  }
+  for (const [name, value] of [
+    ["--embedding-provider", args.embeddingProvider],
+    ["--memory-backend", args.memoryBackend],
+    ["--insights-backend", args.insightsBackend],
+    ["--knowledge-backend", args.knowledgeBackend]
+  ]) {
+    const allowed = ["local", "openrouter", "none", "chroma", "sqlite-vec"];
+    if (!allowed.includes(value)) {
+      throw new Error(`[memory-store/cli] ${name} must be one of: ${allowed.join(", ")} (got "${value}")`);
+    }
+  }
+  return args;
+}
+async function buildUnified(args) {
+  const unified = {};
+  const log = (msg) => console.warn(`[memory-store/cli] ${msg}`);
+  const wantsAiRag = args.embeddingProvider === "local" || args.memoryBackend === "chroma" || args.insightsBackend === "chroma" || args.knowledgeBackend === "chroma";
+  const aiRag = wantsAiRag ? await loadAiRag() : null;
+  if (args.embeddingProvider === "local") {
+    const provider = new aiRag.LocalTransformersEmbeddingProvider({ modelName: args.embeddingModel });
+    unified.embedQuery = async ({ query }) => provider.embedQuery(query);
+    log(
+      `embedQuery wired via LocalTransformersEmbeddingProvider (model=${args.embeddingModel ?? "Xenova/all-MiniLM-L6-v2"})`
+    );
+  } else if (args.embeddingProvider === "openrouter") {
+    const apiKey = process.env.OPENROUTER_API_KEY;
+    if (!apiKey) {
+      throw new Error("--embedding-provider openrouter requires OPENROUTER_API_KEY in the environment");
+    }
+    const model = args.embeddingModel ?? "text-embedding-3-small";
+    const baseURL = "https://openrouter.ai/api/v1";
+    unified.embedQuery = async ({ query }) => {
+      const res = await fetch(`${baseURL}/embeddings`, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          authorization: `Bearer ${apiKey}`,
+          "HTTP-Referer": process.env.NEXT_PUBLIC_APP_URL ?? "https://opencontext.ai",
+          "X-Title": "opencontext AI"
+        },
+        body: JSON.stringify({ model, input: query })
+      });
+      if (!res.ok) {
+        throw new Error(`openrouter embeddings ${res.status}: ${await res.text()}`);
+      }
+      const body = await res.json();
+      const embedding = body.data?.[0]?.embedding;
+      if (!embedding) throw new Error("openrouter embeddings response missing data[0].embedding");
+      return embedding;
+    };
+    log(`embedQuery wired via openrouter (model=${model})`);
+  }
+  if (args.memoryBackend === "sqlite-vec") {
+    const store = createRawMessageStore({ env: void 0 });
+    const manager2 = await store.getManager();
+    if (typeof manager2.searchMessagesSemantically !== "function") {
+      throw new Error(
+        "the active raw-message manager does not implement searchMessagesSemantically; sqlite-vec ANN for the memory source is unavailable in this build"
+      );
+    }
+    unified.searchRawMessagesAnn = async ({ userId, queryEmbedding, limit, threshold, botId }) => {
+      const rows = await manager2.searchMessagesSemantically({
+        userId,
+        queryEmbedding,
+        limit,
+        threshold,
+        botId
+      });
+      return rows.map((r) => ({
+        id: r.id,
+        content: r.content,
+        similarity: r.similarity,
+        metadata: r.metadata ?? {}
+      }));
+    };
+    log("memory backend wired via sqlite-vec (uses the manager's searchMessagesSemantically)");
+  } else if (args.memoryBackend === "chroma") {
+    if (!args.chromaUrl)
+      throw new Error("--memory-backend=chroma requires --chroma-url <url> (or CHROMA_URL env)");
+    const store = new aiRag.ChromaVectorStore({
+      url: args.chromaUrl,
+      collectionName: "opencontext_raw_messages"
+    });
+    unified.searchRawMessagesAnn = async ({ queryEmbedding, limit, threshold }) => {
+      const results = await store.similaritySearchWithOptions(queryEmbedding, { limit: limit + 1 });
+      return results.filter((r) => r.score >= threshold).map((r) => ({ id: r.id, content: r.content, similarity: r.score, metadata: r.metadata ?? {} }));
+    };
+    log("memory backend wired via chroma (collection=opencontext_raw_messages)");
+  }
+  if (args.insightsBackend === "sqlite-vec" && isInsightSQLiteVecEnabled()) {
+    if (!unified.embedQuery) {
+      throw new Error("--insights-backend=sqlite-vec requires --embedding-provider local|openrouter");
+    }
+    const embedQuery = unified.embedQuery;
+    unified.searchInsights = async ({ userId, query, limit, threshold, botIds, includeArchived }) => {
+      const emb = await embedQuery({ userId, query });
+      const rows = await searchInsightsWithSQLiteVec({
+        userId,
+        queryEmbedding: emb,
+        limit,
+        threshold,
+        botIds,
+        includeArchived
+      });
+      return rows.map((r) => ({
+        id: r.id,
+        content: r.content,
+        similarity: r.score,
+        metadata: r.metadata ?? {}
+      }));
+    };
+    log("insights backend wired via sqlite-vec");
+  } else if (args.insightsBackend === "chroma") {
+    if (!args.chromaUrl)
+      throw new Error("--insights-backend=chroma requires --chroma-url <url> (or CHROMA_URL env)");
+    if (!unified.embedQuery)
+      throw new Error("--insights-backend=chroma requires --embedding-provider local|openrouter");
+    const store = new aiRag.ChromaVectorStore({
+      url: args.chromaUrl,
+      collectionName: args.insightsCollection
+    });
+    const embedQuery = unified.embedQuery;
+    unified.searchInsights = async ({ query, limit, threshold }) => {
+      const emb = await embedQuery({ userId: "", query });
+      const results = await store.similaritySearchWithOptions(emb, { limit: limit + 1 });
+      return results.filter((r) => r.score >= threshold).map((r) => ({ id: r.id, content: r.content, similarity: r.score, metadata: r.metadata ?? {} }));
+    };
+    log(`insights backend wired via chroma (collection=${args.insightsCollection})`);
+  }
+  if (args.knowledgeBackend === "chroma") {
+    if (!args.chromaUrl)
+      throw new Error("--knowledge-backend=chroma requires --chroma-url <url> (or CHROMA_URL env)");
+    if (!unified.embedQuery)
+      throw new Error("--knowledge-backend=chroma requires --embedding-provider local|openrouter");
+    const store = new aiRag.ChromaVectorStore({
+      url: args.chromaUrl,
+      collectionName: args.knowledgeCollection
+    });
+    const embedQuery = unified.embedQuery;
+    unified.searchKnowledge = async ({ query, options }) => {
+      const emb = await embedQuery({ userId: "", query });
+      const results = await store.similaritySearchWithOptions(emb, { limit: options.limit + 1 });
+      return results.filter((r) => r.score >= options.threshold).map((r, i) => ({
+        chunkId: r.id,
+        documentId: r.documentId || String(r.metadata?.documentId ?? r.id),
+        documentName: String(r.metadata?.documentName ?? ""),
+        content: r.content,
+        similarity: r.score,
+        chunkIndex: i
+      }));
+    };
+    log(`knowledge backend wired via chroma (collection=${args.knowledgeCollection})`);
+  }
+  return unified;
+}
+function printUnifiedHelp() {
+  console.log(`Embedding (wires unified.embedQuery):
+  --embedding-provider <name>     local | openrouter | none
+                                  (env: EMBEDDING_PROVIDER, default: none)
+  --embedding-model <name>        Model name
+                                  (env: EMBEDDING_MODEL; local \u2192 Xenova/all-MiniLM-L6-v2,
+                                  openrouter \u2192 text-embedding-3-small)
+  Note: "local" dynamically imports @melandlabs/ai-rag (a peer install).
+
+Cross-source search (wires unified.searchKnowledge / searchInsights / searchRawMessagesAnn):
+  --chroma-url <url>              Chroma server URL
+                                  (env: CHROMA_URL; required when any *-backend=chroma)
+  --memory-backend <name>         sqlite-vec | chroma | none
+                                  (env: MEMORY_BACKEND, default: none)
+  --insights-backend <name>       sqlite-vec | chroma | none
+                                  (env: INSIGHTS_BACKEND, default: none)
+  --insights-collection <name>    Chroma collection (default: opencontext_insights)
+  --knowledge-backend <name>      chroma | none
+                                  (env: KNOWLEDGE_BACKEND, default: none)
+  --knowledge-collection <name>   Chroma collection (default: opencontext_knowledge)
+  Note: the chroma backends dynamically import @melandlabs/ai-rag.`);
+}
+
 // src/server/cli-mcp.ts
+function parseArgs(argv) {
+  const env = process.env;
+  const unified = parseUnifiedArgs(argv);
+  const args = {
+    ...unified,
+    name: env.MEMORY_MCP_NAME,
+    version: env.MEMORY_MCP_VERSION
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    const next = argv[i + 1];
+    const takeValue = () => {
+      if (next === void 0) throw new Error(`[memory-store/mcp] ${arg} requires a value`);
+      i += 1;
+      return next;
+    };
+    switch (arg) {
+      case "--name":
+        args.name = takeValue();
+        break;
+      case "--version":
+        args.version = takeValue();
+        break;
+      case "--help":
+      case "-h":
+        printHelp();
+        process.exit(0);
+    }
+  }
+  return args;
+}
+function printHelp() {
+  console.log(`opencontext-memory-mcp \u2014 standalone memory-store MCP daemon.
+
+Usage:
+  opencontext-memory-mcp [options]
+
+Server identity (advertised to MCP clients):
+  --name <name>                   Server name (env: MEMORY_MCP_NAME)
+  --version <version>             Server version (env: MEMORY_MCP_VERSION)
+`);
+  printUnifiedHelp();
+  console.log(`
+Examples:
+  # Minimal \u2014 same as before, all three *not_configured warnings remain
+  opencontext-memory-mcp
+
+  # Local ONNX embedder + sqlite-vec ANN for the memory source
+  opencontext-memory-mcp \\
+    --embedding-provider local \\
+    --memory-backend sqlite-vec
+
+  # Wire everything via a running Chroma server
+  opencontext-memory-mcp \\
+    --embedding-provider openrouter \\
+    --chroma-url http://127.0.0.1:8000 \\
+    --memory-backend chroma \\
+    --insights-backend chroma \\
+    --knowledge-backend chroma`);
+}
 async function main() {
-  const server = await startMcpServer();
+  const args = parseArgs(process.argv.slice(2));
+  const unified = await buildUnified(args);
+  const server = await startMcpServer({
+    unified,
+    name: args.name,
+    version: args.version
+  });
   console.error("[memory-store/mcp] listening on stdio");
   const shutdown = async (signal) => {
     console.error(`[memory-store/mcp] ${signal} received, shutting down\u2026`);