@melandlabs/memory-store 0.1.2 → 0.1.3

package/dist/server/cli-http.js

@@ -462,6 +462,34 @@ function createUnifiedSearch(deps = {}) {
 }
 
 // src/http.ts
+async function embedMissingMessages(messages, deps) {
+  if (typeof deps.embedQuery !== "function") {
+    return messages;
+  }
+  const out = [];
+  for (const message of messages) {
+    if (Array.isArray(message.embedding) && message.embedding.length > 0) {
+      out.push(message);
+      continue;
+    }
+    if (typeof message.content !== "string" || message.content.length === 0) {
+      out.push(message);
+      continue;
+    }
+    const vector = await deps.embedQuery({
+      userId: message.userId,
+      query: message.content
+    });
+    out.push({
+      ...message,
+      embedding: vector,
+      embeddingModel: message.embeddingModel ?? "server",
+      embeddingDimensions: vector.length,
+      embeddingUpdatedAt: Date.now()
+    });
+  }
+  return out;
+}
 async function startHttpServer(options = {}) {
   const port = options.port ?? Number.parseInt(process.env.MEMORY_HTTP_PORT ?? "7421", 10);
   const host = options.host ?? process.env.MEMORY_HTTP_HOST ?? "127.0.0.1";
@@ -497,16 +525,34 @@ async function startHttpServer(options = {}) {
       return c.json({ error: "userId and messages[] required" }, 400);
     }
     const manager2 = await rawStore.getManager();
-    const result = await manager2.upsertRawMessages?.({
-      userId: body.userId,
-      messages: body.messages
-    });
+    const incoming = body.messages;
+    const messages = body.embedOnInsert === true ? await embedMissingMessages(
+      incoming.map((m) => ({ ...m, userId: m.userId ?? body.userId })),
+      options.unified ?? {}
+    ) : incoming.map((m) => ({ ...m, userId: m.userId ?? body.userId }));
+    let result;
+    if (typeof manager2.upsertRawMessages === "function") {
+      result = await manager2.upsertRawMessages({
+        userId: body.userId,
+        messages
+      });
+    } else if (typeof manager2.storeMessages === "function") {
+      const ids = await manager2.storeMessages(messages);
+      result = { inserted: ids.length, ids };
+    } else {
+      return c.json(
+        {
+          error: "active raw-message manager exposes neither upsertRawMessages nor storeMessages"
+        },
+        500
+      );
+    }
     try {
-      await upsertRawMessagesToChroma(body.messages);
+      await upsertRawMessagesToChroma(messages);
     } catch (error) {
       console.warn("[memory-store/http] chroma upsert failed:", error);
     }
-    return c.json({ ok: true, result });
+    return c.json({ ok: true, count: messages.length, result });
   });
   app.get("/v1/raw-messages/:id", async (c) => {
     const id = c.req.param("id");
@@ -528,33 +574,352 @@ async function startHttpServer(options = {}) {
   };
 }
 
+// src/storage/sqlite-vector-index.ts
+function getMemoryVectorStoreBackend() {
+  const backend = (process.env.INSIGHT_VECTOR_STORE_BACKEND || process.env.MEMORY_VECTOR_STORE_BACKEND || process.env.VECTOR_STORE_BACKEND || "sqlite-vec").toLowerCase();
+  return backend === "chroma" ? "chroma" : "sqlite-vec";
+}
+function isInsightSQLiteVecEnabled(_env) {
+  return getMemoryVectorStoreBackend() === "sqlite-vec";
+}
+async function getInsightSQLiteVecStore(_env) {
+  const dbPath = resolveSQLiteRawMessageDbPath();
+  const { getSQLiteVecStore } = await import('@melandlabs/rag/sqlite-vec-store');
+  return await getSQLiteVecStore(dbPath, void 0, {
+    collectionName: process.env.SQLITE_VEC_INSIGHTS_COLLECTION || "opencontext_insights"
+  });
+}
+async function searchInsightsWithSQLiteVec(input, env) {
+  const store = await getInsightSQLiteVecStore();
+  const overfetchLimit = Math.max(input.limit * 8, input.limit);
+  const results = await store.similaritySearchWithOptions(input.queryEmbedding, {
+    limit: overfetchLimit,
+    filter: { userId: input.userId }
+  });
+  return results.filter((result) => result.score >= input.threshold).filter((result) => {
+    const metadata = result.metadata ?? {};
+    if (input.botIds?.length && !input.botIds.includes(String(metadata.botId ?? ""))) {
+      return false;
+    }
+    return input.includeArchived || metadata.archived !== true;
+  }).slice(0, input.limit);
+}
+
+// src/server/cli-shared.ts
+async function loadAiRag() {
+  try {
+    const [local, chroma] = await Promise.all([
+      import('@melandlabs/ai-rag/local-transformers-embedding-provider'),
+      import('@melandlabs/ai-rag/chroma-store')
+    ]);
+    return {
+      LocalTransformersEmbeddingProvider: local.LocalTransformersEmbeddingProvider,
+      ChromaVectorStore: chroma.ChromaVectorStore
+    };
+  } catch (error) {
+    throw new Error(
+      `--embedding-provider local / --*-backend=chroma require @melandlabs/ai-rag to be installed. Run \`pnpm add @melandlabs/ai-rag\` and try again. (${error.message})`
+    );
+  }
+}
+function parseUnifiedArgs(argv) {
+  const env = process.env;
+  const args = {
+    embeddingProvider: env.EMBEDDING_PROVIDER ?? "none",
+    embeddingModel: env.EMBEDDING_MODEL,
+    chromaUrl: env.CHROMA_URL,
+    memoryBackend: env.MEMORY_BACKEND ?? "none",
+    insightsBackend: env.INSIGHTS_BACKEND ?? "none",
+    insightsCollection: env.INSIGHTS_COLLECTION ?? "opencontext_insights",
+    knowledgeBackend: env.KNOWLEDGE_BACKEND ?? "none",
+    knowledgeCollection: env.KNOWLEDGE_COLLECTION ?? "opencontext_knowledge"
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    const next = argv[i + 1];
+    const takeValue = () => {
+      if (next === void 0) throw new Error(`[memory-store/http] ${arg} requires a value`);
+      i += 1;
+      return next;
+    };
+    switch (arg) {
+      case "--embedding-provider":
+        args.embeddingProvider = takeValue();
+        break;
+      case "--embedding-model":
+        args.embeddingModel = takeValue();
+        break;
+      case "--chroma-url":
+        args.chromaUrl = takeValue();
+        break;
+      case "--memory-backend":
+        args.memoryBackend = takeValue();
+        break;
+      case "--insights-backend":
+        args.insightsBackend = takeValue();
+        break;
+      case "--insights-collection":
+        args.insightsCollection = takeValue();
+        break;
+      case "--knowledge-backend":
+        args.knowledgeBackend = takeValue();
+        break;
+      case "--knowledge-collection":
+        args.knowledgeCollection = takeValue();
+        break;
+    }
+  }
+  for (const [name, value] of [
+    ["--embedding-provider", args.embeddingProvider],
+    ["--memory-backend", args.memoryBackend],
+    ["--insights-backend", args.insightsBackend],
+    ["--knowledge-backend", args.knowledgeBackend]
+  ]) {
+    const allowed = ["local", "openrouter", "none", "chroma", "sqlite-vec"];
+    if (!allowed.includes(value)) {
+      throw new Error(`[memory-store/cli] ${name} must be one of: ${allowed.join(", ")} (got "${value}")`);
+    }
+  }
+  return args;
+}
+async function buildUnified(args) {
+  const unified = {};
+  const log = (msg) => console.warn(`[memory-store/cli] ${msg}`);
+  const wantsAiRag = args.embeddingProvider === "local" || args.memoryBackend === "chroma" || args.insightsBackend === "chroma" || args.knowledgeBackend === "chroma";
+  const aiRag = wantsAiRag ? await loadAiRag() : null;
+  if (args.embeddingProvider === "local") {
+    const provider = new aiRag.LocalTransformersEmbeddingProvider({ modelName: args.embeddingModel });
+    unified.embedQuery = async ({ query }) => provider.embedQuery(query);
+    log(
+      `embedQuery wired via LocalTransformersEmbeddingProvider (model=${args.embeddingModel ?? "Xenova/all-MiniLM-L6-v2"})`
+    );
+  } else if (args.embeddingProvider === "openrouter") {
+    const apiKey = process.env.OPENROUTER_API_KEY;
+    if (!apiKey) {
+      throw new Error("--embedding-provider openrouter requires OPENROUTER_API_KEY in the environment");
+    }
+    const model = args.embeddingModel ?? "text-embedding-3-small";
+    const baseURL = "https://openrouter.ai/api/v1";
+    unified.embedQuery = async ({ query }) => {
+      const res = await fetch(`${baseURL}/embeddings`, {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          authorization: `Bearer ${apiKey}`,
+          "HTTP-Referer": process.env.NEXT_PUBLIC_APP_URL ?? "https://opencontext.ai",
+          "X-Title": "opencontext AI"
+        },
+        body: JSON.stringify({ model, input: query })
+      });
+      if (!res.ok) {
+        throw new Error(`openrouter embeddings ${res.status}: ${await res.text()}`);
+      }
+      const body = await res.json();
+      const embedding = body.data?.[0]?.embedding;
+      if (!embedding) throw new Error("openrouter embeddings response missing data[0].embedding");
+      return embedding;
+    };
+    log(`embedQuery wired via openrouter (model=${model})`);
+  }
+  if (args.memoryBackend === "sqlite-vec") {
+    const store = createRawMessageStore({ env: void 0 });
+    const manager2 = await store.getManager();
+    if (typeof manager2.searchMessagesSemantically !== "function") {
+      throw new Error(
+        "the active raw-message manager does not implement searchMessagesSemantically; sqlite-vec ANN for the memory source is unavailable in this build"
+      );
+    }
+    unified.searchRawMessagesAnn = async ({ userId, queryEmbedding, limit, threshold, botId }) => {
+      const rows = await manager2.searchMessagesSemantically({
+        userId,
+        queryEmbedding,
+        limit,
+        threshold,
+        botId
+      });
+      return rows.map((r) => ({
+        id: r.id,
+        content: r.content,
+        similarity: r.similarity,
+        metadata: r.metadata ?? {}
+      }));
+    };
+    log("memory backend wired via sqlite-vec (uses the manager's searchMessagesSemantically)");
+  } else if (args.memoryBackend === "chroma") {
+    if (!args.chromaUrl)
+      throw new Error("--memory-backend=chroma requires --chroma-url <url> (or CHROMA_URL env)");
+    const store = new aiRag.ChromaVectorStore({
+      url: args.chromaUrl,
+      collectionName: "opencontext_raw_messages"
+    });
+    unified.searchRawMessagesAnn = async ({ queryEmbedding, limit, threshold }) => {
+      const results = await store.similaritySearchWithOptions(queryEmbedding, { limit: limit + 1 });
+      return results.filter((r) => r.score >= threshold).map((r) => ({ id: r.id, content: r.content, similarity: r.score, metadata: r.metadata ?? {} }));
+    };
+    log("memory backend wired via chroma (collection=opencontext_raw_messages)");
+  }
+  if (args.insightsBackend === "sqlite-vec" && isInsightSQLiteVecEnabled()) {
+    if (!unified.embedQuery) {
+      throw new Error("--insights-backend=sqlite-vec requires --embedding-provider local|openrouter");
+    }
+    const embedQuery = unified.embedQuery;
+    unified.searchInsights = async ({ userId, query, limit, threshold, botIds, includeArchived }) => {
+      const emb = await embedQuery({ userId, query });
+      const rows = await searchInsightsWithSQLiteVec({
+        userId,
+        queryEmbedding: emb,
+        limit,
+        threshold,
+        botIds,
+        includeArchived
+      });
+      return rows.map((r) => ({
+        id: r.id,
+        content: r.content,
+        similarity: r.score,
+        metadata: r.metadata ?? {}
+      }));
+    };
+    log("insights backend wired via sqlite-vec");
+  } else if (args.insightsBackend === "chroma") {
+    if (!args.chromaUrl)
+      throw new Error("--insights-backend=chroma requires --chroma-url <url> (or CHROMA_URL env)");
+    if (!unified.embedQuery)
+      throw new Error("--insights-backend=chroma requires --embedding-provider local|openrouter");
+    const store = new aiRag.ChromaVectorStore({
+      url: args.chromaUrl,
+      collectionName: args.insightsCollection
+    });
+    const embedQuery = unified.embedQuery;
+    unified.searchInsights = async ({ query, limit, threshold }) => {
+      const emb = await embedQuery({ userId: "", query });
+      const results = await store.similaritySearchWithOptions(emb, { limit: limit + 1 });
+      return results.filter((r) => r.score >= threshold).map((r) => ({ id: r.id, content: r.content, similarity: r.score, metadata: r.metadata ?? {} }));
+    };
+    log(`insights backend wired via chroma (collection=${args.insightsCollection})`);
+  }
+  if (args.knowledgeBackend === "chroma") {
+    if (!args.chromaUrl)
+      throw new Error("--knowledge-backend=chroma requires --chroma-url <url> (or CHROMA_URL env)");
+    if (!unified.embedQuery)
+      throw new Error("--knowledge-backend=chroma requires --embedding-provider local|openrouter");
+    const store = new aiRag.ChromaVectorStore({
+      url: args.chromaUrl,
+      collectionName: args.knowledgeCollection
+    });
+    const embedQuery = unified.embedQuery;
+    unified.searchKnowledge = async ({ query, options }) => {
+      const emb = await embedQuery({ userId: "", query });
+      const results = await store.similaritySearchWithOptions(emb, { limit: options.limit + 1 });
+      return results.filter((r) => r.score >= options.threshold).map((r, i) => ({
+        chunkId: r.id,
+        documentId: r.documentId || String(r.metadata?.documentId ?? r.id),
+        documentName: String(r.metadata?.documentName ?? ""),
+        content: r.content,
+        similarity: r.score,
+        chunkIndex: i
+      }));
+    };
+    log(`knowledge backend wired via chroma (collection=${args.knowledgeCollection})`);
+  }
+  return unified;
+}
+function printUnifiedHelp() {
+  console.log(`Embedding (wires unified.embedQuery):
+  --embedding-provider <name>     local | openrouter | none
+                                  (env: EMBEDDING_PROVIDER, default: none)
+  --embedding-model <name>        Model name
+                                  (env: EMBEDDING_MODEL; local \u2192 Xenova/all-MiniLM-L6-v2,
+                                  openrouter \u2192 text-embedding-3-small)
+  Note: "local" dynamically imports @melandlabs/ai-rag (a peer install).
+
+Cross-source search (wires unified.searchKnowledge / searchInsights / searchRawMessagesAnn):
+  --chroma-url <url>              Chroma server URL
+                                  (env: CHROMA_URL; required when any *-backend=chroma)
+  --memory-backend <name>         sqlite-vec | chroma | none
+                                  (env: MEMORY_BACKEND, default: none)
+  --insights-backend <name>       sqlite-vec | chroma | none
+                                  (env: INSIGHTS_BACKEND, default: none)
+  --insights-collection <name>    Chroma collection (default: opencontext_insights)
+  --knowledge-backend <name>      chroma | none
+                                  (env: KNOWLEDGE_BACKEND, default: none)
+  --knowledge-collection <name>   Chroma collection (default: opencontext_knowledge)
+  Note: the chroma backends dynamically import @melandlabs/ai-rag.`);
+}
+
 // src/server/cli-http.ts
 function parseArgs(argv) {
+  const env = process.env;
+  const unified = parseUnifiedArgs(argv);
   const args = {
-    port: Number.parseInt(process.env.MEMORY_HTTP_PORT ?? "7421", 10),
-    host: process.env.MEMORY_HTTP_HOST ?? "127.0.0.1"
+    ...unified,
+    port: Number.parseInt(env.MEMORY_HTTP_PORT ?? "7421", 10),
+    host: env.MEMORY_HTTP_HOST ?? "127.0.0.1"
   };
   for (let i = 0; i < argv.length; i += 1) {
     const arg = argv[i];
     const next = argv[i + 1];
-    if (arg === "--port" && next) {
-      args.port = Number.parseInt(next, 10);
+    const takeValue = () => {
+      if (next === void 0) throw new Error(`[memory-store/http] ${arg} requires a value`);
       i += 1;
-    } else if (arg === "--host" && next) {
-      args.host = next;
-      i += 1;
-    } else if (arg === "--help" || arg === "-h") {
-      console.log("Usage: opencontext-memory-http [--port <port>] [--host <host>]");
-      process.exit(0);
+      return next;
+    };
+    switch (arg) {
+      case "--port":
+        args.port = Number.parseInt(takeValue(), 10);
+        break;
+      case "--host":
+        args.host = takeValue();
+        break;
+      case "--help":
+      case "-h":
+        printHelp();
+        process.exit(0);
     }
   }
+  if (!Number.isFinite(args.port) || args.port <= 0) {
+    throw new Error(`[memory-store/http] invalid --port: ${args.port}`);
+  }
   return args;
 }
+function printHelp() {
+  console.log(`opencontext-memory-http \u2014 standalone memory-store HTTP daemon.
+
+Usage:
+  opencontext-memory-http [options]
+
+Server:
+  --port <port>                   Port to listen on (default: 7421, env: MEMORY_HTTP_PORT)
+  --host <host>                   Host to bind (default: 127.0.0.1, env: MEMORY_HTTP_HOST)
+`);
+  printUnifiedHelp();
+  console.log(`
+Examples:
+  # Minimal \u2014 same as before, all three *not_configured warnings remain
+  opencontext-memory-http
+
+  # Local ONNX embedder + sqlite-vec ANN for the memory source (no API key,
+  # no extra services). Removes embedQuery + memory warnings; the
+  # insights/knowledge sources still need a backend of their own.
+  opencontext-memory-http \\
+    --embedding-provider local \\
+    --memory-backend sqlite-vec
+
+  # Wire everything via a running Chroma server
+  opencontext-memory-http \\
+    --embedding-provider openrouter \\
+    --chroma-url http://127.0.0.1:8000 \\
+    --memory-backend chroma \\
+    --insights-backend chroma \\
+    --knowledge-backend chroma`);
+}
 async function main() {
   const args = parseArgs(process.argv.slice(2));
-  const { url, port, stop } = await startHttpServer({
+  const unified = await buildUnified(args);
+  const { url, stop } = await startHttpServer({
     port: args.port,
-    host: args.host
+    host: args.host,
+    unified
   });
   console.log(`[memory-store/http] listening at ${url}`);
   const shutdown = async (signal) => {