@melandlabs/memory-store 0.1.0

package/dist/server/cli-http.js

@@ -0,0 +1,584 @@
+#!/usr/bin/env node
+import { serve } from '@hono/node-server';
+import { Hono } from 'hono';
+import { buildMemoryRecordEmbeddingDocument } from '@melandlabs/ai/memory';
+import { ChromaVectorStore } from '@melandlabs/rag';
+import { mkdirSync } from 'fs';
+import { dirname } from 'path';
+import { SQLiteRawMessageManager } from '@melandlabs/sqlite';
+
+// src/contracts.ts
+function normalizeTimestampToMs(value) {
+  if (!Number.isFinite(value)) {
+    return 0;
+  }
+  if (value < 1e11) {
+    return Math.floor(value * 1e3);
+  }
+  return Math.floor(value);
+}
+function rawMessageToMemoryRecord(message) {
+  return {
+    id: message.messageId,
+    userId: message.userId,
+    timestamp: normalizeTimestampToMs(message.timestamp),
+    text: message.archivedAt ? void 0 : message.content,
+    mediaRefs: message.attachments?.map((item) => item.url).filter(Boolean),
+    embedding: message.embedding,
+    embeddingModel: message.embeddingModel,
+    embeddingContentHash: message.embeddingContentHash
+  };
+}
+
+// src/storage/chroma-memory-index.ts
+var DEFAULT_RAW_MESSAGES_COLLECTION = "opencontext_raw_messages";
+var RAW_VECTOR_BACKEND_ENV_KEYS = [
+  "RAW_MESSAGE_VECTOR_STORE_BACKEND",
+  "MEMORY_VECTOR_STORE_BACKEND",
+  "VECTOR_STORE_BACKEND"
+];
+function getBackend(keys) {
+  for (const key of keys) {
+    const value = process.env[key]?.trim().toLowerCase();
+    if (value) {
+      return value;
+    }
+  }
+  return "";
+}
+function isRawMessageChromaEnabled() {
+  return getBackend(RAW_VECTOR_BACKEND_ENV_KEYS) === "chroma";
+}
+function getRawMessagesCollectionName() {
+  return process.env.CHROMA_RAW_MESSAGES_COLLECTION || process.env.CHROMA_MEMORY_COLLECTION || DEFAULT_RAW_MESSAGES_COLLECTION;
+}
+function getChromaStore(collectionName) {
+  return new ChromaVectorStore({
+    url: process.env.CHROMA_URL,
+    collectionName
+  });
+}
+function normalizeTimestampToMs2(value) {
+  if (!Number.isFinite(value)) {
+    return 0;
+  }
+  if (value < 1e11) {
+    return Math.floor(value * 1e3);
+  }
+  return Math.floor(value);
+}
+function toBooleanValue(value) {
+  return value === true || value === "true" || value === 1 || value === "1";
+}
+function rawMessageToChunk(message) {
+  if (!message.embedding || message.embedding.length === 0) {
+    return null;
+  }
+  const record = rawMessageToMemoryRecord(message);
+  const document = buildMemoryRecordEmbeddingDocument(record);
+  if (!document.content) {
+    return null;
+  }
+  return {
+    id: message.messageId,
+    documentId: message.messageId,
+    content: document.content,
+    embedding: message.embedding,
+    metadata: {
+      sourceType: "raw_message",
+      userId: message.userId,
+      platform: message.platform,
+      botId: message.botId,
+      channel: message.channel,
+      person: message.person,
+      timestamp: normalizeTimestampToMs2(message.timestamp),
+      memoryStage: message.memoryStage,
+      embeddingModel: message.embeddingModel,
+      embeddingDimensions: message.embeddingDimensions,
+      contentHash: message.embeddingContentHash,
+      archived: Boolean(message.archivedAt),
+      role: typeof message.metadata?.role === "string" ? message.metadata.role : void 0
+    }
+  };
+}
+async function upsertRawMessagesToChroma(messages) {
+  if (!isRawMessageChromaEnabled() || messages.length === 0) {
+    return 0;
+  }
+  const chunks = messages.map(rawMessageToChunk).filter((chunk) => chunk !== null);
+  if (chunks.length === 0) {
+    return 0;
+  }
+  const store = getChromaStore(getRawMessagesCollectionName());
+  await store.addChunks(chunks);
+  return chunks.length;
+}
+async function searchRawMessagesWithChroma(input) {
+  if (!isRawMessageChromaEnabled() || input.queryEmbedding.length === 0) {
+    return [];
+  }
+  const store = getChromaStore(getRawMessagesCollectionName());
+  const results = await store.similaritySearch(
+    input.queryEmbedding,
+    Math.max(input.limit * 5, input.limit),
+    input.userId
+  );
+  return results.filter((result) => {
+    const metadata = result.metadata ?? {};
+    if (toBooleanValue(metadata.archived)) {
+      return false;
+    }
+    if (input.botId && metadata.botId !== input.botId) {
+      return false;
+    }
+    return result.score >= input.threshold;
+  }).map((result) => ({
+    id: result.id,
+    content: result.content,
+    similarity: result.score,
+    metadata: result.metadata ?? {}
+  })).slice(0, input.limit);
+}
+
+// src/storage/postgres-raw-message-factory.ts
+var factory = null;
+async function resolvePostgresFactory(env) {
+  return null;
+}
+function hasPostgresFactory() {
+  return factory !== null;
+}
+var manager = null;
+function isSQLiteRawMessageStorageAvailable(env) {
+  return resolveEnv(env).isTauriMode();
+}
+async function getSQLiteRawMessageManager(env) {
+  const e = resolveEnv(env);
+  if (!e.isTauriMode()) {
+    throw new Error("SQLite raw message storage is only available in Tauri mode.");
+  }
+  if (!manager) {
+    const dbPath = e.getTauriDbPath?.() ?? process.env.TAURI_DB_PATH ?? "";
+    mkdirSync(dirname(dbPath), { recursive: true });
+    manager = new SQLiteRawMessageManager({
+      dbPath,
+      enableVectorSearch: !isRawMessageChromaEnabled()
+    });
+    await manager.init();
+  }
+  return manager;
+}
+async function closeSQLiteRawMessageManager() {
+  if (!manager) {
+    return;
+  }
+  await manager.close();
+  manager = null;
+}
+function resolveEnv(env) {
+  if (env) return env;
+  return {
+    isTauriMode: () => process.env.IS_TAURI === "true" || typeof process.env.TAURI_MODE === "string",
+    getTauriDbPath: () => process.env.TAURI_DB_PATH ?? "",
+    getTauriDataDir: () => process.env.TAURI_DATA_DIR ?? ""
+  };
+}
+
+// src/storage/raw-message-store.ts
+function resolveEnv2(env) {
+  if (env) return env;
+  return {
+    isTauriMode: () => process.env.IS_TAURI === "true" || typeof process.env.TAURI_MODE === "string",
+    getTauriDbPath: () => process.env.TAURI_DB_PATH ?? "",
+    getTauriDataDir: () => process.env.TAURI_DATA_DIR ?? ""
+  };
+}
+function createRawMessageStore(options = {}) {
+  const env = resolveEnv2(options.env);
+  return {
+    getBackend() {
+      return isSQLiteRawMessageStorageAvailable(env) ? "sqlite" : "postgres";
+    },
+    isAvailable() {
+      return isSQLiteRawMessageStorageAvailable(env) || hasPostgresFactory();
+    },
+    async getManager() {
+      if (isSQLiteRawMessageStorageAvailable(env)) {
+        return await getSQLiteRawMessageManager(env);
+      }
+      const pg = await resolvePostgresFactory();
+      if (!pg) {
+        throw new Error(
+          "No raw-message backend available. SQLite is disabled (not Tauri) and no Postgres factory is registered."
+        );
+      }
+      return pg;
+    },
+    async close() {
+      if (isSQLiteRawMessageStorageAvailable(env)) {
+        await closeSQLiteRawMessageManager();
+      }
+    }
+  };
+}
+var moduleStore = null;
+var moduleConfig = null;
+function isRawMessageStorageAvailable() {
+  if (!moduleStore) {
+    moduleStore = createRawMessageStore({ env: moduleConfig?.env });
+  }
+  return moduleStore.isAvailable();
+}
+
+// src/search/utilities.ts
+var DEFAULT_LIMIT = 10;
+var DEFAULT_THRESHOLD = 0.7;
+var DEFAULT_SOURCES = ["memory", "insights", "knowledge"];
+var SOURCE_SET = new Set(DEFAULT_SOURCES);
+function normalizeUnifiedMemorySearchSources(sources) {
+  if (!Array.isArray(sources) || sources.length === 0) {
+    return [...DEFAULT_SOURCES];
+  }
+  const normalized = sources.filter((source) => typeof source === "string").map((source) => source.trim().toLowerCase()).filter(
+    (source) => SOURCE_SET.has(source)
+  );
+  return normalized.length > 0 ? Array.from(new Set(normalized)) : [...DEFAULT_SOURCES];
+}
+function clampUnifiedMemorySearchLimit(limit) {
+  const parsed = typeof limit === "number" ? limit : Number(limit ?? DEFAULT_LIMIT);
+  if (!Number.isFinite(parsed)) {
+    return DEFAULT_LIMIT;
+  }
+  return Math.min(50, Math.max(1, Math.floor(parsed)));
+}
+function clampUnifiedMemorySearchThreshold(threshold) {
+  const parsed = typeof threshold === "number" ? threshold : Number(threshold ?? DEFAULT_THRESHOLD);
+  if (!Number.isFinite(parsed)) {
+    return DEFAULT_THRESHOLD;
+  }
+  return Math.min(1, Math.max(-1, parsed));
+}
+function mergeUnifiedMemorySearchResults(results, limit) {
+  return [...results].sort((a, b) => {
+    const scoreDelta = b.similarity - a.similarity;
+    if (scoreDelta !== 0) {
+      return scoreDelta;
+    }
+    return a.type.localeCompare(b.type) || a.id.localeCompare(b.id);
+  }).slice(0, limit);
+}
+function toKnowledgeResult(result) {
+  return {
+    type: "knowledge",
+    id: result.chunkId,
+    content: result.content,
+    similarity: result.similarity,
+    metadata: {
+      documentId: result.documentId,
+      documentName: result.documentName,
+      chunkIndex: result.chunkIndex
+    }
+  };
+}
+function toMemoryResult(result) {
+  return {
+    type: "memory",
+    id: result.id,
+    content: result.content,
+    similarity: result.similarity,
+    metadata: result.metadata
+  };
+}
+function isRawMemorySemanticResult(result) {
+  if (!result || typeof result !== "object") {
+    return false;
+  }
+  const item = result;
+  return typeof item.id === "string" && typeof item.content === "string" && typeof item.similarity === "number" && Boolean(item.metadata) && typeof item.metadata === "object";
+}
+
+// src/search/unified-search.ts
+function createUnifiedSearch(deps = {}) {
+  const logger = deps.logger ?? console;
+  async function searchUnifiedMemory(input) {
+    const query = input.query.trim();
+    const sources = normalizeUnifiedMemorySearchSources(input.sources);
+    const limit = clampUnifiedMemorySearchLimit(input.limit);
+    const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
+    const warnings = [];
+    const results = [];
+    if (!query) {
+      return {
+        query,
+        sources,
+        results: [],
+        count: 0,
+        warnings
+      };
+    }
+    if (sources.includes("memory")) {
+      if (isRawMessageStorageAvailable()) {
+        try {
+          const memoryHits = await runMemorySource(input, limit, threshold);
+          results.push(...memoryHits);
+        } catch (error) {
+          logger.warn?.("[memory-store] memory source failed:", error);
+          warnings.push({
+            source: "memory",
+            code: "memory_search_failed",
+            message: error.message ?? "memory_search_failed"
+          });
+        }
+      } else {
+        warnings.push({
+          source: "memory",
+          code: "raw_message_storage_unavailable",
+          message: "Raw memory storage is not available in this environment."
+        });
+      }
+    }
+    if (sources.includes("insights") && typeof deps.searchInsights === "function") {
+      try {
+        const insightHits = await deps.searchInsights({
+          userId: input.userId,
+          query: input.query,
+          limit,
+          threshold,
+          botIds: input.botIds,
+          includeArchived: input.includeArchivedInsights,
+          authToken: input.authToken
+        });
+        for (const hit of insightHits) {
+          results.push({
+            type: "insight",
+            id: hit.id,
+            content: hit.content,
+            similarity: hit.similarity,
+            metadata: hit.metadata
+          });
+        }
+      } catch (error) {
+        logger.warn?.("[memory-store] insights source failed:", error);
+        warnings.push({
+          source: "insights",
+          code: "insights_search_failed",
+          message: error.message ?? "insights_search_failed"
+        });
+      }
+    } else if (sources.includes("insights")) {
+      warnings.push({
+        source: "insights",
+        code: "insights_search_not_configured",
+        message: "Insights search is not configured for this host."
+      });
+    }
+    if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
+      try {
+        const knowledgeHits = await deps.searchKnowledge({
+          userId: input.userId,
+          query: input.query,
+          options: {
+            limit,
+            threshold,
+            documentIds: input.documentIds
+          },
+          authToken: input.authToken
+        });
+        for (const hit of knowledgeHits) {
+          results.push(toKnowledgeResult(hit));
+        }
+      } catch (error) {
+        logger.warn?.("[memory-store] knowledge source failed:", error);
+        warnings.push({
+          source: "knowledge",
+          code: "knowledge_search_failed",
+          message: error.message ?? "knowledge_search_failed"
+        });
+      }
+    } else if (sources.includes("knowledge")) {
+      warnings.push({
+        source: "knowledge",
+        code: "knowledge_search_not_configured",
+        message: "Knowledge search is not configured for this host."
+      });
+    }
+    const merged = mergeUnifiedMemorySearchResults(results, limit);
+    return {
+      query: input.query,
+      sources,
+      results: merged,
+      count: merged.length,
+      warnings
+    };
+  }
+  async function runMemorySource(input, limit, threshold) {
+    if (typeof deps.embedQuery !== "function") {
+      throw new Error("embedQuery is not configured");
+    }
+    const queryEmbedding = await deps.embedQuery({
+      userId: input.userId,
+      query: input.query,
+      authToken: input.authToken
+    });
+    const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
+    if (isRawMessageChromaEnabled()) {
+      try {
+        const chromaHits = (await Promise.all(
+          filters.map((filter) => {
+            const botId = "botId" in filter ? filter.botId : void 0;
+            return searchRawMessagesWithChroma({
+              userId: input.userId,
+              queryEmbedding,
+              limit,
+              threshold,
+              botId
+            });
+          })
+        )).flat().map(toMemoryResult);
+        logger.log?.("[memory-store] Raw message semantic search completed", {
+          backend: "chroma",
+          dimensions: queryEmbedding.length,
+          count: chromaHits.length
+        });
+        return chromaHits;
+      } catch (error) {
+        logger.warn?.(
+          "[memory-store] Chroma raw message search failed; falling back to database search:",
+          error
+        );
+      }
+    }
+    if (typeof deps.searchRawMessagesAnn === "function") {
+      const rows = (await Promise.all(
+        filters.map(
+          (filter) => deps.searchRawMessagesAnn({
+            userId: input.userId,
+            queryEmbedding,
+            limit,
+            threshold,
+            botId: "botId" in filter ? filter.botId : void 0
+          })
+        )
+      )).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
+      return rows;
+    }
+    return [];
+  }
+  async function searchRawMemorySemantically(input) {
+    const limit = clampUnifiedMemorySearchLimit(input.limit);
+    const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
+    return runMemorySource(input, limit, threshold);
+  }
+  return { searchUnifiedMemory, searchRawMemorySemantically };
+}
+
+// src/http.ts
+async function startHttpServer(options = {}) {
+  const port = options.port ?? Number.parseInt(process.env.MEMORY_HTTP_PORT ?? "7421", 10);
+  const host = options.host ?? process.env.MEMORY_HTTP_HOST ?? "127.0.0.1";
+  const rawStore = createRawMessageStore({
+    env: options.env
+  });
+  const search = createUnifiedSearch(options.unified);
+  const app = new Hono();
+  app.get("/health", (c) => c.json({ ok: true, store: "memory", ts: Date.now() }));
+  app.post("/v1/search", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const userId = typeof body.userId === "string" ? body.userId : null;
+    const query = typeof body.query === "string" ? body.query : "";
+    if (!userId || !query) {
+      return c.json({ error: "userId and query are required" }, 400);
+    }
+    const result = await search.searchUnifiedMemory({
+      userId,
+      query,
+      sources: body.sources,
+      limit: body.limit,
+      threshold: body.threshold,
+      botIds: body.botIds,
+      documentIds: body.documentIds,
+      includeArchivedInsights: body.includeArchivedInsights,
+      authToken: body.authToken
+    });
+    return c.json(result);
+  });
+  app.post("/v1/raw-messages", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    if (!Array.isArray(body.messages) || !body.userId) {
+      return c.json({ error: "userId and messages[] required" }, 400);
+    }
+    const manager2 = await rawStore.getManager();
+    const result = await manager2.upsertRawMessages?.({
+      userId: body.userId,
+      messages: body.messages
+    });
+    try {
+      await upsertRawMessagesToChroma(body.messages);
+    } catch (error) {
+      console.warn("[memory-store/http] chroma upsert failed:", error);
+    }
+    return c.json({ ok: true, result });
+  });
+  app.get("/v1/raw-messages/:id", async (c) => {
+    const id = c.req.param("id");
+    const userId = c.req.query("userId");
+    if (!userId) return c.json({ error: "userId query param required" }, 400);
+    const manager2 = await rawStore.getManager();
+    const row = await manager2.getMessageById?.(id);
+    if (!row) return c.json({ error: "not found" }, 404);
+    return c.json({ message: row });
+  });
+  const server = serve({ fetch: app.fetch, port, hostname: host });
+  return {
+    url: `http://${host}:${port}`,
+    port,
+    async stop() {
+      await rawStore.close();
+      server.close();
+    }
+  };
+}
+
+// src/server/cli-http.ts
+function parseArgs(argv) {
+  const args = {
+    port: Number.parseInt(process.env.MEMORY_HTTP_PORT ?? "7421", 10),
+    host: process.env.MEMORY_HTTP_HOST ?? "127.0.0.1"
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    const next = argv[i + 1];
+    if (arg === "--port" && next) {
+      args.port = Number.parseInt(next, 10);
+      i += 1;
+    } else if (arg === "--host" && next) {
+      args.host = next;
+      i += 1;
+    } else if (arg === "--help" || arg === "-h") {
+      console.log("Usage: opencontext-memory-http [--port <port>] [--host <host>]");
+      process.exit(0);
+    }
+  }
+  return args;
+}
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const { url, port, stop } = await startHttpServer({
+    port: args.port,
+    host: args.host
+  });
+  console.log(`[memory-store/http] listening at ${url}`);
+  const shutdown = async (signal) => {
+    console.log(`[memory-store/http] ${signal} received, shutting down\u2026`);
+    await stop();
+    process.exit(0);
+  };
+  process.once("SIGINT", shutdown);
+  process.once("SIGTERM", shutdown);
+}
+main().catch((error) => {
+  console.error("[memory-store/http] fatal:", error);
+  process.exit(1);
+});
+//# sourceMappingURL=cli-http.js.map
+//# sourceMappingURL=cli-http.js.map