@melandlabs/memory-store 1.2.5 → 1.3.0

package/dist/server/cli-shared.d.ts

@@ -1,10 +1,11 @@
 #!/usr/bin/env node
-import { U as UnifiedSearchDeps } from '../config-Dw4aXj2a.js';
+import { U as UnifiedSearchDeps } from '../config-CbE2t4SW.js';
 import '@melandlabs/memory-consolidation';
 import '@melandlabs/contracts/peer';
 import '@melandlabs/contracts';
 import '@melandlabs/contracts/derived-fact';
 import '@melandlabs/contracts/entity-edge';
+import '@melandlabs/indexeddb';
 
 /**
  * Shared argv parsing + `unified.*` wiring for the memory-store CLI bins.
@@ -15,18 +16,30 @@ import '@melandlabs/contracts/entity-edge';
  * `/v1/search` route does, so they must speak the same config. This
  * module owns that surface so the two bins stay in lock-step.
  *
- * `--embedding-provider local` and every `--*-backend=chroma` value
- * require `@melandlabs/ai-rag` (a peer install — pulling it in adds
- * `@huggingface/transformers`, `chromadb`, and ~30 MB of ONNX weights
- * on first run). The bin errors with a clear remediation message if the
- * package is missing.
+ * `--embedding-provider local` and the insight/knowledge Chroma backends
+ * require `@melandlabs/ai-rag`. Raw-message Chroma, LanceDB, and Milvus use
+ * the existing `@melandlabs/rag` stores. Missing optional dependencies fail
+ * at startup with an actionable error.
  */
 
 interface UnifiedArgs {
     embeddingProvider: "local" | "openrouter" | "none";
     embeddingModel?: string;
+    embeddingCacheDir?: string;
+    rerankerProvider: "local" | "none";
+    rerankerModel?: string;
+    rerankerCacheDir?: string;
+    rerankerBatchSize?: number;
+    rerankerMaxTokens?: number;
     chromaUrl?: string;
-    memoryBackend: "sqlite-vec" | "chroma" | "none";
+    memoryBackend: "sqlite-vec" | "chroma" | "lancedb" | "milvus" | "none";
+    lancedbUri?: string;
+    lancedbTable?: string;
+    milvusAddress?: string;
+    milvusToken?: string;
+    milvusDatabase?: string;
+    milvusCollection?: string;
+    milvusDimension?: number;
     insightsBackend: "sqlite-vec" | "chroma" | "none";
     insightsCollection: string;
     knowledgeBackend: "chroma" | "none";

package/dist/server/cli-shared.js

@@ -1,4 +1,5 @@
 #!/usr/bin/env node
+import { ChromaVectorStore } from '@melandlabs/rag/chroma-vector-store';
 import { mkdirSync } from 'fs';
 import { dirname } from 'path';
 import { getOpenContextPath } from '@melandlabs/env-config';
@@ -346,6 +347,121 @@ function createUserVoiceRewriter(options) {
   };
 }
 
+// src/storage/raw-message-child-vector-index.ts
+var RawMessageChildVectorIndex = class {
+  backend;
+  store;
+  catalog;
+  constructor(options) {
+    this.backend = options.backend;
+    this.store = options.store;
+    this.catalog = options.catalog;
+  }
+  async replaceMessages(messages, chunks) {
+    const chunksByMessage = /* @__PURE__ */ new Map();
+    for (const chunk of chunks) {
+      const group = chunksByMessage.get(chunk.messageId) ?? [];
+      group.push(chunk);
+      chunksByMessage.set(chunk.messageId, group);
+    }
+    for (const message of messages) {
+      await this.store.deleteDocument(message.messageId);
+      const vectorChunks = (chunksByMessage.get(message.messageId) ?? []).filter(
+        (chunk) => Boolean(chunk.embedding?.length) && !chunk.embedding?.every((value) => value === 0)
+      ).map((chunk) => ({
+        id: chunk.chunkId,
+        documentId: message.messageId,
+        content: chunk.content,
+        embedding: chunk.embedding,
+        metadata: {
+          userId: message.userId,
+          botId: message.botId,
+          platform: message.platform,
+          timestamp: message.timestamp,
+          chunkIndex: chunk.chunkIndex,
+          chunkCount: chunk.chunkCount,
+          startPosition: chunk.startPosition,
+          endPosition: chunk.endPosition,
+          contentHash: chunk.contentHash,
+          embeddingModel: chunk.embeddingModel
+        }
+      }));
+      if (vectorChunks.length > 0) await this.store.addChunks(vectorChunks);
+    }
+  }
+  async search(input) {
+    const results = input.hybrid && this.store.hybridSearch ? await this.store.hybridSearch({
+      text: input.query ?? "",
+      vector: input.queryEmbedding,
+      limit: input.limit,
+      candidateLimit: input.limit,
+      fusion: "rrf",
+      filter: { userId: input.userId }
+    }) : await this.store.similaritySearch(input.queryEmbedding, input.limit, input.userId);
+    return this.hydrate(results, input);
+  }
+  async getChunkCount() {
+    return this.store.getChunkCount();
+  }
+  async hydrate(results, input) {
+    const matchedChunks = await this.catalog.getRawMessageSearchChunks({
+      chunkIds: results.map((result) => result.id),
+      userId: input.userId
+    });
+    const matchedById = new Map(matchedChunks.map((chunk) => [chunk.chunkId, chunk]));
+    const messageIds = [...new Set(matchedChunks.map((chunk) => chunk.messageId))];
+    const [allChunks, parents] = await Promise.all([
+      this.catalog.getRawMessageSearchChunks({ messageIds, userId: input.userId }),
+      Promise.all(messageIds.map((messageId) => this.catalog.getMessageById(messageId)))
+    ]);
+    const parentById = new Map(
+      parents.filter((parent) => Boolean(parent)).map((parent) => [parent.messageId, parent])
+    );
+    const chunksByMessage = /* @__PURE__ */ new Map();
+    for (const chunk of allChunks) {
+      const group = chunksByMessage.get(chunk.messageId) ?? [];
+      group.push(chunk);
+      chunksByMessage.set(chunk.messageId, group);
+    }
+    const strongest = /* @__PURE__ */ new Map();
+    for (const result of results) {
+      const chunk = matchedById.get(result.id);
+      if (!chunk) continue;
+      const parent = parentById.get(chunk.messageId);
+      if (!parent || input.botId && parent.botId !== input.botId) continue;
+      if (input.threshold !== void 0 && result.score < input.threshold) continue;
+      const siblings = (chunksByMessage.get(parent.messageId) ?? []).sort(
+        (a, b) => a.chunkIndex - b.chunkIndex
+      );
+      const content = chunk.chunkCount <= 1 ? parent.content : parent.content.slice(
+        siblings[Math.max(0, chunk.chunkIndex - 1)]?.startPosition ?? chunk.startPosition,
+        siblings[Math.min(siblings.length - 1, chunk.chunkIndex + 1)]?.endPosition ?? chunk.endPosition
+      );
+      const hit = {
+        id: parent.messageId,
+        content,
+        similarity: result.score,
+        metadata: {
+          ...parent.metadata ?? {},
+          userId: parent.userId,
+          botId: parent.botId,
+          platform: parent.platform,
+          timestamp: parent.timestamp,
+          sourceMessageId: parent.messageId,
+          sourceChunkId: chunk.chunkId,
+          sourceChunkIndex: chunk.chunkIndex,
+          sourceChunkCount: chunk.chunkCount,
+          backend: this.backend,
+          scoring: input.hybrid ? "native-hybrid" : "dense"
+        }
+      };
+      const existing = strongest.get(parent.messageId);
+      if (!existing || hit.similarity > existing.similarity) strongest.set(parent.messageId, hit);
+    }
+    return [...strongest.values()].sort((a, b) => b.similarity - a.similarity).slice(0, input.limit);
+  }
+};
+
 // src/storage/postgres-raw-message-factory.ts
 var factory = null;
 async function resolvePostgresFactory(env) {
@@ -449,7 +565,7 @@ function isInsightSQLiteVecEnabled(_env) {
 }
 async function getInsightSQLiteVecStore(_env) {
   const dbPath = resolveSQLiteRawMessageDbPath();
-  const { getSQLiteVecStore } = await import('@melandlabs/rag/sqlite-vec-store');
+  const { getSQLiteVecStore } = await import('@melandlabs/ai-rag/sqlite-vec-store');
   return await getSQLiteVecStore(dbPath, void 0, {
     collectionName: process.env.SQLITE_VEC_INSIGHTS_COLLECTION || "opencontext_insights"
   });
@@ -473,17 +589,19 @@ async function searchInsightsWithSQLiteVec(input, env) {
 // src/server/cli-shared.ts
 async function loadAiRag() {
   try {
-    const [local, chroma] = await Promise.all([
+    const [local, reranker, chroma] = await Promise.all([
       import('@melandlabs/ai-rag/local-transformers-embedding-provider'),
+      import('@melandlabs/ai-rag/local-transformers-reranker'),
       import('@melandlabs/ai-rag/chroma-store')
     ]);
     return {
       LocalTransformersEmbeddingProvider: local.LocalTransformersEmbeddingProvider,
+      LocalTransformersReranker: reranker.LocalTransformersReranker,
       ChromaVectorStore: chroma.ChromaVectorStore
     };
   } catch (error) {
     throw new Error(
-      `--embedding-provider local / --*-backend=chroma require @melandlabs/ai-rag to be installed. Run \`pnpm add @melandlabs/ai-rag\` and try again. (${error.message})`
+      `Local embedding/reranking and insight/knowledge Chroma require @melandlabs/ai-rag to be installed. Run \`pnpm add @melandlabs/ai-rag\` and try again. (${error.message})`
     );
   }
 }
@@ -492,8 +610,21 @@ function parseUnifiedArgs(argv) {
   const args = {
     embeddingProvider: env.EMBEDDING_PROVIDER ?? "none",
     embeddingModel: env.EMBEDDING_MODEL,
+    embeddingCacheDir: env.LOCAL_EMBEDDING_CACHE_DIR,
+    rerankerProvider: env.RERANKER_PROVIDER ?? "none",
+    rerankerModel: env.LOCAL_RERANKER_MODEL,
+    rerankerCacheDir: env.LOCAL_RERANKER_CACHE_DIR,
+    rerankerBatchSize: env.LOCAL_RERANKER_BATCH_SIZE ? Number.parseInt(env.LOCAL_RERANKER_BATCH_SIZE, 10) : void 0,
+    rerankerMaxTokens: env.LOCAL_RERANKER_MAX_TOKENS ? Number.parseInt(env.LOCAL_RERANKER_MAX_TOKENS, 10) : void 0,
     chromaUrl: env.CHROMA_URL,
     memoryBackend: env.MEMORY_BACKEND ?? "none",
+    lancedbUri: env.LANCEDB_URI,
+    lancedbTable: env.LANCEDB_TABLE,
+    milvusAddress: env.MILVUS_ADDRESS,
+    milvusToken: env.MILVUS_TOKEN,
+    milvusDatabase: env.MILVUS_DATABASE,
+    milvusCollection: env.MILVUS_COLLECTION,
+    milvusDimension: env.MILVUS_DIMENSION ? Number.parseInt(env.MILVUS_DIMENSION, 10) : void 0,
     insightsBackend: env.INSIGHTS_BACKEND ?? "none",
     insightsCollection: env.INSIGHTS_COLLECTION ?? "opencontext_insights",
     knowledgeBackend: env.KNOWLEDGE_BACKEND ?? "none",
@@ -518,12 +649,51 @@ function parseUnifiedArgs(argv) {
       case "--embedding-model":
         args.embeddingModel = takeValue();
         break;
+      case "--embedding-cache-dir":
+        args.embeddingCacheDir = takeValue();
+        break;
+      case "--reranker-provider":
+        args.rerankerProvider = takeValue();
+        break;
+      case "--reranker-model":
+        args.rerankerModel = takeValue();
+        break;
+      case "--reranker-cache-dir":
+        args.rerankerCacheDir = takeValue();
+        break;
+      case "--reranker-batch-size":
+        args.rerankerBatchSize = Number.parseInt(takeValue(), 10);
+        break;
+      case "--reranker-max-tokens":
+        args.rerankerMaxTokens = Number.parseInt(takeValue(), 10);
+        break;
       case "--chroma-url":
         args.chromaUrl = takeValue();
         break;
       case "--memory-backend":
         args.memoryBackend = takeValue();
         break;
+      case "--lancedb-uri":
+        args.lancedbUri = takeValue();
+        break;
+      case "--lancedb-table":
+        args.lancedbTable = takeValue();
+        break;
+      case "--milvus-address":
+        args.milvusAddress = takeValue();
+        break;
+      case "--milvus-token":
+        args.milvusToken = takeValue();
+        break;
+      case "--milvus-database":
+        args.milvusDatabase = takeValue();
+        break;
+      case "--milvus-collection":
+        args.milvusCollection = takeValue();
+        break;
+      case "--milvus-dimension":
+        args.milvusDimension = Number.parseInt(takeValue(), 10);
+        break;
       case "--insights-backend":
         args.insightsBackend = takeValue();
         break;
@@ -553,31 +723,60 @@ function parseUnifiedArgs(argv) {
         break;
     }
   }
-  for (const [name, value] of [
-    ["--embedding-provider", args.embeddingProvider],
-    ["--memory-backend", args.memoryBackend],
-    ["--insights-backend", args.insightsBackend],
-    ["--knowledge-backend", args.knowledgeBackend]
-  ]) {
-    const allowed = ["local", "openrouter", "none", "chroma", "sqlite-vec"];
+  const validate = (name, value, allowed) => {
     if (!allowed.includes(value)) {
       throw new Error(`[memory-store/cli] ${name} must be one of: ${allowed.join(", ")} (got "${value}")`);
     }
+  };
+  validate("--embedding-provider", args.embeddingProvider, ["local", "openrouter", "none"]);
+  validate("--reranker-provider", args.rerankerProvider, ["local", "none"]);
+  validate("--memory-backend", args.memoryBackend, ["sqlite-vec", "chroma", "lancedb", "milvus", "none"]);
+  validate("--insights-backend", args.insightsBackend, ["sqlite-vec", "chroma", "none"]);
+  validate("--knowledge-backend", args.knowledgeBackend, ["chroma", "none"]);
+  if (args.milvusDimension !== void 0 && (!Number.isInteger(args.milvusDimension) || args.milvusDimension <= 0)) {
+    throw new Error("[memory-store/cli] --milvus-dimension must be a positive integer");
+  }
+  for (const [name, value] of [
+    ["--reranker-batch-size", args.rerankerBatchSize],
+    ["--reranker-max-tokens", args.rerankerMaxTokens]
+  ]) {
+    if (value !== void 0 && (!Number.isInteger(value) || value <= 0)) {
+      throw new Error(`[memory-store/cli] ${name} must be a positive integer`);
+    }
   }
   return args;
 }
 async function buildUnified(args) {
   const unified = {};
   const log = (msg) => console.warn(`[memory-store/cli] ${msg}`);
-  const wantsAiRag = args.embeddingProvider === "local" || args.memoryBackend === "chroma" || args.insightsBackend === "chroma" || args.knowledgeBackend === "chroma";
+  const wantsAiRag = args.embeddingProvider === "local" || args.rerankerProvider === "local" || args.insightsBackend === "chroma" || args.knowledgeBackend === "chroma";
   const aiRag = wantsAiRag ? await loadAiRag() : null;
   if (args.embeddingProvider === "local") {
     try {
       if (!aiRag) {
         throw new Error("ai-rag modules not loaded");
       }
-      const provider = new aiRag.LocalTransformersEmbeddingProvider({ modelName: args.embeddingModel });
-      unified.embedQuery = async ({ query }) => provider.embedQuery(query);
+      const provider = new aiRag.LocalTransformersEmbeddingProvider({
+        modelName: args.embeddingModel,
+        cacheDir: args.embeddingCacheDir
+      });
+      unified.embeddingInfo = {
+        provider: "local",
+        model: args.embeddingModel ?? "Xenova/all-MiniLM-L6-v2",
+        maxTokens: 512
+      };
+      unified.embedQuery = async ({ query }) => {
+        const embedding = await provider.embedQuery(query);
+        if (unified.embeddingInfo) unified.embeddingInfo.dimensions = embedding.length;
+        return embedding;
+      };
+      unified.embedDocuments = async ({ texts }) => {
+        const embeddings = await provider.embedDocuments(texts);
+        if (unified.embeddingInfo && embeddings[0]) {
+          unified.embeddingInfo.dimensions = embeddings[0].length;
+        }
+        return embeddings;
+      };
       log(
         `embedQuery wired via LocalTransformersEmbeddingProvider (model=${args.embeddingModel ?? "Xenova/all-MiniLM-L6-v2"})`
       );
@@ -593,6 +792,7 @@ async function buildUnified(args) {
     }
     const model = args.embeddingModel ?? "text-embedding-3-small";
     const baseURL = "https://openrouter.ai/api/v1";
+    unified.embeddingInfo = { provider: "openrouter", model };
     unified.embedQuery = async ({ query }) => {
       const res = await fetch(`${baseURL}/embeddings`, {
         method: "POST",
@@ -610,49 +810,150 @@ async function buildUnified(args) {
       const body = await res.json();
       const embedding = body.data?.[0]?.embedding;
       if (!embedding) throw new Error("openrouter embeddings response missing data[0].embedding");
+      if (unified.embeddingInfo) unified.embeddingInfo.dimensions = embedding.length;
       return embedding;
     };
     log(`embedQuery wired via openrouter (model=${model})`);
   }
-  if (args.memoryBackend === "sqlite-vec") {
-    const store = createRawMessageStore({ env: void 0 });
-    const manager2 = await store.getManager();
-    if (typeof manager2.searchMessagesSemantically !== "function") {
+  if (args.rerankerProvider === "local") {
+    if (!aiRag) throw new Error("Local reranker modules were not loaded");
+    const reranker = new aiRag.LocalTransformersReranker({
+      modelName: args.rerankerModel,
+      cacheDir: args.rerankerCacheDir,
+      batchSize: args.rerankerBatchSize,
+      maxTokens: args.rerankerMaxTokens
+    });
+    await reranker.warmup();
+    unified.reranker = reranker;
+    unified.rerankerInfo = {
+      provider: "local",
+      model: reranker.getModelName(),
+      maxTokens: reranker.getMaxTokens(),
+      ready: true
+    };
+    log(`reranker wired via local Transformers.js (model=${reranker.getModelName()})`);
+  } else {
+    unified.rerankerInfo = { provider: "none", ready: false };
+  }
+  if (args.memoryBackend !== "none") {
+    if (args.memoryBackend === "chroma" && !args.chromaUrl) {
+      throw new Error("--memory-backend=chroma requires --chroma-url <url> (or CHROMA_URL env)");
+    }
+    if (args.memoryBackend === "lancedb" && !args.lancedbUri) {
+      throw new Error("--memory-backend=lancedb requires --lancedb-uri <uri> (or LANCEDB_URI env)");
+    }
+    if (args.memoryBackend === "milvus" && !args.milvusAddress) {
+      throw new Error("--memory-backend=milvus requires --milvus-address <address> (or MILVUS_ADDRESS env)");
+    }
+    if (args.memoryBackend === "lancedb") {
+      try {
+        await import('@lancedb/lancedb');
+      } catch (error) {
+        throw new Error(
+          `LanceDB requires the optional dependency @lancedb/lancedb (${error.message})`
+        );
+      }
+    }
+    if (args.memoryBackend === "milvus") {
+      try {
+        await import('@zilliz/milvus2-sdk-node');
+      } catch (error) {
+        throw new Error(
+          `Milvus requires the optional dependency @zilliz/milvus2-sdk-node (${error.message})`
+        );
+      }
+    }
+    const rawStore = createRawMessageStore({ env: void 0 });
+    const manager2 = await rawStore.getManager();
+    if (typeof manager2.getRawMessageSearchChunks !== "function" || typeof manager2.getRawMessageSearchIndexStats !== "function") {
       throw new Error(
-        "the active raw-message manager does not implement searchMessagesSemantically; sqlite-vec ANN for the memory source is unavailable in this build"
+        `${args.memoryBackend} child retrieval requires the SQLite raw-message catalog; the Postgres manager remains on the legacy parent fallback in this release`
       );
     }
-    unified.searchRawMessagesAnn = async ({ userId, queryEmbedding, limit, threshold, botId }) => {
-      const rows = await manager2.searchMessagesSemantically({
-        userId,
-        queryEmbedding,
-        limit,
-        threshold,
-        botId
+    if (typeof manager2.lexicalSearchMessages === "function") {
+      unified.searchRawMessagesLexical = async (input) => {
+        const rows = await manager2.lexicalSearchMessages(input);
+        return rows.map((row) => ({ ...row, metadata: row.metadata ?? {} }));
+      };
+    }
+    let externalStore;
+    if (args.memoryBackend === "chroma") {
+      externalStore = new ChromaVectorStore({
+        url: args.chromaUrl,
+        collectionName: "opencontext_raw_message_chunks"
+      });
+    } else if (args.memoryBackend === "lancedb") {
+      const { LanceDBStore } = await import('@melandlabs/rag/lancedb-store');
+      externalStore = new LanceDBStore({
+        uri: args.lancedbUri,
+        tableName: args.lancedbTable ?? "opencontext_raw_message_chunks",
+        defaultFusion: "rrf",
+        candidateMultiplier: 4
+      });
+    } else if (args.memoryBackend === "milvus") {
+      const { MilvusStore } = await import('@melandlabs/rag/milvus-store');
+      externalStore = new MilvusStore({
+        address: args.milvusAddress,
+        token: args.milvusToken,
+        database: args.milvusDatabase,
+        collectionName: args.milvusCollection ?? "opencontext_raw_message_chunks",
+        dimension: args.milvusDimension,
+        defaultFusion: "rrf",
+        candidateMultiplier: 4
       });
-      return rows.map((r) => ({
-        id: r.id,
-        content: r.content,
-        similarity: r.similarity,
-        metadata: r.metadata ?? {}
-      }));
-    };
-    log("memory backend wired via sqlite-vec (uses the manager's searchMessagesSemantically)");
-  } else if (args.memoryBackend === "chroma") {
-    if (!args.chromaUrl)
-      throw new Error("--memory-backend=chroma requires --chroma-url <url> (or CHROMA_URL env)");
-    if (!aiRag) {
-      throw new Error("ai-rag modules not loaded");
     }
-    const store = new aiRag.ChromaVectorStore({
-      url: args.chromaUrl,
-      collectionName: "opencontext_raw_messages"
-    });
-    unified.searchRawMessagesAnn = async ({ queryEmbedding, limit, threshold }) => {
-      const results = await store.similaritySearchWithOptions(queryEmbedding, { limit: limit + 1 });
-      return results.filter((r) => r.score >= threshold).map((r) => ({ id: r.id, content: r.content, similarity: r.score, metadata: r.metadata ?? {} }));
+    if (args.memoryBackend === "sqlite-vec") {
+      if (typeof manager2.searchMessagesSemantically !== "function") {
+        throw new Error("sqlite-vec child search is unavailable in the active raw-message manager");
+      }
+      unified.searchRawMessagesAnn = async ({ userId, queryEmbedding, limit, threshold, botId }) => {
+        const rows = await manager2.searchMessagesSemantically({
+          userId,
+          queryEmbedding,
+          limit,
+          threshold,
+          botId
+        });
+        return rows.map((row) => ({ ...row, metadata: row.metadata ?? {} }));
+      };
+    } else if (externalStore) {
+      const childIndex = new RawMessageChildVectorIndex({
+        backend: args.memoryBackend,
+        store: externalStore,
+        catalog: {
+          getMessageById: (messageId) => manager2.getMessageById(messageId),
+          // biome-ignore lint/style/noNonNullAssertion: guarded above
+          getRawMessageSearchChunks: (input) => manager2.getRawMessageSearchChunks(input)
+        }
+      });
+      unified.rawMessageChildIndex = childIndex;
+      unified.searchRawMessagesAnn = (input) => childIndex.search(input);
+      if (args.memoryBackend !== "chroma") {
+        unified.searchRawMessagesHybrid = (input) => childIndex.search({ ...input, hybrid: true });
+      }
+    }
+    unified.getRawMessageRetrievalStatus = async () => {
+      const stats = await manager2.getRawMessageSearchIndexStats();
+      const embeddingDimensions = unified.embeddingInfo?.dimensions;
+      const dimensionMismatch = embeddingDimensions !== void 0 && stats.embeddingDimensions.length > 0 && !stats.embeddingDimensions.includes(embeddingDimensions);
+      const semanticReady = Boolean(unified.embedQuery) && (args.memoryBackend === "sqlite-vec" ? stats.semanticReady : stats.embeddedChunkCount > 0) && !dimensionMismatch;
+      return {
+        backend: args.memoryBackend,
+        embeddingProvider: unified.embeddingInfo?.provider,
+        embeddingModel: unified.embeddingInfo?.model,
+        embeddingDimensions,
+        childCount: stats.chunkCount,
+        embeddedChildCount: stats.embeddedChunkCount,
+        indexedDimensions: stats.embeddingDimensions,
+        semanticReady,
+        lexicalReady: stats.lexicalReady,
+        semanticDegradedReason: dimensionMismatch ? "semantic_dimension_mismatch" : !unified.embedQuery ? "semantic_unavailable" : stats.embeddedChunkCount === 0 ? "semantic_child_vectors_empty" : void 0,
+        rerankerProvider: unified.rerankerInfo?.provider,
+        rerankerModel: unified.rerankerInfo?.model,
+        rerankerReady: unified.rerankerInfo?.ready ?? false
+      };
     };
-    log("memory backend wired via chroma (collection=opencontext_raw_messages)");
+    log(`memory backend wired via ${args.memoryBackend} with SQLite child catalog and lexical search`);
   }
   if (args.insightsBackend === "sqlite-vec" && isInsightSQLiteVecEnabled()) {
     if (!unified.embedQuery) {
@@ -779,20 +1080,41 @@ function printUnifiedHelp() {
   --embedding-model <name>        Model name
                                   (env: EMBEDDING_MODEL; local \u2192 Xenova/all-MiniLM-L6-v2,
                                   openrouter \u2192 text-embedding-3-small)
+	--embedding-cache-dir <path>    Local embedding cache (env: LOCAL_EMBEDDING_CACHE_DIR)
   Note: "local" dynamically imports @melandlabs/ai-rag (a peer install).
 
+Reranking (after RRF/source fusion, before final Top-K):
+  --reranker-provider <name>      local | none
+                                  (env: RERANKER_PROVIDER, default: none)
+  --reranker-model <name>         Local sequence-classification model
+                                  (env: LOCAL_RERANKER_MODEL,
+                                  default: Xenova/ms-marco-MiniLM-L-6-v2)
+  --reranker-cache-dir <path>     Persistent model cache (env: LOCAL_RERANKER_CACHE_DIR)
+  --reranker-batch-size <int>     Pair scoring batch size (env: LOCAL_RERANKER_BATCH_SIZE,
+                                  default: 8)
+  --reranker-max-tokens <int>     Query/document pair token limit
+                                  (env: LOCAL_RERANKER_MAX_TOKENS, default: 512)
+
 Cross-source search (wires unified.searchKnowledge / searchInsights / searchRawMessagesAnn):
   --chroma-url <url>              Chroma server URL
                                   (env: CHROMA_URL; required when any *-backend=chroma)
-  --memory-backend <name>         sqlite-vec | chroma | none
+  --memory-backend <name>         sqlite-vec | chroma | lancedb | milvus | none
                                   (env: MEMORY_BACKEND, default: none)
+  --lancedb-uri <uri>             LanceDB directory or URI (env: LANCEDB_URI)
+  --lancedb-table <name>          Optional table name (env: LANCEDB_TABLE)
+  --milvus-address <address>      Milvus endpoint (env: MILVUS_ADDRESS)
+  --milvus-token <token>          Optional Milvus token (env: MILVUS_TOKEN)
+  --milvus-database <name>        Optional database (env: MILVUS_DATABASE)
+  --milvus-collection <name>      Optional collection (env: MILVUS_COLLECTION)
+  --milvus-dimension <int>        Optional vector dimension (env: MILVUS_DIMENSION)
   --insights-backend <name>       sqlite-vec | chroma | none
                                   (env: INSIGHTS_BACKEND, default: none)
   --insights-collection <name>    Chroma collection (default: opencontext_insights)
   --knowledge-backend <name>      chroma | none
                                   (env: KNOWLEDGE_BACKEND, default: none)
   --knowledge-collection <name>   Chroma collection (default: opencontext_knowledge)
-  Note: the chroma backends dynamically import @melandlabs/ai-rag.
+  Notes: memory Chroma uses @melandlabs/rag; insight/knowledge Chroma use
+  @melandlabs/ai-rag. LanceDB and Milvus require their optional peer dependencies.
 
 Reasoning (wires unified.reasoning.{queryRewriter, iterativePlanner}):
   --reasoning                     Enable the LLM reasoning layer so /v1/search and

package/dist/storage/postgres-raw-message-factory.d.ts

@@ -1,9 +1,10 @@
-import { a as MemoryStoreEnv } from '../config-Dw4aXj2a.js';
+import { a as MemoryStoreEnv } from '../config-CbE2t4SW.js';
 import '@melandlabs/memory-consolidation';
 import '@melandlabs/contracts/peer';
 import '@melandlabs/contracts';
 import '@melandlabs/contracts/derived-fact';
 import '@melandlabs/contracts/entity-edge';
+import '@melandlabs/indexeddb';
 
 /**
  * Lazy registration slot for the host's Postgres-backed raw message

package/dist/storage/raw-message-store.d.ts

@@ -1,5 +1,5 @@
-import { RawMessageStorageManager } from '@melandlabs/indexeddb';
-import { a as MemoryStoreEnv, M as MemoryStoreConfig } from '../config-Dw4aXj2a.js';
+import { RawMessageStorageManager, RawMessageSearchChunk, RawMessageSearchIndexStats } from '@melandlabs/indexeddb';
+import { a as MemoryStoreEnv, M as MemoryStoreConfig } from '../config-CbE2t4SW.js';
 import '@melandlabs/memory-consolidation';
 import '@melandlabs/contracts/peer';
 import '@melandlabs/contracts';
@@ -24,6 +24,12 @@ import '@melandlabs/contracts/entity-edge';
 
 type RawMessageStorageBackend = "sqlite" | "postgres";
 type RawMessageStorageManagerWithSearch = RawMessageStorageManager & {
+    getRawMessageSearchChunks?: (input: {
+        chunkIds?: string[];
+        messageIds?: string[];
+        userId?: string;
+    }) => Promise<RawMessageSearchChunk[]>;
+    getRawMessageSearchIndexStats?: () => Promise<RawMessageSearchIndexStats>;
     searchMessagesSemantically?: (input: {
         userId: string;
         queryEmbedding: number[];
@@ -32,6 +38,12 @@ type RawMessageStorageManagerWithSearch = RawMessageStorageManager & {
         scanLimit?: number;
         threshold?: number;
         includeArchived?: boolean;
+        /**
+         * Forward `includeDeprecated` from the unified search input so
+         * supersession-aware callers can opt into deprecated rows for
+         * audits. Default `false` preserves the current-truth behaviour.
+         */
+        includeDeprecated?: boolean;
         platform?: string;
         botId?: string;
         channel?: string;
@@ -44,6 +56,12 @@ type RawMessageStorageManagerWithSearch = RawMessageStorageManager & {
         keywords: string[];
         limit?: number;
         includeArchived?: boolean;
+        /**
+         * Forward `includeDeprecated` from the unified search input so
+         * supersession-aware callers can opt into deprecated rows for
+         * audits. Default `false` preserves the current-truth behaviour.
+         */
+        includeDeprecated?: boolean;
         platform?: string;
         botId?: string;
     }) => Promise<unknown[]>;

package/dist/storage/sqlite-raw-message-store.d.ts

@@ -1,10 +1,11 @@
 import { SQLiteRawMessageManager } from '@melandlabs/sqlite';
-import { a as MemoryStoreEnv } from '../config-Dw4aXj2a.js';
+import { a as MemoryStoreEnv } from '../config-CbE2t4SW.js';
 import '@melandlabs/memory-consolidation';
 import '@melandlabs/contracts/peer';
 import '@melandlabs/contracts';
 import '@melandlabs/contracts/derived-fact';
 import '@melandlabs/contracts/entity-edge';
+import '@melandlabs/indexeddb';
 
 /**
  * SQLite-backed raw message store.
@@ -48,6 +49,12 @@ declare function lexicalSearchRawMessages(input: {
     keywords: string[];
     limit?: number;
     includeArchived?: boolean;
+    /**
+     * Forward `includeDeprecated` from `UnifiedMemorySearchInput` so
+     * supersession-aware callers can opt into deprecated rows for
+     * audits. Default `false` preserves the current-truth behaviour.
+     */
+    includeDeprecated?: boolean;
     platform?: string;
     botId?: string;
     factTypes?: Array<"world" | "experience" | "mental_model">;