@melandlabs/memory-store 1.1.8 → 1.2.0

package/dist/server/cli-http.js

@@ -1,5 +1,5 @@
 #!/usr/bin/env node
-import { vsaNormalizeVector, vsaAssertSameDim } from '@melandlabs/contracts';
+import { isFactType, vsaNormalizeVector, vsaAssertSameDim } from '@melandlabs/contracts';
 import { buildMemoryRecordEmbeddingDocument } from '@melandlabs/ai/memory';
 import { ChromaVectorStore } from '@melandlabs/rag';
 import { mkdirSync } from 'fs';
@@ -9,6 +9,8 @@ import { getSQLiteVsaStore, closeSQLiteVsaStore, SQLiteRawMessageManager } from
 import { serve } from '@hono/node-server';
 import { Hono } from 'hono';
 import { applyReflectedConsolidationPlan } from '@melandlabs/memory-consolidation';
+import { isDerivedKind } from '@melandlabs/contracts/derived-fact';
+import { isEntityKind } from '@melandlabs/contracts/entity-edge';
 import { superpose, bind, unbind, cleanup, cosineSimilarity } from '@melandlabs/vsa';
 
 var __defProp = Object.defineProperty;
@@ -411,6 +413,9 @@ async function applyReranker(reranker, query, results) {
 }
 
 // src/search/utilities.ts
+function deriveLexicalKeywords(query) {
+  return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
+}
 var DEFAULT_LIMIT = 10;
 var DEFAULT_THRESHOLD = 0.7;
 var DEFAULT_SOURCES = ["memory", "insights", "knowledge"];
@@ -524,6 +529,50 @@ function toMemoryResult(result) {
     metadata: result.metadata
   };
 }
+function materializeSignals(lists, hit) {
+  const channels = [];
+  let semantic;
+  let lexical;
+  let entity;
+  for (const list of lists) {
+    const channel = listNameToChannel(list.name);
+    if (!channel) continue;
+    if (channels.includes(channel)) continue;
+    let found;
+    for (const candidate of list.hits) {
+      if (candidate.type === hit.type && candidate.id === hit.id) {
+        found = candidate;
+        break;
+      }
+    }
+    if (!found) continue;
+    channels.push(channel);
+    if (channel === "semantic" && semantic === void 0) {
+      semantic = found.similarity;
+    } else if (channel === "lexical" && lexical === void 0) {
+      lexical = found.similarity;
+    } else if (channel === "entity" && entity === void 0) {
+      entity = found.similarity;
+    }
+  }
+  if (channels.length === 0) {
+    return void 0;
+  }
+  const out = { channels };
+  if (semantic !== void 0) out.semantic = semantic;
+  if (lexical !== void 0) out.lexical = lexical;
+  if (entity !== void 0) out.entity = entity;
+  if (typeof hit.metadata.rrfScore === "number") {
+    out.rrf = hit.metadata.rrfScore;
+  }
+  return out;
+}
+function listNameToChannel(name) {
+  if (name === "memory-semantic") return "semantic";
+  if (name === "memory-bm25" || name === "memory-lexical") return "lexical";
+  if (name === "memory-entity") return "entity";
+  return void 0;
+}
 function isRawMemorySemanticResult(result) {
   if (!result || typeof result !== "object") {
     return false;
@@ -791,8 +840,65 @@ async function resolveSearchScopePeers(input) {
 }
 
 // src/search/unified-search.ts
-function deriveLexicalKeywords(query) {
-  return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
+function deriveEntityKeywords(query) {
+  const proper = Array.from(new Set(query.match(/\b[A-Z][a-zA-Z]{2,}\b/g) ?? [])).map((w) => w.toLowerCase());
+  const lexical = deriveLexicalKeywords(query);
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const word of [...proper, ...lexical]) {
+    if (word.length < 2) continue;
+    if (seen.has(word)) continue;
+    seen.add(word);
+    out.push(word);
+  }
+  return out.slice(0, 16);
+}
+async function runEntitySearchForQuery(deps, input, limit, logger, peerPeers = []) {
+  if (typeof deps.entitySearch !== "function") {
+    return [];
+  }
+  const keywords = deriveEntityKeywords(input.query);
+  if (keywords.length === 0) {
+    return [];
+  }
+  try {
+    const entitySearch = deps.entitySearch;
+    const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
+    const matches = (await Promise.all(
+      filters.map(
+        (botId) => entitySearch({
+          userId: input.userId,
+          keywords,
+          limit: Math.ceil(limit / filters.length),
+          botId,
+          ...peerPeers.length > 0 ? { peers: peerPeers } : {}
+        })
+      )
+    )).flat();
+    return matches.filter(
+      (m) => typeof m?.messageId === "string" && typeof m?.label === "string" && typeof m?.score === "number"
+    ).map((m) => ({
+      type: "memory",
+      id: m.messageId,
+      // The host's `entitySearch` returns match ids + labels only,
+      // so we surface the entity label as `content`. When the same
+      // `messageId` is also returned by the semantic/lexical channel
+      // (RRF path), the real message content is preserved via the
+      // shared `(type, id)` key; otherwise callers can re-fetch the
+      // original message by `id` (set in metadata as `sourceMessageId`).
+      content: m.label,
+      similarity: m.score,
+      metadata: {
+        scoring: "entity",
+        entityLabel: m.label,
+        isEntityProjection: true,
+        sourceMessageId: m.messageId
+      }
+    }));
+  } catch (error) {
+    logger.warn?.("[memory-store] entity search failed:", error);
+    return [];
+  }
 }
 function parseDateBoundary(value, role) {
   const trimmed = value.trim();
@@ -1232,10 +1338,39 @@ function createUnifiedSearch(deps = {}) {
         });
       }
     }
-    return {
+    let entity;
+    if (typeof deps.entitySearch === "function") {
+      entity = filterByDateRange(
+        await runEntitySearchForQuery(deps, input, limit, logger, peerPeers),
+        input.dateFrom,
+        input.dateTo
+      );
+      if (entity.length === 0) {
+        entity = void 0;
+      } else if (normalizeUnifiedMemoryMergeStrategy(
+        input.mergeStrategy ?? deps.reasoning?.defaultMergeStrategy ?? "rrf"
+      ) !== "rrf") {
+        warnings.push({
+          source: "memory",
+          code: "memory_entity_requires_rrf",
+          message: "Entity channel results are fused via RRF; with mergeStrategy:'similarity' standalone entity hits are suppressed (only the entity channel signal is kept on co-occurring hits). Request mergeStrategy:'rrf' to surface entity matches."
+        });
+      }
+    } else if (input.mergeStrategy === "rrf") {
+      warnings.push({
+        source: "memory",
+        code: "memory_entity_search_not_configured",
+        message: "RRF merge requested but entity search is not configured; falling back to semantic+lexical RRF."
+      });
+    }
+    const out = {
       semantic: filterByDateRange(semantic, input.dateFrom, input.dateTo),
       lexical: filterByDateRange(lexical, input.dateFrom, input.dateTo)
     };
+    if (entity && entity.length > 0) {
+      out.entity = entity;
+    }
+    return out;
   }
   async function searchUnifiedMemory(input) {
     const query = input.query.trim();
@@ -1506,32 +1641,56 @@ function createUnifiedSearch(deps = {}) {
 }
 function mergeAcrossSources(input) {
   const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
+  const lists = buildChannelLists(memorySubs, input.insightHits, input.knowledgeHits);
+  if (input.strategy !== "rrf") {
+    const seen = /* @__PURE__ */ new Map();
+    for (const hit of [
+      ...memorySubs.semantic,
+      ...memorySubs.lexical,
+      ...input.insightHits,
+      ...input.knowledgeHits
+    ]) {
+      const key = `${hit.type}::${hit.id}`;
+      const existing = seen.get(key);
+      if (!existing || hit.similarity > existing.similarity) {
+        seen.set(key, hit);
+      }
+    }
+    const merged2 = mergeUnifiedMemorySearchResults(Array.from(seen.values()), input.limit);
+    return attachSignals(merged2, lists);
+  }
   const all = [
     ...memorySubs.semantic,
     ...memorySubs.lexical,
+    ...memorySubs.entity ?? [],
     ...input.insightHits,
     ...input.knowledgeHits
   ];
-  if (input.strategy !== "rrf") {
-    return mergeUnifiedMemorySearchResults(all, input.limit);
-  }
+  const merged = lists.length <= 1 ? mergeUnifiedMemorySearchResults(all, input.limit) : mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
+  return attachSignals(merged, lists);
+}
+function buildChannelLists(memorySubs, insightHits, knowledgeHits) {
   const lists = [];
-  if (memorySubs.semantic.length > 0) {
-    lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
-  }
-  if (memorySubs.lexical.length > 0) {
-    lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
+  if (memorySubs.semantic.length > 0) lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
+  if (memorySubs.lexical.length > 0) lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
+  if (memorySubs.entity && memorySubs.entity.length > 0) {
+    lists.push({ name: "memory-entity", hits: memorySubs.entity });
   }
-  if (input.insightHits.length > 0) {
-    lists.push({ name: "insights", hits: input.insightHits });
-  }
-  if (input.knowledgeHits.length > 0) {
-    lists.push({ name: "knowledge", hits: input.knowledgeHits });
-  }
-  if (lists.length <= 1) {
-    return mergeUnifiedMemorySearchResults(all, input.limit);
+  if (insightHits.length > 0) lists.push({ name: "insights", hits: insightHits });
+  if (knowledgeHits.length > 0) lists.push({ name: "knowledge", hits: knowledgeHits });
+  return lists;
+}
+function attachSignals(hits, lists) {
+  if (hits.length === 0 || lists.length === 0) {
+    return hits;
   }
-  return mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
+  return hits.map((hit) => {
+    const signals = materializeSignals(lists, hit);
+    if (!signals) {
+      return hit;
+    }
+    return { ...hit, signals };
+  });
 }
 
 // src/search/apply-reflect.ts
@@ -1650,6 +1809,177 @@ function makePlanSummary(plan) {
   }
   return lines.join("\n");
 }
+var DEFAULT_CANDIDATE_LIMIT = 50;
+async function deriveFacts(deps, input, logger = console) {
+  const warnings = [];
+  if (typeof deps.deriver !== "function") {
+    warnings.push({
+      code: "derive_deriver_not_configured",
+      message: "No `deriver` is wired into the unified search deps; returning an empty fact list."
+    });
+    return { facts: [], warnings };
+  }
+  const limit = Math.max(1, Math.min(500, input.candidateLimit ?? DEFAULT_CANDIDATE_LIMIT));
+  if (!input.candidateTexts || input.candidateTexts.length === 0) {
+    if (!input.query) {
+      warnings.push({
+        code: "derive_fallback_query_noise",
+        message: "No `query` supplied and no `candidateTexts`; lexical fallback derives keywords from `userId + botIds`, which is rarely useful. Pass `query` (or `candidateTexts`) for topical derivation."
+      });
+    }
+  }
+  let candidateTexts;
+  if (input.candidateTexts && input.candidateTexts.length > 0) {
+    candidateTexts = input.candidateTexts;
+  } else {
+    candidateTexts = await loadCandidateTexts(deps, input, limit, logger);
+    if (candidateTexts.length === 0) {
+      warnings.push({
+        code: "derive_no_candidates",
+        message: "No candidate fact texts available for derivation"
+      });
+      return { facts: [], warnings };
+    }
+  }
+  let facts;
+  try {
+    facts = await deps.deriver({
+      userId: input.userId,
+      userScope: {
+        userId: input.userId,
+        botIds: input.botIds,
+        dateFrom: input.dateFrom,
+        dateTo: input.dateTo
+      },
+      recentFactTexts: candidateTexts,
+      ...input.window ? { window: input.window } : {}
+    });
+  } catch (error) {
+    logger.warn?.("[memory-store/derive] deriver threw:", error);
+    warnings.push({
+      code: "derive_deriver_failed",
+      message: error.message ?? "deriver failed"
+    });
+    return { facts: [], warnings };
+  }
+  if (!Array.isArray(facts)) {
+    warnings.push({
+      code: "derive_deriver_returned_invalid_shape",
+      message: `deriver returned ${typeof facts}; expected DerivedFact[]`
+    });
+    return { facts: [], warnings };
+  }
+  const normalized = [];
+  const now = Date.now();
+  for (const fact of facts) {
+    if (!fact || typeof fact.text !== "string" || fact.text.length === 0 || typeof fact.kind !== "string" || !isDerivedKind(fact.kind) || !Array.isArray(fact.sources)) {
+      continue;
+    }
+    normalized.push({
+      ...fact,
+      derivedAt: typeof fact.derivedAt === "number" ? fact.derivedAt : now,
+      sources: fact.sources.filter((s) => typeof s === "string")
+    });
+  }
+  if (input.persist && normalized.length > 0) {
+    try {
+      await input.persist(normalized);
+    } catch (error) {
+      logger.warn?.("[memory-store/derive] persist callback threw:", error);
+      warnings.push({
+        code: "derive_persist_failed",
+        message: error.message ?? "derive persist failed"
+      });
+    }
+  }
+  return { facts: normalized, warnings };
+}
+async function loadCandidateTexts(deps, input, limit, logger) {
+  if (typeof deps.searchRawMessagesLexical !== "function") {
+    return [];
+  }
+  const queryText = (input.query ?? `${input.userId} ${input.botIds?.join(" ") ?? ""}`).trim();
+  const keywords = deriveLexicalKeywords(queryText);
+  if (keywords.length === 0) {
+    return [];
+  }
+  try {
+    const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
+    const factTypes = input.factTypes?.length ? input.factTypes : void 0;
+    const hits = (await Promise.all(
+      filters.map(
+        (botId) => deps.searchRawMessagesLexical?.({
+          userId: input.userId,
+          keywords,
+          limit: Math.ceil(limit / filters.length),
+          botId,
+          ...input.peers && input.peers.length > 0 ? { peers: input.peers } : {},
+          ...factTypes ? { factTypes } : {}
+        })
+      )
+    )).flat().filter(isRawMemorySemanticResult);
+    return hits.map((h) => toMemoryResult(h).content);
+  } catch (error) {
+    logger.warn?.("[memory-store/derive] lexical candidate fetch failed:", error);
+    return [];
+  }
+}
+async function distillRawMessage(deps, input, logger = console) {
+  const warnings = [];
+  if (typeof deps.entityExtractor !== "function") {
+    warnings.push({
+      code: "distill_extractor_not_configured",
+      message: "No `entityExtractor` is wired into the unified search deps; returning an empty entity list."
+    });
+    return { edges: [], warnings };
+  }
+  let edges;
+  try {
+    edges = await deps.entityExtractor({
+      userId: input.userId,
+      messageId: input.messageId,
+      content: input.content
+    });
+  } catch (error) {
+    logger.warn?.("[memory-store/distill] extractor threw:", error);
+    warnings.push({
+      code: "distill_extractor_failed",
+      message: error.message ?? "entity extractor failed"
+    });
+    return { edges: [], warnings };
+  }
+  if (!Array.isArray(edges)) {
+    warnings.push({
+      code: "distill_extractor_returned_invalid_shape",
+      message: `entityExtractor returned ${typeof edges}; expected EntityEdge[]`
+    });
+    return { edges: [], warnings };
+  }
+  const normalized = [];
+  const now = Date.now();
+  for (const edge of edges) {
+    if (!edge || typeof edge.label !== "string" || edge.label.length === 0 || typeof edge.kind !== "string" || !isEntityKind(edge.kind) || typeof edge.relation !== "string" || typeof edge.sourceMessageId !== "string") {
+      continue;
+    }
+    normalized.push({
+      ...edge,
+      label: edge.label.trim().toLowerCase(),
+      extractedAt: typeof edge.extractedAt === "number" ? edge.extractedAt : now
+    });
+  }
+  if (input.persist && normalized.length > 0) {
+    try {
+      await input.persist(normalized);
+    } catch (error) {
+      logger.warn?.("[memory-store/distill] persist callback threw:", error);
+      warnings.push({
+        code: "distill_persist_failed",
+        message: error.message ?? "distill persist failed"
+      });
+    }
+  }
+  return { edges: normalized, warnings };
+}
 var DEFAULT_MAX_FACTS = 1e3;
 var LOW_CONFIDENCE_THRESHOLD = 0.1;
 function toHRR(data) {
@@ -1871,6 +2201,56 @@ async function startHttpServer(options = {}) {
     const result = await search.search(input);
     return c.json(result);
   });
+  app.post("/v1/distill", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const userId = typeof body.userId === "string" ? body.userId : null;
+    const messageId = typeof body.messageId === "string" ? body.messageId : null;
+    const content = typeof body.content === "string" ? body.content : null;
+    if (!userId || !messageId || content === null || content.length === 0) {
+      return c.json({ error: "userId, messageId, and a non-empty content are required" }, 400);
+    }
+    const input = { userId, messageId, content };
+    try {
+      const result = await distillRawMessage(options.unified ?? {}, input, console);
+      return c.json(result);
+    } catch (error) {
+      console.error("[memory-store/http] distill failed:", error);
+      return c.json({ error: error.message ?? "distill failed" }, 500);
+    }
+  });
+  app.post("/v1/derive", async (c) => {
+    const body = await c.req.json().catch(() => ({}));
+    const userId = typeof body.userId === "string" ? body.userId : null;
+    if (!userId) {
+      return c.json({ error: "userId is required" }, 400);
+    }
+    const windowFrom = typeof body.windowFrom === "number" ? body.windowFrom : void 0;
+    const windowTo = typeof body.windowTo === "number" ? body.windowTo : void 0;
+    const window = windowFrom !== void 0 && windowTo !== void 0 ? { from: windowFrom, to: windowTo } : void 0;
+    const peers = Array.isArray(body.peers) ? body.peers.filter(
+      (p) => (p?.kind === "user" || p?.kind === "agent") && typeof p?.id === "string" && p.id.length > 0
+    ) : void 0;
+    const factTypes = Array.isArray(body.factTypes) ? body.factTypes.filter(isFactType) : void 0;
+    const input = {
+      userId,
+      ...typeof body.query === "string" ? { query: body.query } : {},
+      botIds: Array.isArray(body.botIds) ? body.botIds : void 0,
+      dateFrom: typeof body.dateFrom === "string" ? body.dateFrom : void 0,
+      dateTo: typeof body.dateTo === "string" ? body.dateTo : void 0,
+      ...window ? { window } : {},
+      ...Array.isArray(body.candidateTexts) ? { candidateTexts: body.candidateTexts } : {},
+      ...typeof body.candidateLimit === "number" ? { candidateLimit: body.candidateLimit } : {},
+      ...peers && peers.length > 0 ? { peers } : {},
+      ...factTypes && factTypes.length > 0 ? { factTypes } : {}
+    };
+    try {
+      const result = await deriveFacts(options.unified ?? {}, input, console);
+      return c.json(result);
+    } catch (error) {
+      console.error("[memory-store/http] derive failed:", error);
+      return c.json({ error: error.message ?? "derive failed" }, 500);
+    }
+  });
   app.post("/v1/consolidate:apply", async (c) => {
     const body = await c.req.json().catch(() => ({}));
     const userId = typeof body.userId === "string" ? body.userId : null;