@melandlabs/memory-store 1.1.8 → 1.2.0

package/dist/search/derive.js

@@ -0,0 +1,145 @@
+import { isDerivedKind } from '@melandlabs/contracts/derived-fact';
+
+// src/search/derive.ts
+
+// src/search/utilities.ts
+function deriveLexicalKeywords(query) {
+  return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
+}
+var DEFAULT_SOURCES = ["memory", "insights", "knowledge"];
+new Set(DEFAULT_SOURCES);
+function toMemoryResult(result) {
+  return {
+    type: "memory",
+    id: result.id,
+    content: result.content,
+    similarity: result.similarity,
+    metadata: result.metadata
+  };
+}
+function isRawMemorySemanticResult(result) {
+  if (!result || typeof result !== "object") {
+    return false;
+  }
+  const item = result;
+  return typeof item.id === "string" && typeof item.content === "string" && typeof item.similarity === "number" && Boolean(item.metadata) && typeof item.metadata === "object";
+}
+
+// src/search/derive.ts
+var DEFAULT_CANDIDATE_LIMIT = 50;
+async function deriveFacts(deps, input, logger = console) {
+  const warnings = [];
+  if (typeof deps.deriver !== "function") {
+    warnings.push({
+      code: "derive_deriver_not_configured",
+      message: "No `deriver` is wired into the unified search deps; returning an empty fact list."
+    });
+    return { facts: [], warnings };
+  }
+  const limit = Math.max(1, Math.min(500, input.candidateLimit ?? DEFAULT_CANDIDATE_LIMIT));
+  if (!input.candidateTexts || input.candidateTexts.length === 0) {
+    if (!input.query) {
+      warnings.push({
+        code: "derive_fallback_query_noise",
+        message: "No `query` supplied and no `candidateTexts`; lexical fallback derives keywords from `userId + botIds`, which is rarely useful. Pass `query` (or `candidateTexts`) for topical derivation."
+      });
+    }
+  }
+  let candidateTexts;
+  if (input.candidateTexts && input.candidateTexts.length > 0) {
+    candidateTexts = input.candidateTexts;
+  } else {
+    candidateTexts = await loadCandidateTexts(deps, input, limit, logger);
+    if (candidateTexts.length === 0) {
+      warnings.push({
+        code: "derive_no_candidates",
+        message: "No candidate fact texts available for derivation"
+      });
+      return { facts: [], warnings };
+    }
+  }
+  let facts;
+  try {
+    facts = await deps.deriver({
+      userId: input.userId,
+      userScope: {
+        userId: input.userId,
+        botIds: input.botIds,
+        dateFrom: input.dateFrom,
+        dateTo: input.dateTo
+      },
+      recentFactTexts: candidateTexts,
+      ...input.window ? { window: input.window } : {}
+    });
+  } catch (error) {
+    logger.warn?.("[memory-store/derive] deriver threw:", error);
+    warnings.push({
+      code: "derive_deriver_failed",
+      message: error.message ?? "deriver failed"
+    });
+    return { facts: [], warnings };
+  }
+  if (!Array.isArray(facts)) {
+    warnings.push({
+      code: "derive_deriver_returned_invalid_shape",
+      message: `deriver returned ${typeof facts}; expected DerivedFact[]`
+    });
+    return { facts: [], warnings };
+  }
+  const normalized = [];
+  const now = Date.now();
+  for (const fact of facts) {
+    if (!fact || typeof fact.text !== "string" || fact.text.length === 0 || typeof fact.kind !== "string" || !isDerivedKind(fact.kind) || !Array.isArray(fact.sources)) {
+      continue;
+    }
+    normalized.push({
+      ...fact,
+      derivedAt: typeof fact.derivedAt === "number" ? fact.derivedAt : now,
+      sources: fact.sources.filter((s) => typeof s === "string")
+    });
+  }
+  if (input.persist && normalized.length > 0) {
+    try {
+      await input.persist(normalized);
+    } catch (error) {
+      logger.warn?.("[memory-store/derive] persist callback threw:", error);
+      warnings.push({
+        code: "derive_persist_failed",
+        message: error.message ?? "derive persist failed"
+      });
+    }
+  }
+  return { facts: normalized, warnings };
+}
+async function loadCandidateTexts(deps, input, limit, logger) {
+  if (typeof deps.searchRawMessagesLexical !== "function") {
+    return [];
+  }
+  const queryText = (input.query ?? `${input.userId} ${input.botIds?.join(" ") ?? ""}`).trim();
+  const keywords = deriveLexicalKeywords(queryText);
+  if (keywords.length === 0) {
+    return [];
+  }
+  try {
+    const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
+    const factTypes = input.factTypes?.length ? input.factTypes : void 0;
+    const hits = (await Promise.all(
+      filters.map(
+        (botId) => deps.searchRawMessagesLexical?.({
+          userId: input.userId,
+          keywords,
+          limit: Math.ceil(limit / filters.length),
+          botId,
+          ...input.peers && input.peers.length > 0 ? { peers: input.peers } : {},
+          ...factTypes ? { factTypes } : {}
+        })
+      )
+    )).flat().filter(isRawMemorySemanticResult);
+    return hits.map((h) => toMemoryResult(h).content);
+  } catch (error) {
+    logger.warn?.("[memory-store/derive] lexical candidate fetch failed:", error);
+    return [];
+  }
+}
+
+export { deriveFacts };

package/dist/search/distill.d.ts

@@ -0,0 +1,60 @@
+import { EntityEdge } from '@melandlabs/contracts/entity-edge';
+import { U as UnifiedSearchDeps } from '../config-DfvVZ--0.js';
+import '@melandlabs/memory-consolidation';
+import '@melandlabs/contracts/peer';
+import '@melandlabs/contracts';
+import '@melandlabs/contracts/derived-fact';
+
+/**
+ * `distill` primitive — extract `EntityEdge`s from a single raw message.
+ *
+ * The host wires in an LLM extractor via `UnifiedSearchDeps.entityExtractor`.
+ * When the dep is absent, the primitive short-circuits with a
+ * `distill_extractor_not_configured` warning — matching the
+ * `memory_iterative_planner_not_configured` precedent so callers can
+ * detect the degraded mode uniformly.
+ *
+ * Persistence is intentionally the host's responsibility: the SDK
+ * surfaces the edges plus an optional `persist` callback that the
+ * caller supplies on the input. This keeps the memory-store decoupled
+ * from any entity-store schema and lets hosts reuse their existing
+ * graph nodes / SQLite tables / Loop-engine queues without an extra
+ * migration.
+ *
+ * The module is intentionally pure (no I/O of its own beyond calling
+ * the extractor and `persist`). That keeps it unit-testable with a
+ * fake extractor and a counter on `persist`.
+ */
+
+interface DistillInput {
+    userId: string;
+    messageId: string;
+    content: string;
+    /**
+     * Optional host-side persistence callback. Invoked exactly once
+     * with the extracted edges when an extractor is wired in. Hosts
+     * that want to batch-distill should call `distillRawMessage` in a
+     * loop and persist themselves.
+     */
+    persist?: (edges: EntityEdge[]) => Promise<void>;
+}
+interface DistillWarning {
+    code: string;
+    message: string;
+}
+interface DistillOutput {
+    edges: EntityEdge[];
+    warnings: DistillWarning[];
+}
+/**
+ * Run the entity extractor over a single raw message.
+ *
+ *   - No extractor configured → `{ edges: [], warnings: [...] }`.
+ *   - Extractor configured → call it, await `persist` if provided,
+ *     return the edges. Errors in the extractor (or the persist
+ *     callback) are surfaced as warnings and the function never
+ *     throws — distill is best-effort by design.
+ */
+declare function distillRawMessage(deps: Pick<UnifiedSearchDeps, "entityExtractor">, input: DistillInput, logger?: Pick<Console, "warn">): Promise<DistillOutput>;
+
+export { type DistillInput, type DistillOutput, type DistillWarning, distillRawMessage };

package/dist/search/distill.js

@@ -0,0 +1,61 @@
+import { isEntityKind } from '@melandlabs/contracts/entity-edge';
+
+// src/search/distill.ts
+async function distillRawMessage(deps, input, logger = console) {
+  const warnings = [];
+  if (typeof deps.entityExtractor !== "function") {
+    warnings.push({
+      code: "distill_extractor_not_configured",
+      message: "No `entityExtractor` is wired into the unified search deps; returning an empty entity list."
+    });
+    return { edges: [], warnings };
+  }
+  let edges;
+  try {
+    edges = await deps.entityExtractor({
+      userId: input.userId,
+      messageId: input.messageId,
+      content: input.content
+    });
+  } catch (error) {
+    logger.warn?.("[memory-store/distill] extractor threw:", error);
+    warnings.push({
+      code: "distill_extractor_failed",
+      message: error.message ?? "entity extractor failed"
+    });
+    return { edges: [], warnings };
+  }
+  if (!Array.isArray(edges)) {
+    warnings.push({
+      code: "distill_extractor_returned_invalid_shape",
+      message: `entityExtractor returned ${typeof edges}; expected EntityEdge[]`
+    });
+    return { edges: [], warnings };
+  }
+  const normalized = [];
+  const now = Date.now();
+  for (const edge of edges) {
+    if (!edge || typeof edge.label !== "string" || edge.label.length === 0 || typeof edge.kind !== "string" || !isEntityKind(edge.kind) || typeof edge.relation !== "string" || typeof edge.sourceMessageId !== "string") {
+      continue;
+    }
+    normalized.push({
+      ...edge,
+      label: edge.label.trim().toLowerCase(),
+      extractedAt: typeof edge.extractedAt === "number" ? edge.extractedAt : now
+    });
+  }
+  if (input.persist && normalized.length > 0) {
+    try {
+      await input.persist(normalized);
+    } catch (error) {
+      logger.warn?.("[memory-store/distill] persist callback threw:", error);
+      warnings.push({
+        code: "distill_persist_failed",
+        message: error.message ?? "distill persist failed"
+      });
+    }
+  }
+  return { edges: normalized, warnings };
+}
+
+export { distillRawMessage };

package/dist/search/unified-search.d.ts

@@ -1,6 +1,8 @@
-export { c as SearchEvidence, d as SearchInput, e as SearchOutput, i as SearchSource, S as SearchTier, j as UnifiedMemoryMergeStrategy, k as UnifiedMemoryRankedList, l as UnifiedMemoryReasoningStrategy, f as UnifiedMemorySearchInput, g as UnifiedMemorySearchOutput, h as UnifiedMemorySearchResult, m as UnifiedMemorySearchSource, b as UnifiedMemorySearchWarning, n as clampUnifiedMemorySearchLimit, o as clampUnifiedMemorySearchThreshold, p as isRawMemorySemanticResult, q as mergeUnifiedMemorySearchResults, r as mergeUnifiedMemorySearchResultsRrf, s as normalizeUnifiedMemoryMergeStrategy, t as normalizeUnifiedMemoryReasoningStrategy, u as normalizeUnifiedMemorySearchSources, v as toKnowledgeResult, w as toMemoryResult } from '../config-CnbqSek5.js';
-export { U as UnifiedSearch, c as createUnifiedSearch } from '../unified-search-Bft57TRH.js';
+export { H as HitChannel, i as HitSignals, c as SearchEvidence, d as SearchInput, e as SearchOutput, j as SearchSource, S as SearchTier, k as UnifiedMemoryMergeStrategy, l as UnifiedMemoryRankedList, m as UnifiedMemoryReasoningStrategy, f as UnifiedMemorySearchInput, g as UnifiedMemorySearchOutput, h as UnifiedMemorySearchResult, n as UnifiedMemorySearchSource, b as UnifiedMemorySearchWarning, o as clampUnifiedMemorySearchLimit, p as clampUnifiedMemorySearchThreshold, q as deriveLexicalKeywords, r as isRawMemorySemanticResult, s as listNameToChannel, t as materializeSignals, u as mergeUnifiedMemorySearchResults, v as mergeUnifiedMemorySearchResultsRrf, w as normalizeUnifiedMemoryMergeStrategy, x as normalizeUnifiedMemoryReasoningStrategy, y as normalizeUnifiedMemorySearchSources, z as toKnowledgeResult, A as toMemoryResult } from '../config-DfvVZ--0.js';
+export { U as UnifiedSearch, c as createUnifiedSearch } from '../unified-search-A8Q3JHwU.js';
 import '@melandlabs/memory-consolidation';
 import '@melandlabs/contracts/peer';
 import '@melandlabs/contracts';
+import '@melandlabs/contracts/derived-fact';
+import '@melandlabs/contracts/entity-edge';
 import '@melandlabs/ai/memory';

package/dist/search/unified-search.js

@@ -250,6 +250,9 @@ init_chroma_memory_index();
 init_raw_message_store();
 
 // src/search/utilities.ts
+function deriveLexicalKeywords(query) {
+  return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
+}
 var DEFAULT_LIMIT = 10;
 var DEFAULT_THRESHOLD = 0.7;
 var DEFAULT_SOURCES = ["memory", "insights", "knowledge"];
@@ -363,6 +366,50 @@ function toMemoryResult(result) {
     metadata: result.metadata
   };
 }
+function materializeSignals(lists, hit) {
+  const channels = [];
+  let semantic;
+  let lexical;
+  let entity;
+  for (const list of lists) {
+    const channel = listNameToChannel(list.name);
+    if (!channel) continue;
+    if (channels.includes(channel)) continue;
+    let found;
+    for (const candidate of list.hits) {
+      if (candidate.type === hit.type && candidate.id === hit.id) {
+        found = candidate;
+        break;
+      }
+    }
+    if (!found) continue;
+    channels.push(channel);
+    if (channel === "semantic" && semantic === void 0) {
+      semantic = found.similarity;
+    } else if (channel === "lexical" && lexical === void 0) {
+      lexical = found.similarity;
+    } else if (channel === "entity" && entity === void 0) {
+      entity = found.similarity;
+    }
+  }
+  if (channels.length === 0) {
+    return void 0;
+  }
+  const out = { channels };
+  if (semantic !== void 0) out.semantic = semantic;
+  if (lexical !== void 0) out.lexical = lexical;
+  if (entity !== void 0) out.entity = entity;
+  if (typeof hit.metadata.rrfScore === "number") {
+    out.rrf = hit.metadata.rrfScore;
+  }
+  return out;
+}
+function listNameToChannel(name) {
+  if (name === "memory-semantic") return "semantic";
+  if (name === "memory-bm25" || name === "memory-lexical") return "lexical";
+  if (name === "memory-entity") return "entity";
+  return void 0;
+}
 function isRawMemorySemanticResult(result) {
   if (!result || typeof result !== "object") {
     return false;
@@ -782,8 +829,65 @@ async function resolveSearchScopePeers(input) {
 }
 
 // src/search/unified-search.ts
-function deriveLexicalKeywords(query) {
-  return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
+function deriveEntityKeywords(query) {
+  const proper = Array.from(new Set(query.match(/\b[A-Z][a-zA-Z]{2,}\b/g) ?? [])).map((w) => w.toLowerCase());
+  const lexical = deriveLexicalKeywords(query);
+  const seen = /* @__PURE__ */ new Set();
+  const out = [];
+  for (const word of [...proper, ...lexical]) {
+    if (word.length < 2) continue;
+    if (seen.has(word)) continue;
+    seen.add(word);
+    out.push(word);
+  }
+  return out.slice(0, 16);
+}
+async function runEntitySearchForQuery(deps, input, limit, logger, peerPeers = []) {
+  if (typeof deps.entitySearch !== "function") {
+    return [];
+  }
+  const keywords = deriveEntityKeywords(input.query);
+  if (keywords.length === 0) {
+    return [];
+  }
+  try {
+    const entitySearch = deps.entitySearch;
+    const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
+    const matches = (await Promise.all(
+      filters.map(
+        (botId) => entitySearch({
+          userId: input.userId,
+          keywords,
+          limit: Math.ceil(limit / filters.length),
+          botId,
+          ...peerPeers.length > 0 ? { peers: peerPeers } : {}
+        })
+      )
+    )).flat();
+    return matches.filter(
+      (m) => typeof m?.messageId === "string" && typeof m?.label === "string" && typeof m?.score === "number"
+    ).map((m) => ({
+      type: "memory",
+      id: m.messageId,
+      // The host's `entitySearch` returns match ids + labels only,
+      // so we surface the entity label as `content`. When the same
+      // `messageId` is also returned by the semantic/lexical channel
+      // (RRF path), the real message content is preserved via the
+      // shared `(type, id)` key; otherwise callers can re-fetch the
+      // original message by `id` (set in metadata as `sourceMessageId`).
+      content: m.label,
+      similarity: m.score,
+      metadata: {
+        scoring: "entity",
+        entityLabel: m.label,
+        isEntityProjection: true,
+        sourceMessageId: m.messageId
+      }
+    }));
+  } catch (error) {
+    logger.warn?.("[memory-store] entity search failed:", error);
+    return [];
+  }
 }
 function parseDateBoundary(value, role) {
   const trimmed = value.trim();
@@ -1223,10 +1327,39 @@ function createUnifiedSearch(deps = {}) {
         });
       }
     }
-    return {
+    let entity;
+    if (typeof deps.entitySearch === "function") {
+      entity = filterByDateRange(
+        await runEntitySearchForQuery(deps, input, limit, logger, peerPeers),
+        input.dateFrom,
+        input.dateTo
+      );
+      if (entity.length === 0) {
+        entity = void 0;
+      } else if (normalizeUnifiedMemoryMergeStrategy(
+        input.mergeStrategy ?? deps.reasoning?.defaultMergeStrategy ?? "rrf"
+      ) !== "rrf") {
+        warnings.push({
+          source: "memory",
+          code: "memory_entity_requires_rrf",
+          message: "Entity channel results are fused via RRF; with mergeStrategy:'similarity' standalone entity hits are suppressed (only the entity channel signal is kept on co-occurring hits). Request mergeStrategy:'rrf' to surface entity matches."
+        });
+      }
+    } else if (input.mergeStrategy === "rrf") {
+      warnings.push({
+        source: "memory",
+        code: "memory_entity_search_not_configured",
+        message: "RRF merge requested but entity search is not configured; falling back to semantic+lexical RRF."
+      });
+    }
+    const out = {
       semantic: filterByDateRange(semantic, input.dateFrom, input.dateTo),
       lexical: filterByDateRange(lexical, input.dateFrom, input.dateTo)
     };
+    if (entity && entity.length > 0) {
+      out.entity = entity;
+    }
+    return out;
   }
   async function searchUnifiedMemory(input) {
     const query = input.query.trim();
@@ -1497,32 +1630,56 @@ function createUnifiedSearch(deps = {}) {
 }
 function mergeAcrossSources(input) {
   const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
+  const lists = buildChannelLists(memorySubs, input.insightHits, input.knowledgeHits);
+  if (input.strategy !== "rrf") {
+    const seen = /* @__PURE__ */ new Map();
+    for (const hit of [
+      ...memorySubs.semantic,
+      ...memorySubs.lexical,
+      ...input.insightHits,
+      ...input.knowledgeHits
+    ]) {
+      const key = `${hit.type}::${hit.id}`;
+      const existing = seen.get(key);
+      if (!existing || hit.similarity > existing.similarity) {
+        seen.set(key, hit);
+      }
+    }
+    const merged2 = mergeUnifiedMemorySearchResults(Array.from(seen.values()), input.limit);
+    return attachSignals(merged2, lists);
+  }
   const all = [
     ...memorySubs.semantic,
     ...memorySubs.lexical,
+    ...memorySubs.entity ?? [],
     ...input.insightHits,
     ...input.knowledgeHits
   ];
-  if (input.strategy !== "rrf") {
-    return mergeUnifiedMemorySearchResults(all, input.limit);
-  }
+  const merged = lists.length <= 1 ? mergeUnifiedMemorySearchResults(all, input.limit) : mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
+  return attachSignals(merged, lists);
+}
+function buildChannelLists(memorySubs, insightHits, knowledgeHits) {
   const lists = [];
-  if (memorySubs.semantic.length > 0) {
-    lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
-  }
-  if (memorySubs.lexical.length > 0) {
-    lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
-  }
-  if (input.insightHits.length > 0) {
-    lists.push({ name: "insights", hits: input.insightHits });
-  }
-  if (input.knowledgeHits.length > 0) {
-    lists.push({ name: "knowledge", hits: input.knowledgeHits });
-  }
-  if (lists.length <= 1) {
-    return mergeUnifiedMemorySearchResults(all, input.limit);
-  }
-  return mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
+  if (memorySubs.semantic.length > 0) lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
+  if (memorySubs.lexical.length > 0) lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
+  if (memorySubs.entity && memorySubs.entity.length > 0) {
+    lists.push({ name: "memory-entity", hits: memorySubs.entity });
+  }
+  if (insightHits.length > 0) lists.push({ name: "insights", hits: insightHits });
+  if (knowledgeHits.length > 0) lists.push({ name: "knowledge", hits: knowledgeHits });
+  return lists;
+}
+function attachSignals(hits, lists) {
+  if (hits.length === 0 || lists.length === 0) {
+    return hits;
+  }
+  return hits.map((hit) => {
+    const signals = materializeSignals(lists, hit);
+    if (!signals) {
+      return hit;
+    }
+    return { ...hit, signals };
+  });
 }
 
-export { clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, mergeUnifiedMemorySearchResultsRrf, normalizeUnifiedMemoryMergeStrategy, normalizeUnifiedMemoryReasoningStrategy, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult };
+export { clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, deriveLexicalKeywords, isRawMemorySemanticResult, listNameToChannel, materializeSignals, mergeUnifiedMemorySearchResults, mergeUnifiedMemorySearchResultsRrf, normalizeUnifiedMemoryMergeStrategy, normalizeUnifiedMemoryReasoningStrategy, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult };