@melandlabs/memory-store 1.1.7 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/{config-CnbqSek5.d.ts → config-DfvVZ--0.d.ts} +137 -1
- package/dist/http.d.ts +11 -2
- package/dist/http.js +466 -56
- package/dist/index.d.ts +10 -5
- package/dist/index.js +351 -21
- package/dist/mcp.d.ts +12 -2
- package/dist/mcp.js +516 -53
- package/dist/search/derive.d.ts +93 -0
- package/dist/search/derive.js +145 -0
- package/dist/search/distill.d.ts +60 -0
- package/dist/search/distill.js +61 -0
- package/dist/search/unified-search.d.ts +4 -2
- package/dist/search/unified-search.js +180 -23
- package/dist/server/cli-http.js +466 -56
- package/dist/server/cli-mcp.js +520 -52
- package/dist/server/cli-shared.d.ts +3 -1
- package/dist/storage/postgres-raw-message-factory.d.ts +3 -1
- package/dist/storage/raw-message-store.d.ts +3 -1
- package/dist/storage/sqlite-raw-message-store.d.ts +3 -1
- package/dist/storage/sqlite-vector-index.d.ts +3 -1
- package/dist/{unified-search-Bft57TRH.d.ts → unified-search-A8Q3JHwU.d.ts} +1 -1
- package/package.json +6 -6
package/dist/mcp.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { vsaNormalizeVector, vsaAssertSameDim } from '@melandlabs/contracts';
|
|
1
|
+
import { isFactType, vsaNormalizeVector, vsaAssertSameDim } from '@melandlabs/contracts';
|
|
2
2
|
import { buildMemoryRecordEmbeddingDocument } from '@melandlabs/ai/memory';
|
|
3
3
|
import { ChromaVectorStore } from '@melandlabs/rag';
|
|
4
4
|
import { mkdirSync } from 'fs';
|
|
@@ -9,6 +9,8 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
|
|
|
9
9
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
10
10
|
import { z } from 'zod';
|
|
11
11
|
import { applyReflectedConsolidationPlan } from '@melandlabs/memory-consolidation';
|
|
12
|
+
import { isDerivedKind } from '@melandlabs/contracts/derived-fact';
|
|
13
|
+
import { isEntityKind } from '@melandlabs/contracts/entity-edge';
|
|
12
14
|
import { superpose, bind, unbind, cleanup, cosineSimilarity } from '@melandlabs/vsa';
|
|
13
15
|
|
|
14
16
|
var __defProp = Object.defineProperty;
|
|
@@ -322,6 +324,55 @@ var init_raw_message_store = __esm({
|
|
|
322
324
|
}
|
|
323
325
|
});
|
|
324
326
|
|
|
327
|
+
// src/embed-on-insert.ts
|
|
328
|
+
async function embedMissingMessages(messages, deps) {
|
|
329
|
+
if (typeof deps.embedQuery !== "function") {
|
|
330
|
+
return messages;
|
|
331
|
+
}
|
|
332
|
+
const out = [];
|
|
333
|
+
for (const message of messages) {
|
|
334
|
+
if (Array.isArray(message.embedding) && message.embedding.length > 0) {
|
|
335
|
+
out.push(message);
|
|
336
|
+
continue;
|
|
337
|
+
}
|
|
338
|
+
if (typeof message.content !== "string" || message.content.length === 0) {
|
|
339
|
+
out.push(message);
|
|
340
|
+
continue;
|
|
341
|
+
}
|
|
342
|
+
const vector = await deps.embedQuery({
|
|
343
|
+
userId: message.userId,
|
|
344
|
+
query: message.content
|
|
345
|
+
});
|
|
346
|
+
out.push({
|
|
347
|
+
...message,
|
|
348
|
+
embedding: vector,
|
|
349
|
+
embeddingModel: message.embeddingModel ?? "server",
|
|
350
|
+
embeddingDimensions: vector.length,
|
|
351
|
+
embeddingUpdatedAt: Date.now()
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
return out;
|
|
355
|
+
}
|
|
356
|
+
var AUTO_APPLIED_WARNING = {
|
|
357
|
+
code: "embed_on_insert_auto_applied",
|
|
358
|
+
message: "Caller set `embedOnInsert` falsy but the active vector backend needs an embedding per row. Server filled in the missing embeddings; pass `embedOnInsert: true` to opt in explicitly."
|
|
359
|
+
};
|
|
360
|
+
async function applyEmbedOnInsertPolicy(incoming, embedOnInsert, unified) {
|
|
361
|
+
const messages = incoming;
|
|
362
|
+
const anyMissing = incoming.some((m) => !Array.isArray(m.embedding) || m.embedding.length === 0);
|
|
363
|
+
const embedderWired = typeof unified?.embedQuery === "function";
|
|
364
|
+
if (embedOnInsert === true && anyMissing) {
|
|
365
|
+
return { messages: await embedMissingMessages(messages, unified ?? {}), warnings: [] };
|
|
366
|
+
}
|
|
367
|
+
if (embedOnInsert !== true && anyMissing && embedderWired) {
|
|
368
|
+
return {
|
|
369
|
+
messages: await embedMissingMessages(messages, unified ?? {}),
|
|
370
|
+
warnings: [AUTO_APPLIED_WARNING]
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
return { messages, warnings: [] };
|
|
374
|
+
}
|
|
375
|
+
|
|
325
376
|
// src/search/unified-search.ts
|
|
326
377
|
init_chroma_memory_index();
|
|
327
378
|
init_raw_message_store();
|
|
@@ -362,6 +413,9 @@ async function applyReranker(reranker, query, results) {
|
|
|
362
413
|
}
|
|
363
414
|
|
|
364
415
|
// src/search/utilities.ts
|
|
416
|
+
function deriveLexicalKeywords(query) {
|
|
417
|
+
return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
|
|
418
|
+
}
|
|
365
419
|
var DEFAULT_LIMIT = 10;
|
|
366
420
|
var DEFAULT_THRESHOLD = 0.7;
|
|
367
421
|
var DEFAULT_SOURCES = ["memory", "insights", "knowledge"];
|
|
@@ -475,6 +529,50 @@ function toMemoryResult(result) {
|
|
|
475
529
|
metadata: result.metadata
|
|
476
530
|
};
|
|
477
531
|
}
|
|
532
|
+
function materializeSignals(lists, hit) {
|
|
533
|
+
const channels = [];
|
|
534
|
+
let semantic;
|
|
535
|
+
let lexical;
|
|
536
|
+
let entity;
|
|
537
|
+
for (const list of lists) {
|
|
538
|
+
const channel = listNameToChannel(list.name);
|
|
539
|
+
if (!channel) continue;
|
|
540
|
+
if (channels.includes(channel)) continue;
|
|
541
|
+
let found;
|
|
542
|
+
for (const candidate of list.hits) {
|
|
543
|
+
if (candidate.type === hit.type && candidate.id === hit.id) {
|
|
544
|
+
found = candidate;
|
|
545
|
+
break;
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
if (!found) continue;
|
|
549
|
+
channels.push(channel);
|
|
550
|
+
if (channel === "semantic" && semantic === void 0) {
|
|
551
|
+
semantic = found.similarity;
|
|
552
|
+
} else if (channel === "lexical" && lexical === void 0) {
|
|
553
|
+
lexical = found.similarity;
|
|
554
|
+
} else if (channel === "entity" && entity === void 0) {
|
|
555
|
+
entity = found.similarity;
|
|
556
|
+
}
|
|
557
|
+
}
|
|
558
|
+
if (channels.length === 0) {
|
|
559
|
+
return void 0;
|
|
560
|
+
}
|
|
561
|
+
const out = { channels };
|
|
562
|
+
if (semantic !== void 0) out.semantic = semantic;
|
|
563
|
+
if (lexical !== void 0) out.lexical = lexical;
|
|
564
|
+
if (entity !== void 0) out.entity = entity;
|
|
565
|
+
if (typeof hit.metadata.rrfScore === "number") {
|
|
566
|
+
out.rrf = hit.metadata.rrfScore;
|
|
567
|
+
}
|
|
568
|
+
return out;
|
|
569
|
+
}
|
|
570
|
+
function listNameToChannel(name) {
|
|
571
|
+
if (name === "memory-semantic") return "semantic";
|
|
572
|
+
if (name === "memory-bm25" || name === "memory-lexical") return "lexical";
|
|
573
|
+
if (name === "memory-entity") return "entity";
|
|
574
|
+
return void 0;
|
|
575
|
+
}
|
|
478
576
|
function isRawMemorySemanticResult(result) {
|
|
479
577
|
if (!result || typeof result !== "object") {
|
|
480
578
|
return false;
|
|
@@ -742,8 +840,65 @@ async function resolveSearchScopePeers(input) {
|
|
|
742
840
|
}
|
|
743
841
|
|
|
744
842
|
// src/search/unified-search.ts
|
|
745
|
-
function
|
|
746
|
-
|
|
843
|
+
function deriveEntityKeywords(query) {
|
|
844
|
+
const proper = Array.from(new Set(query.match(/\b[A-Z][a-zA-Z]{2,}\b/g) ?? [])).map((w) => w.toLowerCase());
|
|
845
|
+
const lexical = deriveLexicalKeywords(query);
|
|
846
|
+
const seen = /* @__PURE__ */ new Set();
|
|
847
|
+
const out = [];
|
|
848
|
+
for (const word of [...proper, ...lexical]) {
|
|
849
|
+
if (word.length < 2) continue;
|
|
850
|
+
if (seen.has(word)) continue;
|
|
851
|
+
seen.add(word);
|
|
852
|
+
out.push(word);
|
|
853
|
+
}
|
|
854
|
+
return out.slice(0, 16);
|
|
855
|
+
}
|
|
856
|
+
async function runEntitySearchForQuery(deps, input, limit, logger, peerPeers = []) {
|
|
857
|
+
if (typeof deps.entitySearch !== "function") {
|
|
858
|
+
return [];
|
|
859
|
+
}
|
|
860
|
+
const keywords = deriveEntityKeywords(input.query);
|
|
861
|
+
if (keywords.length === 0) {
|
|
862
|
+
return [];
|
|
863
|
+
}
|
|
864
|
+
try {
|
|
865
|
+
const entitySearch = deps.entitySearch;
|
|
866
|
+
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
867
|
+
const matches = (await Promise.all(
|
|
868
|
+
filters.map(
|
|
869
|
+
(botId) => entitySearch({
|
|
870
|
+
userId: input.userId,
|
|
871
|
+
keywords,
|
|
872
|
+
limit: Math.ceil(limit / filters.length),
|
|
873
|
+
botId,
|
|
874
|
+
...peerPeers.length > 0 ? { peers: peerPeers } : {}
|
|
875
|
+
})
|
|
876
|
+
)
|
|
877
|
+
)).flat();
|
|
878
|
+
return matches.filter(
|
|
879
|
+
(m) => typeof m?.messageId === "string" && typeof m?.label === "string" && typeof m?.score === "number"
|
|
880
|
+
).map((m) => ({
|
|
881
|
+
type: "memory",
|
|
882
|
+
id: m.messageId,
|
|
883
|
+
// The host's `entitySearch` returns match ids + labels only,
|
|
884
|
+
// so we surface the entity label as `content`. When the same
|
|
885
|
+
// `messageId` is also returned by the semantic/lexical channel
|
|
886
|
+
// (RRF path), the real message content is preserved via the
|
|
887
|
+
// shared `(type, id)` key; otherwise callers can re-fetch the
|
|
888
|
+
// original message by `id` (set in metadata as `sourceMessageId`).
|
|
889
|
+
content: m.label,
|
|
890
|
+
similarity: m.score,
|
|
891
|
+
metadata: {
|
|
892
|
+
scoring: "entity",
|
|
893
|
+
entityLabel: m.label,
|
|
894
|
+
isEntityProjection: true,
|
|
895
|
+
sourceMessageId: m.messageId
|
|
896
|
+
}
|
|
897
|
+
}));
|
|
898
|
+
} catch (error) {
|
|
899
|
+
logger.warn?.("[memory-store] entity search failed:", error);
|
|
900
|
+
return [];
|
|
901
|
+
}
|
|
747
902
|
}
|
|
748
903
|
function parseDateBoundary(value, role) {
|
|
749
904
|
const trimmed = value.trim();
|
|
@@ -1183,10 +1338,39 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1183
1338
|
});
|
|
1184
1339
|
}
|
|
1185
1340
|
}
|
|
1186
|
-
|
|
1341
|
+
let entity;
|
|
1342
|
+
if (typeof deps.entitySearch === "function") {
|
|
1343
|
+
entity = filterByDateRange(
|
|
1344
|
+
await runEntitySearchForQuery(deps, input, limit, logger, peerPeers),
|
|
1345
|
+
input.dateFrom,
|
|
1346
|
+
input.dateTo
|
|
1347
|
+
);
|
|
1348
|
+
if (entity.length === 0) {
|
|
1349
|
+
entity = void 0;
|
|
1350
|
+
} else if (normalizeUnifiedMemoryMergeStrategy(
|
|
1351
|
+
input.mergeStrategy ?? deps.reasoning?.defaultMergeStrategy ?? "rrf"
|
|
1352
|
+
) !== "rrf") {
|
|
1353
|
+
warnings.push({
|
|
1354
|
+
source: "memory",
|
|
1355
|
+
code: "memory_entity_requires_rrf",
|
|
1356
|
+
message: "Entity channel results are fused via RRF; with mergeStrategy:'similarity' standalone entity hits are suppressed (only the entity channel signal is kept on co-occurring hits). Request mergeStrategy:'rrf' to surface entity matches."
|
|
1357
|
+
});
|
|
1358
|
+
}
|
|
1359
|
+
} else if (input.mergeStrategy === "rrf") {
|
|
1360
|
+
warnings.push({
|
|
1361
|
+
source: "memory",
|
|
1362
|
+
code: "memory_entity_search_not_configured",
|
|
1363
|
+
message: "RRF merge requested but entity search is not configured; falling back to semantic+lexical RRF."
|
|
1364
|
+
});
|
|
1365
|
+
}
|
|
1366
|
+
const out = {
|
|
1187
1367
|
semantic: filterByDateRange(semantic, input.dateFrom, input.dateTo),
|
|
1188
1368
|
lexical: filterByDateRange(lexical, input.dateFrom, input.dateTo)
|
|
1189
1369
|
};
|
|
1370
|
+
if (entity && entity.length > 0) {
|
|
1371
|
+
out.entity = entity;
|
|
1372
|
+
}
|
|
1373
|
+
return out;
|
|
1190
1374
|
}
|
|
1191
1375
|
async function searchUnifiedMemory(input) {
|
|
1192
1376
|
const query = input.query.trim();
|
|
@@ -1457,32 +1641,56 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1457
1641
|
}
|
|
1458
1642
|
function mergeAcrossSources(input) {
|
|
1459
1643
|
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|
|
1644
|
+
const lists = buildChannelLists(memorySubs, input.insightHits, input.knowledgeHits);
|
|
1645
|
+
if (input.strategy !== "rrf") {
|
|
1646
|
+
const seen = /* @__PURE__ */ new Map();
|
|
1647
|
+
for (const hit of [
|
|
1648
|
+
...memorySubs.semantic,
|
|
1649
|
+
...memorySubs.lexical,
|
|
1650
|
+
...input.insightHits,
|
|
1651
|
+
...input.knowledgeHits
|
|
1652
|
+
]) {
|
|
1653
|
+
const key = `${hit.type}::${hit.id}`;
|
|
1654
|
+
const existing = seen.get(key);
|
|
1655
|
+
if (!existing || hit.similarity > existing.similarity) {
|
|
1656
|
+
seen.set(key, hit);
|
|
1657
|
+
}
|
|
1658
|
+
}
|
|
1659
|
+
const merged2 = mergeUnifiedMemorySearchResults(Array.from(seen.values()), input.limit);
|
|
1660
|
+
return attachSignals(merged2, lists);
|
|
1661
|
+
}
|
|
1460
1662
|
const all = [
|
|
1461
1663
|
...memorySubs.semantic,
|
|
1462
1664
|
...memorySubs.lexical,
|
|
1665
|
+
...memorySubs.entity ?? [],
|
|
1463
1666
|
...input.insightHits,
|
|
1464
1667
|
...input.knowledgeHits
|
|
1465
1668
|
];
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1669
|
+
const merged = lists.length <= 1 ? mergeUnifiedMemorySearchResults(all, input.limit) : mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
|
|
1670
|
+
return attachSignals(merged, lists);
|
|
1671
|
+
}
|
|
1672
|
+
function buildChannelLists(memorySubs, insightHits, knowledgeHits) {
|
|
1469
1673
|
const lists = [];
|
|
1470
|
-
if (memorySubs.semantic.length > 0) {
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
}
|
|
1476
|
-
if (
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1674
|
+
if (memorySubs.semantic.length > 0) lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
|
|
1675
|
+
if (memorySubs.lexical.length > 0) lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
|
|
1676
|
+
if (memorySubs.entity && memorySubs.entity.length > 0) {
|
|
1677
|
+
lists.push({ name: "memory-entity", hits: memorySubs.entity });
|
|
1678
|
+
}
|
|
1679
|
+
if (insightHits.length > 0) lists.push({ name: "insights", hits: insightHits });
|
|
1680
|
+
if (knowledgeHits.length > 0) lists.push({ name: "knowledge", hits: knowledgeHits });
|
|
1681
|
+
return lists;
|
|
1682
|
+
}
|
|
1683
|
+
function attachSignals(hits, lists) {
|
|
1684
|
+
if (hits.length === 0 || lists.length === 0) {
|
|
1685
|
+
return hits;
|
|
1686
|
+
}
|
|
1687
|
+
return hits.map((hit) => {
|
|
1688
|
+
const signals = materializeSignals(lists, hit);
|
|
1689
|
+
if (!signals) {
|
|
1690
|
+
return hit;
|
|
1691
|
+
}
|
|
1692
|
+
return { ...hit, signals };
|
|
1693
|
+
});
|
|
1486
1694
|
}
|
|
1487
1695
|
|
|
1488
1696
|
// src/search/apply-reflect.ts
|
|
@@ -1601,6 +1809,177 @@ function makePlanSummary(plan) {
|
|
|
1601
1809
|
}
|
|
1602
1810
|
return lines.join("\n");
|
|
1603
1811
|
}
|
|
1812
|
+
var DEFAULT_CANDIDATE_LIMIT = 50;
|
|
1813
|
+
async function deriveFacts(deps, input, logger = console) {
|
|
1814
|
+
const warnings = [];
|
|
1815
|
+
if (typeof deps.deriver !== "function") {
|
|
1816
|
+
warnings.push({
|
|
1817
|
+
code: "derive_deriver_not_configured",
|
|
1818
|
+
message: "No `deriver` is wired into the unified search deps; returning an empty fact list."
|
|
1819
|
+
});
|
|
1820
|
+
return { facts: [], warnings };
|
|
1821
|
+
}
|
|
1822
|
+
const limit = Math.max(1, Math.min(500, input.candidateLimit ?? DEFAULT_CANDIDATE_LIMIT));
|
|
1823
|
+
if (!input.candidateTexts || input.candidateTexts.length === 0) {
|
|
1824
|
+
if (!input.query) {
|
|
1825
|
+
warnings.push({
|
|
1826
|
+
code: "derive_fallback_query_noise",
|
|
1827
|
+
message: "No `query` supplied and no `candidateTexts`; lexical fallback derives keywords from `userId + botIds`, which is rarely useful. Pass `query` (or `candidateTexts`) for topical derivation."
|
|
1828
|
+
});
|
|
1829
|
+
}
|
|
1830
|
+
}
|
|
1831
|
+
let candidateTexts;
|
|
1832
|
+
if (input.candidateTexts && input.candidateTexts.length > 0) {
|
|
1833
|
+
candidateTexts = input.candidateTexts;
|
|
1834
|
+
} else {
|
|
1835
|
+
candidateTexts = await loadCandidateTexts(deps, input, limit, logger);
|
|
1836
|
+
if (candidateTexts.length === 0) {
|
|
1837
|
+
warnings.push({
|
|
1838
|
+
code: "derive_no_candidates",
|
|
1839
|
+
message: "No candidate fact texts available for derivation"
|
|
1840
|
+
});
|
|
1841
|
+
return { facts: [], warnings };
|
|
1842
|
+
}
|
|
1843
|
+
}
|
|
1844
|
+
let facts;
|
|
1845
|
+
try {
|
|
1846
|
+
facts = await deps.deriver({
|
|
1847
|
+
userId: input.userId,
|
|
1848
|
+
userScope: {
|
|
1849
|
+
userId: input.userId,
|
|
1850
|
+
botIds: input.botIds,
|
|
1851
|
+
dateFrom: input.dateFrom,
|
|
1852
|
+
dateTo: input.dateTo
|
|
1853
|
+
},
|
|
1854
|
+
recentFactTexts: candidateTexts,
|
|
1855
|
+
...input.window ? { window: input.window } : {}
|
|
1856
|
+
});
|
|
1857
|
+
} catch (error) {
|
|
1858
|
+
logger.warn?.("[memory-store/derive] deriver threw:", error);
|
|
1859
|
+
warnings.push({
|
|
1860
|
+
code: "derive_deriver_failed",
|
|
1861
|
+
message: error.message ?? "deriver failed"
|
|
1862
|
+
});
|
|
1863
|
+
return { facts: [], warnings };
|
|
1864
|
+
}
|
|
1865
|
+
if (!Array.isArray(facts)) {
|
|
1866
|
+
warnings.push({
|
|
1867
|
+
code: "derive_deriver_returned_invalid_shape",
|
|
1868
|
+
message: `deriver returned ${typeof facts}; expected DerivedFact[]`
|
|
1869
|
+
});
|
|
1870
|
+
return { facts: [], warnings };
|
|
1871
|
+
}
|
|
1872
|
+
const normalized = [];
|
|
1873
|
+
const now = Date.now();
|
|
1874
|
+
for (const fact of facts) {
|
|
1875
|
+
if (!fact || typeof fact.text !== "string" || fact.text.length === 0 || typeof fact.kind !== "string" || !isDerivedKind(fact.kind) || !Array.isArray(fact.sources)) {
|
|
1876
|
+
continue;
|
|
1877
|
+
}
|
|
1878
|
+
normalized.push({
|
|
1879
|
+
...fact,
|
|
1880
|
+
derivedAt: typeof fact.derivedAt === "number" ? fact.derivedAt : now,
|
|
1881
|
+
sources: fact.sources.filter((s) => typeof s === "string")
|
|
1882
|
+
});
|
|
1883
|
+
}
|
|
1884
|
+
if (input.persist && normalized.length > 0) {
|
|
1885
|
+
try {
|
|
1886
|
+
await input.persist(normalized);
|
|
1887
|
+
} catch (error) {
|
|
1888
|
+
logger.warn?.("[memory-store/derive] persist callback threw:", error);
|
|
1889
|
+
warnings.push({
|
|
1890
|
+
code: "derive_persist_failed",
|
|
1891
|
+
message: error.message ?? "derive persist failed"
|
|
1892
|
+
});
|
|
1893
|
+
}
|
|
1894
|
+
}
|
|
1895
|
+
return { facts: normalized, warnings };
|
|
1896
|
+
}
|
|
1897
|
+
async function loadCandidateTexts(deps, input, limit, logger) {
|
|
1898
|
+
if (typeof deps.searchRawMessagesLexical !== "function") {
|
|
1899
|
+
return [];
|
|
1900
|
+
}
|
|
1901
|
+
const queryText = (input.query ?? `${input.userId} ${input.botIds?.join(" ") ?? ""}`).trim();
|
|
1902
|
+
const keywords = deriveLexicalKeywords(queryText);
|
|
1903
|
+
if (keywords.length === 0) {
|
|
1904
|
+
return [];
|
|
1905
|
+
}
|
|
1906
|
+
try {
|
|
1907
|
+
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
1908
|
+
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
1909
|
+
const hits = (await Promise.all(
|
|
1910
|
+
filters.map(
|
|
1911
|
+
(botId) => deps.searchRawMessagesLexical?.({
|
|
1912
|
+
userId: input.userId,
|
|
1913
|
+
keywords,
|
|
1914
|
+
limit: Math.ceil(limit / filters.length),
|
|
1915
|
+
botId,
|
|
1916
|
+
...input.peers && input.peers.length > 0 ? { peers: input.peers } : {},
|
|
1917
|
+
...factTypes ? { factTypes } : {}
|
|
1918
|
+
})
|
|
1919
|
+
)
|
|
1920
|
+
)).flat().filter(isRawMemorySemanticResult);
|
|
1921
|
+
return hits.map((h) => toMemoryResult(h).content);
|
|
1922
|
+
} catch (error) {
|
|
1923
|
+
logger.warn?.("[memory-store/derive] lexical candidate fetch failed:", error);
|
|
1924
|
+
return [];
|
|
1925
|
+
}
|
|
1926
|
+
}
|
|
1927
|
+
async function distillRawMessage(deps, input, logger = console) {
|
|
1928
|
+
const warnings = [];
|
|
1929
|
+
if (typeof deps.entityExtractor !== "function") {
|
|
1930
|
+
warnings.push({
|
|
1931
|
+
code: "distill_extractor_not_configured",
|
|
1932
|
+
message: "No `entityExtractor` is wired into the unified search deps; returning an empty entity list."
|
|
1933
|
+
});
|
|
1934
|
+
return { edges: [], warnings };
|
|
1935
|
+
}
|
|
1936
|
+
let edges;
|
|
1937
|
+
try {
|
|
1938
|
+
edges = await deps.entityExtractor({
|
|
1939
|
+
userId: input.userId,
|
|
1940
|
+
messageId: input.messageId,
|
|
1941
|
+
content: input.content
|
|
1942
|
+
});
|
|
1943
|
+
} catch (error) {
|
|
1944
|
+
logger.warn?.("[memory-store/distill] extractor threw:", error);
|
|
1945
|
+
warnings.push({
|
|
1946
|
+
code: "distill_extractor_failed",
|
|
1947
|
+
message: error.message ?? "entity extractor failed"
|
|
1948
|
+
});
|
|
1949
|
+
return { edges: [], warnings };
|
|
1950
|
+
}
|
|
1951
|
+
if (!Array.isArray(edges)) {
|
|
1952
|
+
warnings.push({
|
|
1953
|
+
code: "distill_extractor_returned_invalid_shape",
|
|
1954
|
+
message: `entityExtractor returned ${typeof edges}; expected EntityEdge[]`
|
|
1955
|
+
});
|
|
1956
|
+
return { edges: [], warnings };
|
|
1957
|
+
}
|
|
1958
|
+
const normalized = [];
|
|
1959
|
+
const now = Date.now();
|
|
1960
|
+
for (const edge of edges) {
|
|
1961
|
+
if (!edge || typeof edge.label !== "string" || edge.label.length === 0 || typeof edge.kind !== "string" || !isEntityKind(edge.kind) || typeof edge.relation !== "string" || typeof edge.sourceMessageId !== "string") {
|
|
1962
|
+
continue;
|
|
1963
|
+
}
|
|
1964
|
+
normalized.push({
|
|
1965
|
+
...edge,
|
|
1966
|
+
label: edge.label.trim().toLowerCase(),
|
|
1967
|
+
extractedAt: typeof edge.extractedAt === "number" ? edge.extractedAt : now
|
|
1968
|
+
});
|
|
1969
|
+
}
|
|
1970
|
+
if (input.persist && normalized.length > 0) {
|
|
1971
|
+
try {
|
|
1972
|
+
await input.persist(normalized);
|
|
1973
|
+
} catch (error) {
|
|
1974
|
+
logger.warn?.("[memory-store/distill] persist callback threw:", error);
|
|
1975
|
+
warnings.push({
|
|
1976
|
+
code: "distill_persist_failed",
|
|
1977
|
+
message: error.message ?? "distill persist failed"
|
|
1978
|
+
});
|
|
1979
|
+
}
|
|
1980
|
+
}
|
|
1981
|
+
return { edges: normalized, warnings };
|
|
1982
|
+
}
|
|
1604
1983
|
var DEFAULT_MAX_FACTS = 1e3;
|
|
1605
1984
|
var LOW_CONFIDENCE_THRESHOLD = 0.1;
|
|
1606
1985
|
function toHRR(data) {
|
|
@@ -1764,34 +2143,6 @@ init_raw_message_store();
|
|
|
1764
2143
|
init_sqlite_raw_message_store();
|
|
1765
2144
|
var DEFAULT_NAME = "@melandlabs/memory-store";
|
|
1766
2145
|
var DEFAULT_VERSION = "0.1.0";
|
|
1767
|
-
async function embedMissingMessages(messages, deps) {
|
|
1768
|
-
if (typeof deps.embedQuery !== "function") {
|
|
1769
|
-
return messages;
|
|
1770
|
-
}
|
|
1771
|
-
const out = [];
|
|
1772
|
-
for (const message of messages) {
|
|
1773
|
-
if (Array.isArray(message.embedding) && message.embedding.length > 0) {
|
|
1774
|
-
out.push(message);
|
|
1775
|
-
continue;
|
|
1776
|
-
}
|
|
1777
|
-
if (typeof message.content !== "string" || message.content.length === 0) {
|
|
1778
|
-
out.push(message);
|
|
1779
|
-
continue;
|
|
1780
|
-
}
|
|
1781
|
-
const vector = await deps.embedQuery({
|
|
1782
|
-
userId: message.userId,
|
|
1783
|
-
query: message.content
|
|
1784
|
-
});
|
|
1785
|
-
out.push({
|
|
1786
|
-
...message,
|
|
1787
|
-
embedding: vector,
|
|
1788
|
-
embeddingModel: message.embeddingModel ?? "server",
|
|
1789
|
-
embeddingDimensions: vector.length,
|
|
1790
|
-
embeddingUpdatedAt: Date.now()
|
|
1791
|
-
});
|
|
1792
|
-
}
|
|
1793
|
-
return out;
|
|
1794
|
-
}
|
|
1795
2146
|
async function startMcpServer(options = {}) {
|
|
1796
2147
|
const server = new McpServer({
|
|
1797
2148
|
name: options.name ?? DEFAULT_NAME,
|
|
@@ -1918,7 +2269,11 @@ async function startMcpServer(options = {}) {
|
|
|
1918
2269
|
const a = args;
|
|
1919
2270
|
const manager2 = await rawStore.getManager();
|
|
1920
2271
|
const incoming = [a.message].map((m) => ({ ...m, userId: m.userId ?? a.userId }));
|
|
1921
|
-
const messages
|
|
2272
|
+
const { messages, warnings } = await applyEmbedOnInsertPolicy(
|
|
2273
|
+
incoming,
|
|
2274
|
+
a.embedOnInsert,
|
|
2275
|
+
options.unified
|
|
2276
|
+
);
|
|
1922
2277
|
let result;
|
|
1923
2278
|
if (typeof manager2.upsertRawMessages === "function") {
|
|
1924
2279
|
result = await manager2.upsertRawMessages({
|
|
@@ -1940,7 +2295,12 @@ async function startMcpServer(options = {}) {
|
|
|
1940
2295
|
content: [
|
|
1941
2296
|
{
|
|
1942
2297
|
type: "text",
|
|
1943
|
-
text: JSON.stringify({
|
|
2298
|
+
text: JSON.stringify({
|
|
2299
|
+
ok: true,
|
|
2300
|
+
count: messages.length,
|
|
2301
|
+
result,
|
|
2302
|
+
...warnings.length > 0 ? { warnings } : {}
|
|
2303
|
+
})
|
|
1944
2304
|
}
|
|
1945
2305
|
]
|
|
1946
2306
|
};
|
|
@@ -2002,6 +2362,109 @@ async function startMcpServer(options = {}) {
|
|
|
2002
2362
|
};
|
|
2003
2363
|
}
|
|
2004
2364
|
);
|
|
2365
|
+
const distillSchema = {
|
|
2366
|
+
userId: z.string(),
|
|
2367
|
+
messageId: z.string().min(1),
|
|
2368
|
+
content: z.string().min(1)
|
|
2369
|
+
};
|
|
2370
|
+
server.registerTool(
|
|
2371
|
+
"memory.distill",
|
|
2372
|
+
{
|
|
2373
|
+
title: "Distill Entities",
|
|
2374
|
+
description: "Extract `EntityEdge`s from a single raw message via the host's `unified.entityExtractor`. Returns `{ edges, warnings }`. Without the extractor wired in, returns an empty list with a `distill_extractor_not_configured` warning. Persistence is the host's responsibility \u2014 the MCP transport cannot carry host-side `persist` callbacks. Call `distillRawMessage` directly from host code if you need round-trip persistence.",
|
|
2375
|
+
// biome-ignore lint/suspicious/noExplicitAny: MCP SDK expects Zod schema type
|
|
2376
|
+
inputSchema: distillSchema
|
|
2377
|
+
},
|
|
2378
|
+
async (args) => {
|
|
2379
|
+
const a = args;
|
|
2380
|
+
const input = {
|
|
2381
|
+
userId: a.userId,
|
|
2382
|
+
messageId: a.messageId,
|
|
2383
|
+
content: a.content
|
|
2384
|
+
};
|
|
2385
|
+
try {
|
|
2386
|
+
const result = await distillRawMessage(options.unified ?? {}, input, console);
|
|
2387
|
+
return {
|
|
2388
|
+
content: [{ type: "text", text: JSON.stringify(result) }]
|
|
2389
|
+
};
|
|
2390
|
+
} catch (error) {
|
|
2391
|
+
console.error("[memory-store/mcp] distill failed:", error);
|
|
2392
|
+
return {
|
|
2393
|
+
content: [
|
|
2394
|
+
{
|
|
2395
|
+
type: "text",
|
|
2396
|
+
text: JSON.stringify({ error: error.message ?? "distill failed" })
|
|
2397
|
+
}
|
|
2398
|
+
],
|
|
2399
|
+
isError: true
|
|
2400
|
+
};
|
|
2401
|
+
}
|
|
2402
|
+
}
|
|
2403
|
+
);
|
|
2404
|
+
const deriveSchema = {
|
|
2405
|
+
userId: z.string(),
|
|
2406
|
+
query: z.string().optional().describe(
|
|
2407
|
+
"Optional topical query used to derive lexical keywords when `candidateTexts` is omitted. Loop-engine schedulers should pass the topic they want synthesized facts about (e.g. 'cat preferences'). Without it the lexical fallback uses `userId + botIds` which is rarely useful."
|
|
2408
|
+
),
|
|
2409
|
+
botIds: z.array(z.string()).optional(),
|
|
2410
|
+
peers: z.array(
|
|
2411
|
+
z.object({
|
|
2412
|
+
kind: z.enum(["user", "agent"]),
|
|
2413
|
+
id: z.string().min(1)
|
|
2414
|
+
})
|
|
2415
|
+
).optional().describe("Optional peer scope forwarded to the lexical candidate-fetch fallback."),
|
|
2416
|
+
factTypes: z.array(z.string()).optional().describe("Optional FactType filter forwarded to the lexical candidate-fetch fallback."),
|
|
2417
|
+
dateFrom: z.string().optional(),
|
|
2418
|
+
dateTo: z.string().optional(),
|
|
2419
|
+
windowFrom: z.number().optional().describe("Optional time-window start (ms) forwarded to the deriver."),
|
|
2420
|
+
windowTo: z.number().optional().describe("Optional time-window end (ms) forwarded to the deriver."),
|
|
2421
|
+
candidateTexts: z.array(z.string()).optional().describe(
|
|
2422
|
+
"Pre-computed candidate fact texts. When omitted, the SDK pulls up to `candidateLimit` (default 50) texts via the lexical sub-query."
|
|
2423
|
+
),
|
|
2424
|
+
candidateLimit: z.number().int().min(1).max(500).optional()
|
|
2425
|
+
};
|
|
2426
|
+
server.registerTool(
|
|
2427
|
+
"memory.derive",
|
|
2428
|
+
{
|
|
2429
|
+
title: "Derive Facts",
|
|
2430
|
+
description: "Synthesize `DerivedFact`s over a window of candidate fact texts via the host's `unified.deriver`. Returns `{ facts, warnings }`. Without the deriver wired in, returns an empty list with a `derive_deriver_not_configured` warning. Persistence is the host's responsibility \u2014 the MCP transport cannot carry host-side `persist` callbacks. Call `deriveFacts` directly from host code if you need round-trip persistence.",
|
|
2431
|
+
// biome-ignore lint/suspicious/noExplicitAny: MCP SDK expects Zod schema type
|
|
2432
|
+
inputSchema: deriveSchema
|
|
2433
|
+
},
|
|
2434
|
+
async (args) => {
|
|
2435
|
+
const a = args;
|
|
2436
|
+
const window = a.windowFrom !== void 0 && a.windowTo !== void 0 ? { from: a.windowFrom, to: a.windowTo } : void 0;
|
|
2437
|
+
const input = {
|
|
2438
|
+
userId: a.userId,
|
|
2439
|
+
...a.query ? { query: a.query } : {},
|
|
2440
|
+
botIds: a.botIds,
|
|
2441
|
+
...a.peers && a.peers.length > 0 ? { peers: a.peers } : {},
|
|
2442
|
+
...a.factTypes && a.factTypes.length > 0 ? { factTypes: a.factTypes.filter(isFactType) } : {},
|
|
2443
|
+
dateFrom: a.dateFrom,
|
|
2444
|
+
dateTo: a.dateTo,
|
|
2445
|
+
...window ? { window } : {},
|
|
2446
|
+
...a.candidateTexts ? { candidateTexts: a.candidateTexts } : {},
|
|
2447
|
+
...a.candidateLimit !== void 0 ? { candidateLimit: a.candidateLimit } : {}
|
|
2448
|
+
};
|
|
2449
|
+
try {
|
|
2450
|
+
const result = await deriveFacts(options.unified ?? {}, input, console);
|
|
2451
|
+
return {
|
|
2452
|
+
content: [{ type: "text", text: JSON.stringify(result) }]
|
|
2453
|
+
};
|
|
2454
|
+
} catch (error) {
|
|
2455
|
+
console.error("[memory-store/mcp] derive failed:", error);
|
|
2456
|
+
return {
|
|
2457
|
+
content: [
|
|
2458
|
+
{
|
|
2459
|
+
type: "text",
|
|
2460
|
+
text: JSON.stringify({ error: error.message ?? "derive failed" })
|
|
2461
|
+
}
|
|
2462
|
+
],
|
|
2463
|
+
isError: true
|
|
2464
|
+
};
|
|
2465
|
+
}
|
|
2466
|
+
}
|
|
2467
|
+
);
|
|
2005
2468
|
const vsaVocabularyEntry = {
|
|
2006
2469
|
label: z.string().min(1),
|
|
2007
2470
|
vector: z.array(z.number()).describe("Float32 vector for the vocabulary entry.")
|