@melandlabs/memory-store 1.1.7 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/dist/{config-CnbqSek5.d.ts → config-DfvVZ--0.d.ts} +137 -1
- package/dist/http.d.ts +11 -2
- package/dist/http.js +466 -56
- package/dist/index.d.ts +10 -5
- package/dist/index.js +351 -21
- package/dist/mcp.d.ts +12 -2
- package/dist/mcp.js +516 -53
- package/dist/search/derive.d.ts +93 -0
- package/dist/search/derive.js +145 -0
- package/dist/search/distill.d.ts +60 -0
- package/dist/search/distill.js +61 -0
- package/dist/search/unified-search.d.ts +4 -2
- package/dist/search/unified-search.js +180 -23
- package/dist/server/cli-http.js +466 -56
- package/dist/server/cli-mcp.js +520 -52
- package/dist/server/cli-shared.d.ts +3 -1
- package/dist/storage/postgres-raw-message-factory.d.ts +3 -1
- package/dist/storage/raw-message-store.d.ts +3 -1
- package/dist/storage/sqlite-raw-message-store.d.ts +3 -1
- package/dist/storage/sqlite-vector-index.d.ts +3 -1
- package/dist/{unified-search-Bft57TRH.d.ts → unified-search-A8Q3JHwU.d.ts} +1 -1
- package/package.json +6 -6
package/dist/http.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { vsaNormalizeVector, vsaAssertSameDim } from '@melandlabs/contracts';
|
|
1
|
+
import { isFactType, vsaNormalizeVector, vsaAssertSameDim } from '@melandlabs/contracts';
|
|
2
2
|
import { buildMemoryRecordEmbeddingDocument } from '@melandlabs/ai/memory';
|
|
3
3
|
import { ChromaVectorStore } from '@melandlabs/rag';
|
|
4
4
|
import { mkdirSync } from 'fs';
|
|
@@ -8,6 +8,8 @@ import { SQLiteRawMessageManager, getSQLiteVsaStore, closeSQLiteVsaStore } from
|
|
|
8
8
|
import { serve } from '@hono/node-server';
|
|
9
9
|
import { Hono } from 'hono';
|
|
10
10
|
import { applyReflectedConsolidationPlan } from '@melandlabs/memory-consolidation';
|
|
11
|
+
import { isDerivedKind } from '@melandlabs/contracts/derived-fact';
|
|
12
|
+
import { isEntityKind } from '@melandlabs/contracts/entity-edge';
|
|
11
13
|
import { superpose, bind, unbind, cleanup, cosineSimilarity } from '@melandlabs/vsa';
|
|
12
14
|
|
|
13
15
|
var __defProp = Object.defineProperty;
|
|
@@ -321,6 +323,55 @@ var init_raw_message_store = __esm({
|
|
|
321
323
|
}
|
|
322
324
|
});
|
|
323
325
|
|
|
326
|
+
// src/embed-on-insert.ts
|
|
327
|
+
async function embedMissingMessages(messages, deps) {
|
|
328
|
+
if (typeof deps.embedQuery !== "function") {
|
|
329
|
+
return messages;
|
|
330
|
+
}
|
|
331
|
+
const out = [];
|
|
332
|
+
for (const message of messages) {
|
|
333
|
+
if (Array.isArray(message.embedding) && message.embedding.length > 0) {
|
|
334
|
+
out.push(message);
|
|
335
|
+
continue;
|
|
336
|
+
}
|
|
337
|
+
if (typeof message.content !== "string" || message.content.length === 0) {
|
|
338
|
+
out.push(message);
|
|
339
|
+
continue;
|
|
340
|
+
}
|
|
341
|
+
const vector = await deps.embedQuery({
|
|
342
|
+
userId: message.userId,
|
|
343
|
+
query: message.content
|
|
344
|
+
});
|
|
345
|
+
out.push({
|
|
346
|
+
...message,
|
|
347
|
+
embedding: vector,
|
|
348
|
+
embeddingModel: message.embeddingModel ?? "server",
|
|
349
|
+
embeddingDimensions: vector.length,
|
|
350
|
+
embeddingUpdatedAt: Date.now()
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
return out;
|
|
354
|
+
}
|
|
355
|
+
var AUTO_APPLIED_WARNING = {
|
|
356
|
+
code: "embed_on_insert_auto_applied",
|
|
357
|
+
message: "Caller set `embedOnInsert` falsy but the active vector backend needs an embedding per row. Server filled in the missing embeddings; pass `embedOnInsert: true` to opt in explicitly."
|
|
358
|
+
};
|
|
359
|
+
async function applyEmbedOnInsertPolicy(incoming, embedOnInsert, unified) {
|
|
360
|
+
const messages = incoming;
|
|
361
|
+
const anyMissing = incoming.some((m) => !Array.isArray(m.embedding) || m.embedding.length === 0);
|
|
362
|
+
const embedderWired = typeof unified?.embedQuery === "function";
|
|
363
|
+
if (embedOnInsert === true && anyMissing) {
|
|
364
|
+
return { messages: await embedMissingMessages(messages, unified ?? {}), warnings: [] };
|
|
365
|
+
}
|
|
366
|
+
if (embedOnInsert !== true && anyMissing && embedderWired) {
|
|
367
|
+
return {
|
|
368
|
+
messages: await embedMissingMessages(messages, unified ?? {}),
|
|
369
|
+
warnings: [AUTO_APPLIED_WARNING]
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
return { messages, warnings: [] };
|
|
373
|
+
}
|
|
374
|
+
|
|
324
375
|
// src/search/unified-search.ts
|
|
325
376
|
init_chroma_memory_index();
|
|
326
377
|
init_raw_message_store();
|
|
@@ -361,6 +412,9 @@ async function applyReranker(reranker, query, results) {
|
|
|
361
412
|
}
|
|
362
413
|
|
|
363
414
|
// src/search/utilities.ts
|
|
415
|
+
function deriveLexicalKeywords(query) {
|
|
416
|
+
return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
|
|
417
|
+
}
|
|
364
418
|
var DEFAULT_LIMIT = 10;
|
|
365
419
|
var DEFAULT_THRESHOLD = 0.7;
|
|
366
420
|
var DEFAULT_SOURCES = ["memory", "insights", "knowledge"];
|
|
@@ -474,6 +528,50 @@ function toMemoryResult(result) {
|
|
|
474
528
|
metadata: result.metadata
|
|
475
529
|
};
|
|
476
530
|
}
|
|
531
|
+
function materializeSignals(lists, hit) {
|
|
532
|
+
const channels = [];
|
|
533
|
+
let semantic;
|
|
534
|
+
let lexical;
|
|
535
|
+
let entity;
|
|
536
|
+
for (const list of lists) {
|
|
537
|
+
const channel = listNameToChannel(list.name);
|
|
538
|
+
if (!channel) continue;
|
|
539
|
+
if (channels.includes(channel)) continue;
|
|
540
|
+
let found;
|
|
541
|
+
for (const candidate of list.hits) {
|
|
542
|
+
if (candidate.type === hit.type && candidate.id === hit.id) {
|
|
543
|
+
found = candidate;
|
|
544
|
+
break;
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
if (!found) continue;
|
|
548
|
+
channels.push(channel);
|
|
549
|
+
if (channel === "semantic" && semantic === void 0) {
|
|
550
|
+
semantic = found.similarity;
|
|
551
|
+
} else if (channel === "lexical" && lexical === void 0) {
|
|
552
|
+
lexical = found.similarity;
|
|
553
|
+
} else if (channel === "entity" && entity === void 0) {
|
|
554
|
+
entity = found.similarity;
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
if (channels.length === 0) {
|
|
558
|
+
return void 0;
|
|
559
|
+
}
|
|
560
|
+
const out = { channels };
|
|
561
|
+
if (semantic !== void 0) out.semantic = semantic;
|
|
562
|
+
if (lexical !== void 0) out.lexical = lexical;
|
|
563
|
+
if (entity !== void 0) out.entity = entity;
|
|
564
|
+
if (typeof hit.metadata.rrfScore === "number") {
|
|
565
|
+
out.rrf = hit.metadata.rrfScore;
|
|
566
|
+
}
|
|
567
|
+
return out;
|
|
568
|
+
}
|
|
569
|
+
function listNameToChannel(name) {
|
|
570
|
+
if (name === "memory-semantic") return "semantic";
|
|
571
|
+
if (name === "memory-bm25" || name === "memory-lexical") return "lexical";
|
|
572
|
+
if (name === "memory-entity") return "entity";
|
|
573
|
+
return void 0;
|
|
574
|
+
}
|
|
477
575
|
function isRawMemorySemanticResult(result) {
|
|
478
576
|
if (!result || typeof result !== "object") {
|
|
479
577
|
return false;
|
|
@@ -741,8 +839,65 @@ async function resolveSearchScopePeers(input) {
|
|
|
741
839
|
}
|
|
742
840
|
|
|
743
841
|
// src/search/unified-search.ts
|
|
744
|
-
function
|
|
745
|
-
|
|
842
|
+
function deriveEntityKeywords(query) {
|
|
843
|
+
const proper = Array.from(new Set(query.match(/\b[A-Z][a-zA-Z]{2,}\b/g) ?? [])).map((w) => w.toLowerCase());
|
|
844
|
+
const lexical = deriveLexicalKeywords(query);
|
|
845
|
+
const seen = /* @__PURE__ */ new Set();
|
|
846
|
+
const out = [];
|
|
847
|
+
for (const word of [...proper, ...lexical]) {
|
|
848
|
+
if (word.length < 2) continue;
|
|
849
|
+
if (seen.has(word)) continue;
|
|
850
|
+
seen.add(word);
|
|
851
|
+
out.push(word);
|
|
852
|
+
}
|
|
853
|
+
return out.slice(0, 16);
|
|
854
|
+
}
|
|
855
|
+
async function runEntitySearchForQuery(deps, input, limit, logger, peerPeers = []) {
|
|
856
|
+
if (typeof deps.entitySearch !== "function") {
|
|
857
|
+
return [];
|
|
858
|
+
}
|
|
859
|
+
const keywords = deriveEntityKeywords(input.query);
|
|
860
|
+
if (keywords.length === 0) {
|
|
861
|
+
return [];
|
|
862
|
+
}
|
|
863
|
+
try {
|
|
864
|
+
const entitySearch = deps.entitySearch;
|
|
865
|
+
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
866
|
+
const matches = (await Promise.all(
|
|
867
|
+
filters.map(
|
|
868
|
+
(botId) => entitySearch({
|
|
869
|
+
userId: input.userId,
|
|
870
|
+
keywords,
|
|
871
|
+
limit: Math.ceil(limit / filters.length),
|
|
872
|
+
botId,
|
|
873
|
+
...peerPeers.length > 0 ? { peers: peerPeers } : {}
|
|
874
|
+
})
|
|
875
|
+
)
|
|
876
|
+
)).flat();
|
|
877
|
+
return matches.filter(
|
|
878
|
+
(m) => typeof m?.messageId === "string" && typeof m?.label === "string" && typeof m?.score === "number"
|
|
879
|
+
).map((m) => ({
|
|
880
|
+
type: "memory",
|
|
881
|
+
id: m.messageId,
|
|
882
|
+
// The host's `entitySearch` returns match ids + labels only,
|
|
883
|
+
// so we surface the entity label as `content`. When the same
|
|
884
|
+
// `messageId` is also returned by the semantic/lexical channel
|
|
885
|
+
// (RRF path), the real message content is preserved via the
|
|
886
|
+
// shared `(type, id)` key; otherwise callers can re-fetch the
|
|
887
|
+
// original message by `id` (set in metadata as `sourceMessageId`).
|
|
888
|
+
content: m.label,
|
|
889
|
+
similarity: m.score,
|
|
890
|
+
metadata: {
|
|
891
|
+
scoring: "entity",
|
|
892
|
+
entityLabel: m.label,
|
|
893
|
+
isEntityProjection: true,
|
|
894
|
+
sourceMessageId: m.messageId
|
|
895
|
+
}
|
|
896
|
+
}));
|
|
897
|
+
} catch (error) {
|
|
898
|
+
logger.warn?.("[memory-store] entity search failed:", error);
|
|
899
|
+
return [];
|
|
900
|
+
}
|
|
746
901
|
}
|
|
747
902
|
function parseDateBoundary(value, role) {
|
|
748
903
|
const trimmed = value.trim();
|
|
@@ -1182,10 +1337,39 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1182
1337
|
});
|
|
1183
1338
|
}
|
|
1184
1339
|
}
|
|
1185
|
-
|
|
1340
|
+
let entity;
|
|
1341
|
+
if (typeof deps.entitySearch === "function") {
|
|
1342
|
+
entity = filterByDateRange(
|
|
1343
|
+
await runEntitySearchForQuery(deps, input, limit, logger, peerPeers),
|
|
1344
|
+
input.dateFrom,
|
|
1345
|
+
input.dateTo
|
|
1346
|
+
);
|
|
1347
|
+
if (entity.length === 0) {
|
|
1348
|
+
entity = void 0;
|
|
1349
|
+
} else if (normalizeUnifiedMemoryMergeStrategy(
|
|
1350
|
+
input.mergeStrategy ?? deps.reasoning?.defaultMergeStrategy ?? "rrf"
|
|
1351
|
+
) !== "rrf") {
|
|
1352
|
+
warnings.push({
|
|
1353
|
+
source: "memory",
|
|
1354
|
+
code: "memory_entity_requires_rrf",
|
|
1355
|
+
message: "Entity channel results are fused via RRF; with mergeStrategy:'similarity' standalone entity hits are suppressed (only the entity channel signal is kept on co-occurring hits). Request mergeStrategy:'rrf' to surface entity matches."
|
|
1356
|
+
});
|
|
1357
|
+
}
|
|
1358
|
+
} else if (input.mergeStrategy === "rrf") {
|
|
1359
|
+
warnings.push({
|
|
1360
|
+
source: "memory",
|
|
1361
|
+
code: "memory_entity_search_not_configured",
|
|
1362
|
+
message: "RRF merge requested but entity search is not configured; falling back to semantic+lexical RRF."
|
|
1363
|
+
});
|
|
1364
|
+
}
|
|
1365
|
+
const out = {
|
|
1186
1366
|
semantic: filterByDateRange(semantic, input.dateFrom, input.dateTo),
|
|
1187
1367
|
lexical: filterByDateRange(lexical, input.dateFrom, input.dateTo)
|
|
1188
1368
|
};
|
|
1369
|
+
if (entity && entity.length > 0) {
|
|
1370
|
+
out.entity = entity;
|
|
1371
|
+
}
|
|
1372
|
+
return out;
|
|
1189
1373
|
}
|
|
1190
1374
|
async function searchUnifiedMemory(input) {
|
|
1191
1375
|
const query = input.query.trim();
|
|
@@ -1456,32 +1640,56 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1456
1640
|
}
|
|
1457
1641
|
function mergeAcrossSources(input) {
|
|
1458
1642
|
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|
|
1643
|
+
const lists = buildChannelLists(memorySubs, input.insightHits, input.knowledgeHits);
|
|
1644
|
+
if (input.strategy !== "rrf") {
|
|
1645
|
+
const seen = /* @__PURE__ */ new Map();
|
|
1646
|
+
for (const hit of [
|
|
1647
|
+
...memorySubs.semantic,
|
|
1648
|
+
...memorySubs.lexical,
|
|
1649
|
+
...input.insightHits,
|
|
1650
|
+
...input.knowledgeHits
|
|
1651
|
+
]) {
|
|
1652
|
+
const key = `${hit.type}::${hit.id}`;
|
|
1653
|
+
const existing = seen.get(key);
|
|
1654
|
+
if (!existing || hit.similarity > existing.similarity) {
|
|
1655
|
+
seen.set(key, hit);
|
|
1656
|
+
}
|
|
1657
|
+
}
|
|
1658
|
+
const merged2 = mergeUnifiedMemorySearchResults(Array.from(seen.values()), input.limit);
|
|
1659
|
+
return attachSignals(merged2, lists);
|
|
1660
|
+
}
|
|
1459
1661
|
const all = [
|
|
1460
1662
|
...memorySubs.semantic,
|
|
1461
1663
|
...memorySubs.lexical,
|
|
1664
|
+
...memorySubs.entity ?? [],
|
|
1462
1665
|
...input.insightHits,
|
|
1463
1666
|
...input.knowledgeHits
|
|
1464
1667
|
];
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1668
|
+
const merged = lists.length <= 1 ? mergeUnifiedMemorySearchResults(all, input.limit) : mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
|
|
1669
|
+
return attachSignals(merged, lists);
|
|
1670
|
+
}
|
|
1671
|
+
function buildChannelLists(memorySubs, insightHits, knowledgeHits) {
|
|
1468
1672
|
const lists = [];
|
|
1469
|
-
if (memorySubs.semantic.length > 0) {
|
|
1470
|
-
|
|
1673
|
+
if (memorySubs.semantic.length > 0) lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
|
|
1674
|
+
if (memorySubs.lexical.length > 0) lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
|
|
1675
|
+
if (memorySubs.entity && memorySubs.entity.length > 0) {
|
|
1676
|
+
lists.push({ name: "memory-entity", hits: memorySubs.entity });
|
|
1471
1677
|
}
|
|
1472
|
-
if (
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
lists.push({ name: "knowledge", hits: input.knowledgeHits });
|
|
1480
|
-
}
|
|
1481
|
-
if (lists.length <= 1) {
|
|
1482
|
-
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
1678
|
+
if (insightHits.length > 0) lists.push({ name: "insights", hits: insightHits });
|
|
1679
|
+
if (knowledgeHits.length > 0) lists.push({ name: "knowledge", hits: knowledgeHits });
|
|
1680
|
+
return lists;
|
|
1681
|
+
}
|
|
1682
|
+
function attachSignals(hits, lists) {
|
|
1683
|
+
if (hits.length === 0 || lists.length === 0) {
|
|
1684
|
+
return hits;
|
|
1483
1685
|
}
|
|
1484
|
-
return
|
|
1686
|
+
return hits.map((hit) => {
|
|
1687
|
+
const signals = materializeSignals(lists, hit);
|
|
1688
|
+
if (!signals) {
|
|
1689
|
+
return hit;
|
|
1690
|
+
}
|
|
1691
|
+
return { ...hit, signals };
|
|
1692
|
+
});
|
|
1485
1693
|
}
|
|
1486
1694
|
|
|
1487
1695
|
// src/search/apply-reflect.ts
|
|
@@ -1600,6 +1808,177 @@ function makePlanSummary(plan) {
|
|
|
1600
1808
|
}
|
|
1601
1809
|
return lines.join("\n");
|
|
1602
1810
|
}
|
|
1811
|
+
var DEFAULT_CANDIDATE_LIMIT = 50;
|
|
1812
|
+
async function deriveFacts(deps, input, logger = console) {
|
|
1813
|
+
const warnings = [];
|
|
1814
|
+
if (typeof deps.deriver !== "function") {
|
|
1815
|
+
warnings.push({
|
|
1816
|
+
code: "derive_deriver_not_configured",
|
|
1817
|
+
message: "No `deriver` is wired into the unified search deps; returning an empty fact list."
|
|
1818
|
+
});
|
|
1819
|
+
return { facts: [], warnings };
|
|
1820
|
+
}
|
|
1821
|
+
const limit = Math.max(1, Math.min(500, input.candidateLimit ?? DEFAULT_CANDIDATE_LIMIT));
|
|
1822
|
+
if (!input.candidateTexts || input.candidateTexts.length === 0) {
|
|
1823
|
+
if (!input.query) {
|
|
1824
|
+
warnings.push({
|
|
1825
|
+
code: "derive_fallback_query_noise",
|
|
1826
|
+
message: "No `query` supplied and no `candidateTexts`; lexical fallback derives keywords from `userId + botIds`, which is rarely useful. Pass `query` (or `candidateTexts`) for topical derivation."
|
|
1827
|
+
});
|
|
1828
|
+
}
|
|
1829
|
+
}
|
|
1830
|
+
let candidateTexts;
|
|
1831
|
+
if (input.candidateTexts && input.candidateTexts.length > 0) {
|
|
1832
|
+
candidateTexts = input.candidateTexts;
|
|
1833
|
+
} else {
|
|
1834
|
+
candidateTexts = await loadCandidateTexts(deps, input, limit, logger);
|
|
1835
|
+
if (candidateTexts.length === 0) {
|
|
1836
|
+
warnings.push({
|
|
1837
|
+
code: "derive_no_candidates",
|
|
1838
|
+
message: "No candidate fact texts available for derivation"
|
|
1839
|
+
});
|
|
1840
|
+
return { facts: [], warnings };
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
let facts;
|
|
1844
|
+
try {
|
|
1845
|
+
facts = await deps.deriver({
|
|
1846
|
+
userId: input.userId,
|
|
1847
|
+
userScope: {
|
|
1848
|
+
userId: input.userId,
|
|
1849
|
+
botIds: input.botIds,
|
|
1850
|
+
dateFrom: input.dateFrom,
|
|
1851
|
+
dateTo: input.dateTo
|
|
1852
|
+
},
|
|
1853
|
+
recentFactTexts: candidateTexts,
|
|
1854
|
+
...input.window ? { window: input.window } : {}
|
|
1855
|
+
});
|
|
1856
|
+
} catch (error) {
|
|
1857
|
+
logger.warn?.("[memory-store/derive] deriver threw:", error);
|
|
1858
|
+
warnings.push({
|
|
1859
|
+
code: "derive_deriver_failed",
|
|
1860
|
+
message: error.message ?? "deriver failed"
|
|
1861
|
+
});
|
|
1862
|
+
return { facts: [], warnings };
|
|
1863
|
+
}
|
|
1864
|
+
if (!Array.isArray(facts)) {
|
|
1865
|
+
warnings.push({
|
|
1866
|
+
code: "derive_deriver_returned_invalid_shape",
|
|
1867
|
+
message: `deriver returned ${typeof facts}; expected DerivedFact[]`
|
|
1868
|
+
});
|
|
1869
|
+
return { facts: [], warnings };
|
|
1870
|
+
}
|
|
1871
|
+
const normalized = [];
|
|
1872
|
+
const now = Date.now();
|
|
1873
|
+
for (const fact of facts) {
|
|
1874
|
+
if (!fact || typeof fact.text !== "string" || fact.text.length === 0 || typeof fact.kind !== "string" || !isDerivedKind(fact.kind) || !Array.isArray(fact.sources)) {
|
|
1875
|
+
continue;
|
|
1876
|
+
}
|
|
1877
|
+
normalized.push({
|
|
1878
|
+
...fact,
|
|
1879
|
+
derivedAt: typeof fact.derivedAt === "number" ? fact.derivedAt : now,
|
|
1880
|
+
sources: fact.sources.filter((s) => typeof s === "string")
|
|
1881
|
+
});
|
|
1882
|
+
}
|
|
1883
|
+
if (input.persist && normalized.length > 0) {
|
|
1884
|
+
try {
|
|
1885
|
+
await input.persist(normalized);
|
|
1886
|
+
} catch (error) {
|
|
1887
|
+
logger.warn?.("[memory-store/derive] persist callback threw:", error);
|
|
1888
|
+
warnings.push({
|
|
1889
|
+
code: "derive_persist_failed",
|
|
1890
|
+
message: error.message ?? "derive persist failed"
|
|
1891
|
+
});
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
return { facts: normalized, warnings };
|
|
1895
|
+
}
|
|
1896
|
+
async function loadCandidateTexts(deps, input, limit, logger) {
|
|
1897
|
+
if (typeof deps.searchRawMessagesLexical !== "function") {
|
|
1898
|
+
return [];
|
|
1899
|
+
}
|
|
1900
|
+
const queryText = (input.query ?? `${input.userId} ${input.botIds?.join(" ") ?? ""}`).trim();
|
|
1901
|
+
const keywords = deriveLexicalKeywords(queryText);
|
|
1902
|
+
if (keywords.length === 0) {
|
|
1903
|
+
return [];
|
|
1904
|
+
}
|
|
1905
|
+
try {
|
|
1906
|
+
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
1907
|
+
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
1908
|
+
const hits = (await Promise.all(
|
|
1909
|
+
filters.map(
|
|
1910
|
+
(botId) => deps.searchRawMessagesLexical?.({
|
|
1911
|
+
userId: input.userId,
|
|
1912
|
+
keywords,
|
|
1913
|
+
limit: Math.ceil(limit / filters.length),
|
|
1914
|
+
botId,
|
|
1915
|
+
...input.peers && input.peers.length > 0 ? { peers: input.peers } : {},
|
|
1916
|
+
...factTypes ? { factTypes } : {}
|
|
1917
|
+
})
|
|
1918
|
+
)
|
|
1919
|
+
)).flat().filter(isRawMemorySemanticResult);
|
|
1920
|
+
return hits.map((h) => toMemoryResult(h).content);
|
|
1921
|
+
} catch (error) {
|
|
1922
|
+
logger.warn?.("[memory-store/derive] lexical candidate fetch failed:", error);
|
|
1923
|
+
return [];
|
|
1924
|
+
}
|
|
1925
|
+
}
|
|
1926
|
+
async function distillRawMessage(deps, input, logger = console) {
|
|
1927
|
+
const warnings = [];
|
|
1928
|
+
if (typeof deps.entityExtractor !== "function") {
|
|
1929
|
+
warnings.push({
|
|
1930
|
+
code: "distill_extractor_not_configured",
|
|
1931
|
+
message: "No `entityExtractor` is wired into the unified search deps; returning an empty entity list."
|
|
1932
|
+
});
|
|
1933
|
+
return { edges: [], warnings };
|
|
1934
|
+
}
|
|
1935
|
+
let edges;
|
|
1936
|
+
try {
|
|
1937
|
+
edges = await deps.entityExtractor({
|
|
1938
|
+
userId: input.userId,
|
|
1939
|
+
messageId: input.messageId,
|
|
1940
|
+
content: input.content
|
|
1941
|
+
});
|
|
1942
|
+
} catch (error) {
|
|
1943
|
+
logger.warn?.("[memory-store/distill] extractor threw:", error);
|
|
1944
|
+
warnings.push({
|
|
1945
|
+
code: "distill_extractor_failed",
|
|
1946
|
+
message: error.message ?? "entity extractor failed"
|
|
1947
|
+
});
|
|
1948
|
+
return { edges: [], warnings };
|
|
1949
|
+
}
|
|
1950
|
+
if (!Array.isArray(edges)) {
|
|
1951
|
+
warnings.push({
|
|
1952
|
+
code: "distill_extractor_returned_invalid_shape",
|
|
1953
|
+
message: `entityExtractor returned ${typeof edges}; expected EntityEdge[]`
|
|
1954
|
+
});
|
|
1955
|
+
return { edges: [], warnings };
|
|
1956
|
+
}
|
|
1957
|
+
const normalized = [];
|
|
1958
|
+
const now = Date.now();
|
|
1959
|
+
for (const edge of edges) {
|
|
1960
|
+
if (!edge || typeof edge.label !== "string" || edge.label.length === 0 || typeof edge.kind !== "string" || !isEntityKind(edge.kind) || typeof edge.relation !== "string" || typeof edge.sourceMessageId !== "string") {
|
|
1961
|
+
continue;
|
|
1962
|
+
}
|
|
1963
|
+
normalized.push({
|
|
1964
|
+
...edge,
|
|
1965
|
+
label: edge.label.trim().toLowerCase(),
|
|
1966
|
+
extractedAt: typeof edge.extractedAt === "number" ? edge.extractedAt : now
|
|
1967
|
+
});
|
|
1968
|
+
}
|
|
1969
|
+
if (input.persist && normalized.length > 0) {
|
|
1970
|
+
try {
|
|
1971
|
+
await input.persist(normalized);
|
|
1972
|
+
} catch (error) {
|
|
1973
|
+
logger.warn?.("[memory-store/distill] persist callback threw:", error);
|
|
1974
|
+
warnings.push({
|
|
1975
|
+
code: "distill_persist_failed",
|
|
1976
|
+
message: error.message ?? "distill persist failed"
|
|
1977
|
+
});
|
|
1978
|
+
}
|
|
1979
|
+
}
|
|
1980
|
+
return { edges: normalized, warnings };
|
|
1981
|
+
}
|
|
1603
1982
|
var DEFAULT_MAX_FACTS = 1e3;
|
|
1604
1983
|
var LOW_CONFIDENCE_THRESHOLD = 0.1;
|
|
1605
1984
|
function toHRR(data) {
|
|
@@ -1761,34 +2140,6 @@ function createVsaRecall(storage) {
|
|
|
1761
2140
|
init_chroma_memory_index();
|
|
1762
2141
|
init_raw_message_store();
|
|
1763
2142
|
init_sqlite_raw_message_store();
|
|
1764
|
-
async function embedMissingMessages(messages, deps) {
|
|
1765
|
-
if (typeof deps.embedQuery !== "function") {
|
|
1766
|
-
return messages;
|
|
1767
|
-
}
|
|
1768
|
-
const out = [];
|
|
1769
|
-
for (const message of messages) {
|
|
1770
|
-
if (Array.isArray(message.embedding) && message.embedding.length > 0) {
|
|
1771
|
-
out.push(message);
|
|
1772
|
-
continue;
|
|
1773
|
-
}
|
|
1774
|
-
if (typeof message.content !== "string" || message.content.length === 0) {
|
|
1775
|
-
out.push(message);
|
|
1776
|
-
continue;
|
|
1777
|
-
}
|
|
1778
|
-
const vector = await deps.embedQuery({
|
|
1779
|
-
userId: message.userId,
|
|
1780
|
-
query: message.content
|
|
1781
|
-
});
|
|
1782
|
-
out.push({
|
|
1783
|
-
...message,
|
|
1784
|
-
embedding: vector,
|
|
1785
|
-
embeddingModel: message.embeddingModel ?? "server",
|
|
1786
|
-
embeddingDimensions: vector.length,
|
|
1787
|
-
embeddingUpdatedAt: Date.now()
|
|
1788
|
-
});
|
|
1789
|
-
}
|
|
1790
|
-
return out;
|
|
1791
|
-
}
|
|
1792
2143
|
async function startHttpServer(options = {}) {
|
|
1793
2144
|
const port = options.port ?? Number.parseInt(process.env.MEMORY_HTTP_PORT ?? "7421", 10);
|
|
1794
2145
|
const host = options.host ?? process.env.MEMORY_HTTP_HOST ?? "127.0.0.1";
|
|
@@ -1849,6 +2200,56 @@ async function startHttpServer(options = {}) {
|
|
|
1849
2200
|
const result = await search.search(input);
|
|
1850
2201
|
return c.json(result);
|
|
1851
2202
|
});
|
|
2203
|
+
app.post("/v1/distill", async (c) => {
|
|
2204
|
+
const body = await c.req.json().catch(() => ({}));
|
|
2205
|
+
const userId = typeof body.userId === "string" ? body.userId : null;
|
|
2206
|
+
const messageId = typeof body.messageId === "string" ? body.messageId : null;
|
|
2207
|
+
const content = typeof body.content === "string" ? body.content : null;
|
|
2208
|
+
if (!userId || !messageId || content === null || content.length === 0) {
|
|
2209
|
+
return c.json({ error: "userId, messageId, and a non-empty content are required" }, 400);
|
|
2210
|
+
}
|
|
2211
|
+
const input = { userId, messageId, content };
|
|
2212
|
+
try {
|
|
2213
|
+
const result = await distillRawMessage(options.unified ?? {}, input, console);
|
|
2214
|
+
return c.json(result);
|
|
2215
|
+
} catch (error) {
|
|
2216
|
+
console.error("[memory-store/http] distill failed:", error);
|
|
2217
|
+
return c.json({ error: error.message ?? "distill failed" }, 500);
|
|
2218
|
+
}
|
|
2219
|
+
});
|
|
2220
|
+
app.post("/v1/derive", async (c) => {
|
|
2221
|
+
const body = await c.req.json().catch(() => ({}));
|
|
2222
|
+
const userId = typeof body.userId === "string" ? body.userId : null;
|
|
2223
|
+
if (!userId) {
|
|
2224
|
+
return c.json({ error: "userId is required" }, 400);
|
|
2225
|
+
}
|
|
2226
|
+
const windowFrom = typeof body.windowFrom === "number" ? body.windowFrom : void 0;
|
|
2227
|
+
const windowTo = typeof body.windowTo === "number" ? body.windowTo : void 0;
|
|
2228
|
+
const window = windowFrom !== void 0 && windowTo !== void 0 ? { from: windowFrom, to: windowTo } : void 0;
|
|
2229
|
+
const peers = Array.isArray(body.peers) ? body.peers.filter(
|
|
2230
|
+
(p) => (p?.kind === "user" || p?.kind === "agent") && typeof p?.id === "string" && p.id.length > 0
|
|
2231
|
+
) : void 0;
|
|
2232
|
+
const factTypes = Array.isArray(body.factTypes) ? body.factTypes.filter(isFactType) : void 0;
|
|
2233
|
+
const input = {
|
|
2234
|
+
userId,
|
|
2235
|
+
...typeof body.query === "string" ? { query: body.query } : {},
|
|
2236
|
+
botIds: Array.isArray(body.botIds) ? body.botIds : void 0,
|
|
2237
|
+
dateFrom: typeof body.dateFrom === "string" ? body.dateFrom : void 0,
|
|
2238
|
+
dateTo: typeof body.dateTo === "string" ? body.dateTo : void 0,
|
|
2239
|
+
...window ? { window } : {},
|
|
2240
|
+
...Array.isArray(body.candidateTexts) ? { candidateTexts: body.candidateTexts } : {},
|
|
2241
|
+
...typeof body.candidateLimit === "number" ? { candidateLimit: body.candidateLimit } : {},
|
|
2242
|
+
...peers && peers.length > 0 ? { peers } : {},
|
|
2243
|
+
...factTypes && factTypes.length > 0 ? { factTypes } : {}
|
|
2244
|
+
};
|
|
2245
|
+
try {
|
|
2246
|
+
const result = await deriveFacts(options.unified ?? {}, input, console);
|
|
2247
|
+
return c.json(result);
|
|
2248
|
+
} catch (error) {
|
|
2249
|
+
console.error("[memory-store/http] derive failed:", error);
|
|
2250
|
+
return c.json({ error: error.message ?? "derive failed" }, 500);
|
|
2251
|
+
}
|
|
2252
|
+
});
|
|
1852
2253
|
app.post("/v1/consolidate:apply", async (c) => {
|
|
1853
2254
|
const body = await c.req.json().catch(() => ({}));
|
|
1854
2255
|
const userId = typeof body.userId === "string" ? body.userId : null;
|
|
@@ -1889,11 +2290,15 @@ async function startHttpServer(options = {}) {
|
|
|
1889
2290
|
return c.json({ error: "userId and messages[] required" }, 400);
|
|
1890
2291
|
}
|
|
1891
2292
|
const manager3 = await rawStore.getManager();
|
|
1892
|
-
const incoming = body.messages
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
2293
|
+
const incoming = body.messages.map((m) => ({
|
|
2294
|
+
...m,
|
|
2295
|
+
userId: m.userId ?? body.userId
|
|
2296
|
+
}));
|
|
2297
|
+
const { messages, warnings } = await applyEmbedOnInsertPolicy(
|
|
2298
|
+
incoming,
|
|
2299
|
+
body.embedOnInsert,
|
|
2300
|
+
options.unified
|
|
2301
|
+
);
|
|
1897
2302
|
let result;
|
|
1898
2303
|
if (typeof manager3.upsertRawMessages === "function") {
|
|
1899
2304
|
result = await manager3.upsertRawMessages({
|
|
@@ -1935,7 +2340,12 @@ async function startHttpServer(options = {}) {
|
|
|
1935
2340
|
} catch (error) {
|
|
1936
2341
|
console.warn("[memory-store/http] sqlite-vec vector update failed:", error);
|
|
1937
2342
|
}
|
|
1938
|
-
return c.json({
|
|
2343
|
+
return c.json({
|
|
2344
|
+
ok: true,
|
|
2345
|
+
count: messages.length,
|
|
2346
|
+
result,
|
|
2347
|
+
...warnings.length > 0 ? { warnings } : {}
|
|
2348
|
+
});
|
|
1939
2349
|
});
|
|
1940
2350
|
app.get("/v1/raw-messages/:id", async (c) => {
|
|
1941
2351
|
const id = c.req.param("id");
|
|
@@ -2039,9 +2449,9 @@ async function startHttpServer(options = {}) {
|
|
|
2039
2449
|
url: `http://${host}:${port}`,
|
|
2040
2450
|
port,
|
|
2041
2451
|
async stop() {
|
|
2452
|
+
await new Promise((resolve) => server.close(() => resolve()));
|
|
2042
2453
|
await rawStore.close();
|
|
2043
2454
|
await closeSQLiteVsaStore();
|
|
2044
|
-
server.close();
|
|
2045
2455
|
}
|
|
2046
2456
|
};
|
|
2047
2457
|
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,19 +1,24 @@
|
|
|
1
1
|
import * as _melandlabs_memory_consolidation from '@melandlabs/memory-consolidation';
|
|
2
2
|
import { OwnerScope } from '@melandlabs/memory-consolidation';
|
|
3
|
-
import { d as SearchInput, e as SearchOutput, M as MemoryStoreConfig } from './config-
|
|
4
|
-
export { E as EmbedQueryFn, I as IterativeRecallCandidate,
|
|
5
|
-
import { U as UnifiedSearch, A as ApplyConsolidateInput, a as ApplyConsolidateOutput } from './unified-search-
|
|
6
|
-
export { b as applyReflectedPlan, c as createUnifiedSearch } from './unified-search-
|
|
3
|
+
import { d as SearchInput, e as SearchOutput, M as MemoryStoreConfig } from './config-DfvVZ--0.js';
|
|
4
|
+
export { E as EmbedQueryFn, H as HitChannel, i as HitSignals, I as IterativeRecallCandidate, B as IterativeRecallExecutor, C as IterativeRecallPlanner, D as IterativeRecallPlannerOptions, F as IterativeRecallResult, G as IterativeRecallSearchRequest, J as IterativeRecallSearchResult, K as IterativeRecallStats, L as MemoryStoreDb, a as MemoryStoreEnv, Q as QueryRewriter, N as QueryRewriterInput, O as QueryRewriterOptions, c as SearchEvidence, j as SearchSource, S as SearchTier, k as UnifiedMemoryMergeStrategy, l as UnifiedMemoryRankedList, P as UnifiedMemoryReasoningInfo, m as UnifiedMemoryReasoningStrategy, f as UnifiedMemorySearchInput, g as UnifiedMemorySearchOutput, h as UnifiedMemorySearchResult, n as UnifiedMemorySearchSource, b as UnifiedMemorySearchWarning, R as UnifiedSearchInsightsResult, T as UnifiedSearchKnowledgeResult, V as UnifiedSearchReasoningDeps, W as VectorBackend, o as clampUnifiedMemorySearchLimit, p as clampUnifiedMemorySearchThreshold, X as createIdentityIterativePlanner, Y as createIdentityRewriter, Z as createIterativeRecallPlanner, _ as createUserVoiceRewriter, r as isRawMemorySemanticResult, s as listNameToChannel, t as materializeSignals, u as mergeUnifiedMemorySearchResults, v as mergeUnifiedMemorySearchResultsRrf, x as normalizeUnifiedMemoryReasoningStrategy, y as normalizeUnifiedMemorySearchSources, z as toKnowledgeResult, A as toMemoryResult } from './config-DfvVZ--0.js';
|
|
5
|
+
import { U as UnifiedSearch, A as ApplyConsolidateInput, a as ApplyConsolidateOutput } from './unified-search-A8Q3JHwU.js';
|
|
6
|
+
export { b as applyReflectedPlan, c as createUnifiedSearch } from './unified-search-A8Q3JHwU.js';
|
|
7
7
|
import { StoreVsaFactInput, StoreVsaFactOutput, VsaRecallInput, VsaRecallOutput, VsaListInput, VsaFactSummary, VsaForgetInput, VsaForgetOutput, VsaFactStorage } from '@melandlabs/contracts';
|
|
8
|
-
export { StoreVsaFactInput, StoreVsaFactOutput, VsaFact, VsaFactStorage, VsaFactSummary, VsaForgetInput, VsaForgetOutput, VsaListInput, VsaRecallInput, VsaRecallOutput, VsaRecallScore, VsaVocabularyEntry } from '@melandlabs/contracts';
|
|
8
|
+
export { DerivedFact, DerivedKind, EntityEdge, EntityKind, StoreVsaFactInput, StoreVsaFactOutput, VsaFact, VsaFactStorage, VsaFactSummary, VsaForgetInput, VsaForgetOutput, VsaListInput, VsaRecallInput, VsaRecallOutput, VsaRecallScore, VsaVocabularyEntry } from '@melandlabs/contracts';
|
|
9
9
|
import { RawMessageStore, getRawMessageManager } from './storage/raw-message-store.js';
|
|
10
10
|
export { RawMessageStorageBackend, RawMessageStorageManagerWithSearch, closeRawMessageStore, createRawMessageStore, getRawMessageStorageBackend, isRawMessageStorageAvailable } from './storage/raw-message-store.js';
|
|
11
11
|
export { PostgresFactoryFn, PostgresRawMessageManagerLike, clearPostgresFactory, hasPostgresFactory, registerPostgresFactory, resolvePostgresFactory } from './storage/postgres-raw-message-factory.js';
|
|
12
12
|
import { RawMessageGraphEvolutionStorage } from '@melandlabs/indexeddb';
|
|
13
|
+
export { DeriveInput, DeriveOutput, DeriveWarning, deriveFacts } from './search/derive.js';
|
|
14
|
+
export { DistillInput, DistillOutput, DistillWarning, distillRawMessage } from './search/distill.js';
|
|
13
15
|
import { Peer } from '@melandlabs/contracts/peer';
|
|
14
16
|
export { SQLiteVsaStore, SQLiteVsaStoreOptions, closeSQLiteVsaStore, getSQLiteVsaStore } from '@melandlabs/sqlite';
|
|
15
17
|
export { RawMessage } from './contracts.js';
|
|
18
|
+
import '@melandlabs/contracts/derived-fact';
|
|
19
|
+
import '@melandlabs/contracts/entity-edge';
|
|
16
20
|
import '@melandlabs/ai/memory';
|
|
21
|
+
import '@melandlabs/contracts/fact-type';
|
|
17
22
|
|
|
18
23
|
/**
|
|
19
24
|
* `@melandlabs/memory-store` — VSA (Vector Symbolic Architecture) facade.
|