@melandlabs/memory-store 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{config-D6MzksfT.d.ts → config-BYfTzG3h.d.ts} +17 -0
- package/dist/contracts.d.ts +7 -0
- package/dist/contracts.js +0 -2
- package/dist/http.d.ts +1 -1
- package/dist/http.js +144 -23
- package/dist/index.d.ts +2 -2
- package/dist/index.js +144 -23
- package/dist/mcp.d.ts +1 -1
- package/dist/mcp.js +144 -23
- package/dist/policies/memory-graph-correction-policy.js +0 -2
- package/dist/policies/memory-graph-write-policy.js +0 -2
- package/dist/search/unified-search.d.ts +39 -3
- package/dist/search/unified-search.js +145 -24
- package/dist/server/cli-http.js +144 -23
- package/dist/server/cli-mcp.js +144 -23
- package/dist/server/cli-shared.d.ts +1 -1
- package/dist/server/cli-shared.js +0 -2
- package/dist/storage/chroma-memory-index.js +0 -2
- package/dist/storage/postgres-raw-message-factory.d.ts +1 -1
- package/dist/storage/postgres-raw-message-factory.js +0 -2
- package/dist/storage/raw-message-store.d.ts +9 -1
- package/dist/storage/raw-message-store.js +0 -2
- package/dist/storage/sqlite-raw-message-store.d.ts +15 -2
- package/dist/storage/sqlite-raw-message-store.js +8 -3
- package/dist/storage/sqlite-vector-index.d.ts +1 -1
- package/dist/storage/sqlite-vector-index.js +0 -2
- package/package.json +8 -8
- package/dist/contracts.js.map +0 -1
- package/dist/http.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/mcp.js.map +0 -1
- package/dist/policies/memory-graph-correction-policy.js.map +0 -1
- package/dist/policies/memory-graph-write-policy.js.map +0 -1
- package/dist/search/unified-search.js.map +0 -1
- package/dist/server/cli-http.js.map +0 -1
- package/dist/server/cli-mcp.js.map +0 -1
- package/dist/server/cli-shared.js.map +0 -1
- package/dist/storage/chroma-memory-index.js.map +0 -1
- package/dist/storage/postgres-raw-message-factory.js.map +0 -1
- package/dist/storage/raw-message-store.js.map +0 -1
- package/dist/storage/sqlite-raw-message-store.js.map +0 -1
- package/dist/storage/sqlite-vector-index.js.map +0 -1
package/dist/mcp.js
CHANGED
|
@@ -249,7 +249,14 @@ function clampUnifiedMemorySearchThreshold(threshold) {
|
|
|
249
249
|
}
|
|
250
250
|
return Math.min(1, Math.max(-1, parsed));
|
|
251
251
|
}
|
|
252
|
-
function mergeUnifiedMemorySearchResults(results, limit) {
|
|
252
|
+
function mergeUnifiedMemorySearchResults(results, limit, options = {}) {
|
|
253
|
+
const strategy = normalizeUnifiedMemoryMergeStrategy(options.strategy);
|
|
254
|
+
if (strategy === "rrf") {
|
|
255
|
+
const lists = options.rankedLists ?? [
|
|
256
|
+
{ name: "memory-semantic", hits: results }
|
|
257
|
+
];
|
|
258
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, limit);
|
|
259
|
+
}
|
|
253
260
|
return [...results].sort((a, b) => {
|
|
254
261
|
const scoreDelta = b.similarity - a.similarity;
|
|
255
262
|
if (scoreDelta !== 0) {
|
|
@@ -258,6 +265,42 @@ function mergeUnifiedMemorySearchResults(results, limit) {
|
|
|
258
265
|
return a.type.localeCompare(b.type) || a.id.localeCompare(b.id);
|
|
259
266
|
}).slice(0, limit);
|
|
260
267
|
}
|
|
268
|
+
function normalizeUnifiedMemoryMergeStrategy(value) {
|
|
269
|
+
return value === "rrf" ? "rrf" : "similarity";
|
|
270
|
+
}
|
|
271
|
+
var DEFAULT_RRF_K = 60;
|
|
272
|
+
function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K) {
|
|
273
|
+
if (!Array.isArray(lists) || lists.length === 0) {
|
|
274
|
+
return [];
|
|
275
|
+
}
|
|
276
|
+
const safeK = Number.isFinite(k) && k > 0 ? k : DEFAULT_RRF_K;
|
|
277
|
+
const scores = /* @__PURE__ */ new Map();
|
|
278
|
+
const order = [];
|
|
279
|
+
for (const list of lists) {
|
|
280
|
+
for (let index = 0; index < list.hits.length; index += 1) {
|
|
281
|
+
const hit = list.hits[index];
|
|
282
|
+
const key = `${hit.type}::${hit.id}`;
|
|
283
|
+
const contribution = 1 / (safeK + index + 1);
|
|
284
|
+
const existing = scores.get(key);
|
|
285
|
+
if (existing) {
|
|
286
|
+
existing.rrf += contribution;
|
|
287
|
+
} else {
|
|
288
|
+
scores.set(key, { hit, rrf: contribution });
|
|
289
|
+
order.push(key);
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
const ranked = order.map((key) => scores.get(key)).sort((a, b) => {
|
|
294
|
+
if (b.rrf !== a.rrf) {
|
|
295
|
+
return b.rrf - a.rrf;
|
|
296
|
+
}
|
|
297
|
+
return a.hit.type.localeCompare(b.hit.type) || a.hit.id.localeCompare(b.hit.id);
|
|
298
|
+
}).slice(0, limit);
|
|
299
|
+
return ranked.map(({ hit, rrf }) => ({
|
|
300
|
+
...hit,
|
|
301
|
+
metadata: { ...hit.metadata, rrfScore: rrf }
|
|
302
|
+
}));
|
|
303
|
+
}
|
|
261
304
|
function toKnowledgeResult(result) {
|
|
262
305
|
return {
|
|
263
306
|
type: "knowledge",
|
|
@@ -289,6 +332,9 @@ function isRawMemorySemanticResult(result) {
|
|
|
289
332
|
}
|
|
290
333
|
|
|
291
334
|
// src/search/unified-search.ts
|
|
335
|
+
function deriveLexicalKeywords(query) {
|
|
336
|
+
return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
|
|
337
|
+
}
|
|
292
338
|
function createUnifiedSearch(deps = {}) {
|
|
293
339
|
const logger = deps.logger ?? console;
|
|
294
340
|
async function searchUnifiedMemory(input) {
|
|
@@ -296,8 +342,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
296
342
|
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
297
343
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
298
344
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
345
|
+
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy);
|
|
299
346
|
const warnings = [];
|
|
300
|
-
const results = [];
|
|
301
347
|
if (!query) {
|
|
302
348
|
return {
|
|
303
349
|
query,
|
|
@@ -307,11 +353,11 @@ function createUnifiedSearch(deps = {}) {
|
|
|
307
353
|
warnings
|
|
308
354
|
};
|
|
309
355
|
}
|
|
356
|
+
let memorySubs;
|
|
310
357
|
if (sources.includes("memory")) {
|
|
311
358
|
if (isRawMessageStorageAvailable()) {
|
|
312
359
|
try {
|
|
313
|
-
|
|
314
|
-
results.push(...memoryHits);
|
|
360
|
+
memorySubs = await runMemorySource(input, limit, threshold, warnings);
|
|
315
361
|
} catch (error) {
|
|
316
362
|
logger.warn?.("[memory-store] memory source failed:", error);
|
|
317
363
|
warnings.push({
|
|
@@ -328,9 +374,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
328
374
|
});
|
|
329
375
|
}
|
|
330
376
|
}
|
|
377
|
+
const insightHits = [];
|
|
331
378
|
if (sources.includes("insights") && typeof deps.searchInsights === "function") {
|
|
332
379
|
try {
|
|
333
|
-
const
|
|
380
|
+
const hits = await deps.searchInsights({
|
|
334
381
|
userId: input.userId,
|
|
335
382
|
query: input.query,
|
|
336
383
|
limit,
|
|
@@ -339,8 +386,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
339
386
|
includeArchived: input.includeArchivedInsights,
|
|
340
387
|
authToken: input.authToken
|
|
341
388
|
});
|
|
342
|
-
for (const hit of
|
|
343
|
-
|
|
389
|
+
for (const hit of hits) {
|
|
390
|
+
insightHits.push({
|
|
344
391
|
type: "insight",
|
|
345
392
|
id: hit.id,
|
|
346
393
|
content: hit.content,
|
|
@@ -363,9 +410,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
363
410
|
message: "Insights search is not configured for this host."
|
|
364
411
|
});
|
|
365
412
|
}
|
|
413
|
+
const knowledgeHits = [];
|
|
366
414
|
if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
|
|
367
415
|
try {
|
|
368
|
-
const
|
|
416
|
+
const hits = await deps.searchKnowledge({
|
|
369
417
|
userId: input.userId,
|
|
370
418
|
query: input.query,
|
|
371
419
|
options: {
|
|
@@ -375,8 +423,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
375
423
|
},
|
|
376
424
|
authToken: input.authToken
|
|
377
425
|
});
|
|
378
|
-
for (const hit of
|
|
379
|
-
|
|
426
|
+
for (const hit of hits) {
|
|
427
|
+
knowledgeHits.push(toKnowledgeResult(hit));
|
|
380
428
|
}
|
|
381
429
|
} catch (error) {
|
|
382
430
|
logger.warn?.("[memory-store] knowledge source failed:", error);
|
|
@@ -393,7 +441,13 @@ function createUnifiedSearch(deps = {}) {
|
|
|
393
441
|
message: "Knowledge search is not configured for this host."
|
|
394
442
|
});
|
|
395
443
|
}
|
|
396
|
-
const merged =
|
|
444
|
+
const merged = mergeAcrossSources({
|
|
445
|
+
memorySubs,
|
|
446
|
+
insightHits,
|
|
447
|
+
knowledgeHits,
|
|
448
|
+
limit,
|
|
449
|
+
strategy: mergeStrategy
|
|
450
|
+
});
|
|
397
451
|
return {
|
|
398
452
|
query: input.query,
|
|
399
453
|
sources,
|
|
@@ -402,7 +456,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
402
456
|
warnings
|
|
403
457
|
};
|
|
404
458
|
}
|
|
405
|
-
async function runMemorySource(input, limit, threshold) {
|
|
459
|
+
async function runMemorySource(input, limit, threshold, warnings) {
|
|
406
460
|
if (typeof deps.embedQuery !== "function") {
|
|
407
461
|
throw new Error("embedQuery is not configured");
|
|
408
462
|
}
|
|
@@ -412,9 +466,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
412
466
|
authToken: input.authToken
|
|
413
467
|
});
|
|
414
468
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
469
|
+
let semantic = [];
|
|
415
470
|
if (isRawMessageChromaEnabled()) {
|
|
416
471
|
try {
|
|
417
|
-
|
|
472
|
+
semantic = (await Promise.all(
|
|
418
473
|
filters.map((filter) => {
|
|
419
474
|
const botId = "botId" in filter ? filter.botId : void 0;
|
|
420
475
|
return searchRawMessagesWithChroma({
|
|
@@ -429,18 +484,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
429
484
|
logger.log?.("[memory-store] Raw message semantic search completed", {
|
|
430
485
|
backend: "chroma",
|
|
431
486
|
dimensions: queryEmbedding.length,
|
|
432
|
-
count:
|
|
487
|
+
count: semantic.length
|
|
433
488
|
});
|
|
434
|
-
return chromaHits;
|
|
435
489
|
} catch (error) {
|
|
436
490
|
logger.warn?.(
|
|
437
491
|
"[memory-store] Chroma raw message search failed; falling back to database search:",
|
|
438
492
|
error
|
|
439
493
|
);
|
|
440
494
|
}
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
const rows = (await Promise.all(
|
|
495
|
+
} else if (typeof deps.searchRawMessagesAnn === "function") {
|
|
496
|
+
semantic = (await Promise.all(
|
|
444
497
|
filters.map(
|
|
445
498
|
(filter) => deps.searchRawMessagesAnn({
|
|
446
499
|
userId: input.userId,
|
|
@@ -451,17 +504,87 @@ function createUnifiedSearch(deps = {}) {
|
|
|
451
504
|
})
|
|
452
505
|
)
|
|
453
506
|
)).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
454
|
-
return rows;
|
|
455
507
|
}
|
|
456
|
-
|
|
508
|
+
let lexical = [];
|
|
509
|
+
if (typeof deps.searchRawMessagesLexical === "function") {
|
|
510
|
+
const keywords = deriveLexicalKeywords(input.query);
|
|
511
|
+
if (keywords.length > 0) {
|
|
512
|
+
try {
|
|
513
|
+
const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
514
|
+
lexical = (await Promise.all(
|
|
515
|
+
lexFilters.map(
|
|
516
|
+
(botId) => deps.searchRawMessagesLexical({
|
|
517
|
+
userId: input.userId,
|
|
518
|
+
keywords,
|
|
519
|
+
limit,
|
|
520
|
+
botId
|
|
521
|
+
})
|
|
522
|
+
)
|
|
523
|
+
)).flat().filter(isRawMemorySemanticResult).map((hit) => ({
|
|
524
|
+
...toMemoryResult(hit),
|
|
525
|
+
metadata: { ...hit.metadata, scoring: "bm25" }
|
|
526
|
+
}));
|
|
527
|
+
} catch (error) {
|
|
528
|
+
logger.warn?.("[memory-store] lexical memory search failed:", error);
|
|
529
|
+
warnings.push({
|
|
530
|
+
source: "memory",
|
|
531
|
+
code: "memory_lexical_search_failed",
|
|
532
|
+
message: error.message ?? "memory_lexical_search_failed"
|
|
533
|
+
});
|
|
534
|
+
}
|
|
535
|
+
}
|
|
536
|
+
} else if (input.mergeStrategy === "rrf") {
|
|
537
|
+
warnings.push({
|
|
538
|
+
source: "memory",
|
|
539
|
+
code: "memory_lexical_search_not_configured",
|
|
540
|
+
message: "RRF merge requested but lexical search is not configured; falling back to semantic-only RRF."
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
return { semantic, lexical };
|
|
457
544
|
}
|
|
458
545
|
async function searchRawMemorySemantically(input) {
|
|
459
546
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
460
547
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
461
|
-
|
|
548
|
+
const sub = await runMemorySource(input, limit, threshold, []);
|
|
549
|
+
return mergeAcrossSources({
|
|
550
|
+
memorySubs: sub,
|
|
551
|
+
insightHits: [],
|
|
552
|
+
knowledgeHits: [],
|
|
553
|
+
limit,
|
|
554
|
+
strategy: normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy)
|
|
555
|
+
});
|
|
462
556
|
}
|
|
463
557
|
return { searchUnifiedMemory, searchRawMemorySemantically };
|
|
464
558
|
}
|
|
559
|
+
function mergeAcrossSources(input) {
|
|
560
|
+
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|
|
561
|
+
const all = [
|
|
562
|
+
...memorySubs.semantic,
|
|
563
|
+
...memorySubs.lexical,
|
|
564
|
+
...input.insightHits,
|
|
565
|
+
...input.knowledgeHits
|
|
566
|
+
];
|
|
567
|
+
if (input.strategy !== "rrf") {
|
|
568
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
569
|
+
}
|
|
570
|
+
const lists = [];
|
|
571
|
+
if (memorySubs.semantic.length > 0) {
|
|
572
|
+
lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
|
|
573
|
+
}
|
|
574
|
+
if (memorySubs.lexical.length > 0) {
|
|
575
|
+
lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
|
|
576
|
+
}
|
|
577
|
+
if (input.insightHits.length > 0) {
|
|
578
|
+
lists.push({ name: "insights", hits: input.insightHits });
|
|
579
|
+
}
|
|
580
|
+
if (input.knowledgeHits.length > 0) {
|
|
581
|
+
lists.push({ name: "knowledge", hits: input.knowledgeHits });
|
|
582
|
+
}
|
|
583
|
+
if (lists.length <= 1) {
|
|
584
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
585
|
+
}
|
|
586
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
|
|
587
|
+
}
|
|
465
588
|
|
|
466
589
|
// src/mcp.ts
|
|
467
590
|
var DEFAULT_NAME = "@melandlabs/memory-store";
|
|
@@ -623,5 +746,3 @@ async function startMcpServer(options = {}) {
|
|
|
623
746
|
}
|
|
624
747
|
|
|
625
748
|
export { startMcpServer };
|
|
626
|
-
//# sourceMappingURL=mcp.js.map
|
|
627
|
-
//# sourceMappingURL=mcp.js.map
|
|
@@ -74,5 +74,3 @@ function resolveUntrustedRawMemoryGraphWritePolicy() {
|
|
|
74
74
|
}
|
|
75
75
|
|
|
76
76
|
export { CHAT_MEMORY_EVIDENCE_ID_PREFIX, MEMORY_GRAPH_SUMMARY_ID_PREFIXES, isReservedChatMemoryEvidenceId, isReservedMemoryGraphSummaryId, resolveAllowlistedMemoryGraphPolicy, resolveMemoryGraphWritePolicy, resolveUntrustedRawMemoryGraphWritePolicy, sanitizeUntrustedMemoryMetadata };
|
|
77
|
-
//# sourceMappingURL=memory-graph-write-policy.js.map
|
|
78
|
-
//# sourceMappingURL=memory-graph-write-policy.js.map
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { d as UnifiedSearchDeps } from '../config-
|
|
1
|
+
import { d as UnifiedSearchDeps } from '../config-BYfTzG3h.js';
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
4
|
* Pure helpers used by the unified memory search pipeline.
|
|
@@ -23,6 +23,30 @@ interface UnifiedMemorySearchInput {
|
|
|
23
23
|
includeArchivedInsights?: boolean;
|
|
24
24
|
botIds?: string[];
|
|
25
25
|
documentIds?: string[];
|
|
26
|
+
/**
|
|
27
|
+
* Optional ISO-8601 timestamp. When set, graph-aware retrieval and other
|
|
28
|
+
* snapshot consumers drop nodes / edges whose `applicability.validFrom` /
|
|
29
|
+
* `validUntil` window is closed at that instant. Backward-compatible:
|
|
30
|
+
* omitted `asOf` keeps the legacy behaviour exactly.
|
|
31
|
+
*/
|
|
32
|
+
asOf?: string;
|
|
33
|
+
/**
|
|
34
|
+
* How to merge per-source result lists. Defaults to `"similarity"` —
|
|
35
|
+
* the legacy global sort by similarity, then type/id tie-break. The
|
|
36
|
+
* alternate value `"rrf"` invokes reciprocal-rank fusion across the
|
|
37
|
+
* lists fed via the (internal) `rankedLists` option.
|
|
38
|
+
*/
|
|
39
|
+
mergeStrategy?: UnifiedMemoryMergeStrategy;
|
|
40
|
+
}
|
|
41
|
+
type UnifiedMemoryMergeStrategy = "similarity" | "rrf";
|
|
42
|
+
/**
|
|
43
|
+
* A single ranked list from one retrieval channel. Used by the RRF merge to
|
|
44
|
+
* combine multiple signals (semantic + lexical + insights + knowledge) without
|
|
45
|
+
* flattening them into a single similarity score first.
|
|
46
|
+
*/
|
|
47
|
+
interface UnifiedMemoryRankedList {
|
|
48
|
+
name: "memory-bm25" | "memory-semantic" | "insights" | "knowledge";
|
|
49
|
+
hits: UnifiedMemorySearchResult[];
|
|
26
50
|
}
|
|
27
51
|
interface UnifiedMemorySearchResult {
|
|
28
52
|
type: "memory" | "insight" | "knowledge";
|
|
@@ -41,7 +65,19 @@ interface UnifiedMemorySearchOutput {
|
|
|
41
65
|
declare function normalizeUnifiedMemorySearchSources(sources: unknown): UnifiedMemorySearchSource[];
|
|
42
66
|
declare function clampUnifiedMemorySearchLimit(limit: unknown): number;
|
|
43
67
|
declare function clampUnifiedMemorySearchThreshold(threshold: unknown): number;
|
|
44
|
-
declare function mergeUnifiedMemorySearchResults(results: UnifiedMemorySearchResult[], limit: number
|
|
68
|
+
declare function mergeUnifiedMemorySearchResults(results: UnifiedMemorySearchResult[], limit: number, options?: {
|
|
69
|
+
strategy?: UnifiedMemoryMergeStrategy;
|
|
70
|
+
rankedLists?: UnifiedMemoryRankedList[];
|
|
71
|
+
}): UnifiedMemorySearchResult[];
|
|
72
|
+
declare function normalizeUnifiedMemoryMergeStrategy(value: unknown): UnifiedMemoryMergeStrategy;
|
|
73
|
+
/**
|
|
74
|
+
* Reciprocal Rank Fusion across multiple ranked lists. Each list contributes
|
|
75
|
+
* `1 / (k + rank + 1)` to a hit's accumulated score. Hits are de-duplicated by
|
|
76
|
+
* `(type, id)` so the same message surfaced by both lexical and semantic
|
|
77
|
+
* channels naturally outranks single-channel matches. Tie-break is
|
|
78
|
+
* deterministic: descending RRF score, then `(type, id)` lexical.
|
|
79
|
+
*/
|
|
80
|
+
declare function mergeUnifiedMemorySearchResultsRrf(lists: UnifiedMemoryRankedList[], limit: number, k?: number): UnifiedMemorySearchResult[];
|
|
45
81
|
declare function toKnowledgeResult(result: {
|
|
46
82
|
chunkId: string;
|
|
47
83
|
documentId: string;
|
|
@@ -78,4 +114,4 @@ interface UnifiedSearch {
|
|
|
78
114
|
}
|
|
79
115
|
declare function createUnifiedSearch(deps?: UnifiedSearchDeps): UnifiedSearch;
|
|
80
116
|
|
|
81
|
-
export { type UnifiedMemorySearchInput, type UnifiedMemorySearchOutput, type UnifiedMemorySearchResult, type UnifiedMemorySearchSource, type UnifiedMemorySearchWarning, type UnifiedSearch, clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult };
|
|
117
|
+
export { type UnifiedMemoryMergeStrategy, type UnifiedMemoryRankedList, type UnifiedMemorySearchInput, type UnifiedMemorySearchOutput, type UnifiedMemorySearchResult, type UnifiedMemorySearchSource, type UnifiedMemorySearchWarning, type UnifiedSearch, clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, mergeUnifiedMemorySearchResultsRrf, normalizeUnifiedMemoryMergeStrategy, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult };
|
|
@@ -169,7 +169,14 @@ function clampUnifiedMemorySearchThreshold(threshold) {
|
|
|
169
169
|
}
|
|
170
170
|
return Math.min(1, Math.max(-1, parsed));
|
|
171
171
|
}
|
|
172
|
-
function mergeUnifiedMemorySearchResults(results, limit) {
|
|
172
|
+
function mergeUnifiedMemorySearchResults(results, limit, options = {}) {
|
|
173
|
+
const strategy = normalizeUnifiedMemoryMergeStrategy(options.strategy);
|
|
174
|
+
if (strategy === "rrf") {
|
|
175
|
+
const lists = options.rankedLists ?? [
|
|
176
|
+
{ name: "memory-semantic", hits: results }
|
|
177
|
+
];
|
|
178
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, limit);
|
|
179
|
+
}
|
|
173
180
|
return [...results].sort((a, b) => {
|
|
174
181
|
const scoreDelta = b.similarity - a.similarity;
|
|
175
182
|
if (scoreDelta !== 0) {
|
|
@@ -178,6 +185,42 @@ function mergeUnifiedMemorySearchResults(results, limit) {
|
|
|
178
185
|
return a.type.localeCompare(b.type) || a.id.localeCompare(b.id);
|
|
179
186
|
}).slice(0, limit);
|
|
180
187
|
}
|
|
188
|
+
function normalizeUnifiedMemoryMergeStrategy(value) {
|
|
189
|
+
return value === "rrf" ? "rrf" : "similarity";
|
|
190
|
+
}
|
|
191
|
+
var DEFAULT_RRF_K = 60;
|
|
192
|
+
function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K) {
|
|
193
|
+
if (!Array.isArray(lists) || lists.length === 0) {
|
|
194
|
+
return [];
|
|
195
|
+
}
|
|
196
|
+
const safeK = Number.isFinite(k) && k > 0 ? k : DEFAULT_RRF_K;
|
|
197
|
+
const scores = /* @__PURE__ */ new Map();
|
|
198
|
+
const order = [];
|
|
199
|
+
for (const list of lists) {
|
|
200
|
+
for (let index = 0; index < list.hits.length; index += 1) {
|
|
201
|
+
const hit = list.hits[index];
|
|
202
|
+
const key = `${hit.type}::${hit.id}`;
|
|
203
|
+
const contribution = 1 / (safeK + index + 1);
|
|
204
|
+
const existing = scores.get(key);
|
|
205
|
+
if (existing) {
|
|
206
|
+
existing.rrf += contribution;
|
|
207
|
+
} else {
|
|
208
|
+
scores.set(key, { hit, rrf: contribution });
|
|
209
|
+
order.push(key);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
const ranked = order.map((key) => scores.get(key)).sort((a, b) => {
|
|
214
|
+
if (b.rrf !== a.rrf) {
|
|
215
|
+
return b.rrf - a.rrf;
|
|
216
|
+
}
|
|
217
|
+
return a.hit.type.localeCompare(b.hit.type) || a.hit.id.localeCompare(b.hit.id);
|
|
218
|
+
}).slice(0, limit);
|
|
219
|
+
return ranked.map(({ hit, rrf }) => ({
|
|
220
|
+
...hit,
|
|
221
|
+
metadata: { ...hit.metadata, rrfScore: rrf }
|
|
222
|
+
}));
|
|
223
|
+
}
|
|
181
224
|
function toKnowledgeResult(result) {
|
|
182
225
|
return {
|
|
183
226
|
type: "knowledge",
|
|
@@ -209,6 +252,9 @@ function isRawMemorySemanticResult(result) {
|
|
|
209
252
|
}
|
|
210
253
|
|
|
211
254
|
// src/search/unified-search.ts
|
|
255
|
+
function deriveLexicalKeywords(query) {
|
|
256
|
+
return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
|
|
257
|
+
}
|
|
212
258
|
function createUnifiedSearch(deps = {}) {
|
|
213
259
|
const logger = deps.logger ?? console;
|
|
214
260
|
async function searchUnifiedMemory(input) {
|
|
@@ -216,8 +262,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
216
262
|
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
217
263
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
218
264
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
265
|
+
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy);
|
|
219
266
|
const warnings = [];
|
|
220
|
-
const results = [];
|
|
221
267
|
if (!query) {
|
|
222
268
|
return {
|
|
223
269
|
query,
|
|
@@ -227,11 +273,11 @@ function createUnifiedSearch(deps = {}) {
|
|
|
227
273
|
warnings
|
|
228
274
|
};
|
|
229
275
|
}
|
|
276
|
+
let memorySubs;
|
|
230
277
|
if (sources.includes("memory")) {
|
|
231
278
|
if (isRawMessageStorageAvailable()) {
|
|
232
279
|
try {
|
|
233
|
-
|
|
234
|
-
results.push(...memoryHits);
|
|
280
|
+
memorySubs = await runMemorySource(input, limit, threshold, warnings);
|
|
235
281
|
} catch (error) {
|
|
236
282
|
logger.warn?.("[memory-store] memory source failed:", error);
|
|
237
283
|
warnings.push({
|
|
@@ -248,9 +294,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
248
294
|
});
|
|
249
295
|
}
|
|
250
296
|
}
|
|
297
|
+
const insightHits = [];
|
|
251
298
|
if (sources.includes("insights") && typeof deps.searchInsights === "function") {
|
|
252
299
|
try {
|
|
253
|
-
const
|
|
300
|
+
const hits = await deps.searchInsights({
|
|
254
301
|
userId: input.userId,
|
|
255
302
|
query: input.query,
|
|
256
303
|
limit,
|
|
@@ -259,8 +306,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
259
306
|
includeArchived: input.includeArchivedInsights,
|
|
260
307
|
authToken: input.authToken
|
|
261
308
|
});
|
|
262
|
-
for (const hit of
|
|
263
|
-
|
|
309
|
+
for (const hit of hits) {
|
|
310
|
+
insightHits.push({
|
|
264
311
|
type: "insight",
|
|
265
312
|
id: hit.id,
|
|
266
313
|
content: hit.content,
|
|
@@ -283,9 +330,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
283
330
|
message: "Insights search is not configured for this host."
|
|
284
331
|
});
|
|
285
332
|
}
|
|
333
|
+
const knowledgeHits = [];
|
|
286
334
|
if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
|
|
287
335
|
try {
|
|
288
|
-
const
|
|
336
|
+
const hits = await deps.searchKnowledge({
|
|
289
337
|
userId: input.userId,
|
|
290
338
|
query: input.query,
|
|
291
339
|
options: {
|
|
@@ -295,8 +343,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
295
343
|
},
|
|
296
344
|
authToken: input.authToken
|
|
297
345
|
});
|
|
298
|
-
for (const hit of
|
|
299
|
-
|
|
346
|
+
for (const hit of hits) {
|
|
347
|
+
knowledgeHits.push(toKnowledgeResult(hit));
|
|
300
348
|
}
|
|
301
349
|
} catch (error) {
|
|
302
350
|
logger.warn?.("[memory-store] knowledge source failed:", error);
|
|
@@ -313,7 +361,13 @@ function createUnifiedSearch(deps = {}) {
|
|
|
313
361
|
message: "Knowledge search is not configured for this host."
|
|
314
362
|
});
|
|
315
363
|
}
|
|
316
|
-
const merged =
|
|
364
|
+
const merged = mergeAcrossSources({
|
|
365
|
+
memorySubs,
|
|
366
|
+
insightHits,
|
|
367
|
+
knowledgeHits,
|
|
368
|
+
limit,
|
|
369
|
+
strategy: mergeStrategy
|
|
370
|
+
});
|
|
317
371
|
return {
|
|
318
372
|
query: input.query,
|
|
319
373
|
sources,
|
|
@@ -322,7 +376,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
322
376
|
warnings
|
|
323
377
|
};
|
|
324
378
|
}
|
|
325
|
-
async function runMemorySource(input, limit, threshold) {
|
|
379
|
+
async function runMemorySource(input, limit, threshold, warnings) {
|
|
326
380
|
if (typeof deps.embedQuery !== "function") {
|
|
327
381
|
throw new Error("embedQuery is not configured");
|
|
328
382
|
}
|
|
@@ -332,9 +386,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
332
386
|
authToken: input.authToken
|
|
333
387
|
});
|
|
334
388
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
389
|
+
let semantic = [];
|
|
335
390
|
if (isRawMessageChromaEnabled()) {
|
|
336
391
|
try {
|
|
337
|
-
|
|
392
|
+
semantic = (await Promise.all(
|
|
338
393
|
filters.map((filter) => {
|
|
339
394
|
const botId = "botId" in filter ? filter.botId : void 0;
|
|
340
395
|
return searchRawMessagesWithChroma({
|
|
@@ -349,18 +404,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
349
404
|
logger.log?.("[memory-store] Raw message semantic search completed", {
|
|
350
405
|
backend: "chroma",
|
|
351
406
|
dimensions: queryEmbedding.length,
|
|
352
|
-
count:
|
|
407
|
+
count: semantic.length
|
|
353
408
|
});
|
|
354
|
-
return chromaHits;
|
|
355
409
|
} catch (error) {
|
|
356
410
|
logger.warn?.(
|
|
357
411
|
"[memory-store] Chroma raw message search failed; falling back to database search:",
|
|
358
412
|
error
|
|
359
413
|
);
|
|
360
414
|
}
|
|
361
|
-
}
|
|
362
|
-
|
|
363
|
-
const rows = (await Promise.all(
|
|
415
|
+
} else if (typeof deps.searchRawMessagesAnn === "function") {
|
|
416
|
+
semantic = (await Promise.all(
|
|
364
417
|
filters.map(
|
|
365
418
|
(filter) => deps.searchRawMessagesAnn({
|
|
366
419
|
userId: input.userId,
|
|
@@ -371,18 +424,86 @@ function createUnifiedSearch(deps = {}) {
|
|
|
371
424
|
})
|
|
372
425
|
)
|
|
373
426
|
)).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
374
|
-
return rows;
|
|
375
427
|
}
|
|
376
|
-
|
|
428
|
+
let lexical = [];
|
|
429
|
+
if (typeof deps.searchRawMessagesLexical === "function") {
|
|
430
|
+
const keywords = deriveLexicalKeywords(input.query);
|
|
431
|
+
if (keywords.length > 0) {
|
|
432
|
+
try {
|
|
433
|
+
const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
434
|
+
lexical = (await Promise.all(
|
|
435
|
+
lexFilters.map(
|
|
436
|
+
(botId) => deps.searchRawMessagesLexical({
|
|
437
|
+
userId: input.userId,
|
|
438
|
+
keywords,
|
|
439
|
+
limit,
|
|
440
|
+
botId
|
|
441
|
+
})
|
|
442
|
+
)
|
|
443
|
+
)).flat().filter(isRawMemorySemanticResult).map((hit) => ({
|
|
444
|
+
...toMemoryResult(hit),
|
|
445
|
+
metadata: { ...hit.metadata, scoring: "bm25" }
|
|
446
|
+
}));
|
|
447
|
+
} catch (error) {
|
|
448
|
+
logger.warn?.("[memory-store] lexical memory search failed:", error);
|
|
449
|
+
warnings.push({
|
|
450
|
+
source: "memory",
|
|
451
|
+
code: "memory_lexical_search_failed",
|
|
452
|
+
message: error.message ?? "memory_lexical_search_failed"
|
|
453
|
+
});
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
} else if (input.mergeStrategy === "rrf") {
|
|
457
|
+
warnings.push({
|
|
458
|
+
source: "memory",
|
|
459
|
+
code: "memory_lexical_search_not_configured",
|
|
460
|
+
message: "RRF merge requested but lexical search is not configured; falling back to semantic-only RRF."
|
|
461
|
+
});
|
|
462
|
+
}
|
|
463
|
+
return { semantic, lexical };
|
|
377
464
|
}
|
|
378
465
|
async function searchRawMemorySemantically(input) {
|
|
379
466
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
380
467
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
381
|
-
|
|
468
|
+
const sub = await runMemorySource(input, limit, threshold, []);
|
|
469
|
+
return mergeAcrossSources({
|
|
470
|
+
memorySubs: sub,
|
|
471
|
+
insightHits: [],
|
|
472
|
+
knowledgeHits: [],
|
|
473
|
+
limit,
|
|
474
|
+
strategy: normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy)
|
|
475
|
+
});
|
|
382
476
|
}
|
|
383
477
|
return { searchUnifiedMemory, searchRawMemorySemantically };
|
|
384
478
|
}
|
|
479
|
+
function mergeAcrossSources(input) {
|
|
480
|
+
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|
|
481
|
+
const all = [
|
|
482
|
+
...memorySubs.semantic,
|
|
483
|
+
...memorySubs.lexical,
|
|
484
|
+
...input.insightHits,
|
|
485
|
+
...input.knowledgeHits
|
|
486
|
+
];
|
|
487
|
+
if (input.strategy !== "rrf") {
|
|
488
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
489
|
+
}
|
|
490
|
+
const lists = [];
|
|
491
|
+
if (memorySubs.semantic.length > 0) {
|
|
492
|
+
lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
|
|
493
|
+
}
|
|
494
|
+
if (memorySubs.lexical.length > 0) {
|
|
495
|
+
lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
|
|
496
|
+
}
|
|
497
|
+
if (input.insightHits.length > 0) {
|
|
498
|
+
lists.push({ name: "insights", hits: input.insightHits });
|
|
499
|
+
}
|
|
500
|
+
if (input.knowledgeHits.length > 0) {
|
|
501
|
+
lists.push({ name: "knowledge", hits: input.knowledgeHits });
|
|
502
|
+
}
|
|
503
|
+
if (lists.length <= 1) {
|
|
504
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
505
|
+
}
|
|
506
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
|
|
507
|
+
}
|
|
385
508
|
|
|
386
|
-
export { clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult };
|
|
387
|
-
//# sourceMappingURL=unified-search.js.map
|
|
388
|
-
//# sourceMappingURL=unified-search.js.map
|
|
509
|
+
export { clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, mergeUnifiedMemorySearchResultsRrf, normalizeUnifiedMemoryMergeStrategy, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult };
|