@melandlabs/memory-store 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{config-D6MzksfT.d.ts → config-BYfTzG3h.d.ts} +17 -0
- package/dist/contracts.d.ts +7 -0
- package/dist/contracts.js +0 -2
- package/dist/http.d.ts +1 -1
- package/dist/http.js +144 -23
- package/dist/index.d.ts +2 -2
- package/dist/index.js +144 -23
- package/dist/mcp.d.ts +1 -1
- package/dist/mcp.js +144 -23
- package/dist/policies/memory-graph-correction-policy.js +0 -2
- package/dist/policies/memory-graph-write-policy.js +0 -2
- package/dist/search/unified-search.d.ts +39 -3
- package/dist/search/unified-search.js +145 -24
- package/dist/server/cli-http.js +144 -23
- package/dist/server/cli-mcp.js +144 -23
- package/dist/server/cli-shared.d.ts +1 -1
- package/dist/server/cli-shared.js +0 -2
- package/dist/storage/chroma-memory-index.js +0 -2
- package/dist/storage/postgres-raw-message-factory.d.ts +1 -1
- package/dist/storage/postgres-raw-message-factory.js +0 -2
- package/dist/storage/raw-message-store.d.ts +9 -1
- package/dist/storage/raw-message-store.js +0 -2
- package/dist/storage/sqlite-raw-message-store.d.ts +15 -2
- package/dist/storage/sqlite-raw-message-store.js +8 -3
- package/dist/storage/sqlite-vector-index.d.ts +1 -1
- package/dist/storage/sqlite-vector-index.js +0 -2
- package/package.json +8 -8
- package/dist/contracts.js.map +0 -1
- package/dist/http.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/mcp.js.map +0 -1
- package/dist/policies/memory-graph-correction-policy.js.map +0 -1
- package/dist/policies/memory-graph-write-policy.js.map +0 -1
- package/dist/search/unified-search.js.map +0 -1
- package/dist/server/cli-http.js.map +0 -1
- package/dist/server/cli-mcp.js.map +0 -1
- package/dist/server/cli-shared.js.map +0 -1
- package/dist/storage/chroma-memory-index.js.map +0 -1
- package/dist/storage/postgres-raw-message-factory.js.map +0 -1
- package/dist/storage/raw-message-store.js.map +0 -1
- package/dist/storage/sqlite-raw-message-store.js.map +0 -1
- package/dist/storage/sqlite-vector-index.js.map +0 -1
|
@@ -136,6 +136,23 @@ interface UnifiedSearchDeps {
|
|
|
136
136
|
includeArchived?: boolean;
|
|
137
137
|
authToken?: string;
|
|
138
138
|
}) => Promise<UnifiedSearchInsightsResult[]>;
|
|
139
|
+
/**
|
|
140
|
+
* Optional BM25 (lexical) sub-query for the memory source. Mirrors the
|
|
141
|
+
* `searchRawMessagesAnn` shape but operates on FTS5 keywords instead of
|
|
142
|
+
* dense embeddings. When absent, the unified pipeline skips the lexical
|
|
143
|
+
* path and emits a `memory_lexical_search_not_configured` warning.
|
|
144
|
+
*/
|
|
145
|
+
searchRawMessagesLexical?: (input: {
|
|
146
|
+
userId: string;
|
|
147
|
+
keywords: string[];
|
|
148
|
+
limit: number;
|
|
149
|
+
botId?: string;
|
|
150
|
+
}) => Promise<Array<{
|
|
151
|
+
id: string;
|
|
152
|
+
content: string;
|
|
153
|
+
similarity: number;
|
|
154
|
+
metadata: Record<string, unknown>;
|
|
155
|
+
}>>;
|
|
139
156
|
}
|
|
140
157
|
interface MemoryStoreConfig {
|
|
141
158
|
/** Database handle factory. Required for any persistence. */
|
package/dist/contracts.d.ts
CHANGED
|
@@ -53,6 +53,13 @@ interface RawMessage {
|
|
|
53
53
|
deprecatedAt?: number;
|
|
54
54
|
deprecationReason?: string;
|
|
55
55
|
supersededBySummaryId?: string;
|
|
56
|
+
/**
|
|
57
|
+
* Optional back-pointer to the originating {@link Episode} envelope. When
|
|
58
|
+
* ingest adapters group raw messages by transcript / meeting / voice memo
|
|
59
|
+
* the episode id is recorded here so that downstream retrieval can lift
|
|
60
|
+
* whole episodes atomically.
|
|
61
|
+
*/
|
|
62
|
+
sourceEpisodeId?: string;
|
|
56
63
|
}
|
|
57
64
|
/** Convert a RawMessage to a MemoryRecord for embedding. Local copy of the
|
|
58
65
|
* helper that previously lived in `@melandlabs/indexeddb/embedding` and pulled
|
package/dist/contracts.js
CHANGED
package/dist/http.d.ts
CHANGED
package/dist/http.js
CHANGED
|
@@ -248,7 +248,14 @@ function clampUnifiedMemorySearchThreshold(threshold) {
|
|
|
248
248
|
}
|
|
249
249
|
return Math.min(1, Math.max(-1, parsed));
|
|
250
250
|
}
|
|
251
|
-
function mergeUnifiedMemorySearchResults(results, limit) {
|
|
251
|
+
function mergeUnifiedMemorySearchResults(results, limit, options = {}) {
|
|
252
|
+
const strategy = normalizeUnifiedMemoryMergeStrategy(options.strategy);
|
|
253
|
+
if (strategy === "rrf") {
|
|
254
|
+
const lists = options.rankedLists ?? [
|
|
255
|
+
{ name: "memory-semantic", hits: results }
|
|
256
|
+
];
|
|
257
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, limit);
|
|
258
|
+
}
|
|
252
259
|
return [...results].sort((a, b) => {
|
|
253
260
|
const scoreDelta = b.similarity - a.similarity;
|
|
254
261
|
if (scoreDelta !== 0) {
|
|
@@ -257,6 +264,42 @@ function mergeUnifiedMemorySearchResults(results, limit) {
|
|
|
257
264
|
return a.type.localeCompare(b.type) || a.id.localeCompare(b.id);
|
|
258
265
|
}).slice(0, limit);
|
|
259
266
|
}
|
|
267
|
+
function normalizeUnifiedMemoryMergeStrategy(value) {
|
|
268
|
+
return value === "rrf" ? "rrf" : "similarity";
|
|
269
|
+
}
|
|
270
|
+
var DEFAULT_RRF_K = 60;
|
|
271
|
+
function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K) {
|
|
272
|
+
if (!Array.isArray(lists) || lists.length === 0) {
|
|
273
|
+
return [];
|
|
274
|
+
}
|
|
275
|
+
const safeK = Number.isFinite(k) && k > 0 ? k : DEFAULT_RRF_K;
|
|
276
|
+
const scores = /* @__PURE__ */ new Map();
|
|
277
|
+
const order = [];
|
|
278
|
+
for (const list of lists) {
|
|
279
|
+
for (let index = 0; index < list.hits.length; index += 1) {
|
|
280
|
+
const hit = list.hits[index];
|
|
281
|
+
const key = `${hit.type}::${hit.id}`;
|
|
282
|
+
const contribution = 1 / (safeK + index + 1);
|
|
283
|
+
const existing = scores.get(key);
|
|
284
|
+
if (existing) {
|
|
285
|
+
existing.rrf += contribution;
|
|
286
|
+
} else {
|
|
287
|
+
scores.set(key, { hit, rrf: contribution });
|
|
288
|
+
order.push(key);
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
const ranked = order.map((key) => scores.get(key)).sort((a, b) => {
|
|
293
|
+
if (b.rrf !== a.rrf) {
|
|
294
|
+
return b.rrf - a.rrf;
|
|
295
|
+
}
|
|
296
|
+
return a.hit.type.localeCompare(b.hit.type) || a.hit.id.localeCompare(b.hit.id);
|
|
297
|
+
}).slice(0, limit);
|
|
298
|
+
return ranked.map(({ hit, rrf }) => ({
|
|
299
|
+
...hit,
|
|
300
|
+
metadata: { ...hit.metadata, rrfScore: rrf }
|
|
301
|
+
}));
|
|
302
|
+
}
|
|
260
303
|
function toKnowledgeResult(result) {
|
|
261
304
|
return {
|
|
262
305
|
type: "knowledge",
|
|
@@ -288,6 +331,9 @@ function isRawMemorySemanticResult(result) {
|
|
|
288
331
|
}
|
|
289
332
|
|
|
290
333
|
// src/search/unified-search.ts
|
|
334
|
+
function deriveLexicalKeywords(query) {
|
|
335
|
+
return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
|
|
336
|
+
}
|
|
291
337
|
function createUnifiedSearch(deps = {}) {
|
|
292
338
|
const logger = deps.logger ?? console;
|
|
293
339
|
async function searchUnifiedMemory(input) {
|
|
@@ -295,8 +341,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
295
341
|
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
296
342
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
297
343
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
344
|
+
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy);
|
|
298
345
|
const warnings = [];
|
|
299
|
-
const results = [];
|
|
300
346
|
if (!query) {
|
|
301
347
|
return {
|
|
302
348
|
query,
|
|
@@ -306,11 +352,11 @@ function createUnifiedSearch(deps = {}) {
|
|
|
306
352
|
warnings
|
|
307
353
|
};
|
|
308
354
|
}
|
|
355
|
+
let memorySubs;
|
|
309
356
|
if (sources.includes("memory")) {
|
|
310
357
|
if (isRawMessageStorageAvailable()) {
|
|
311
358
|
try {
|
|
312
|
-
|
|
313
|
-
results.push(...memoryHits);
|
|
359
|
+
memorySubs = await runMemorySource(input, limit, threshold, warnings);
|
|
314
360
|
} catch (error) {
|
|
315
361
|
logger.warn?.("[memory-store] memory source failed:", error);
|
|
316
362
|
warnings.push({
|
|
@@ -327,9 +373,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
327
373
|
});
|
|
328
374
|
}
|
|
329
375
|
}
|
|
376
|
+
const insightHits = [];
|
|
330
377
|
if (sources.includes("insights") && typeof deps.searchInsights === "function") {
|
|
331
378
|
try {
|
|
332
|
-
const
|
|
379
|
+
const hits = await deps.searchInsights({
|
|
333
380
|
userId: input.userId,
|
|
334
381
|
query: input.query,
|
|
335
382
|
limit,
|
|
@@ -338,8 +385,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
338
385
|
includeArchived: input.includeArchivedInsights,
|
|
339
386
|
authToken: input.authToken
|
|
340
387
|
});
|
|
341
|
-
for (const hit of
|
|
342
|
-
|
|
388
|
+
for (const hit of hits) {
|
|
389
|
+
insightHits.push({
|
|
343
390
|
type: "insight",
|
|
344
391
|
id: hit.id,
|
|
345
392
|
content: hit.content,
|
|
@@ -362,9 +409,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
362
409
|
message: "Insights search is not configured for this host."
|
|
363
410
|
});
|
|
364
411
|
}
|
|
412
|
+
const knowledgeHits = [];
|
|
365
413
|
if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
|
|
366
414
|
try {
|
|
367
|
-
const
|
|
415
|
+
const hits = await deps.searchKnowledge({
|
|
368
416
|
userId: input.userId,
|
|
369
417
|
query: input.query,
|
|
370
418
|
options: {
|
|
@@ -374,8 +422,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
374
422
|
},
|
|
375
423
|
authToken: input.authToken
|
|
376
424
|
});
|
|
377
|
-
for (const hit of
|
|
378
|
-
|
|
425
|
+
for (const hit of hits) {
|
|
426
|
+
knowledgeHits.push(toKnowledgeResult(hit));
|
|
379
427
|
}
|
|
380
428
|
} catch (error) {
|
|
381
429
|
logger.warn?.("[memory-store] knowledge source failed:", error);
|
|
@@ -392,7 +440,13 @@ function createUnifiedSearch(deps = {}) {
|
|
|
392
440
|
message: "Knowledge search is not configured for this host."
|
|
393
441
|
});
|
|
394
442
|
}
|
|
395
|
-
const merged =
|
|
443
|
+
const merged = mergeAcrossSources({
|
|
444
|
+
memorySubs,
|
|
445
|
+
insightHits,
|
|
446
|
+
knowledgeHits,
|
|
447
|
+
limit,
|
|
448
|
+
strategy: mergeStrategy
|
|
449
|
+
});
|
|
396
450
|
return {
|
|
397
451
|
query: input.query,
|
|
398
452
|
sources,
|
|
@@ -401,7 +455,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
401
455
|
warnings
|
|
402
456
|
};
|
|
403
457
|
}
|
|
404
|
-
async function runMemorySource(input, limit, threshold) {
|
|
458
|
+
async function runMemorySource(input, limit, threshold, warnings) {
|
|
405
459
|
if (typeof deps.embedQuery !== "function") {
|
|
406
460
|
throw new Error("embedQuery is not configured");
|
|
407
461
|
}
|
|
@@ -411,9 +465,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
411
465
|
authToken: input.authToken
|
|
412
466
|
});
|
|
413
467
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
468
|
+
let semantic = [];
|
|
414
469
|
if (isRawMessageChromaEnabled()) {
|
|
415
470
|
try {
|
|
416
|
-
|
|
471
|
+
semantic = (await Promise.all(
|
|
417
472
|
filters.map((filter) => {
|
|
418
473
|
const botId = "botId" in filter ? filter.botId : void 0;
|
|
419
474
|
return searchRawMessagesWithChroma({
|
|
@@ -428,18 +483,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
428
483
|
logger.log?.("[memory-store] Raw message semantic search completed", {
|
|
429
484
|
backend: "chroma",
|
|
430
485
|
dimensions: queryEmbedding.length,
|
|
431
|
-
count:
|
|
486
|
+
count: semantic.length
|
|
432
487
|
});
|
|
433
|
-
return chromaHits;
|
|
434
488
|
} catch (error) {
|
|
435
489
|
logger.warn?.(
|
|
436
490
|
"[memory-store] Chroma raw message search failed; falling back to database search:",
|
|
437
491
|
error
|
|
438
492
|
);
|
|
439
493
|
}
|
|
440
|
-
}
|
|
441
|
-
|
|
442
|
-
const rows = (await Promise.all(
|
|
494
|
+
} else if (typeof deps.searchRawMessagesAnn === "function") {
|
|
495
|
+
semantic = (await Promise.all(
|
|
443
496
|
filters.map(
|
|
444
497
|
(filter) => deps.searchRawMessagesAnn({
|
|
445
498
|
userId: input.userId,
|
|
@@ -450,17 +503,87 @@ function createUnifiedSearch(deps = {}) {
|
|
|
450
503
|
})
|
|
451
504
|
)
|
|
452
505
|
)).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
453
|
-
return rows;
|
|
454
506
|
}
|
|
455
|
-
|
|
507
|
+
let lexical = [];
|
|
508
|
+
if (typeof deps.searchRawMessagesLexical === "function") {
|
|
509
|
+
const keywords = deriveLexicalKeywords(input.query);
|
|
510
|
+
if (keywords.length > 0) {
|
|
511
|
+
try {
|
|
512
|
+
const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
513
|
+
lexical = (await Promise.all(
|
|
514
|
+
lexFilters.map(
|
|
515
|
+
(botId) => deps.searchRawMessagesLexical({
|
|
516
|
+
userId: input.userId,
|
|
517
|
+
keywords,
|
|
518
|
+
limit,
|
|
519
|
+
botId
|
|
520
|
+
})
|
|
521
|
+
)
|
|
522
|
+
)).flat().filter(isRawMemorySemanticResult).map((hit) => ({
|
|
523
|
+
...toMemoryResult(hit),
|
|
524
|
+
metadata: { ...hit.metadata, scoring: "bm25" }
|
|
525
|
+
}));
|
|
526
|
+
} catch (error) {
|
|
527
|
+
logger.warn?.("[memory-store] lexical memory search failed:", error);
|
|
528
|
+
warnings.push({
|
|
529
|
+
source: "memory",
|
|
530
|
+
code: "memory_lexical_search_failed",
|
|
531
|
+
message: error.message ?? "memory_lexical_search_failed"
|
|
532
|
+
});
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
} else if (input.mergeStrategy === "rrf") {
|
|
536
|
+
warnings.push({
|
|
537
|
+
source: "memory",
|
|
538
|
+
code: "memory_lexical_search_not_configured",
|
|
539
|
+
message: "RRF merge requested but lexical search is not configured; falling back to semantic-only RRF."
|
|
540
|
+
});
|
|
541
|
+
}
|
|
542
|
+
return { semantic, lexical };
|
|
456
543
|
}
|
|
457
544
|
async function searchRawMemorySemantically(input) {
|
|
458
545
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
459
546
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
460
|
-
|
|
547
|
+
const sub = await runMemorySource(input, limit, threshold, []);
|
|
548
|
+
return mergeAcrossSources({
|
|
549
|
+
memorySubs: sub,
|
|
550
|
+
insightHits: [],
|
|
551
|
+
knowledgeHits: [],
|
|
552
|
+
limit,
|
|
553
|
+
strategy: normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy)
|
|
554
|
+
});
|
|
461
555
|
}
|
|
462
556
|
return { searchUnifiedMemory, searchRawMemorySemantically };
|
|
463
557
|
}
|
|
558
|
+
function mergeAcrossSources(input) {
|
|
559
|
+
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|
|
560
|
+
const all = [
|
|
561
|
+
...memorySubs.semantic,
|
|
562
|
+
...memorySubs.lexical,
|
|
563
|
+
...input.insightHits,
|
|
564
|
+
...input.knowledgeHits
|
|
565
|
+
];
|
|
566
|
+
if (input.strategy !== "rrf") {
|
|
567
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
568
|
+
}
|
|
569
|
+
const lists = [];
|
|
570
|
+
if (memorySubs.semantic.length > 0) {
|
|
571
|
+
lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
|
|
572
|
+
}
|
|
573
|
+
if (memorySubs.lexical.length > 0) {
|
|
574
|
+
lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
|
|
575
|
+
}
|
|
576
|
+
if (input.insightHits.length > 0) {
|
|
577
|
+
lists.push({ name: "insights", hits: input.insightHits });
|
|
578
|
+
}
|
|
579
|
+
if (input.knowledgeHits.length > 0) {
|
|
580
|
+
lists.push({ name: "knowledge", hits: input.knowledgeHits });
|
|
581
|
+
}
|
|
582
|
+
if (lists.length <= 1) {
|
|
583
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
584
|
+
}
|
|
585
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
|
|
586
|
+
}
|
|
464
587
|
|
|
465
588
|
// src/http.ts
|
|
466
589
|
async function embedMissingMessages(messages, deps) {
|
|
@@ -576,5 +699,3 @@ async function startHttpServer(options = {}) {
|
|
|
576
699
|
}
|
|
577
700
|
|
|
578
701
|
export { startHttpServer };
|
|
579
|
-
//# sourceMappingURL=http.js.map
|
|
580
|
-
//# sourceMappingURL=http.js.map
|
package/dist/index.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
import { M as MemoryStoreConfig } from './config-
|
|
2
|
-
export { E as EmbedQueryFn, a as MemoryStoreDb, b as MemoryStoreEnv, U as UnifiedSearchInsightsResult, c as UnifiedSearchKnowledgeResult, V as VectorBackend } from './config-
|
|
1
|
+
import { M as MemoryStoreConfig } from './config-BYfTzG3h.js';
|
|
2
|
+
export { E as EmbedQueryFn, a as MemoryStoreDb, b as MemoryStoreEnv, U as UnifiedSearchInsightsResult, c as UnifiedSearchKnowledgeResult, V as VectorBackend } from './config-BYfTzG3h.js';
|
|
3
3
|
import { UnifiedSearch } from './search/unified-search.js';
|
|
4
4
|
export { UnifiedMemorySearchInput, UnifiedMemorySearchOutput, UnifiedMemorySearchResult, UnifiedMemorySearchSource, UnifiedMemorySearchWarning, clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult } from './search/unified-search.js';
|
|
5
5
|
import { RawMessageStore, getRawMessageManager } from './storage/raw-message-store.js';
|
package/dist/index.js
CHANGED
|
@@ -198,7 +198,14 @@ function clampUnifiedMemorySearchThreshold(threshold) {
|
|
|
198
198
|
}
|
|
199
199
|
return Math.min(1, Math.max(-1, parsed));
|
|
200
200
|
}
|
|
201
|
-
function mergeUnifiedMemorySearchResults(results, limit) {
|
|
201
|
+
function mergeUnifiedMemorySearchResults(results, limit, options = {}) {
|
|
202
|
+
const strategy = normalizeUnifiedMemoryMergeStrategy(options.strategy);
|
|
203
|
+
if (strategy === "rrf") {
|
|
204
|
+
const lists = options.rankedLists ?? [
|
|
205
|
+
{ name: "memory-semantic", hits: results }
|
|
206
|
+
];
|
|
207
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, limit);
|
|
208
|
+
}
|
|
202
209
|
return [...results].sort((a, b) => {
|
|
203
210
|
const scoreDelta = b.similarity - a.similarity;
|
|
204
211
|
if (scoreDelta !== 0) {
|
|
@@ -207,6 +214,42 @@ function mergeUnifiedMemorySearchResults(results, limit) {
|
|
|
207
214
|
return a.type.localeCompare(b.type) || a.id.localeCompare(b.id);
|
|
208
215
|
}).slice(0, limit);
|
|
209
216
|
}
|
|
217
|
+
function normalizeUnifiedMemoryMergeStrategy(value) {
|
|
218
|
+
return value === "rrf" ? "rrf" : "similarity";
|
|
219
|
+
}
|
|
220
|
+
var DEFAULT_RRF_K = 60;
|
|
221
|
+
function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K) {
|
|
222
|
+
if (!Array.isArray(lists) || lists.length === 0) {
|
|
223
|
+
return [];
|
|
224
|
+
}
|
|
225
|
+
const safeK = Number.isFinite(k) && k > 0 ? k : DEFAULT_RRF_K;
|
|
226
|
+
const scores = /* @__PURE__ */ new Map();
|
|
227
|
+
const order = [];
|
|
228
|
+
for (const list of lists) {
|
|
229
|
+
for (let index = 0; index < list.hits.length; index += 1) {
|
|
230
|
+
const hit = list.hits[index];
|
|
231
|
+
const key = `${hit.type}::${hit.id}`;
|
|
232
|
+
const contribution = 1 / (safeK + index + 1);
|
|
233
|
+
const existing = scores.get(key);
|
|
234
|
+
if (existing) {
|
|
235
|
+
existing.rrf += contribution;
|
|
236
|
+
} else {
|
|
237
|
+
scores.set(key, { hit, rrf: contribution });
|
|
238
|
+
order.push(key);
|
|
239
|
+
}
|
|
240
|
+
}
|
|
241
|
+
}
|
|
242
|
+
const ranked = order.map((key) => scores.get(key)).sort((a, b) => {
|
|
243
|
+
if (b.rrf !== a.rrf) {
|
|
244
|
+
return b.rrf - a.rrf;
|
|
245
|
+
}
|
|
246
|
+
return a.hit.type.localeCompare(b.hit.type) || a.hit.id.localeCompare(b.hit.id);
|
|
247
|
+
}).slice(0, limit);
|
|
248
|
+
return ranked.map(({ hit, rrf }) => ({
|
|
249
|
+
...hit,
|
|
250
|
+
metadata: { ...hit.metadata, rrfScore: rrf }
|
|
251
|
+
}));
|
|
252
|
+
}
|
|
210
253
|
function toKnowledgeResult(result) {
|
|
211
254
|
return {
|
|
212
255
|
type: "knowledge",
|
|
@@ -238,6 +281,9 @@ function isRawMemorySemanticResult(result) {
|
|
|
238
281
|
}
|
|
239
282
|
|
|
240
283
|
// src/search/unified-search.ts
|
|
284
|
+
function deriveLexicalKeywords(query) {
|
|
285
|
+
return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
|
|
286
|
+
}
|
|
241
287
|
function createUnifiedSearch(deps = {}) {
|
|
242
288
|
const logger = deps.logger ?? console;
|
|
243
289
|
async function searchUnifiedMemory(input) {
|
|
@@ -245,8 +291,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
245
291
|
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
246
292
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
247
293
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
294
|
+
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy);
|
|
248
295
|
const warnings = [];
|
|
249
|
-
const results = [];
|
|
250
296
|
if (!query) {
|
|
251
297
|
return {
|
|
252
298
|
query,
|
|
@@ -256,11 +302,11 @@ function createUnifiedSearch(deps = {}) {
|
|
|
256
302
|
warnings
|
|
257
303
|
};
|
|
258
304
|
}
|
|
305
|
+
let memorySubs;
|
|
259
306
|
if (sources.includes("memory")) {
|
|
260
307
|
if (isRawMessageStorageAvailable()) {
|
|
261
308
|
try {
|
|
262
|
-
|
|
263
|
-
results.push(...memoryHits);
|
|
309
|
+
memorySubs = await runMemorySource(input, limit, threshold, warnings);
|
|
264
310
|
} catch (error) {
|
|
265
311
|
logger.warn?.("[memory-store] memory source failed:", error);
|
|
266
312
|
warnings.push({
|
|
@@ -277,9 +323,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
277
323
|
});
|
|
278
324
|
}
|
|
279
325
|
}
|
|
326
|
+
const insightHits = [];
|
|
280
327
|
if (sources.includes("insights") && typeof deps.searchInsights === "function") {
|
|
281
328
|
try {
|
|
282
|
-
const
|
|
329
|
+
const hits = await deps.searchInsights({
|
|
283
330
|
userId: input.userId,
|
|
284
331
|
query: input.query,
|
|
285
332
|
limit,
|
|
@@ -288,8 +335,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
288
335
|
includeArchived: input.includeArchivedInsights,
|
|
289
336
|
authToken: input.authToken
|
|
290
337
|
});
|
|
291
|
-
for (const hit of
|
|
292
|
-
|
|
338
|
+
for (const hit of hits) {
|
|
339
|
+
insightHits.push({
|
|
293
340
|
type: "insight",
|
|
294
341
|
id: hit.id,
|
|
295
342
|
content: hit.content,
|
|
@@ -312,9 +359,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
312
359
|
message: "Insights search is not configured for this host."
|
|
313
360
|
});
|
|
314
361
|
}
|
|
362
|
+
const knowledgeHits = [];
|
|
315
363
|
if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
|
|
316
364
|
try {
|
|
317
|
-
const
|
|
365
|
+
const hits = await deps.searchKnowledge({
|
|
318
366
|
userId: input.userId,
|
|
319
367
|
query: input.query,
|
|
320
368
|
options: {
|
|
@@ -324,8 +372,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
324
372
|
},
|
|
325
373
|
authToken: input.authToken
|
|
326
374
|
});
|
|
327
|
-
for (const hit of
|
|
328
|
-
|
|
375
|
+
for (const hit of hits) {
|
|
376
|
+
knowledgeHits.push(toKnowledgeResult(hit));
|
|
329
377
|
}
|
|
330
378
|
} catch (error) {
|
|
331
379
|
logger.warn?.("[memory-store] knowledge source failed:", error);
|
|
@@ -342,7 +390,13 @@ function createUnifiedSearch(deps = {}) {
|
|
|
342
390
|
message: "Knowledge search is not configured for this host."
|
|
343
391
|
});
|
|
344
392
|
}
|
|
345
|
-
const merged =
|
|
393
|
+
const merged = mergeAcrossSources({
|
|
394
|
+
memorySubs,
|
|
395
|
+
insightHits,
|
|
396
|
+
knowledgeHits,
|
|
397
|
+
limit,
|
|
398
|
+
strategy: mergeStrategy
|
|
399
|
+
});
|
|
346
400
|
return {
|
|
347
401
|
query: input.query,
|
|
348
402
|
sources,
|
|
@@ -351,7 +405,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
351
405
|
warnings
|
|
352
406
|
};
|
|
353
407
|
}
|
|
354
|
-
async function runMemorySource(input, limit, threshold) {
|
|
408
|
+
async function runMemorySource(input, limit, threshold, warnings) {
|
|
355
409
|
if (typeof deps.embedQuery !== "function") {
|
|
356
410
|
throw new Error("embedQuery is not configured");
|
|
357
411
|
}
|
|
@@ -361,9 +415,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
361
415
|
authToken: input.authToken
|
|
362
416
|
});
|
|
363
417
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
418
|
+
let semantic = [];
|
|
364
419
|
if (isRawMessageChromaEnabled()) {
|
|
365
420
|
try {
|
|
366
|
-
|
|
421
|
+
semantic = (await Promise.all(
|
|
367
422
|
filters.map((filter) => {
|
|
368
423
|
const botId = "botId" in filter ? filter.botId : void 0;
|
|
369
424
|
return searchRawMessagesWithChroma({
|
|
@@ -378,18 +433,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
378
433
|
logger.log?.("[memory-store] Raw message semantic search completed", {
|
|
379
434
|
backend: "chroma",
|
|
380
435
|
dimensions: queryEmbedding.length,
|
|
381
|
-
count:
|
|
436
|
+
count: semantic.length
|
|
382
437
|
});
|
|
383
|
-
return chromaHits;
|
|
384
438
|
} catch (error) {
|
|
385
439
|
logger.warn?.(
|
|
386
440
|
"[memory-store] Chroma raw message search failed; falling back to database search:",
|
|
387
441
|
error
|
|
388
442
|
);
|
|
389
443
|
}
|
|
390
|
-
}
|
|
391
|
-
|
|
392
|
-
const rows = (await Promise.all(
|
|
444
|
+
} else if (typeof deps.searchRawMessagesAnn === "function") {
|
|
445
|
+
semantic = (await Promise.all(
|
|
393
446
|
filters.map(
|
|
394
447
|
(filter) => deps.searchRawMessagesAnn({
|
|
395
448
|
userId: input.userId,
|
|
@@ -400,17 +453,87 @@ function createUnifiedSearch(deps = {}) {
|
|
|
400
453
|
})
|
|
401
454
|
)
|
|
402
455
|
)).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
403
|
-
return rows;
|
|
404
456
|
}
|
|
405
|
-
|
|
457
|
+
let lexical = [];
|
|
458
|
+
if (typeof deps.searchRawMessagesLexical === "function") {
|
|
459
|
+
const keywords = deriveLexicalKeywords(input.query);
|
|
460
|
+
if (keywords.length > 0) {
|
|
461
|
+
try {
|
|
462
|
+
const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
463
|
+
lexical = (await Promise.all(
|
|
464
|
+
lexFilters.map(
|
|
465
|
+
(botId) => deps.searchRawMessagesLexical({
|
|
466
|
+
userId: input.userId,
|
|
467
|
+
keywords,
|
|
468
|
+
limit,
|
|
469
|
+
botId
|
|
470
|
+
})
|
|
471
|
+
)
|
|
472
|
+
)).flat().filter(isRawMemorySemanticResult).map((hit) => ({
|
|
473
|
+
...toMemoryResult(hit),
|
|
474
|
+
metadata: { ...hit.metadata, scoring: "bm25" }
|
|
475
|
+
}));
|
|
476
|
+
} catch (error) {
|
|
477
|
+
logger.warn?.("[memory-store] lexical memory search failed:", error);
|
|
478
|
+
warnings.push({
|
|
479
|
+
source: "memory",
|
|
480
|
+
code: "memory_lexical_search_failed",
|
|
481
|
+
message: error.message ?? "memory_lexical_search_failed"
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
} else if (input.mergeStrategy === "rrf") {
|
|
486
|
+
warnings.push({
|
|
487
|
+
source: "memory",
|
|
488
|
+
code: "memory_lexical_search_not_configured",
|
|
489
|
+
message: "RRF merge requested but lexical search is not configured; falling back to semantic-only RRF."
|
|
490
|
+
});
|
|
491
|
+
}
|
|
492
|
+
return { semantic, lexical };
|
|
406
493
|
}
|
|
407
494
|
async function searchRawMemorySemantically(input) {
|
|
408
495
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
409
496
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
410
|
-
|
|
497
|
+
const sub = await runMemorySource(input, limit, threshold, []);
|
|
498
|
+
return mergeAcrossSources({
|
|
499
|
+
memorySubs: sub,
|
|
500
|
+
insightHits: [],
|
|
501
|
+
knowledgeHits: [],
|
|
502
|
+
limit,
|
|
503
|
+
strategy: normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy)
|
|
504
|
+
});
|
|
411
505
|
}
|
|
412
506
|
return { searchUnifiedMemory, searchRawMemorySemantically };
|
|
413
507
|
}
|
|
508
|
+
function mergeAcrossSources(input) {
|
|
509
|
+
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|
|
510
|
+
const all = [
|
|
511
|
+
...memorySubs.semantic,
|
|
512
|
+
...memorySubs.lexical,
|
|
513
|
+
...input.insightHits,
|
|
514
|
+
...input.knowledgeHits
|
|
515
|
+
];
|
|
516
|
+
if (input.strategy !== "rrf") {
|
|
517
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
518
|
+
}
|
|
519
|
+
const lists = [];
|
|
520
|
+
if (memorySubs.semantic.length > 0) {
|
|
521
|
+
lists.push({ name: "memory-semantic", hits: memorySubs.semantic });
|
|
522
|
+
}
|
|
523
|
+
if (memorySubs.lexical.length > 0) {
|
|
524
|
+
lists.push({ name: "memory-bm25", hits: memorySubs.lexical });
|
|
525
|
+
}
|
|
526
|
+
if (input.insightHits.length > 0) {
|
|
527
|
+
lists.push({ name: "insights", hits: input.insightHits });
|
|
528
|
+
}
|
|
529
|
+
if (input.knowledgeHits.length > 0) {
|
|
530
|
+
lists.push({ name: "knowledge", hits: input.knowledgeHits });
|
|
531
|
+
}
|
|
532
|
+
if (lists.length <= 1) {
|
|
533
|
+
return mergeUnifiedMemorySearchResults(all, input.limit);
|
|
534
|
+
}
|
|
535
|
+
return mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
|
|
536
|
+
}
|
|
414
537
|
|
|
415
538
|
// src/index.ts
|
|
416
539
|
async function createMemoryStore(config = {}) {
|
|
@@ -427,5 +550,3 @@ async function createMemoryStore(config = {}) {
|
|
|
427
550
|
}
|
|
428
551
|
|
|
429
552
|
export { clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, clearPostgresFactory, closeRawMessageStore, createMemoryStore, createRawMessageStore, createUnifiedSearch, getRawMessageManager, getRawMessageStorageBackend, hasPostgresFactory, isRawMemorySemanticResult, isRawMessageStorageAvailable, mergeUnifiedMemorySearchResults, normalizeUnifiedMemorySearchSources, registerPostgresFactory, resolvePostgresFactory, toKnowledgeResult, toMemoryResult };
|
|
430
|
-
//# sourceMappingURL=index.js.map
|
|
431
|
-
//# sourceMappingURL=index.js.map
|
package/dist/mcp.d.ts
CHANGED