@melandlabs/memory-store 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/config-2Q092MUz.d.ts +470 -0
- package/dist/http.d.ts +1 -1
- package/dist/http.js +367 -98
- package/dist/index.d.ts +3 -3
- package/dist/index.js +724 -99
- package/dist/mcp.d.ts +1 -1
- package/dist/mcp.js +370 -98
- package/dist/search/unified-search.d.ts +3 -101
- package/dist/search/unified-search.js +368 -99
- package/dist/server/cli-http.js +376 -98
- package/dist/server/cli-mcp.js +379 -98
- package/dist/server/cli-shared.d.ts +1 -1
- package/dist/server/cli-shared.js +10 -1
- package/dist/storage/postgres-raw-message-factory.d.ts +1 -1
- package/dist/storage/raw-message-store.d.ts +1 -1
- package/dist/storage/raw-message-store.js +1 -1
- package/dist/storage/sqlite-raw-message-store.d.ts +1 -1
- package/dist/storage/sqlite-vector-index.d.ts +1 -1
- package/package.json +10 -9
- package/dist/config-Ll6ktm4D.d.ts +0 -183
|
@@ -1,103 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Pure helpers used by the unified memory search pipeline.
|
|
5
|
-
*
|
|
6
|
-
* Kept side-effect free so they can be reused by tests, MCP tools,
|
|
7
|
-
* and the HTTP daemon without pulling in any of the heavier
|
|
8
|
-
* storage/search implementations.
|
|
9
|
-
*/
|
|
10
|
-
type UnifiedMemorySearchSource = "memory" | "insights" | "knowledge";
|
|
11
|
-
interface UnifiedMemorySearchWarning {
|
|
12
|
-
source: UnifiedMemorySearchSource;
|
|
13
|
-
code: string;
|
|
14
|
-
message: string;
|
|
15
|
-
}
|
|
16
|
-
interface UnifiedMemorySearchInput {
|
|
17
|
-
userId: string;
|
|
18
|
-
query: string;
|
|
19
|
-
sources?: UnifiedMemorySearchSource[];
|
|
20
|
-
limit?: number;
|
|
21
|
-
threshold?: number;
|
|
22
|
-
authToken?: string;
|
|
23
|
-
includeArchivedInsights?: boolean;
|
|
24
|
-
botIds?: string[];
|
|
25
|
-
documentIds?: string[];
|
|
26
|
-
/**
|
|
27
|
-
* Optional ISO-8601 timestamp. When set, graph-aware retrieval and other
|
|
28
|
-
* snapshot consumers drop nodes / edges whose `applicability.validFrom` /
|
|
29
|
-
* `validUntil` window is closed at that instant. Backward-compatible:
|
|
30
|
-
* omitted `asOf` keeps the legacy behaviour exactly.
|
|
31
|
-
*/
|
|
32
|
-
asOf?: string;
|
|
33
|
-
/**
|
|
34
|
-
* How to merge per-source result lists. Defaults to `"similarity"` —
|
|
35
|
-
* the legacy global sort by similarity, then type/id tie-break. The
|
|
36
|
-
* alternate value `"rrf"` invokes reciprocal-rank fusion across the
|
|
37
|
-
* lists fed via the (internal) `rankedLists` option.
|
|
38
|
-
*/
|
|
39
|
-
mergeStrategy?: UnifiedMemoryMergeStrategy;
|
|
40
|
-
}
|
|
41
|
-
type UnifiedMemoryMergeStrategy = "similarity" | "rrf";
|
|
42
|
-
/**
|
|
43
|
-
* A single ranked list from one retrieval channel. Used by the RRF merge to
|
|
44
|
-
* combine multiple signals (semantic + lexical + insights + knowledge) without
|
|
45
|
-
* flattening them into a single similarity score first.
|
|
46
|
-
*/
|
|
47
|
-
interface UnifiedMemoryRankedList {
|
|
48
|
-
name: "memory-bm25" | "memory-semantic" | "insights" | "knowledge";
|
|
49
|
-
hits: UnifiedMemorySearchResult[];
|
|
50
|
-
}
|
|
51
|
-
interface UnifiedMemorySearchResult {
|
|
52
|
-
type: "memory" | "insight" | "knowledge";
|
|
53
|
-
id: string;
|
|
54
|
-
content: string;
|
|
55
|
-
similarity: number;
|
|
56
|
-
metadata: Record<string, unknown>;
|
|
57
|
-
}
|
|
58
|
-
interface UnifiedMemorySearchOutput {
|
|
59
|
-
query: string;
|
|
60
|
-
sources: UnifiedMemorySearchSource[];
|
|
61
|
-
results: UnifiedMemorySearchResult[];
|
|
62
|
-
count: number;
|
|
63
|
-
warnings: UnifiedMemorySearchWarning[];
|
|
64
|
-
}
|
|
65
|
-
declare function normalizeUnifiedMemorySearchSources(sources: unknown): UnifiedMemorySearchSource[];
|
|
66
|
-
declare function clampUnifiedMemorySearchLimit(limit: unknown): number;
|
|
67
|
-
declare function clampUnifiedMemorySearchThreshold(threshold: unknown): number;
|
|
68
|
-
declare function mergeUnifiedMemorySearchResults(results: UnifiedMemorySearchResult[], limit: number, options?: {
|
|
69
|
-
strategy?: UnifiedMemoryMergeStrategy;
|
|
70
|
-
rankedLists?: UnifiedMemoryRankedList[];
|
|
71
|
-
}): UnifiedMemorySearchResult[];
|
|
72
|
-
declare function normalizeUnifiedMemoryMergeStrategy(value: unknown): UnifiedMemoryMergeStrategy;
|
|
73
|
-
/**
|
|
74
|
-
* Reciprocal Rank Fusion across multiple ranked lists. Each list contributes
|
|
75
|
-
* `1 / (k + rank + 1)` to a hit's accumulated score. Hits are de-duplicated by
|
|
76
|
-
* `(type, id)` so the same message surfaced by both lexical and semantic
|
|
77
|
-
* channels naturally outranks single-channel matches. Tie-break is
|
|
78
|
-
* deterministic: descending RRF score, then `(type, id)` lexical.
|
|
79
|
-
*/
|
|
80
|
-
declare function mergeUnifiedMemorySearchResultsRrf(lists: UnifiedMemoryRankedList[], limit: number, k?: number): UnifiedMemorySearchResult[];
|
|
81
|
-
declare function toKnowledgeResult(result: {
|
|
82
|
-
chunkId: string;
|
|
83
|
-
documentId: string;
|
|
84
|
-
documentName: string;
|
|
85
|
-
content: string;
|
|
86
|
-
similarity: number;
|
|
87
|
-
chunkIndex: number;
|
|
88
|
-
}): UnifiedMemorySearchResult;
|
|
89
|
-
declare function toMemoryResult(result: {
|
|
90
|
-
id: string;
|
|
91
|
-
content: string;
|
|
92
|
-
similarity: number;
|
|
93
|
-
metadata: Record<string, unknown>;
|
|
94
|
-
}): UnifiedMemorySearchResult;
|
|
95
|
-
declare function isRawMemorySemanticResult(result: unknown): result is {
|
|
96
|
-
id: string;
|
|
97
|
-
content: string;
|
|
98
|
-
similarity: number;
|
|
99
|
-
metadata: Record<string, unknown>;
|
|
100
|
-
};
|
|
1
|
+
import { m as UnifiedMemorySearchInput, n as UnifiedMemorySearchOutput, o as UnifiedMemorySearchResult, H as UnifiedSearchDeps } from '../config-2Q092MUz.js';
|
|
2
|
+
export { J as UnifiedMemoryMergeStrategy, K as UnifiedMemoryRankedList, l as UnifiedMemoryReasoningStrategy, p as UnifiedMemorySearchSource, q as UnifiedMemorySearchWarning, u as clampUnifiedMemorySearchLimit, v as clampUnifiedMemorySearchThreshold, A as isRawMemorySemanticResult, B as mergeUnifiedMemorySearchResults, L as mergeUnifiedMemorySearchResultsRrf, N as normalizeUnifiedMemoryMergeStrategy, C as normalizeUnifiedMemoryReasoningStrategy, D as normalizeUnifiedMemorySearchSources, F as toKnowledgeResult, G as toMemoryResult } from '../config-2Q092MUz.js';
|
|
101
3
|
|
|
102
4
|
/**
|
|
103
5
|
* Unified semantic search facade.
|
|
@@ -114,4 +16,4 @@ interface UnifiedSearch {
|
|
|
114
16
|
}
|
|
115
17
|
declare function createUnifiedSearch(deps?: UnifiedSearchDeps): UnifiedSearch;
|
|
116
18
|
|
|
117
|
-
export {
|
|
19
|
+
export { UnifiedMemorySearchInput, UnifiedMemorySearchOutput, UnifiedMemorySearchResult, type UnifiedSearch, createUnifiedSearch };
|
|
@@ -164,7 +164,7 @@ __export(raw_message_store_exports, {
|
|
|
164
164
|
getRawMessageStorageBackend: () => getRawMessageStorageBackend,
|
|
165
165
|
isRawMessageStorageAvailable: () => isRawMessageStorageAvailable
|
|
166
166
|
});
|
|
167
|
-
function resolveBackend(
|
|
167
|
+
function resolveBackend(_env) {
|
|
168
168
|
const explicit = process.env.OPENCONTEXT_MEMORY_STORE_BACKEND?.trim().toLowerCase();
|
|
169
169
|
if (explicit === "postgres") return "postgres";
|
|
170
170
|
return "sqlite";
|
|
@@ -297,6 +297,12 @@ function mergeUnifiedMemorySearchResults(results, limit, options = {}) {
|
|
|
297
297
|
function normalizeUnifiedMemoryMergeStrategy(value) {
|
|
298
298
|
return value === "rrf" ? "rrf" : "similarity";
|
|
299
299
|
}
|
|
300
|
+
function normalizeUnifiedMemoryReasoningStrategy(value, defaultStrategy = "none") {
|
|
301
|
+
if (value === "rewrite" || value === "iterative") {
|
|
302
|
+
return value;
|
|
303
|
+
}
|
|
304
|
+
return defaultStrategy;
|
|
305
|
+
}
|
|
300
306
|
var DEFAULT_RRF_K = 60;
|
|
301
307
|
function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K) {
|
|
302
308
|
if (!Array.isArray(lists) || lists.length === 0) {
|
|
@@ -319,7 +325,13 @@ function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K) {
|
|
|
319
325
|
}
|
|
320
326
|
}
|
|
321
327
|
}
|
|
322
|
-
const ranked = order.map((key) =>
|
|
328
|
+
const ranked = order.map((key) => {
|
|
329
|
+
const entry = scores.get(key);
|
|
330
|
+
if (!entry) {
|
|
331
|
+
throw new Error(`Invariant violation: missing RRF score for key "${key}"`);
|
|
332
|
+
}
|
|
333
|
+
return entry;
|
|
334
|
+
}).sort((a, b) => {
|
|
323
335
|
if (b.rrf !== a.rrf) {
|
|
324
336
|
return b.rrf - a.rrf;
|
|
325
337
|
}
|
|
@@ -364,14 +376,222 @@ function isRawMemorySemanticResult(result) {
|
|
|
364
376
|
function deriveLexicalKeywords(query) {
|
|
365
377
|
return query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
|
|
366
378
|
}
|
|
379
|
+
function parseDateBoundary(value, role) {
|
|
380
|
+
const trimmed = value.trim();
|
|
381
|
+
if (!trimmed) {
|
|
382
|
+
return void 0;
|
|
383
|
+
}
|
|
384
|
+
const dateOnlyMatch = trimmed.match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
|
385
|
+
if (dateOnlyMatch) {
|
|
386
|
+
const year = Number.parseInt(dateOnlyMatch[1], 10);
|
|
387
|
+
const month = Number.parseInt(dateOnlyMatch[2], 10) - 1;
|
|
388
|
+
const day = Number.parseInt(dateOnlyMatch[3], 10);
|
|
389
|
+
if (role === "from") {
|
|
390
|
+
return Date.UTC(year, month, day);
|
|
391
|
+
}
|
|
392
|
+
return Date.UTC(year, month, day + 1) - 1;
|
|
393
|
+
}
|
|
394
|
+
const parsed = Date.parse(trimmed);
|
|
395
|
+
if (Number.isNaN(parsed)) {
|
|
396
|
+
return void 0;
|
|
397
|
+
}
|
|
398
|
+
return parsed;
|
|
399
|
+
}
|
|
400
|
+
function getCandidateTimestamp(metadata) {
|
|
401
|
+
for (const key of ["timestamp", "createdAt", "time"]) {
|
|
402
|
+
const value = metadata[key];
|
|
403
|
+
if (typeof value === "number") {
|
|
404
|
+
return value;
|
|
405
|
+
}
|
|
406
|
+
if (typeof value === "string") {
|
|
407
|
+
const parsed = Date.parse(value);
|
|
408
|
+
if (!Number.isNaN(parsed)) {
|
|
409
|
+
return parsed;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
return void 0;
|
|
414
|
+
}
|
|
415
|
+
function filterByDateRange(items, dateFrom, dateTo) {
|
|
416
|
+
if (!dateFrom && !dateTo) {
|
|
417
|
+
return items;
|
|
418
|
+
}
|
|
419
|
+
const fromMs = dateFrom ? parseDateBoundary(dateFrom, "from") : void 0;
|
|
420
|
+
const toMs = dateTo ? parseDateBoundary(dateTo, "to") : void 0;
|
|
421
|
+
if (fromMs === void 0 && toMs === void 0) {
|
|
422
|
+
return items;
|
|
423
|
+
}
|
|
424
|
+
return items.filter((item) => {
|
|
425
|
+
const ts = getCandidateTimestamp(item.metadata);
|
|
426
|
+
if (ts === void 0) {
|
|
427
|
+
return true;
|
|
428
|
+
}
|
|
429
|
+
if (fromMs !== void 0 && ts < fromMs) {
|
|
430
|
+
return false;
|
|
431
|
+
}
|
|
432
|
+
if (toMs !== void 0 && ts > toMs) {
|
|
433
|
+
return false;
|
|
434
|
+
}
|
|
435
|
+
return true;
|
|
436
|
+
});
|
|
437
|
+
}
|
|
438
|
+
async function embedQueryVariant(embedQuery, input, query) {
|
|
439
|
+
return embedQuery({
|
|
440
|
+
userId: input.userId,
|
|
441
|
+
query,
|
|
442
|
+
authToken: input.authToken
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit, threshold, logger) {
|
|
446
|
+
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
447
|
+
let semantic = [];
|
|
448
|
+
if (isRawMessageChromaEnabled()) {
|
|
449
|
+
try {
|
|
450
|
+
semantic = (await Promise.all(
|
|
451
|
+
filters.map((filter) => {
|
|
452
|
+
const botId = "botId" in filter ? filter.botId : void 0;
|
|
453
|
+
return searchRawMessagesWithChroma({
|
|
454
|
+
userId: input.userId,
|
|
455
|
+
queryEmbedding,
|
|
456
|
+
limit,
|
|
457
|
+
threshold,
|
|
458
|
+
botId
|
|
459
|
+
});
|
|
460
|
+
})
|
|
461
|
+
)).flat().map(toMemoryResult);
|
|
462
|
+
logger.log?.("[memory-store] Raw message semantic search completed", {
|
|
463
|
+
backend: "chroma",
|
|
464
|
+
dimensions: queryEmbedding.length,
|
|
465
|
+
count: semantic.length
|
|
466
|
+
});
|
|
467
|
+
} catch (error) {
|
|
468
|
+
logger.warn?.(
|
|
469
|
+
"[memory-store] Chroma raw message search failed; falling back to database search:",
|
|
470
|
+
error
|
|
471
|
+
);
|
|
472
|
+
}
|
|
473
|
+
} else if (typeof deps.searchRawMessagesAnn === "function") {
|
|
474
|
+
const searchRawMessagesAnn = deps.searchRawMessagesAnn;
|
|
475
|
+
semantic = (await Promise.all(
|
|
476
|
+
filters.map(
|
|
477
|
+
(filter) => searchRawMessagesAnn({
|
|
478
|
+
userId: input.userId,
|
|
479
|
+
queryEmbedding,
|
|
480
|
+
limit,
|
|
481
|
+
threshold,
|
|
482
|
+
botId: "botId" in filter ? filter.botId : void 0
|
|
483
|
+
})
|
|
484
|
+
)
|
|
485
|
+
)).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
486
|
+
}
|
|
487
|
+
if (semantic.length === 0) {
|
|
488
|
+
try {
|
|
489
|
+
const { getRawMessageManager: getRawMessageManager2 } = await Promise.resolve().then(() => (init_raw_message_store(), raw_message_store_exports));
|
|
490
|
+
const manager2 = await getRawMessageManager2();
|
|
491
|
+
if (typeof manager2.searchMessagesSemantically === "function") {
|
|
492
|
+
const results = await manager2.searchMessagesSemantically({
|
|
493
|
+
userId: input.userId,
|
|
494
|
+
queryEmbedding,
|
|
495
|
+
limit,
|
|
496
|
+
threshold
|
|
497
|
+
});
|
|
498
|
+
semantic = results.map((r) => ({
|
|
499
|
+
type: "memory",
|
|
500
|
+
id: r.id,
|
|
501
|
+
content: r.content,
|
|
502
|
+
similarity: r.similarity,
|
|
503
|
+
metadata: r.metadata ?? {}
|
|
504
|
+
}));
|
|
505
|
+
}
|
|
506
|
+
} catch (error) {
|
|
507
|
+
logger.warn?.("[memory-store] SQLite semantic search failed:", error);
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
return semantic;
|
|
511
|
+
}
|
|
512
|
+
function mergeByMaxScore(lists) {
|
|
513
|
+
const best = /* @__PURE__ */ new Map();
|
|
514
|
+
for (const list of lists) {
|
|
515
|
+
for (const hit of list) {
|
|
516
|
+
const existing = best.get(hit.id);
|
|
517
|
+
if (!existing || hit.similarity > existing.similarity) {
|
|
518
|
+
best.set(hit.id, hit);
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
return Array.from(best.values()).sort((a, b) => b.similarity - a.similarity);
|
|
523
|
+
}
|
|
524
|
+
function toIterativeRecallCandidate(hit) {
|
|
525
|
+
return {
|
|
526
|
+
id: hit.id,
|
|
527
|
+
content: hit.content,
|
|
528
|
+
similarity: hit.similarity,
|
|
529
|
+
metadata: hit.metadata
|
|
530
|
+
};
|
|
531
|
+
}
|
|
532
|
+
async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger) {
|
|
533
|
+
if (keywords.length === 0) {
|
|
534
|
+
return [];
|
|
535
|
+
}
|
|
536
|
+
if (typeof deps.searchRawMessagesLexical === "function") {
|
|
537
|
+
try {
|
|
538
|
+
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
539
|
+
const searchRawMessagesLexical = deps.searchRawMessagesLexical;
|
|
540
|
+
return (await Promise.all(
|
|
541
|
+
filters.map(
|
|
542
|
+
(botId) => searchRawMessagesLexical({
|
|
543
|
+
userId: input.userId,
|
|
544
|
+
keywords,
|
|
545
|
+
limit: Math.ceil(limit / filters.length),
|
|
546
|
+
botId
|
|
547
|
+
})
|
|
548
|
+
)
|
|
549
|
+
)).flat().filter(isRawMemorySemanticResult).map((hit) => ({
|
|
550
|
+
...toMemoryResult(hit),
|
|
551
|
+
metadata: { ...hit.metadata, scoring: "bm25" }
|
|
552
|
+
}));
|
|
553
|
+
} catch (error) {
|
|
554
|
+
logger.warn?.("[memory-store] lexical memory search failed:", error);
|
|
555
|
+
}
|
|
556
|
+
}
|
|
557
|
+
try {
|
|
558
|
+
const { lexicalSearchRawMessages: lexicalSearchRawMessages2 } = await Promise.resolve().then(() => (init_sqlite_raw_message_store(), sqlite_raw_message_store_exports));
|
|
559
|
+
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
560
|
+
return (await Promise.all(
|
|
561
|
+
filters.map(
|
|
562
|
+
(botId) => lexicalSearchRawMessages2({
|
|
563
|
+
userId: input.userId,
|
|
564
|
+
keywords,
|
|
565
|
+
limit: Math.ceil(limit / filters.length),
|
|
566
|
+
botId
|
|
567
|
+
})
|
|
568
|
+
)
|
|
569
|
+
)).flat().filter(Boolean).map(
|
|
570
|
+
(r) => toMemoryResult(
|
|
571
|
+
r
|
|
572
|
+
)
|
|
573
|
+
);
|
|
574
|
+
} catch (error) {
|
|
575
|
+
logger.warn?.("[memory-store] SQLite lexical search failed:", error);
|
|
576
|
+
}
|
|
577
|
+
return [];
|
|
578
|
+
}
|
|
367
579
|
function createUnifiedSearch(deps = {}) {
|
|
368
580
|
const logger = deps.logger ?? console;
|
|
581
|
+
function resolveReasoningStrategy(input) {
|
|
582
|
+
const requested = normalizeUnifiedMemoryReasoningStrategy(input.reasoningStrategy);
|
|
583
|
+
if (requested !== "none") {
|
|
584
|
+
return requested;
|
|
585
|
+
}
|
|
586
|
+
return normalizeUnifiedMemoryReasoningStrategy(deps.reasoning?.defaultStrategy, "none");
|
|
587
|
+
}
|
|
369
588
|
async function searchUnifiedMemory(input) {
|
|
370
589
|
const query = input.query.trim();
|
|
371
590
|
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
372
591
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
373
592
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
374
593
|
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy);
|
|
594
|
+
const reasoningStrategy = resolveReasoningStrategy(input);
|
|
375
595
|
const warnings = [];
|
|
376
596
|
if (!query) {
|
|
377
597
|
return {
|
|
@@ -383,10 +603,21 @@ function createUnifiedSearch(deps = {}) {
|
|
|
383
603
|
};
|
|
384
604
|
}
|
|
385
605
|
let memorySubs;
|
|
606
|
+
const reasoningInfo = { strategy: reasoningStrategy };
|
|
607
|
+
if (input.dateFrom || input.dateTo) {
|
|
608
|
+
reasoningInfo.dateRange = { from: input.dateFrom, to: input.dateTo };
|
|
609
|
+
}
|
|
386
610
|
if (sources.includes("memory")) {
|
|
387
611
|
if (isRawMessageStorageAvailable()) {
|
|
388
612
|
try {
|
|
389
|
-
memorySubs = await runMemorySource(
|
|
613
|
+
memorySubs = await runMemorySource(
|
|
614
|
+
input,
|
|
615
|
+
limit,
|
|
616
|
+
threshold,
|
|
617
|
+
warnings,
|
|
618
|
+
reasoningStrategy,
|
|
619
|
+
reasoningInfo
|
|
620
|
+
);
|
|
390
621
|
} catch (error) {
|
|
391
622
|
logger.warn?.("[memory-store] memory source failed:", error);
|
|
392
623
|
warnings.push({
|
|
@@ -465,121 +696,155 @@ function createUnifiedSearch(deps = {}) {
|
|
|
465
696
|
limit,
|
|
466
697
|
strategy: mergeStrategy
|
|
467
698
|
});
|
|
468
|
-
|
|
699
|
+
const output = {
|
|
469
700
|
query: input.query,
|
|
470
701
|
sources,
|
|
471
702
|
results: merged,
|
|
472
703
|
count: merged.length,
|
|
473
704
|
warnings
|
|
474
705
|
};
|
|
706
|
+
if (reasoningInfo && (reasoningInfo.strategy !== "none" || reasoningInfo.dateRange !== void 0)) {
|
|
707
|
+
output.reasoning = reasoningInfo;
|
|
708
|
+
}
|
|
709
|
+
return output;
|
|
475
710
|
}
|
|
476
|
-
async function runMemorySource(input, limit, threshold, warnings) {
|
|
711
|
+
async function runMemorySource(input, limit, threshold, warnings, reasoningStrategy = "none", reasoningInfo) {
|
|
712
|
+
if (reasoningStrategy === "iterative" && !deps.reasoning?.iterativePlanner) {
|
|
713
|
+
warnings.push({
|
|
714
|
+
source: "memory",
|
|
715
|
+
code: "memory_iterative_planner_not_configured",
|
|
716
|
+
message: "Iterative reasoning requested but no iterativePlanner is configured; falling back to default search."
|
|
717
|
+
});
|
|
718
|
+
if (reasoningInfo) reasoningInfo.degraded = true;
|
|
719
|
+
}
|
|
720
|
+
if (reasoningStrategy === "rewrite" && !deps.reasoning?.queryRewriter) {
|
|
721
|
+
warnings.push({
|
|
722
|
+
source: "memory",
|
|
723
|
+
code: "memory_query_rewrite_not_configured",
|
|
724
|
+
message: "Rewrite reasoning requested but no queryRewriter is configured; falling back to default search."
|
|
725
|
+
});
|
|
726
|
+
if (reasoningInfo) reasoningInfo.degraded = true;
|
|
727
|
+
}
|
|
728
|
+
if (reasoningStrategy === "iterative" && deps.reasoning?.iterativePlanner) {
|
|
729
|
+
const planner = deps.reasoning.iterativePlanner;
|
|
730
|
+
const result = await planner.plan({
|
|
731
|
+
query: input.query,
|
|
732
|
+
dateFrom: input.dateFrom,
|
|
733
|
+
dateTo: input.dateTo,
|
|
734
|
+
executor: {
|
|
735
|
+
search: async (request) => {
|
|
736
|
+
const keywords2 = request.keywords.length > 0 ? request.keywords : deriveLexicalKeywords(input.query);
|
|
737
|
+
const lexicalHits = await runLexicalSearchForKeywords(deps, input, keywords2, limit, logger);
|
|
738
|
+
let semanticHits = [];
|
|
739
|
+
if (typeof deps.embedQuery === "function") {
|
|
740
|
+
try {
|
|
741
|
+
const semanticQuery = keywords2.join(" ");
|
|
742
|
+
const queryEmbedding = await embedQueryVariant(deps.embedQuery, input, semanticQuery);
|
|
743
|
+
semanticHits = await runSemanticSearchForEmbedding(
|
|
744
|
+
deps,
|
|
745
|
+
input,
|
|
746
|
+
queryEmbedding,
|
|
747
|
+
limit,
|
|
748
|
+
threshold,
|
|
749
|
+
logger
|
|
750
|
+
);
|
|
751
|
+
} catch (error) {
|
|
752
|
+
logger.warn?.(
|
|
753
|
+
"[memory-store] Iterative semantic search failed; using lexical results only:",
|
|
754
|
+
error
|
|
755
|
+
);
|
|
756
|
+
}
|
|
757
|
+
}
|
|
758
|
+
const candidates = filterByDateRange(
|
|
759
|
+
mergeByMaxScore([
|
|
760
|
+
lexicalHits.map(toIterativeRecallCandidate),
|
|
761
|
+
semanticHits.map(toIterativeRecallCandidate)
|
|
762
|
+
]),
|
|
763
|
+
request.dateFrom,
|
|
764
|
+
request.dateTo
|
|
765
|
+
);
|
|
766
|
+
return { candidates };
|
|
767
|
+
}
|
|
768
|
+
}
|
|
769
|
+
});
|
|
770
|
+
if (reasoningInfo) {
|
|
771
|
+
reasoningInfo.iterations = result.stats.iterations;
|
|
772
|
+
reasoningInfo.evidenceCount = result.evidence.length;
|
|
773
|
+
if (planner.lastDegraded?.()) {
|
|
774
|
+
reasoningInfo.degraded = true;
|
|
775
|
+
}
|
|
776
|
+
}
|
|
777
|
+
const semantic2 = result.evidence.map((candidate) => ({
|
|
778
|
+
type: "memory",
|
|
779
|
+
id: candidate.id,
|
|
780
|
+
content: candidate.content,
|
|
781
|
+
similarity: candidate.similarity,
|
|
782
|
+
metadata: { ...candidate.metadata, reasoning: "iterative" }
|
|
783
|
+
}));
|
|
784
|
+
return { semantic: semantic2, lexical: [] };
|
|
785
|
+
}
|
|
477
786
|
if (typeof deps.embedQuery !== "function") {
|
|
478
787
|
warnings.push({
|
|
479
788
|
source: "memory",
|
|
480
789
|
code: "memory_lexical_search_fallback",
|
|
481
790
|
message: "Semantic search not configured, using keyword search as fallback"
|
|
482
791
|
});
|
|
483
|
-
const
|
|
484
|
-
const
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
const results = await Promise.all(
|
|
490
|
-
filters2.map(
|
|
491
|
-
(botId) => lexicalSearchRawMessages2({
|
|
492
|
-
userId: input.userId,
|
|
493
|
-
keywords,
|
|
494
|
-
limit: Math.ceil(limit / filters2.length),
|
|
495
|
-
botId
|
|
496
|
-
})
|
|
497
|
-
)
|
|
498
|
-
);
|
|
499
|
-
lexical2 = results.flat().filter(Boolean).map(toMemoryResult);
|
|
500
|
-
} catch (error) {
|
|
501
|
-
logger.warn?.("[memory-store] Default lexical search failed:", error);
|
|
502
|
-
}
|
|
503
|
-
}
|
|
792
|
+
const keywords2 = deriveLexicalKeywords(input.query);
|
|
793
|
+
const lexical2 = filterByDateRange(
|
|
794
|
+
await runLexicalSearchForKeywords(deps, input, keywords2, limit, logger),
|
|
795
|
+
input.dateFrom,
|
|
796
|
+
input.dateTo
|
|
797
|
+
);
|
|
504
798
|
return { semantic: [], lexical: lexical2 };
|
|
505
799
|
}
|
|
506
|
-
const queryEmbedding = await deps.embedQuery({
|
|
507
|
-
userId: input.userId,
|
|
508
|
-
query: input.query,
|
|
509
|
-
authToken: input.authToken
|
|
510
|
-
});
|
|
511
|
-
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
512
800
|
let semantic = [];
|
|
513
|
-
|
|
801
|
+
const embedQuery = deps.embedQuery;
|
|
802
|
+
if (reasoningStrategy === "rewrite" && deps.reasoning?.queryRewriter) {
|
|
514
803
|
try {
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
userId: input.userId,
|
|
520
|
-
queryEmbedding,
|
|
521
|
-
limit,
|
|
522
|
-
threshold,
|
|
523
|
-
botId
|
|
524
|
-
});
|
|
525
|
-
})
|
|
526
|
-
)).flat().map(toMemoryResult);
|
|
527
|
-
logger.log?.("[memory-store] Raw message semantic search completed", {
|
|
528
|
-
backend: "chroma",
|
|
529
|
-
dimensions: queryEmbedding.length,
|
|
530
|
-
count: semantic.length
|
|
804
|
+
const variants = await deps.reasoning.queryRewriter.rewrite({
|
|
805
|
+
query: input.query,
|
|
806
|
+
userId: input.userId,
|
|
807
|
+
authToken: input.authToken
|
|
531
808
|
});
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
809
|
+
if (reasoningInfo) {
|
|
810
|
+
reasoningInfo.rewrittenQueries = variants;
|
|
811
|
+
if (deps.reasoning.queryRewriter.lastDegraded?.()) {
|
|
812
|
+
reasoningInfo.degraded = true;
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
const embeddings = await Promise.all(
|
|
816
|
+
variants.map((variant) => embedQueryVariant(embedQuery, input, variant))
|
|
536
817
|
);
|
|
818
|
+
const lists = await Promise.all(
|
|
819
|
+
embeddings.map(
|
|
820
|
+
(embedding) => runSemanticSearchForEmbedding(deps, input, embedding, limit, threshold, logger)
|
|
821
|
+
)
|
|
822
|
+
);
|
|
823
|
+
semantic = mergeByMaxScore(lists);
|
|
824
|
+
} catch (error) {
|
|
825
|
+
logger.warn?.("[memory-store] Query rewriting failed; falling back to original query:", error);
|
|
826
|
+
warnings.push({
|
|
827
|
+
source: "memory",
|
|
828
|
+
code: "memory_query_rewrite_failed",
|
|
829
|
+
message: `Query rewriting failed: ${error.message ?? "Unknown error"}. Using original query.`
|
|
830
|
+
});
|
|
831
|
+
if (reasoningInfo) reasoningInfo.degraded = true;
|
|
537
832
|
}
|
|
538
|
-
} else if (typeof deps.searchRawMessagesAnn === "function") {
|
|
539
|
-
semantic = (await Promise.all(
|
|
540
|
-
filters.map(
|
|
541
|
-
(filter) => deps.searchRawMessagesAnn({
|
|
542
|
-
userId: input.userId,
|
|
543
|
-
queryEmbedding,
|
|
544
|
-
limit,
|
|
545
|
-
threshold,
|
|
546
|
-
botId: "botId" in filter ? filter.botId : void 0
|
|
547
|
-
})
|
|
548
|
-
)
|
|
549
|
-
)).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
550
833
|
}
|
|
551
834
|
if (semantic.length === 0) {
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
const manager2 = await getRawMessageManager2();
|
|
555
|
-
if (typeof manager2.searchMessagesSemantically === "function") {
|
|
556
|
-
const results = await manager2.searchMessagesSemantically({
|
|
557
|
-
userId: input.userId,
|
|
558
|
-
queryEmbedding,
|
|
559
|
-
limit,
|
|
560
|
-
threshold
|
|
561
|
-
});
|
|
562
|
-
semantic = results.map((r) => ({
|
|
563
|
-
type: "memory",
|
|
564
|
-
id: r.id,
|
|
565
|
-
content: r.content,
|
|
566
|
-
similarity: r.similarity,
|
|
567
|
-
metadata: r.metadata ?? {}
|
|
568
|
-
}));
|
|
569
|
-
}
|
|
570
|
-
} catch (error) {
|
|
571
|
-
logger.warn?.("[memory-store] SQLite semantic search failed:", error);
|
|
572
|
-
}
|
|
835
|
+
const queryEmbedding = await embedQueryVariant(embedQuery, input, input.query);
|
|
836
|
+
semantic = await runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit, threshold, logger);
|
|
573
837
|
}
|
|
574
838
|
let lexical = [];
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
if (
|
|
839
|
+
const keywords = deriveLexicalKeywords(input.query);
|
|
840
|
+
if (keywords.length > 0) {
|
|
841
|
+
if (typeof deps.searchRawMessagesLexical === "function") {
|
|
578
842
|
try {
|
|
579
843
|
const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
844
|
+
const searchRawMessagesLexical = deps.searchRawMessagesLexical;
|
|
580
845
|
lexical = (await Promise.all(
|
|
581
846
|
lexFilters.map(
|
|
582
|
-
(botId) =>
|
|
847
|
+
(botId) => searchRawMessagesLexical({
|
|
583
848
|
userId: input.userId,
|
|
584
849
|
keywords,
|
|
585
850
|
limit,
|
|
@@ -598,20 +863,24 @@ function createUnifiedSearch(deps = {}) {
|
|
|
598
863
|
message: error.message ?? "memory_lexical_search_failed"
|
|
599
864
|
});
|
|
600
865
|
}
|
|
866
|
+
} else if (input.mergeStrategy === "rrf") {
|
|
867
|
+
warnings.push({
|
|
868
|
+
source: "memory",
|
|
869
|
+
code: "memory_lexical_search_not_configured",
|
|
870
|
+
message: "RRF merge requested but lexical search is not configured; falling back to semantic-only RRF."
|
|
871
|
+
});
|
|
601
872
|
}
|
|
602
|
-
} else if (input.mergeStrategy === "rrf") {
|
|
603
|
-
warnings.push({
|
|
604
|
-
source: "memory",
|
|
605
|
-
code: "memory_lexical_search_not_configured",
|
|
606
|
-
message: "RRF merge requested but lexical search is not configured; falling back to semantic-only RRF."
|
|
607
|
-
});
|
|
608
873
|
}
|
|
609
|
-
return {
|
|
874
|
+
return {
|
|
875
|
+
semantic: filterByDateRange(semantic, input.dateFrom, input.dateTo),
|
|
876
|
+
lexical: filterByDateRange(lexical, input.dateFrom, input.dateTo)
|
|
877
|
+
};
|
|
610
878
|
}
|
|
611
879
|
async function searchRawMemorySemantically(input) {
|
|
612
880
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
613
881
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
614
|
-
const
|
|
882
|
+
const reasoningStrategy = resolveReasoningStrategy(input);
|
|
883
|
+
const sub = await runMemorySource(input, limit, threshold, [], reasoningStrategy);
|
|
615
884
|
return mergeAcrossSources({
|
|
616
885
|
memorySubs: sub,
|
|
617
886
|
insightHits: [],
|
|
@@ -652,4 +921,4 @@ function mergeAcrossSources(input) {
|
|
|
652
921
|
return mergeUnifiedMemorySearchResultsRrf(lists, input.limit);
|
|
653
922
|
}
|
|
654
923
|
|
|
655
|
-
export { clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, mergeUnifiedMemorySearchResultsRrf, normalizeUnifiedMemoryMergeStrategy, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult };
|
|
924
|
+
export { clampUnifiedMemorySearchLimit, clampUnifiedMemorySearchThreshold, createUnifiedSearch, isRawMemorySemanticResult, mergeUnifiedMemorySearchResults, mergeUnifiedMemorySearchResultsRrf, normalizeUnifiedMemoryMergeStrategy, normalizeUnifiedMemoryReasoningStrategy, normalizeUnifiedMemorySearchSources, toKnowledgeResult, toMemoryResult };
|