@melandlabs/memory-store 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +209 -56
- package/dist/{config-2Q092MUz.d.ts → config-CnbqSek5.d.ts} +215 -2
- package/dist/contracts.d.ts +1 -0
- package/dist/contracts.js +2 -0
- package/dist/http.d.ts +21 -5
- package/dist/http.js +1051 -155
- package/dist/index.d.ts +230 -15
- package/dist/index.js +1087 -152
- package/dist/mcp.d.ts +14 -5
- package/dist/mcp.js +1188 -174
- package/dist/policies/memory-graph-correction-policy.d.ts +1 -0
- package/dist/policies/memory-graph-correction-policy.js +4 -0
- package/dist/policies/memory-graph-write-policy.d.ts +1 -0
- package/dist/policies/memory-graph-write-policy.js +2 -0
- package/dist/search/unified-search.d.ts +6 -19
- package/dist/search/unified-search.js +749 -145
- package/dist/server/cli-http.js +1051 -155
- package/dist/server/cli-mcp.js +1185 -171
- package/dist/server/cli-shared.d.ts +5 -2
- package/dist/server/cli-shared.js +6 -4
- package/dist/storage/chroma-memory-index.d.ts +1 -0
- package/dist/storage/chroma-memory-index.js +1 -2
- package/dist/storage/postgres-raw-message-factory.d.ts +4 -1
- package/dist/storage/raw-message-store.d.ts +5 -2
- package/dist/storage/raw-message-store.js +6 -4
- package/dist/storage/sqlite-raw-message-store.d.ts +5 -1
- package/dist/storage/sqlite-raw-message-store.js +6 -4
- package/dist/storage/sqlite-vector-index.d.ts +4 -1
- package/dist/storage/sqlite-vector-index.js +4 -4
- package/dist/unified-search-Bft57TRH.d.ts +122 -0
- package/package.json +11 -7
|
@@ -1,9 +1,11 @@
|
|
|
1
|
+
import '@melandlabs/contracts';
|
|
1
2
|
import '@melandlabs/ai/memory';
|
|
2
3
|
import { ChromaVectorStore } from '@melandlabs/rag';
|
|
3
4
|
import { mkdirSync } from 'fs';
|
|
4
|
-
import {
|
|
5
|
-
import {
|
|
5
|
+
import { dirname } from 'path';
|
|
6
|
+
import { getOpenContextPath } from '@melandlabs/env-config';
|
|
6
7
|
import { SQLiteRawMessageManager } from '@melandlabs/sqlite';
|
|
8
|
+
import { applyReflectedConsolidationPlan } from '@melandlabs/memory-consolidation';
|
|
7
9
|
|
|
8
10
|
var __defProp = Object.defineProperty;
|
|
9
11
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
@@ -14,8 +16,6 @@ var __export = (target, all) => {
|
|
|
14
16
|
for (var name in all)
|
|
15
17
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
16
18
|
};
|
|
17
|
-
|
|
18
|
-
// src/contracts.ts
|
|
19
19
|
var init_contracts = __esm({
|
|
20
20
|
"src/contracts.ts"() {
|
|
21
21
|
}
|
|
@@ -111,7 +111,7 @@ __export(sqlite_raw_message_store_exports, {
|
|
|
111
111
|
function resolveSQLiteRawMessageDbPath(dbPath) {
|
|
112
112
|
if (dbPath && dbPath.length > 0) return dbPath;
|
|
113
113
|
const fromEnv = process.env.MEMORY_STORE_DB_PATH?.trim();
|
|
114
|
-
return fromEnv && fromEnv.length > 0 ? fromEnv :
|
|
114
|
+
return fromEnv && fromEnv.length > 0 ? fromEnv : getOpenContextPath("memory", "store.db");
|
|
115
115
|
}
|
|
116
116
|
function isSQLiteRawMessageStorageAvailable(_env) {
|
|
117
117
|
return true;
|
|
@@ -145,12 +145,11 @@ async function lexicalSearchRawMessages(input) {
|
|
|
145
145
|
}
|
|
146
146
|
return mgr.lexicalSearchMessages(input);
|
|
147
147
|
}
|
|
148
|
-
var manager
|
|
148
|
+
var manager;
|
|
149
149
|
var init_sqlite_raw_message_store = __esm({
|
|
150
150
|
"src/storage/sqlite-raw-message-store.ts"() {
|
|
151
151
|
init_chroma_memory_index();
|
|
152
152
|
manager = null;
|
|
153
|
-
DEFAULT_SQLITE_PATH = join(homedir(), ".opencontext", "memory", "store.db");
|
|
154
153
|
}
|
|
155
154
|
});
|
|
156
155
|
|
|
@@ -371,6 +370,416 @@ function isRawMemorySemanticResult(result) {
|
|
|
371
370
|
const item = result;
|
|
372
371
|
return typeof item.id === "string" && typeof item.content === "string" && typeof item.similarity === "number" && Boolean(item.metadata) && typeof item.metadata === "object";
|
|
373
372
|
}
|
|
373
|
+
async function resolveScopePeer(input) {
|
|
374
|
+
const peers = input.peerFilter ?? [];
|
|
375
|
+
if (peers.length === 0) {
|
|
376
|
+
return { peers: [], warnings: [] };
|
|
377
|
+
}
|
|
378
|
+
if (typeof input.scopeCheck !== "function") {
|
|
379
|
+
return { peers, warnings: [] };
|
|
380
|
+
}
|
|
381
|
+
const retained = [];
|
|
382
|
+
const dropped = [];
|
|
383
|
+
for (const peer of peers) {
|
|
384
|
+
const ok = await input.scopeCheck({ userId: input.userId, peers: [peer] });
|
|
385
|
+
if (ok) {
|
|
386
|
+
retained.push(peer);
|
|
387
|
+
} else {
|
|
388
|
+
dropped.push(peer);
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
if (dropped.length === 0) {
|
|
392
|
+
return { peers, warnings: [] };
|
|
393
|
+
}
|
|
394
|
+
if (retained.length === 0) {
|
|
395
|
+
return {
|
|
396
|
+
peers: [],
|
|
397
|
+
warnings: [
|
|
398
|
+
{
|
|
399
|
+
source: "memory",
|
|
400
|
+
code: "peer_filter_outside_user_scope",
|
|
401
|
+
message: `${dropped.map((peer) => `${peer.kind}:${peer.id}`).join(",")} ignored; falling back to userId scope`
|
|
402
|
+
}
|
|
403
|
+
]
|
|
404
|
+
};
|
|
405
|
+
}
|
|
406
|
+
return {
|
|
407
|
+
peers: retained,
|
|
408
|
+
warnings: [
|
|
409
|
+
{
|
|
410
|
+
source: "memory",
|
|
411
|
+
code: "peer_filter_outside_user_scope",
|
|
412
|
+
message: `${dropped.map((peer) => `${peer.kind}:${peer.id}`).join(",")} ignored because they fall outside userId ${input.userId}`
|
|
413
|
+
}
|
|
414
|
+
]
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
// src/search/apply-reflect.ts
|
|
419
|
+
var DEFAULT_TIERS = ["summary", "raw", "insight", "knowledge"];
|
|
420
|
+
function reflectTierToEvidenceTier(source) {
|
|
421
|
+
switch (source) {
|
|
422
|
+
case "raw":
|
|
423
|
+
return "short";
|
|
424
|
+
case "summary":
|
|
425
|
+
return "long";
|
|
426
|
+
default:
|
|
427
|
+
return "mid";
|
|
428
|
+
}
|
|
429
|
+
}
|
|
430
|
+
function toMemoryEvidenceRecord(input, fallbackUserId) {
|
|
431
|
+
const tier = reflectTierToEvidenceTier(input.source);
|
|
432
|
+
const timestamp = input.timestamp ?? Date.now();
|
|
433
|
+
return {
|
|
434
|
+
id: input.id,
|
|
435
|
+
userId: fallbackUserId,
|
|
436
|
+
timestamp,
|
|
437
|
+
tier,
|
|
438
|
+
text: input.snippet,
|
|
439
|
+
metadata: {
|
|
440
|
+
source: input.source,
|
|
441
|
+
score: input.score,
|
|
442
|
+
...input.timestamp !== void 0 ? { originalTimestamp: input.timestamp } : {}
|
|
443
|
+
}
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
async function applyReflectedPlan(deps, store, input, _logger) {
|
|
447
|
+
const query = input.query.trim();
|
|
448
|
+
const warnings = [];
|
|
449
|
+
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
450
|
+
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
451
|
+
const tiers = input.tiers && input.tiers.length > 0 ? input.tiers : DEFAULT_TIERS;
|
|
452
|
+
const searchInput = {
|
|
453
|
+
userId: input.userId,
|
|
454
|
+
query,
|
|
455
|
+
botIds: input.botIds,
|
|
456
|
+
dateFrom: input.dateFrom,
|
|
457
|
+
dateTo: input.dateTo,
|
|
458
|
+
tiers,
|
|
459
|
+
limit,
|
|
460
|
+
threshold,
|
|
461
|
+
authToken: input.authToken,
|
|
462
|
+
peerFilter: input.peerFilter,
|
|
463
|
+
synthesize: true
|
|
464
|
+
};
|
|
465
|
+
const search = createUnifiedSearch(deps);
|
|
466
|
+
const result = await search.search(searchInput);
|
|
467
|
+
warnings.push(...result.warnings);
|
|
468
|
+
const evidence = result.evidence.map((item) => ({
|
|
469
|
+
id: item.id,
|
|
470
|
+
source: item.source ?? "raw",
|
|
471
|
+
snippet: item.snippet,
|
|
472
|
+
score: item.score,
|
|
473
|
+
timestamp: item.timestamp
|
|
474
|
+
}));
|
|
475
|
+
const records = evidence.map((item) => toMemoryEvidenceRecord(item, input.userId));
|
|
476
|
+
const applyInput = {
|
|
477
|
+
userId: input.userId,
|
|
478
|
+
ownerScope: input.ownerScope,
|
|
479
|
+
records,
|
|
480
|
+
plan: input.plan,
|
|
481
|
+
dryRun: input.dryRun,
|
|
482
|
+
expectedVersion: input.expectedVersion,
|
|
483
|
+
getClusterKey: input.getClusterKey ?? defaultClusterKey,
|
|
484
|
+
llmReview: deps.reasoning?.complete ? {
|
|
485
|
+
complete: deps.reasoning.complete,
|
|
486
|
+
maxTokens: input.llmPlanReview?.maxTokens
|
|
487
|
+
} : void 0,
|
|
488
|
+
graphStore: input.graphStore ?? store.graphStore,
|
|
489
|
+
storage: input.storage ?? store.storage
|
|
490
|
+
};
|
|
491
|
+
const applyOutput = await applyReflectedConsolidationPlan(applyInput);
|
|
492
|
+
for (const warning of applyOutput.warnings) {
|
|
493
|
+
warnings.push({
|
|
494
|
+
source: "memory",
|
|
495
|
+
code: warning.code,
|
|
496
|
+
message: warning.message
|
|
497
|
+
});
|
|
498
|
+
}
|
|
499
|
+
const summaryAnswer = makePlanSummary(applyOutput.plan);
|
|
500
|
+
return {
|
|
501
|
+
answer: result.answer ? `${result.answer}
|
|
502
|
+
|
|
503
|
+
Plan summary:
|
|
504
|
+
${summaryAnswer}` : summaryAnswer,
|
|
505
|
+
plan: applyOutput.plan,
|
|
506
|
+
persistenceResult: applyOutput.persistenceResult,
|
|
507
|
+
deprecationCounts: applyOutput.deprecationCounts,
|
|
508
|
+
warnings,
|
|
509
|
+
applied: applyOutput.applied,
|
|
510
|
+
evidence
|
|
511
|
+
};
|
|
512
|
+
}
|
|
513
|
+
function defaultClusterKey(record) {
|
|
514
|
+
return record.text ?? record.id;
|
|
515
|
+
}
|
|
516
|
+
function makePlanSummary(plan) {
|
|
517
|
+
const lines = [];
|
|
518
|
+
const counts = {
|
|
519
|
+
preserve: plan.actions.preserve.length,
|
|
520
|
+
observe: plan.actions.observe.length,
|
|
521
|
+
decay: plan.actions.decay.length,
|
|
522
|
+
deprecate: plan.actions.deprecate.length
|
|
523
|
+
};
|
|
524
|
+
lines.push(
|
|
525
|
+
`- preserve=${counts.preserve} observe=${counts.observe} decay=${counts.decay} deprecate=${counts.deprecate}`
|
|
526
|
+
);
|
|
527
|
+
for (const entry of plan.actions.deprecate.slice(0, 5)) {
|
|
528
|
+
lines.push(
|
|
529
|
+
` deprecate ${entry.recordIds.length} record(s) from cluster ${entry.clusterKey} -> ${entry.supersededBySummaryId ?? entry.winningClusterKey}`
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
return lines.join("\n");
|
|
533
|
+
}
|
|
534
|
+
|
|
535
|
+
// src/search/reranker.ts
|
|
536
|
+
async function applyReranker(reranker, query, results) {
|
|
537
|
+
if (!reranker || results.length === 0) {
|
|
538
|
+
return results;
|
|
539
|
+
}
|
|
540
|
+
const scores = await reranker.rerank({
|
|
541
|
+
query,
|
|
542
|
+
candidates: results.map((result) => ({
|
|
543
|
+
id: result.id,
|
|
544
|
+
content: result.content,
|
|
545
|
+
metadata: result.metadata
|
|
546
|
+
}))
|
|
547
|
+
});
|
|
548
|
+
if (scores.length === 0) {
|
|
549
|
+
return results;
|
|
550
|
+
}
|
|
551
|
+
const byId = new Map(results.map((result) => [result.id, result]));
|
|
552
|
+
const reordered = [];
|
|
553
|
+
const seen = /* @__PURE__ */ new Set();
|
|
554
|
+
for (const { id, score } of scores) {
|
|
555
|
+
const hit = byId.get(id);
|
|
556
|
+
if (!hit || seen.has(id)) {
|
|
557
|
+
continue;
|
|
558
|
+
}
|
|
559
|
+
seen.add(id);
|
|
560
|
+
reordered.push({ ...hit, metadata: { ...hit.metadata, rerankerScore: score } });
|
|
561
|
+
}
|
|
562
|
+
for (const hit of results) {
|
|
563
|
+
if (!seen.has(hit.id)) {
|
|
564
|
+
reordered.push(hit);
|
|
565
|
+
}
|
|
566
|
+
}
|
|
567
|
+
return reordered;
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
// src/search/gather-evidence.ts
|
|
571
|
+
var DEFAULT_TIERS2 = ["summary", "raw", "insight", "knowledge"];
|
|
572
|
+
var MAX_EVIDENCE_CHARS = 600;
|
|
573
|
+
function toSummaryResult(row) {
|
|
574
|
+
const start = typeof row.startTimestamp === "number" ? row.startTimestamp : void 0;
|
|
575
|
+
const end = typeof row.endTimestamp === "number" ? row.endTimestamp : void 0;
|
|
576
|
+
return {
|
|
577
|
+
type: "memory",
|
|
578
|
+
id: row.summaryId,
|
|
579
|
+
content: row.summaryText,
|
|
580
|
+
similarity: 1,
|
|
581
|
+
metadata: {
|
|
582
|
+
tier: "summary",
|
|
583
|
+
summaryTier: row.summaryTier,
|
|
584
|
+
keywords: row.keywords,
|
|
585
|
+
startTimestamp: start,
|
|
586
|
+
endTimestamp: end
|
|
587
|
+
}
|
|
588
|
+
};
|
|
589
|
+
}
|
|
590
|
+
async function gatherSummaries(deps, input, limit, threshold, logger, peerPeers) {
|
|
591
|
+
if (typeof deps.searchSummaries !== "function") {
|
|
592
|
+
return { hits: [], warnings: [] };
|
|
593
|
+
}
|
|
594
|
+
const keywords = input.query.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((token) => token.length >= 2).slice(0, 16);
|
|
595
|
+
try {
|
|
596
|
+
const rows = await deps.searchSummaries({
|
|
597
|
+
userId: input.userId,
|
|
598
|
+
query: input.query,
|
|
599
|
+
keywords,
|
|
600
|
+
limit,
|
|
601
|
+
threshold,
|
|
602
|
+
authToken: input.authToken,
|
|
603
|
+
...peerPeers.length > 0 ? { peers: peerPeers } : {}
|
|
604
|
+
});
|
|
605
|
+
const hits = rows.map(toSummaryResult);
|
|
606
|
+
return { hits, warnings: [] };
|
|
607
|
+
} catch (error) {
|
|
608
|
+
logger.warn?.("[memory-store] gather-evidence: summary search failed:", error);
|
|
609
|
+
return {
|
|
610
|
+
hits: [],
|
|
611
|
+
warnings: [
|
|
612
|
+
{
|
|
613
|
+
source: "memory",
|
|
614
|
+
code: "reflect_summaries_failed",
|
|
615
|
+
message: error.message ?? "reflect_summaries_failed"
|
|
616
|
+
}
|
|
617
|
+
]
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
function truncate(input, max = MAX_EVIDENCE_CHARS) {
|
|
622
|
+
if (input.length <= max) {
|
|
623
|
+
return input;
|
|
624
|
+
}
|
|
625
|
+
return `${input.slice(0, max - 1)}\u2026`;
|
|
626
|
+
}
|
|
627
|
+
function buildSynthesisPrompt(input) {
|
|
628
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
629
|
+
for (const item of input.evidence) {
|
|
630
|
+
const bucket = grouped.get(item.source) ?? [];
|
|
631
|
+
bucket.push(item);
|
|
632
|
+
grouped.set(item.source, bucket);
|
|
633
|
+
}
|
|
634
|
+
const sections = [];
|
|
635
|
+
for (const tier of DEFAULT_TIERS2) {
|
|
636
|
+
const items = grouped.get(tier) ?? [];
|
|
637
|
+
if (items.length === 0) {
|
|
638
|
+
continue;
|
|
639
|
+
}
|
|
640
|
+
const lines = items.map(
|
|
641
|
+
(item, index) => ` [${index + 1}] (${item.id}, score=${item.score.toFixed(4)}) ${item.snippet}`
|
|
642
|
+
);
|
|
643
|
+
sections.push(`## ${tier}
|
|
644
|
+
${lines.join("\n")}`);
|
|
645
|
+
}
|
|
646
|
+
const schemaDirective = input.responseSchema ? `
|
|
647
|
+
Respond with a JSON object that conforms to the following schema:
|
|
648
|
+
${JSON.stringify(input.responseSchema, null, 2)}
|
|
649
|
+
Wrap your JSON in a single fenced \`\`\`json code block; do not add any prose outside the code block.` : "\nRespond in plain prose. Be concise; cite evidence by its [n] number.";
|
|
650
|
+
if (sections.length === 0) {
|
|
651
|
+
return `You are a reflection assistant.
|
|
652
|
+
|
|
653
|
+
Question: ${input.query}
|
|
654
|
+
|
|
655
|
+
No evidence was found across any tier. State that explicitly.${schemaDirective}`;
|
|
656
|
+
}
|
|
657
|
+
return `You are a reflection assistant. Synthesise the user's question using only the evidence below. Do not invent facts.
|
|
658
|
+
|
|
659
|
+
Question: ${input.query}
|
|
660
|
+
|
|
661
|
+
Evidence (grouped by tier):
|
|
662
|
+
${sections.join("\n\n")}${schemaDirective}`;
|
|
663
|
+
}
|
|
664
|
+
function extractJsonPayload(text) {
|
|
665
|
+
const fenced = text.match(/```(?:json)?\s*([\s\S]*?)\s*```/);
|
|
666
|
+
if (fenced?.[1]) {
|
|
667
|
+
return fenced[1];
|
|
668
|
+
}
|
|
669
|
+
const start = text.indexOf("{");
|
|
670
|
+
if (start === -1) {
|
|
671
|
+
return void 0;
|
|
672
|
+
}
|
|
673
|
+
let depth = 0;
|
|
674
|
+
let inString = false;
|
|
675
|
+
let escaped = false;
|
|
676
|
+
for (let i = start; i < text.length; i += 1) {
|
|
677
|
+
const ch = text[i];
|
|
678
|
+
if (escaped) {
|
|
679
|
+
escaped = false;
|
|
680
|
+
continue;
|
|
681
|
+
}
|
|
682
|
+
if (inString) {
|
|
683
|
+
if (ch === "\\") {
|
|
684
|
+
escaped = true;
|
|
685
|
+
} else if (ch === '"') {
|
|
686
|
+
inString = false;
|
|
687
|
+
}
|
|
688
|
+
continue;
|
|
689
|
+
}
|
|
690
|
+
if (ch === '"') {
|
|
691
|
+
inString = true;
|
|
692
|
+
} else if (ch === "{") {
|
|
693
|
+
depth += 1;
|
|
694
|
+
} else if (ch === "}") {
|
|
695
|
+
depth -= 1;
|
|
696
|
+
if (depth === 0) {
|
|
697
|
+
return text.slice(start, i + 1);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
return void 0;
|
|
702
|
+
}
|
|
703
|
+
function coerceAnswer(text, schema) {
|
|
704
|
+
if (!schema) {
|
|
705
|
+
return text;
|
|
706
|
+
}
|
|
707
|
+
const payload = extractJsonPayload(text);
|
|
708
|
+
if (!payload) {
|
|
709
|
+
return text;
|
|
710
|
+
}
|
|
711
|
+
try {
|
|
712
|
+
const parsed = JSON.parse(payload);
|
|
713
|
+
if (parsed && typeof parsed === "object") {
|
|
714
|
+
const answer = parsed.answer;
|
|
715
|
+
if (typeof answer === "string") {
|
|
716
|
+
return answer;
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
} catch {
|
|
720
|
+
}
|
|
721
|
+
return text;
|
|
722
|
+
}
|
|
723
|
+
async function synthesizeAnswer(input) {
|
|
724
|
+
const { query, hits, responseSchema, deps, logger } = input;
|
|
725
|
+
const evidence = hits.map((hit) => ({
|
|
726
|
+
id: hit.id,
|
|
727
|
+
source: mapHitToTier(hit),
|
|
728
|
+
snippet: truncate(hit.content),
|
|
729
|
+
score: hit.similarity
|
|
730
|
+
}));
|
|
731
|
+
const warnings = [];
|
|
732
|
+
const complete = deps.reasoning?.complete;
|
|
733
|
+
if (!complete) {
|
|
734
|
+
warnings.push({
|
|
735
|
+
source: "memory",
|
|
736
|
+
code: "reflect_llm_not_configured",
|
|
737
|
+
message: "No `reasoning.complete` callback is configured; search() returned evidence without an LLM synthesis."
|
|
738
|
+
});
|
|
739
|
+
return { answer: "", warnings };
|
|
740
|
+
}
|
|
741
|
+
const prompt = buildSynthesisPrompt({ query, evidence, responseSchema });
|
|
742
|
+
try {
|
|
743
|
+
const raw = await complete(prompt);
|
|
744
|
+
const answer = coerceAnswer(raw, responseSchema);
|
|
745
|
+
logger.log?.("[memory-store] search synthesis completed", {
|
|
746
|
+
evidenceCount: evidence.length,
|
|
747
|
+
answerChars: answer.length
|
|
748
|
+
});
|
|
749
|
+
return { answer, warnings };
|
|
750
|
+
} catch (error) {
|
|
751
|
+
logger.warn?.("[memory-store] search synthesis failed:", error);
|
|
752
|
+
warnings.push({
|
|
753
|
+
source: "memory",
|
|
754
|
+
code: "reflect_llm_failed",
|
|
755
|
+
message: error.message ?? "reflect_llm_failed"
|
|
756
|
+
});
|
|
757
|
+
return { answer: "", warnings };
|
|
758
|
+
}
|
|
759
|
+
}
|
|
760
|
+
function mapHitToTier(hit) {
|
|
761
|
+
const tierMarker = hit.metadata?.tier;
|
|
762
|
+
if (tierMarker === "summary") {
|
|
763
|
+
return "summary";
|
|
764
|
+
}
|
|
765
|
+
if (hit.type === "memory") {
|
|
766
|
+
return "raw";
|
|
767
|
+
}
|
|
768
|
+
if (hit.type === "insight") {
|
|
769
|
+
return "insight";
|
|
770
|
+
}
|
|
771
|
+
if (hit.type === "knowledge") {
|
|
772
|
+
return "knowledge";
|
|
773
|
+
}
|
|
774
|
+
return hit.type;
|
|
775
|
+
}
|
|
776
|
+
async function resolveSearchScopePeers(input) {
|
|
777
|
+
return resolveScopePeer({
|
|
778
|
+
userId: input.userId,
|
|
779
|
+
peerFilter: input.peerFilter,
|
|
780
|
+
scopeCheck: input.deps.peerScopeCheck
|
|
781
|
+
});
|
|
782
|
+
}
|
|
374
783
|
|
|
375
784
|
// src/search/unified-search.ts
|
|
376
785
|
function deriveLexicalKeywords(query) {
|
|
@@ -442,7 +851,7 @@ async function embedQueryVariant(embedQuery, input, query) {
|
|
|
442
851
|
authToken: input.authToken
|
|
443
852
|
});
|
|
444
853
|
}
|
|
445
|
-
async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit, threshold, logger) {
|
|
854
|
+
async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit, threshold, logger, peerPeers = []) {
|
|
446
855
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
447
856
|
let semantic = [];
|
|
448
857
|
if (isRawMessageChromaEnabled()) {
|
|
@@ -472,6 +881,7 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
472
881
|
}
|
|
473
882
|
} else if (typeof deps.searchRawMessagesAnn === "function") {
|
|
474
883
|
const searchRawMessagesAnn = deps.searchRawMessagesAnn;
|
|
884
|
+
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
475
885
|
semantic = (await Promise.all(
|
|
476
886
|
filters.map(
|
|
477
887
|
(filter) => searchRawMessagesAnn({
|
|
@@ -479,7 +889,9 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
479
889
|
queryEmbedding,
|
|
480
890
|
limit,
|
|
481
891
|
threshold,
|
|
482
|
-
botId: "botId" in filter ? filter.botId : void 0
|
|
892
|
+
botId: "botId" in filter ? filter.botId : void 0,
|
|
893
|
+
...peerPeers.length > 0 ? { peers: peerPeers } : {},
|
|
894
|
+
...factTypes ? { factTypes } : {}
|
|
483
895
|
})
|
|
484
896
|
)
|
|
485
897
|
)).flat().filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
@@ -489,11 +901,14 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
489
901
|
const { getRawMessageManager: getRawMessageManager2 } = await Promise.resolve().then(() => (init_raw_message_store(), raw_message_store_exports));
|
|
490
902
|
const manager2 = await getRawMessageManager2();
|
|
491
903
|
if (typeof manager2.searchMessagesSemantically === "function") {
|
|
904
|
+
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
492
905
|
const results = await manager2.searchMessagesSemantically({
|
|
493
906
|
userId: input.userId,
|
|
494
907
|
queryEmbedding,
|
|
495
908
|
limit,
|
|
496
|
-
threshold
|
|
909
|
+
threshold,
|
|
910
|
+
...peerPeers.length > 0 ? { peers: peerPeers } : {},
|
|
911
|
+
...factTypes ? { factTypes } : {}
|
|
497
912
|
});
|
|
498
913
|
semantic = results.map((r) => ({
|
|
499
914
|
type: "memory",
|
|
@@ -529,7 +944,7 @@ function toIterativeRecallCandidate(hit) {
|
|
|
529
944
|
metadata: hit.metadata
|
|
530
945
|
};
|
|
531
946
|
}
|
|
532
|
-
async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger) {
|
|
947
|
+
async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger, peerPeers = []) {
|
|
533
948
|
if (keywords.length === 0) {
|
|
534
949
|
return [];
|
|
535
950
|
}
|
|
@@ -537,13 +952,16 @@ async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger)
|
|
|
537
952
|
try {
|
|
538
953
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
539
954
|
const searchRawMessagesLexical = deps.searchRawMessagesLexical;
|
|
955
|
+
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
540
956
|
return (await Promise.all(
|
|
541
957
|
filters.map(
|
|
542
958
|
(botId) => searchRawMessagesLexical({
|
|
543
959
|
userId: input.userId,
|
|
544
960
|
keywords,
|
|
545
961
|
limit: Math.ceil(limit / filters.length),
|
|
546
|
-
botId
|
|
962
|
+
botId,
|
|
963
|
+
...peerPeers.length > 0 ? { peers: peerPeers } : {},
|
|
964
|
+
...factTypes ? { factTypes } : {}
|
|
547
965
|
})
|
|
548
966
|
)
|
|
549
967
|
)).flat().filter(isRawMemorySemanticResult).map((hit) => ({
|
|
@@ -557,13 +975,15 @@ async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger)
|
|
|
557
975
|
try {
|
|
558
976
|
const { lexicalSearchRawMessages: lexicalSearchRawMessages2 } = await Promise.resolve().then(() => (init_sqlite_raw_message_store(), sqlite_raw_message_store_exports));
|
|
559
977
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
978
|
+
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
560
979
|
return (await Promise.all(
|
|
561
980
|
filters.map(
|
|
562
981
|
(botId) => lexicalSearchRawMessages2({
|
|
563
982
|
userId: input.userId,
|
|
564
983
|
keywords,
|
|
565
984
|
limit: Math.ceil(limit / filters.length),
|
|
566
|
-
botId
|
|
985
|
+
botId,
|
|
986
|
+
...factTypes ? { factTypes } : {}
|
|
567
987
|
})
|
|
568
988
|
)
|
|
569
989
|
)).flat().filter(Boolean).map(
|
|
@@ -576,6 +996,42 @@ async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger)
|
|
|
576
996
|
}
|
|
577
997
|
return [];
|
|
578
998
|
}
|
|
999
|
+
function searchInputToUnified(input) {
|
|
1000
|
+
return {
|
|
1001
|
+
userId: input.userId,
|
|
1002
|
+
query: input.query,
|
|
1003
|
+
sources: input.sources ? [...input.sources] : void 0,
|
|
1004
|
+
limit: input.limit,
|
|
1005
|
+
threshold: input.threshold,
|
|
1006
|
+
authToken: input.authToken,
|
|
1007
|
+
includeArchivedInsights: input.includeArchivedInsights,
|
|
1008
|
+
botIds: input.botIds,
|
|
1009
|
+
documentIds: input.documentIds,
|
|
1010
|
+
asOf: input.asOf,
|
|
1011
|
+
dateFrom: input.dateFrom,
|
|
1012
|
+
dateTo: input.dateTo,
|
|
1013
|
+
mergeStrategy: input.mergeStrategy,
|
|
1014
|
+
peerFilter: input.peerFilter,
|
|
1015
|
+
reasoningStrategy: input.reasoningStrategy,
|
|
1016
|
+
factTypes: input.factTypes
|
|
1017
|
+
};
|
|
1018
|
+
}
|
|
1019
|
+
function mapHitToEvidenceSource(hit) {
|
|
1020
|
+
const tierMarker = hit.metadata?.tier;
|
|
1021
|
+
if (tierMarker === "summary") {
|
|
1022
|
+
return "summary";
|
|
1023
|
+
}
|
|
1024
|
+
if (hit.type === "memory") {
|
|
1025
|
+
return "raw";
|
|
1026
|
+
}
|
|
1027
|
+
if (hit.type === "insight") {
|
|
1028
|
+
return "insight";
|
|
1029
|
+
}
|
|
1030
|
+
if (hit.type === "knowledge") {
|
|
1031
|
+
return "knowledge";
|
|
1032
|
+
}
|
|
1033
|
+
return hit.type;
|
|
1034
|
+
}
|
|
579
1035
|
function createUnifiedSearch(deps = {}) {
|
|
580
1036
|
const logger = deps.logger ?? console;
|
|
581
1037
|
function resolveReasoningStrategy(input) {
|
|
@@ -585,130 +1041,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
585
1041
|
}
|
|
586
1042
|
return normalizeUnifiedMemoryReasoningStrategy(deps.reasoning?.defaultStrategy, "none");
|
|
587
1043
|
}
|
|
588
|
-
async function
|
|
589
|
-
const query = input.query.trim();
|
|
590
|
-
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
591
|
-
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
592
|
-
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
593
|
-
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(input.mergeStrategy);
|
|
594
|
-
const reasoningStrategy = resolveReasoningStrategy(input);
|
|
595
|
-
const warnings = [];
|
|
596
|
-
if (!query) {
|
|
597
|
-
return {
|
|
598
|
-
query,
|
|
599
|
-
sources,
|
|
600
|
-
results: [],
|
|
601
|
-
count: 0,
|
|
602
|
-
warnings
|
|
603
|
-
};
|
|
604
|
-
}
|
|
605
|
-
let memorySubs;
|
|
606
|
-
const reasoningInfo = { strategy: reasoningStrategy };
|
|
607
|
-
if (input.dateFrom || input.dateTo) {
|
|
608
|
-
reasoningInfo.dateRange = { from: input.dateFrom, to: input.dateTo };
|
|
609
|
-
}
|
|
610
|
-
if (sources.includes("memory")) {
|
|
611
|
-
if (isRawMessageStorageAvailable()) {
|
|
612
|
-
try {
|
|
613
|
-
memorySubs = await runMemorySource(
|
|
614
|
-
input,
|
|
615
|
-
limit,
|
|
616
|
-
threshold,
|
|
617
|
-
warnings,
|
|
618
|
-
reasoningStrategy,
|
|
619
|
-
reasoningInfo
|
|
620
|
-
);
|
|
621
|
-
} catch (error) {
|
|
622
|
-
logger.warn?.("[memory-store] memory source failed:", error);
|
|
623
|
-
warnings.push({
|
|
624
|
-
source: "memory",
|
|
625
|
-
code: "memory_search_failed",
|
|
626
|
-
message: `Memory search failed: ${error.message ?? "Unknown error"}. Using keyword search fallback.`
|
|
627
|
-
});
|
|
628
|
-
}
|
|
629
|
-
} else {
|
|
630
|
-
warnings.push({
|
|
631
|
-
source: "memory",
|
|
632
|
-
code: "raw_message_storage_unavailable",
|
|
633
|
-
message: "Raw memory storage is not available in this environment."
|
|
634
|
-
});
|
|
635
|
-
}
|
|
636
|
-
}
|
|
637
|
-
const insightHits = [];
|
|
638
|
-
if (sources.includes("insights") && typeof deps.searchInsights === "function") {
|
|
639
|
-
try {
|
|
640
|
-
const hits = await deps.searchInsights({
|
|
641
|
-
userId: input.userId,
|
|
642
|
-
query: input.query,
|
|
643
|
-
limit,
|
|
644
|
-
threshold,
|
|
645
|
-
botIds: input.botIds,
|
|
646
|
-
includeArchived: input.includeArchivedInsights,
|
|
647
|
-
authToken: input.authToken
|
|
648
|
-
});
|
|
649
|
-
for (const hit of hits) {
|
|
650
|
-
insightHits.push({
|
|
651
|
-
type: "insight",
|
|
652
|
-
id: hit.id,
|
|
653
|
-
content: hit.content,
|
|
654
|
-
similarity: hit.similarity,
|
|
655
|
-
metadata: hit.metadata
|
|
656
|
-
});
|
|
657
|
-
}
|
|
658
|
-
} catch (error) {
|
|
659
|
-
logger.warn?.("[memory-store] insights source failed:", error);
|
|
660
|
-
warnings.push({
|
|
661
|
-
source: "insights",
|
|
662
|
-
code: "insights_search_failed",
|
|
663
|
-
message: error.message ?? "insights_search_failed"
|
|
664
|
-
});
|
|
665
|
-
}
|
|
666
|
-
} else if (sources.includes("insights")) ;
|
|
667
|
-
const knowledgeHits = [];
|
|
668
|
-
if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
|
|
669
|
-
try {
|
|
670
|
-
const hits = await deps.searchKnowledge({
|
|
671
|
-
userId: input.userId,
|
|
672
|
-
query: input.query,
|
|
673
|
-
options: {
|
|
674
|
-
limit,
|
|
675
|
-
threshold,
|
|
676
|
-
documentIds: input.documentIds
|
|
677
|
-
},
|
|
678
|
-
authToken: input.authToken
|
|
679
|
-
});
|
|
680
|
-
for (const hit of hits) {
|
|
681
|
-
knowledgeHits.push(toKnowledgeResult(hit));
|
|
682
|
-
}
|
|
683
|
-
} catch (error) {
|
|
684
|
-
logger.warn?.("[memory-store] knowledge source failed:", error);
|
|
685
|
-
warnings.push({
|
|
686
|
-
source: "knowledge",
|
|
687
|
-
code: "knowledge_search_failed",
|
|
688
|
-
message: error.message ?? "knowledge_search_failed"
|
|
689
|
-
});
|
|
690
|
-
}
|
|
691
|
-
} else if (sources.includes("knowledge")) ;
|
|
692
|
-
const merged = mergeAcrossSources({
|
|
693
|
-
memorySubs,
|
|
694
|
-
insightHits,
|
|
695
|
-
knowledgeHits,
|
|
696
|
-
limit,
|
|
697
|
-
strategy: mergeStrategy
|
|
698
|
-
});
|
|
699
|
-
const output = {
|
|
700
|
-
query: input.query,
|
|
701
|
-
sources,
|
|
702
|
-
results: merged,
|
|
703
|
-
count: merged.length,
|
|
704
|
-
warnings
|
|
705
|
-
};
|
|
706
|
-
if (reasoningInfo && (reasoningInfo.strategy !== "none" || reasoningInfo.dateRange !== void 0)) {
|
|
707
|
-
output.reasoning = reasoningInfo;
|
|
708
|
-
}
|
|
709
|
-
return output;
|
|
710
|
-
}
|
|
711
|
-
async function runMemorySource(input, limit, threshold, warnings, reasoningStrategy = "none", reasoningInfo) {
|
|
1044
|
+
async function runMemorySource(input, limit, threshold, warnings, reasoningStrategy = "none", reasoningInfo, peerPeers = []) {
|
|
712
1045
|
if (reasoningStrategy === "iterative" && !deps.reasoning?.iterativePlanner) {
|
|
713
1046
|
warnings.push({
|
|
714
1047
|
source: "memory",
|
|
@@ -734,7 +1067,14 @@ function createUnifiedSearch(deps = {}) {
|
|
|
734
1067
|
executor: {
|
|
735
1068
|
search: async (request) => {
|
|
736
1069
|
const keywords2 = request.keywords.length > 0 ? request.keywords : deriveLexicalKeywords(input.query);
|
|
737
|
-
const lexicalHits = await runLexicalSearchForKeywords(
|
|
1070
|
+
const lexicalHits = await runLexicalSearchForKeywords(
|
|
1071
|
+
deps,
|
|
1072
|
+
input,
|
|
1073
|
+
keywords2,
|
|
1074
|
+
limit,
|
|
1075
|
+
logger,
|
|
1076
|
+
peerPeers
|
|
1077
|
+
);
|
|
738
1078
|
let semanticHits = [];
|
|
739
1079
|
if (typeof deps.embedQuery === "function") {
|
|
740
1080
|
try {
|
|
@@ -746,7 +1086,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
746
1086
|
queryEmbedding,
|
|
747
1087
|
limit,
|
|
748
1088
|
threshold,
|
|
749
|
-
logger
|
|
1089
|
+
logger,
|
|
1090
|
+
peerPeers
|
|
750
1091
|
);
|
|
751
1092
|
} catch (error) {
|
|
752
1093
|
logger.warn?.(
|
|
@@ -791,7 +1132,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
791
1132
|
});
|
|
792
1133
|
const keywords2 = deriveLexicalKeywords(input.query);
|
|
793
1134
|
const lexical2 = filterByDateRange(
|
|
794
|
-
await runLexicalSearchForKeywords(deps, input, keywords2, limit, logger),
|
|
1135
|
+
await runLexicalSearchForKeywords(deps, input, keywords2, limit, logger, peerPeers),
|
|
795
1136
|
input.dateFrom,
|
|
796
1137
|
input.dateTo
|
|
797
1138
|
);
|
|
@@ -817,7 +1158,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
817
1158
|
);
|
|
818
1159
|
const lists = await Promise.all(
|
|
819
1160
|
embeddings.map(
|
|
820
|
-
(embedding) => runSemanticSearchForEmbedding(deps, input, embedding, limit, threshold, logger)
|
|
1161
|
+
(embedding) => runSemanticSearchForEmbedding(deps, input, embedding, limit, threshold, logger, peerPeers)
|
|
821
1162
|
)
|
|
822
1163
|
);
|
|
823
1164
|
semantic = mergeByMaxScore(lists);
|
|
@@ -833,7 +1174,15 @@ function createUnifiedSearch(deps = {}) {
|
|
|
833
1174
|
}
|
|
834
1175
|
if (semantic.length === 0) {
|
|
835
1176
|
const queryEmbedding = await embedQueryVariant(embedQuery, input, input.query);
|
|
836
|
-
semantic = await runSemanticSearchForEmbedding(
|
|
1177
|
+
semantic = await runSemanticSearchForEmbedding(
|
|
1178
|
+
deps,
|
|
1179
|
+
input,
|
|
1180
|
+
queryEmbedding,
|
|
1181
|
+
limit,
|
|
1182
|
+
threshold,
|
|
1183
|
+
logger,
|
|
1184
|
+
peerPeers
|
|
1185
|
+
);
|
|
837
1186
|
}
|
|
838
1187
|
let lexical = [];
|
|
839
1188
|
const keywords = deriveLexicalKeywords(input.query);
|
|
@@ -842,13 +1191,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
842
1191
|
try {
|
|
843
1192
|
const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
844
1193
|
const searchRawMessagesLexical = deps.searchRawMessagesLexical;
|
|
1194
|
+
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
845
1195
|
lexical = (await Promise.all(
|
|
846
1196
|
lexFilters.map(
|
|
847
1197
|
(botId) => searchRawMessagesLexical({
|
|
848
1198
|
userId: input.userId,
|
|
849
1199
|
keywords,
|
|
850
1200
|
limit,
|
|
851
|
-
botId
|
|
1201
|
+
botId,
|
|
1202
|
+
...peerPeers.length > 0 ? { peers: peerPeers } : {},
|
|
1203
|
+
...factTypes ? { factTypes } : {}
|
|
852
1204
|
})
|
|
853
1205
|
)
|
|
854
1206
|
)).flat().filter(isRawMemorySemanticResult).map((hit) => ({
|
|
@@ -876,20 +1228,272 @@ function createUnifiedSearch(deps = {}) {
|
|
|
876
1228
|
lexical: filterByDateRange(lexical, input.dateFrom, input.dateTo)
|
|
877
1229
|
};
|
|
878
1230
|
}
|
|
1231
|
+
async function searchUnifiedMemory(input) {
|
|
1232
|
+
const query = input.query.trim();
|
|
1233
|
+
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
1234
|
+
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
1235
|
+
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
1236
|
+
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(
|
|
1237
|
+
input.mergeStrategy ?? deps.reasoning?.defaultMergeStrategy ?? "rrf"
|
|
1238
|
+
);
|
|
1239
|
+
const reasoningStrategy = resolveReasoningStrategy(input);
|
|
1240
|
+
const warnings = [];
|
|
1241
|
+
const peerScope = input.peerFilter && input.peerFilter.length > 0 ? await resolveScopePeer({
|
|
1242
|
+
userId: input.userId,
|
|
1243
|
+
peerFilter: input.peerFilter,
|
|
1244
|
+
scopeCheck: deps.peerScopeCheck
|
|
1245
|
+
}) : { peers: [], warnings: [] };
|
|
1246
|
+
warnings.push(...peerScope.warnings);
|
|
1247
|
+
const peerPeers = peerScope.peers;
|
|
1248
|
+
if (!query) {
|
|
1249
|
+
return {
|
|
1250
|
+
query,
|
|
1251
|
+
sources,
|
|
1252
|
+
results: [],
|
|
1253
|
+
count: 0,
|
|
1254
|
+
warnings
|
|
1255
|
+
};
|
|
1256
|
+
}
|
|
1257
|
+
let memorySubs;
|
|
1258
|
+
const reasoningInfo = { strategy: reasoningStrategy };
|
|
1259
|
+
if (input.dateFrom || input.dateTo) {
|
|
1260
|
+
reasoningInfo.dateRange = { from: input.dateFrom, to: input.dateTo };
|
|
1261
|
+
}
|
|
1262
|
+
if (sources.includes("memory")) {
|
|
1263
|
+
const hasRawProviders = typeof deps.searchRawMessagesAnn === "function" || typeof deps.searchRawMessagesLexical === "function";
|
|
1264
|
+
if (isRawMessageStorageAvailable() || hasRawProviders) {
|
|
1265
|
+
try {
|
|
1266
|
+
memorySubs = await runMemorySource(
|
|
1267
|
+
input,
|
|
1268
|
+
limit,
|
|
1269
|
+
threshold,
|
|
1270
|
+
warnings,
|
|
1271
|
+
reasoningStrategy,
|
|
1272
|
+
reasoningInfo,
|
|
1273
|
+
peerPeers
|
|
1274
|
+
);
|
|
1275
|
+
} catch (error) {
|
|
1276
|
+
logger.warn?.("[memory-store] memory source failed:", error);
|
|
1277
|
+
warnings.push({
|
|
1278
|
+
source: "memory",
|
|
1279
|
+
code: "memory_search_failed",
|
|
1280
|
+
message: `Memory search failed: ${error.message ?? "Unknown error"}. Using keyword search fallback.`
|
|
1281
|
+
});
|
|
1282
|
+
}
|
|
1283
|
+
} else {
|
|
1284
|
+
warnings.push({
|
|
1285
|
+
source: "memory",
|
|
1286
|
+
code: "raw_message_storage_unavailable",
|
|
1287
|
+
message: "Raw memory storage is not available in this environment."
|
|
1288
|
+
});
|
|
1289
|
+
}
|
|
1290
|
+
}
|
|
1291
|
+
const insightHits = [];
|
|
1292
|
+
if (sources.includes("insights") && typeof deps.searchInsights === "function") {
|
|
1293
|
+
try {
|
|
1294
|
+
const hits = await deps.searchInsights({
|
|
1295
|
+
userId: input.userId,
|
|
1296
|
+
query: input.query,
|
|
1297
|
+
limit,
|
|
1298
|
+
threshold,
|
|
1299
|
+
botIds: input.botIds,
|
|
1300
|
+
includeArchived: input.includeArchivedInsights,
|
|
1301
|
+
authToken: input.authToken,
|
|
1302
|
+
...peerPeers.length > 0 ? { peers: peerPeers } : {}
|
|
1303
|
+
});
|
|
1304
|
+
for (const hit of hits) {
|
|
1305
|
+
insightHits.push({
|
|
1306
|
+
type: "insight",
|
|
1307
|
+
id: hit.id,
|
|
1308
|
+
content: hit.content,
|
|
1309
|
+
similarity: hit.similarity,
|
|
1310
|
+
metadata: hit.metadata
|
|
1311
|
+
});
|
|
1312
|
+
}
|
|
1313
|
+
} catch (error) {
|
|
1314
|
+
logger.warn?.("[memory-store] insights source failed:", error);
|
|
1315
|
+
warnings.push({
|
|
1316
|
+
source: "insights",
|
|
1317
|
+
code: "insights_search_failed",
|
|
1318
|
+
message: error.message ?? "insights_search_failed"
|
|
1319
|
+
});
|
|
1320
|
+
}
|
|
1321
|
+
} else if (sources.includes("insights")) ;
|
|
1322
|
+
const knowledgeHits = [];
|
|
1323
|
+
if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
|
|
1324
|
+
try {
|
|
1325
|
+
const hits = await deps.searchKnowledge({
|
|
1326
|
+
userId: input.userId,
|
|
1327
|
+
query: input.query,
|
|
1328
|
+
options: {
|
|
1329
|
+
limit,
|
|
1330
|
+
threshold,
|
|
1331
|
+
documentIds: input.documentIds
|
|
1332
|
+
},
|
|
1333
|
+
authToken: input.authToken,
|
|
1334
|
+
...peerPeers.length > 0 ? { peers: peerPeers } : {}
|
|
1335
|
+
});
|
|
1336
|
+
for (const hit of hits) {
|
|
1337
|
+
knowledgeHits.push(toKnowledgeResult(hit));
|
|
1338
|
+
}
|
|
1339
|
+
} catch (error) {
|
|
1340
|
+
logger.warn?.("[memory-store] knowledge source failed:", error);
|
|
1341
|
+
warnings.push({
|
|
1342
|
+
source: "knowledge",
|
|
1343
|
+
code: "knowledge_search_failed",
|
|
1344
|
+
message: error.message ?? "knowledge_search_failed"
|
|
1345
|
+
});
|
|
1346
|
+
}
|
|
1347
|
+
} else if (sources.includes("knowledge")) ;
|
|
1348
|
+
const merged = mergeAcrossSources({
|
|
1349
|
+
memorySubs,
|
|
1350
|
+
insightHits,
|
|
1351
|
+
knowledgeHits,
|
|
1352
|
+
limit,
|
|
1353
|
+
strategy: mergeStrategy
|
|
1354
|
+
});
|
|
1355
|
+
const ranked = await applyReranker(deps.reranker, input.query, merged);
|
|
1356
|
+
const output = {
|
|
1357
|
+
query: input.query,
|
|
1358
|
+
sources,
|
|
1359
|
+
results: ranked,
|
|
1360
|
+
count: ranked.length,
|
|
1361
|
+
warnings
|
|
1362
|
+
};
|
|
1363
|
+
if (reasoningInfo && (reasoningInfo.strategy !== "none" || reasoningInfo.dateRange !== void 0)) {
|
|
1364
|
+
output.reasoning = reasoningInfo;
|
|
1365
|
+
}
|
|
1366
|
+
return output;
|
|
1367
|
+
}
|
|
879
1368
|
async function searchRawMemorySemantically(input) {
|
|
880
1369
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
881
1370
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
882
1371
|
const reasoningStrategy = resolveReasoningStrategy(input);
|
|
1372
|
+
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(
|
|
1373
|
+
input.mergeStrategy ?? deps.reasoning?.defaultMergeStrategy ?? "rrf"
|
|
1374
|
+
);
|
|
883
1375
|
const sub = await runMemorySource(input, limit, threshold, [], reasoningStrategy);
|
|
884
1376
|
return mergeAcrossSources({
|
|
885
1377
|
memorySubs: sub,
|
|
886
1378
|
insightHits: [],
|
|
887
1379
|
knowledgeHits: [],
|
|
888
1380
|
limit,
|
|
889
|
-
strategy:
|
|
1381
|
+
strategy: mergeStrategy
|
|
890
1382
|
});
|
|
891
1383
|
}
|
|
892
|
-
|
|
1384
|
+
async function search(input) {
|
|
1385
|
+
const query = input.query.trim();
|
|
1386
|
+
const wantsSynthesis = typeof input.synthesize === "boolean" ? input.synthesize : typeof input.synthesize === "object" && input.synthesize !== null;
|
|
1387
|
+
const synthesisSchema = typeof input.synthesize === "object" && input.synthesize !== null ? input.synthesize.responseSchema ?? input.responseSchema : input.responseSchema;
|
|
1388
|
+
const peerScope = await resolveSearchScopePeers({
|
|
1389
|
+
deps,
|
|
1390
|
+
userId: input.userId,
|
|
1391
|
+
peerFilter: input.peerFilter
|
|
1392
|
+
});
|
|
1393
|
+
if (!query) {
|
|
1394
|
+
const empty = {
|
|
1395
|
+
query,
|
|
1396
|
+
sources: input.sources ? [...input.sources] : [],
|
|
1397
|
+
results: [],
|
|
1398
|
+
evidence: [],
|
|
1399
|
+
count: 0,
|
|
1400
|
+
warnings: peerScope.warnings,
|
|
1401
|
+
answer: ""
|
|
1402
|
+
};
|
|
1403
|
+
return empty;
|
|
1404
|
+
}
|
|
1405
|
+
const tiers = input.tiers ? [...input.tiers] : wantsSynthesis ? ["summary", "raw", "insight", "knowledge"] : ["raw", "insight", "knowledge"];
|
|
1406
|
+
const wantsSummaryTier = tiers.includes("summary");
|
|
1407
|
+
const wantsRawTier = tiers.includes("raw");
|
|
1408
|
+
const wantsInsightTier = tiers.includes("insight");
|
|
1409
|
+
const wantsKnowledgeTier = tiers.includes("knowledge");
|
|
1410
|
+
const warnings = [...peerScope.warnings];
|
|
1411
|
+
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
1412
|
+
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
1413
|
+
let unifiedHits = [];
|
|
1414
|
+
let sources = input.sources ? [...input.sources] : [];
|
|
1415
|
+
let unifiedReasoning;
|
|
1416
|
+
if (wantsRawTier || wantsInsightTier || wantsKnowledgeTier) {
|
|
1417
|
+
const inferredSources = sources.length > 0 ? sources : [
|
|
1418
|
+
...wantsRawTier ? ["memory"] : [],
|
|
1419
|
+
...wantsInsightTier ? ["insights"] : [],
|
|
1420
|
+
...wantsKnowledgeTier ? ["knowledge"] : []
|
|
1421
|
+
];
|
|
1422
|
+
const unified = await searchUnifiedMemory({
|
|
1423
|
+
...searchInputToUnified(input),
|
|
1424
|
+
sources: inferredSources.length > 0 ? inferredSources : void 0
|
|
1425
|
+
});
|
|
1426
|
+
unifiedHits = unified.results;
|
|
1427
|
+
sources = unified.sources;
|
|
1428
|
+
unifiedReasoning = unified.reasoning;
|
|
1429
|
+
warnings.push(...unified.warnings);
|
|
1430
|
+
}
|
|
1431
|
+
let summaryHits = [];
|
|
1432
|
+
if (wantsSummaryTier && typeof deps.searchSummaries === "function") {
|
|
1433
|
+
const summaryBucket = await gatherSummaries(
|
|
1434
|
+
deps,
|
|
1435
|
+
{ userId: input.userId, query: input.query, authToken: input.authToken },
|
|
1436
|
+
limit,
|
|
1437
|
+
threshold,
|
|
1438
|
+
logger,
|
|
1439
|
+
peerScope.peers
|
|
1440
|
+
);
|
|
1441
|
+
warnings.push(...summaryBucket.warnings);
|
|
1442
|
+
summaryHits = summaryBucket.hits;
|
|
1443
|
+
}
|
|
1444
|
+
const tierLists = [];
|
|
1445
|
+
if (unifiedHits.length > 0) {
|
|
1446
|
+
tierLists.push({ name: "memory", hits: unifiedHits });
|
|
1447
|
+
}
|
|
1448
|
+
if (summaryHits.length > 0) {
|
|
1449
|
+
tierLists.push({ name: "summary", hits: summaryHits });
|
|
1450
|
+
}
|
|
1451
|
+
let hits;
|
|
1452
|
+
if (tierLists.length === 0) {
|
|
1453
|
+
hits = [];
|
|
1454
|
+
} else if (tierLists.length === 1) {
|
|
1455
|
+
hits = tierLists[0].hits.slice(0, limit);
|
|
1456
|
+
} else {
|
|
1457
|
+
hits = mergeUnifiedMemorySearchResultsRrf(tierLists, limit);
|
|
1458
|
+
}
|
|
1459
|
+
const evidence = hits.map((hit) => ({
|
|
1460
|
+
id: hit.id,
|
|
1461
|
+
source: mapHitToEvidenceSource(hit),
|
|
1462
|
+
snippet: hit.content,
|
|
1463
|
+
score: hit.similarity,
|
|
1464
|
+
timestamp: getCandidateTimestamp(hit.metadata)
|
|
1465
|
+
}));
|
|
1466
|
+
const base = {
|
|
1467
|
+
query,
|
|
1468
|
+
sources,
|
|
1469
|
+
results: hits,
|
|
1470
|
+
evidence,
|
|
1471
|
+
count: hits.length,
|
|
1472
|
+
warnings
|
|
1473
|
+
};
|
|
1474
|
+
if (unifiedReasoning) {
|
|
1475
|
+
base.reasoning = unifiedReasoning;
|
|
1476
|
+
}
|
|
1477
|
+
if (!wantsSynthesis) {
|
|
1478
|
+
return base;
|
|
1479
|
+
}
|
|
1480
|
+
const { answer, warnings: synthWarnings } = await synthesizeAnswer({
|
|
1481
|
+
query,
|
|
1482
|
+
hits,
|
|
1483
|
+
responseSchema: synthesisSchema,
|
|
1484
|
+
deps,
|
|
1485
|
+
logger
|
|
1486
|
+
});
|
|
1487
|
+
base.answer = answer;
|
|
1488
|
+
base.warnings = [...base.warnings, ...synthWarnings];
|
|
1489
|
+
return base;
|
|
1490
|
+
}
|
|
1491
|
+
return {
|
|
1492
|
+
search,
|
|
1493
|
+
searchUnifiedMemory,
|
|
1494
|
+
searchRawMemorySemantically,
|
|
1495
|
+
consolidate: (input) => applyReflectedPlan(deps, {}, input)
|
|
1496
|
+
};
|
|
893
1497
|
}
|
|
894
1498
|
function mergeAcrossSources(input) {
|
|
895
1499
|
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|