@melandlabs/memory-store 1.3.8 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{config-9RzSCdXG.d.ts → config-Cnk2b7fH.d.ts} +25 -4
- package/dist/{evidence-selector-uOHsxwch.d.ts → evidence-selector-C_ZUQsio.d.ts} +34 -5
- package/dist/http.d.ts +2 -2
- package/dist/http.js +356 -112
- package/dist/index.d.ts +6 -6
- package/dist/index.js +349 -111
- package/dist/mcp.d.ts +2 -2
- package/dist/mcp.js +352 -114
- package/dist/search/derive.d.ts +2 -2
- package/dist/search/derive.js +1 -1
- package/dist/search/distill.d.ts +2 -2
- package/dist/search/evidence-selector.d.ts +2 -1
- package/dist/search/unified-search.d.ts +3 -3
- package/dist/search/unified-search.js +349 -111
- package/dist/server/cli-http.js +356 -112
- package/dist/server/cli-mcp.js +352 -114
- package/dist/server/cli-shared.d.ts +2 -2
- package/dist/storage/postgres-raw-message-factory.d.ts +2 -2
- package/dist/storage/raw-message-store.d.ts +2 -2
- package/dist/storage/sqlite-raw-message-store.d.ts +2 -2
- package/dist/storage/sqlite-vector-index.d.ts +2 -2
- package/dist/{unified-search-BBC3jkPJ.d.ts → unified-search-DfAYG3S9.d.ts} +3 -3
- package/package.json +6 -5
package/dist/mcp.js
CHANGED
|
@@ -319,6 +319,46 @@ function resolveSearchRuntimeContext(input, runtimeContext, now = Date.now) {
|
|
|
319
319
|
};
|
|
320
320
|
}
|
|
321
321
|
|
|
322
|
+
// src/search/execution-budget.ts
|
|
323
|
+
function boundedSearchOperation(operation, budget) {
|
|
324
|
+
const timeout = budget.sourceTimeoutMs ?? 5e3;
|
|
325
|
+
if (!Number.isFinite(timeout) || timeout <= 0)
|
|
326
|
+
throw new TypeError("sourceTimeoutMs must be positive and finite");
|
|
327
|
+
return (...args) => new Promise((resolve, reject) => {
|
|
328
|
+
const signal = budget.signal;
|
|
329
|
+
if (signal?.aborted) {
|
|
330
|
+
reject(signal.reason);
|
|
331
|
+
return;
|
|
332
|
+
}
|
|
333
|
+
const cleanup2 = () => {
|
|
334
|
+
clearTimeout(timer);
|
|
335
|
+
signal?.removeEventListener("abort", onAbort);
|
|
336
|
+
};
|
|
337
|
+
const onAbort = () => {
|
|
338
|
+
cleanup2();
|
|
339
|
+
reject(signal?.reason);
|
|
340
|
+
};
|
|
341
|
+
const timer = setTimeout(() => {
|
|
342
|
+
cleanup2();
|
|
343
|
+
reject(new Error(`Search operation exceeded ${timeout}ms waiting budget`));
|
|
344
|
+
}, timeout);
|
|
345
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
346
|
+
Promise.resolve().then(() => {
|
|
347
|
+
signal?.throwIfAborted();
|
|
348
|
+
return operation(...args);
|
|
349
|
+
}).then(
|
|
350
|
+
(value) => {
|
|
351
|
+
cleanup2();
|
|
352
|
+
resolve(value);
|
|
353
|
+
},
|
|
354
|
+
(error) => {
|
|
355
|
+
cleanup2();
|
|
356
|
+
reject(error);
|
|
357
|
+
}
|
|
358
|
+
);
|
|
359
|
+
});
|
|
360
|
+
}
|
|
361
|
+
|
|
322
362
|
// src/search/reranker.ts
|
|
323
363
|
async function applyReranker(reranker, query, results) {
|
|
324
364
|
if (!reranker || results.length === 0) {
|
|
@@ -465,16 +505,21 @@ function deriveLexicalKeywords(query) {
|
|
|
465
505
|
}
|
|
466
506
|
var DEFAULT_LIMIT = 10;
|
|
467
507
|
var DEFAULT_THRESHOLD = 0.7;
|
|
468
|
-
var DEFAULT_SOURCES = ["memory", "insights", "knowledge"];
|
|
508
|
+
var DEFAULT_SOURCES = ["memory", "insights", "knowledge", "workspace"];
|
|
469
509
|
var SOURCE_SET = new Set(DEFAULT_SOURCES);
|
|
470
510
|
function normalizeUnifiedMemorySearchSources(sources) {
|
|
471
|
-
if (
|
|
511
|
+
if (sources === void 0) {
|
|
472
512
|
return [...DEFAULT_SOURCES];
|
|
473
513
|
}
|
|
514
|
+
if (!Array.isArray(sources) || sources.length === 0) {
|
|
515
|
+
throw new TypeError("Search sources must be a non-empty array when specified");
|
|
516
|
+
}
|
|
474
517
|
const normalized = sources.filter((source) => typeof source === "string").map((source) => source.trim().toLowerCase()).filter(
|
|
475
518
|
(source) => SOURCE_SET.has(source)
|
|
476
519
|
);
|
|
477
|
-
|
|
520
|
+
if (normalized.length !== sources.length)
|
|
521
|
+
throw new TypeError("Search sources contain an unsupported source");
|
|
522
|
+
return Array.from(new Set(normalized));
|
|
478
523
|
}
|
|
479
524
|
function clampUnifiedMemorySearchLimit(limit) {
|
|
480
525
|
const parsed = typeof limit === "number" ? limit : Number(limit ?? DEFAULT_LIMIT);
|
|
@@ -516,6 +561,20 @@ function normalizeUnifiedMemoryReasoningStrategy(value, defaultStrategy = "none"
|
|
|
516
561
|
return defaultStrategy;
|
|
517
562
|
}
|
|
518
563
|
var DEFAULT_RRF_K = 60;
|
|
564
|
+
function mergeSearchResultProvenance(preferred, other) {
|
|
565
|
+
const first = preferred.provenance;
|
|
566
|
+
const second = other.provenance;
|
|
567
|
+
const firstVersion = first?.version ?? first?.citation?.version_id;
|
|
568
|
+
const secondVersion = second?.version ?? second?.citation?.version_id;
|
|
569
|
+
const firstHash = first?.contentHash ?? first?.citation?.content_hash;
|
|
570
|
+
const secondHash = second?.contentHash ?? second?.citation?.content_hash;
|
|
571
|
+
if (firstVersion != null && secondVersion != null && firstVersion !== secondVersion || firstHash != null && secondHash != null && firstHash !== secondHash || first?.citation && second?.citation && first.citation.id !== second.citation.id || preferred.content !== other.content && !(firstHash && firstHash === secondHash))
|
|
572
|
+
return preferred;
|
|
573
|
+
const metadata = { ...other.metadata, ...preferred.metadata };
|
|
574
|
+
if (!second) return { ...preferred, metadata };
|
|
575
|
+
const knownFirst = Object.fromEntries(Object.entries(first ?? {}).filter(([, value]) => value != null));
|
|
576
|
+
return { ...preferred, metadata, provenance: { ...second, ...knownFirst } };
|
|
577
|
+
}
|
|
519
578
|
function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K, weights = {}) {
|
|
520
579
|
if (!Array.isArray(lists) || lists.length === 0) {
|
|
521
580
|
return [];
|
|
@@ -529,6 +588,7 @@ function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K, wei
|
|
|
529
588
|
const scores = /* @__PURE__ */ new Map();
|
|
530
589
|
const order = [];
|
|
531
590
|
for (const list of lists) {
|
|
591
|
+
const seenInList = /* @__PURE__ */ new Set();
|
|
532
592
|
const channel = listNameToChannel(list.name);
|
|
533
593
|
const configuredWeight = channel ? effectiveWeights[channel] : void 0;
|
|
534
594
|
const weight = typeof configuredWeight === "number" && Number.isFinite(configuredWeight) && configuredWeight >= 0 ? configuredWeight : 1;
|
|
@@ -536,7 +596,8 @@ function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K, wei
|
|
|
536
596
|
for (let index = 0; index < list.hits.length; index += 1) {
|
|
537
597
|
const hit = list.hits[index];
|
|
538
598
|
const key = `${hit.type}::${hit.id}`;
|
|
539
|
-
const contribution = weight / (safeK +
|
|
599
|
+
const contribution = seenInList.has(key) ? 0 : weight / (safeK + seenInList.size + 1);
|
|
600
|
+
seenInList.add(key);
|
|
540
601
|
const existing = scores.get(key);
|
|
541
602
|
const rankedHit = hit.type === "memory" ? withMatchedEvidence(hit, list.name, index + 1) : hit;
|
|
542
603
|
if (existing) {
|
|
@@ -544,6 +605,7 @@ function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K, wei
|
|
|
544
605
|
if (hit.type === "memory" && existing.hit.type === "memory") {
|
|
545
606
|
existing.hit = mergeMatchedEvidence(existing.hit, rankedHit);
|
|
546
607
|
}
|
|
608
|
+
existing.hit = mergeSearchResultProvenance(existing.hit, rankedHit);
|
|
547
609
|
} else {
|
|
548
610
|
scores.set(key, { hit: rankedHit, rrf: contribution });
|
|
549
611
|
order.push(key);
|
|
@@ -750,14 +812,15 @@ function buildSynthesisPrompt(input) {
|
|
|
750
812
|
grouped.set(item.source, bucket);
|
|
751
813
|
}
|
|
752
814
|
const sections = [];
|
|
753
|
-
for (const tier of DEFAULT_TIERS) {
|
|
815
|
+
for (const tier of [...DEFAULT_TIERS, "workspace"]) {
|
|
754
816
|
const items = grouped.get(tier) ?? [];
|
|
755
817
|
if (items.length === 0) {
|
|
756
818
|
continue;
|
|
757
819
|
}
|
|
758
820
|
const lines = items.map((item, index) => {
|
|
759
821
|
const factSource = typeof item.metadata?.source === "string" && item.metadata.source.length > 0 ? item.metadata.source : item.source ?? "unknown";
|
|
760
|
-
|
|
822
|
+
const provenance = item.provenance ? ` provenance=${JSON.stringify(item.provenance)}` : "";
|
|
823
|
+
return ` [${index + 1}] (${item.id}, source=${factSource}, score=${item.score.toFixed(4)}) ${item.snippet}${provenance}`;
|
|
761
824
|
});
|
|
762
825
|
sections.push(`## ${tier}
|
|
763
826
|
${lines.join("\n")}`);
|
|
@@ -848,7 +911,8 @@ async function synthesizeAnswer(input) {
|
|
|
848
911
|
score: item.score,
|
|
849
912
|
// Forward per-fact provenance (e.g. `metadata.source` from
|
|
850
913
|
// `opencontext add --source`) so the synthesis prompt can cite it.
|
|
851
|
-
metadata: item.metadata
|
|
914
|
+
metadata: item.metadata,
|
|
915
|
+
provenance: item.provenance
|
|
852
916
|
}));
|
|
853
917
|
const warnings = [];
|
|
854
918
|
const complete = deps.reasoning?.complete;
|
|
@@ -1069,7 +1133,7 @@ async function embedQueryVariant(embedQuery, input, query) {
|
|
|
1069
1133
|
authToken: input.authToken
|
|
1070
1134
|
});
|
|
1071
1135
|
}
|
|
1072
|
-
async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit, threshold, logger, peerPeers, warnings, runtimeContext) {
|
|
1136
|
+
async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit, threshold, logger, peerPeers, warnings, runtimeContext, execution) {
|
|
1073
1137
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
1074
1138
|
let semantic = [];
|
|
1075
1139
|
if (typeof deps.searchRawMessagesAnn === "function") {
|
|
@@ -1097,7 +1161,10 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
1097
1161
|
semantic = (await Promise.all(
|
|
1098
1162
|
filters.map((filter) => {
|
|
1099
1163
|
const botId = "botId" in filter ? filter.botId : void 0;
|
|
1100
|
-
return
|
|
1164
|
+
return boundedSearchOperation(
|
|
1165
|
+
searchRawMessagesWithChroma,
|
|
1166
|
+
deps
|
|
1167
|
+
)({
|
|
1101
1168
|
userId: input.userId,
|
|
1102
1169
|
queryEmbedding,
|
|
1103
1170
|
limit,
|
|
@@ -1106,19 +1173,21 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
1106
1173
|
});
|
|
1107
1174
|
})
|
|
1108
1175
|
)).flat().map(toMemoryResult);
|
|
1176
|
+
if (execution) execution.completed++;
|
|
1109
1177
|
logger.log?.("[memory-store] Raw message semantic search completed", {
|
|
1110
1178
|
backend: "chroma",
|
|
1111
1179
|
dimensions: queryEmbedding.length,
|
|
1112
1180
|
count: semantic.length
|
|
1113
1181
|
});
|
|
1114
1182
|
} catch (error) {
|
|
1183
|
+
if (execution) execution.failed++;
|
|
1115
1184
|
logger.warn?.(
|
|
1116
1185
|
"[memory-store] Chroma raw message search failed; falling back to database search:",
|
|
1117
1186
|
error
|
|
1118
1187
|
);
|
|
1119
1188
|
}
|
|
1120
1189
|
}
|
|
1121
|
-
if (semantic.length === 0 && runtimeContext !== void 0) {
|
|
1190
|
+
if (semantic.length === 0 && runtimeContext !== void 0 && !deps.searchRawMessagesAnn) {
|
|
1122
1191
|
warnApplicabilityNotEnforced(warnings);
|
|
1123
1192
|
return [];
|
|
1124
1193
|
}
|
|
@@ -1128,7 +1197,10 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
1128
1197
|
const manager2 = await getRawMessageManager2();
|
|
1129
1198
|
if (typeof manager2.searchMessagesSemantically === "function") {
|
|
1130
1199
|
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
1131
|
-
const results = await
|
|
1200
|
+
const results = await boundedSearchOperation(
|
|
1201
|
+
manager2.searchMessagesSemantically.bind(manager2),
|
|
1202
|
+
deps
|
|
1203
|
+
)({
|
|
1132
1204
|
userId: input.userId,
|
|
1133
1205
|
queryEmbedding,
|
|
1134
1206
|
limit,
|
|
@@ -1138,6 +1210,7 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
1138
1210
|
...peerPeers.length > 0 ? { peers: peerPeers } : {},
|
|
1139
1211
|
...factTypes ? { factTypes } : {}
|
|
1140
1212
|
});
|
|
1213
|
+
if (execution) execution.completed++;
|
|
1141
1214
|
semantic = results.map((r) => ({
|
|
1142
1215
|
type: "memory",
|
|
1143
1216
|
id: r.id,
|
|
@@ -1147,6 +1220,7 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
1147
1220
|
}));
|
|
1148
1221
|
}
|
|
1149
1222
|
} catch (error) {
|
|
1223
|
+
if (execution) execution.failed++;
|
|
1150
1224
|
logger.warn?.("[memory-store] SQLite semantic search failed:", error);
|
|
1151
1225
|
}
|
|
1152
1226
|
}
|
|
@@ -1197,7 +1271,7 @@ function toIterativeRecallCandidate(hit) {
|
|
|
1197
1271
|
metadata: hit.metadata
|
|
1198
1272
|
};
|
|
1199
1273
|
}
|
|
1200
|
-
async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger, peerPeers, warnings, runtimeContext) {
|
|
1274
|
+
async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger, peerPeers, warnings, runtimeContext, execution) {
|
|
1201
1275
|
if (keywords.length === 0) {
|
|
1202
1276
|
return [];
|
|
1203
1277
|
}
|
|
@@ -1247,9 +1321,12 @@ async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger,
|
|
|
1247
1321
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
1248
1322
|
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
1249
1323
|
const includeDeprecated = input.includeDeprecated === true;
|
|
1250
|
-
|
|
1324
|
+
const results = (await Promise.all(
|
|
1251
1325
|
filters.map(
|
|
1252
|
-
(botId) =>
|
|
1326
|
+
(botId) => boundedSearchOperation(
|
|
1327
|
+
lexicalSearchRawMessages2,
|
|
1328
|
+
deps
|
|
1329
|
+
)({
|
|
1253
1330
|
userId: input.userId,
|
|
1254
1331
|
keywords,
|
|
1255
1332
|
limit: Math.ceil(limit / filters.length),
|
|
@@ -1264,7 +1341,10 @@ async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger,
|
|
|
1264
1341
|
r
|
|
1265
1342
|
)
|
|
1266
1343
|
);
|
|
1344
|
+
if (execution) execution.completed++;
|
|
1345
|
+
return results;
|
|
1267
1346
|
} catch (error) {
|
|
1347
|
+
if (execution) execution.failed++;
|
|
1268
1348
|
logger.warn?.("[memory-store] SQLite lexical search failed:", error);
|
|
1269
1349
|
}
|
|
1270
1350
|
return [];
|
|
@@ -1311,7 +1391,21 @@ function mapHitToEvidenceSource(hit) {
|
|
|
1311
1391
|
}
|
|
1312
1392
|
return hit.type;
|
|
1313
1393
|
}
|
|
1314
|
-
function createUnifiedSearch(
|
|
1394
|
+
function createUnifiedSearch(inputDeps = {}) {
|
|
1395
|
+
const deps = {
|
|
1396
|
+
...inputDeps,
|
|
1397
|
+
embedQuery: inputDeps.embedQuery && boundedSearchOperation(inputDeps.embedQuery, inputDeps),
|
|
1398
|
+
searchRawMessagesAnn: inputDeps.searchRawMessagesAnn && boundedSearchOperation(inputDeps.searchRawMessagesAnn, inputDeps),
|
|
1399
|
+
searchRawMessagesHybrid: inputDeps.searchRawMessagesHybrid && boundedSearchOperation(inputDeps.searchRawMessagesHybrid, inputDeps),
|
|
1400
|
+
searchRawMessagesLexical: inputDeps.searchRawMessagesLexical && boundedSearchOperation(inputDeps.searchRawMessagesLexical, inputDeps),
|
|
1401
|
+
searchWorkspace: inputDeps.searchWorkspace && boundedSearchOperation(inputDeps.searchWorkspace, inputDeps),
|
|
1402
|
+
searchInsights: inputDeps.searchInsights && boundedSearchOperation(inputDeps.searchInsights, inputDeps),
|
|
1403
|
+
searchKnowledge: inputDeps.searchKnowledge && boundedSearchOperation(inputDeps.searchKnowledge, inputDeps),
|
|
1404
|
+
searchSummaries: inputDeps.searchSummaries && boundedSearchOperation(inputDeps.searchSummaries, inputDeps),
|
|
1405
|
+
getRawMessageRetrievalStatus: inputDeps.getRawMessageRetrievalStatus && boundedSearchOperation(inputDeps.getRawMessageRetrievalStatus, inputDeps),
|
|
1406
|
+
entitySearch: inputDeps.entitySearch && boundedSearchOperation(inputDeps.entitySearch, inputDeps),
|
|
1407
|
+
searchRawMessageNeighbors: inputDeps.searchRawMessageNeighbors && boundedSearchOperation(inputDeps.searchRawMessageNeighbors, inputDeps)
|
|
1408
|
+
};
|
|
1315
1409
|
const logger = deps.logger ?? console;
|
|
1316
1410
|
function resolveReasoningStrategy(input) {
|
|
1317
1411
|
const requested = normalizeUnifiedMemoryReasoningStrategy(input.reasoningStrategy);
|
|
@@ -1321,6 +1415,25 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1321
1415
|
return normalizeUnifiedMemoryReasoningStrategy(deps.reasoning?.defaultStrategy, "none");
|
|
1322
1416
|
}
|
|
1323
1417
|
async function runMemorySource(input, limit, threshold, warnings, reasoningStrategy = "none", reasoningInfo, peerPeers = [], runtimeContext) {
|
|
1418
|
+
const execution = { completed: 0, failed: 0 };
|
|
1419
|
+
const observe = (operation, preparation = false) => operation && (async (...args) => {
|
|
1420
|
+
try {
|
|
1421
|
+
const result = await operation(...args);
|
|
1422
|
+
if (!preparation) execution.completed++;
|
|
1423
|
+
return result;
|
|
1424
|
+
} catch (error) {
|
|
1425
|
+
execution.failed++;
|
|
1426
|
+
throw error;
|
|
1427
|
+
}
|
|
1428
|
+
});
|
|
1429
|
+
const sourceDeps = {
|
|
1430
|
+
...deps,
|
|
1431
|
+
embedQuery: observe(deps.embedQuery, true),
|
|
1432
|
+
searchRawMessagesAnn: observe(deps.searchRawMessagesAnn),
|
|
1433
|
+
searchRawMessagesHybrid: observe(deps.searchRawMessagesHybrid),
|
|
1434
|
+
searchRawMessagesLexical: observe(deps.searchRawMessagesLexical),
|
|
1435
|
+
entitySearch: observe(deps.entitySearch)
|
|
1436
|
+
};
|
|
1324
1437
|
const sourceStartedAt = monotonicNow();
|
|
1325
1438
|
const timings = {
|
|
1326
1439
|
semanticMs: 0,
|
|
@@ -1338,9 +1451,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1338
1451
|
};
|
|
1339
1452
|
const finish = (results) => ({
|
|
1340
1453
|
results,
|
|
1341
|
-
timings: { ...timings, memorySourceMs: monotonicNow() - sourceStartedAt }
|
|
1454
|
+
timings: { ...timings, memorySourceMs: monotonicNow() - sourceStartedAt },
|
|
1455
|
+
availability: execution.failed > 0 ? execution.completed > 0 ? "partial" : "failed" : execution.completed > 0 || Object.values(results).some((hits) => hits.length > 0) ? "complete" : "not_ready"
|
|
1342
1456
|
});
|
|
1343
|
-
if ((reasoningStrategy === "iterative" || reasoningStrategy === "union") && !
|
|
1457
|
+
if ((reasoningStrategy === "iterative" || reasoningStrategy === "union") && !sourceDeps.reasoning?.iterativePlanner) {
|
|
1344
1458
|
warnings.push({
|
|
1345
1459
|
source: "memory",
|
|
1346
1460
|
code: "memory_iterative_planner_not_configured",
|
|
@@ -1348,7 +1462,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1348
1462
|
});
|
|
1349
1463
|
if (reasoningInfo) reasoningInfo.degraded = true;
|
|
1350
1464
|
}
|
|
1351
|
-
if (reasoningStrategy === "rewrite" && !
|
|
1465
|
+
if (reasoningStrategy === "rewrite" && !sourceDeps.reasoning?.queryRewriter) {
|
|
1352
1466
|
warnings.push({
|
|
1353
1467
|
source: "memory",
|
|
1354
1468
|
code: "memory_query_rewrite_not_configured",
|
|
@@ -1357,8 +1471,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1357
1471
|
if (reasoningInfo) reasoningInfo.degraded = true;
|
|
1358
1472
|
}
|
|
1359
1473
|
let unionEvidence = null;
|
|
1360
|
-
if ((reasoningStrategy === "iterative" || reasoningStrategy === "union") &&
|
|
1361
|
-
const planner =
|
|
1474
|
+
if ((reasoningStrategy === "iterative" || reasoningStrategy === "union") && sourceDeps.reasoning?.iterativePlanner) {
|
|
1475
|
+
const planner = sourceDeps.reasoning.iterativePlanner;
|
|
1362
1476
|
const result = await planner.plan({
|
|
1363
1477
|
query: input.query,
|
|
1364
1478
|
...input.includeRetrievalDiagnostics ? { options: { collectDiagnostics: true } } : {},
|
|
@@ -1370,26 +1484,27 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1370
1484
|
const lexicalHits = await measure(
|
|
1371
1485
|
"lexicalMs",
|
|
1372
1486
|
() => runLexicalSearchForKeywords(
|
|
1373
|
-
|
|
1487
|
+
sourceDeps,
|
|
1374
1488
|
input,
|
|
1375
1489
|
keywords2,
|
|
1376
1490
|
limit,
|
|
1377
1491
|
logger,
|
|
1378
1492
|
peerPeers,
|
|
1379
1493
|
warnings,
|
|
1380
|
-
runtimeContext
|
|
1494
|
+
runtimeContext,
|
|
1495
|
+
execution
|
|
1381
1496
|
)
|
|
1382
1497
|
);
|
|
1383
1498
|
let semanticHits = [];
|
|
1384
|
-
if (typeof
|
|
1499
|
+
if (typeof sourceDeps.embedQuery === "function") {
|
|
1385
1500
|
try {
|
|
1386
1501
|
semanticHits = await measure("semanticMs", async () => {
|
|
1387
1502
|
const semanticQuery = keywords2.join(" ");
|
|
1388
|
-
const queryEmbedder =
|
|
1503
|
+
const queryEmbedder = sourceDeps.embedQuery;
|
|
1389
1504
|
if (typeof queryEmbedder !== "function") return [];
|
|
1390
1505
|
const queryEmbedding = await embedQueryVariant(queryEmbedder, input, semanticQuery);
|
|
1391
1506
|
return runSemanticSearchForEmbedding(
|
|
1392
|
-
|
|
1507
|
+
sourceDeps,
|
|
1393
1508
|
input,
|
|
1394
1509
|
queryEmbedding,
|
|
1395
1510
|
limit,
|
|
@@ -1397,7 +1512,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1397
1512
|
logger,
|
|
1398
1513
|
peerPeers,
|
|
1399
1514
|
warnings,
|
|
1400
|
-
runtimeContext
|
|
1515
|
+
runtimeContext,
|
|
1516
|
+
execution
|
|
1401
1517
|
);
|
|
1402
1518
|
});
|
|
1403
1519
|
} catch (error) {
|
|
@@ -1450,66 +1566,74 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1450
1566
|
}
|
|
1451
1567
|
unionEvidence = evidence;
|
|
1452
1568
|
}
|
|
1453
|
-
|
|
1569
|
+
const lexicalFallback = async () => {
|
|
1570
|
+
const lexical2 = await runLexicalSearchForKeywords(
|
|
1571
|
+
sourceDeps,
|
|
1572
|
+
input,
|
|
1573
|
+
deriveLexicalKeywords(input.query),
|
|
1574
|
+
limit,
|
|
1575
|
+
logger,
|
|
1576
|
+
peerPeers,
|
|
1577
|
+
warnings,
|
|
1578
|
+
runtimeContext,
|
|
1579
|
+
execution
|
|
1580
|
+
);
|
|
1581
|
+
return finish({ semantic: [], lexical: filterByDateRange(lexical2, input.dateFrom, input.dateTo) });
|
|
1582
|
+
};
|
|
1583
|
+
if (typeof sourceDeps.embedQuery !== "function") {
|
|
1454
1584
|
warnings.push({
|
|
1455
1585
|
source: "memory",
|
|
1456
1586
|
code: "memory_lexical_search_fallback",
|
|
1457
1587
|
message: "No embedding provider is configured; using lexical (BM25) search as fallback. Pass an `embedQuery` function via `unified.embedQuery` to enable semantic retrieval."
|
|
1458
1588
|
});
|
|
1459
|
-
|
|
1460
|
-
const lexical2 = filterByDateRange(
|
|
1461
|
-
await runLexicalSearchForKeywords(
|
|
1462
|
-
deps,
|
|
1463
|
-
input,
|
|
1464
|
-
keywords2,
|
|
1465
|
-
limit,
|
|
1466
|
-
logger,
|
|
1467
|
-
peerPeers,
|
|
1468
|
-
warnings,
|
|
1469
|
-
runtimeContext
|
|
1470
|
-
),
|
|
1471
|
-
input.dateFrom,
|
|
1472
|
-
input.dateTo
|
|
1473
|
-
);
|
|
1474
|
-
return finish({ semantic: [], lexical: lexical2 });
|
|
1589
|
+
return lexicalFallback();
|
|
1475
1590
|
}
|
|
1476
1591
|
let semantic = [];
|
|
1477
1592
|
let lexicalQueries = [input.query];
|
|
1478
|
-
const embedQuery =
|
|
1593
|
+
const embedQuery = sourceDeps.embedQuery;
|
|
1479
1594
|
const activeMergeStrategy = normalizeUnifiedMemoryMergeStrategy(
|
|
1480
|
-
input.mergeStrategy ??
|
|
1595
|
+
input.mergeStrategy ?? sourceDeps.reasoning?.defaultMergeStrategy ?? "rrf"
|
|
1481
1596
|
);
|
|
1482
|
-
if (reasoningStrategy === "none" && activeMergeStrategy === "rrf" && input.threshold === void 0 && typeof
|
|
1483
|
-
|
|
1484
|
-
const
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1597
|
+
if (reasoningStrategy === "none" && activeMergeStrategy === "rrf" && input.threshold === void 0 && typeof sourceDeps.searchRawMessagesHybrid === "function") {
|
|
1598
|
+
try {
|
|
1599
|
+
const hybrid = await measure("hybridMs", async () => {
|
|
1600
|
+
const queryEmbedding = await embedQueryVariant(embedQuery, input, input.query);
|
|
1601
|
+
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
1602
|
+
return (await Promise.all(
|
|
1603
|
+
filters.map(
|
|
1604
|
+
(botId) => sourceDeps.searchRawMessagesHybrid?.({
|
|
1605
|
+
userId: input.userId,
|
|
1606
|
+
query: input.query,
|
|
1607
|
+
queryEmbedding,
|
|
1608
|
+
limit: Math.ceil(limit / filters.length),
|
|
1609
|
+
botId,
|
|
1610
|
+
...runtimeContext ?? {}
|
|
1611
|
+
})
|
|
1612
|
+
)
|
|
1613
|
+
)).flatMap((items) => items ?? []).filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
1614
|
+
});
|
|
1615
|
+
return finish({
|
|
1616
|
+
semantic: [],
|
|
1617
|
+
lexical: [],
|
|
1618
|
+
hybrid: filterByDateRange(hybrid, input.dateFrom, input.dateTo)
|
|
1619
|
+
});
|
|
1620
|
+
} catch (error) {
|
|
1621
|
+
warnings.push({
|
|
1622
|
+
source: "memory",
|
|
1623
|
+
code: "memory_hybrid_search_failed",
|
|
1624
|
+
message: error instanceof Error ? error.message : String(error)
|
|
1625
|
+
});
|
|
1626
|
+
return lexicalFallback();
|
|
1627
|
+
}
|
|
1504
1628
|
}
|
|
1505
|
-
if (reasoningStrategy === "rewrite" &&
|
|
1629
|
+
if (reasoningStrategy === "rewrite" && sourceDeps.reasoning?.queryRewriter) {
|
|
1506
1630
|
try {
|
|
1507
|
-
const variants = await
|
|
1631
|
+
const variants = await sourceDeps.reasoning.queryRewriter.rewrite({
|
|
1508
1632
|
query: input.query,
|
|
1509
1633
|
userId: input.userId,
|
|
1510
1634
|
authToken: input.authToken
|
|
1511
1635
|
});
|
|
1512
|
-
if (
|
|
1636
|
+
if (sourceDeps.reasoning.rewriteLexical === true) {
|
|
1513
1637
|
const seen = /* @__PURE__ */ new Set();
|
|
1514
1638
|
lexicalQueries = [input.query, ...variants].map((query) => query.trim()).filter((query) => {
|
|
1515
1639
|
const key = query.toLowerCase();
|
|
@@ -1520,7 +1644,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1520
1644
|
}
|
|
1521
1645
|
if (reasoningInfo) {
|
|
1522
1646
|
reasoningInfo.rewrittenQueries = variants;
|
|
1523
|
-
if (
|
|
1647
|
+
if (sourceDeps.reasoning.queryRewriter.lastDegraded?.()) {
|
|
1524
1648
|
reasoningInfo.degraded = true;
|
|
1525
1649
|
}
|
|
1526
1650
|
}
|
|
@@ -1531,7 +1655,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1531
1655
|
return Promise.all(
|
|
1532
1656
|
embeddings.map(
|
|
1533
1657
|
(embedding) => runSemanticSearchForEmbedding(
|
|
1534
|
-
|
|
1658
|
+
sourceDeps,
|
|
1535
1659
|
input,
|
|
1536
1660
|
embedding,
|
|
1537
1661
|
limit,
|
|
@@ -1539,12 +1663,13 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1539
1663
|
logger,
|
|
1540
1664
|
peerPeers,
|
|
1541
1665
|
warnings,
|
|
1542
|
-
runtimeContext
|
|
1666
|
+
runtimeContext,
|
|
1667
|
+
execution
|
|
1543
1668
|
)
|
|
1544
1669
|
)
|
|
1545
1670
|
);
|
|
1546
1671
|
});
|
|
1547
|
-
if (activeMergeStrategy === "rrf" &&
|
|
1672
|
+
if (activeMergeStrategy === "rrf" && sourceDeps.reasoning.rewriteSemanticMerge === "rrf" && lists.length > 1 && lists.some((hits) => hits.length > 0)) {
|
|
1548
1673
|
semantic = mergeUnifiedMemorySearchResultsRrf(
|
|
1549
1674
|
lists.map((hits, index) => ({
|
|
1550
1675
|
name: `memory-semantic-query:${index}`,
|
|
@@ -1567,24 +1692,33 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1567
1692
|
}
|
|
1568
1693
|
}
|
|
1569
1694
|
if (semantic.length === 0) {
|
|
1570
|
-
|
|
1571
|
-
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1695
|
+
try {
|
|
1696
|
+
semantic = await measure("semanticMs", async () => {
|
|
1697
|
+
const queryEmbedding = await embedQueryVariant(embedQuery, input, input.query);
|
|
1698
|
+
return runSemanticSearchForEmbedding(
|
|
1699
|
+
sourceDeps,
|
|
1700
|
+
input,
|
|
1701
|
+
queryEmbedding,
|
|
1702
|
+
limit,
|
|
1703
|
+
threshold,
|
|
1704
|
+
logger,
|
|
1705
|
+
peerPeers,
|
|
1706
|
+
warnings,
|
|
1707
|
+
runtimeContext,
|
|
1708
|
+
execution
|
|
1709
|
+
);
|
|
1710
|
+
});
|
|
1711
|
+
} catch (error) {
|
|
1712
|
+
warnings.push({
|
|
1713
|
+
source: "memory",
|
|
1714
|
+
code: "memory_semantic_search_failed",
|
|
1715
|
+
message: error instanceof Error ? error.message : String(error)
|
|
1716
|
+
});
|
|
1717
|
+
}
|
|
1584
1718
|
}
|
|
1585
1719
|
let lexical = [];
|
|
1586
1720
|
const keywords = deriveLexicalKeywords(input.query);
|
|
1587
|
-
if (activeMergeStrategy === "rrf" && lexicalQueries.length > 1 && typeof
|
|
1721
|
+
if (activeMergeStrategy === "rrf" && lexicalQueries.length > 1 && typeof sourceDeps.searchRawMessagesLexical === "function") {
|
|
1588
1722
|
const lists = await Promise.all(
|
|
1589
1723
|
lexicalQueries.map(async (query) => ({
|
|
1590
1724
|
name: "memory-bm25",
|
|
@@ -1592,14 +1726,15 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1592
1726
|
await measure(
|
|
1593
1727
|
"lexicalMs",
|
|
1594
1728
|
() => runLexicalSearchForKeywords(
|
|
1595
|
-
|
|
1729
|
+
sourceDeps,
|
|
1596
1730
|
input,
|
|
1597
1731
|
deriveLexicalKeywords(query),
|
|
1598
1732
|
limit,
|
|
1599
1733
|
logger,
|
|
1600
1734
|
peerPeers,
|
|
1601
1735
|
warnings,
|
|
1602
|
-
runtimeContext
|
|
1736
|
+
runtimeContext,
|
|
1737
|
+
execution
|
|
1603
1738
|
)
|
|
1604
1739
|
),
|
|
1605
1740
|
"memory-bm25"
|
|
@@ -1612,11 +1747,11 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1612
1747
|
}));
|
|
1613
1748
|
if (reasoningInfo) reasoningInfo.lexicalRewrittenQueries = lexicalQueries;
|
|
1614
1749
|
} else if (keywords.length > 0) {
|
|
1615
|
-
if (typeof
|
|
1750
|
+
if (typeof sourceDeps.searchRawMessagesLexical === "function") {
|
|
1616
1751
|
const lexicalStartedAt = monotonicNow();
|
|
1617
1752
|
try {
|
|
1618
1753
|
const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
1619
|
-
const searchRawMessagesLexical =
|
|
1754
|
+
const searchRawMessagesLexical = sourceDeps.searchRawMessagesLexical;
|
|
1620
1755
|
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
1621
1756
|
const includeDeprecated = input.includeDeprecated === true;
|
|
1622
1757
|
lexical = (await Promise.all(
|
|
@@ -1656,16 +1791,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1656
1791
|
}
|
|
1657
1792
|
}
|
|
1658
1793
|
let entity;
|
|
1659
|
-
if (typeof
|
|
1794
|
+
if (typeof sourceDeps.entitySearch === "function") {
|
|
1660
1795
|
entity = filterByDateRange(
|
|
1661
|
-
await runEntitySearchForQuery(
|
|
1796
|
+
await runEntitySearchForQuery(sourceDeps, input, limit, logger, peerPeers, runtimeContext),
|
|
1662
1797
|
input.dateFrom,
|
|
1663
1798
|
input.dateTo
|
|
1664
1799
|
);
|
|
1665
1800
|
if (entity.length === 0) {
|
|
1666
1801
|
entity = void 0;
|
|
1667
1802
|
} else if (normalizeUnifiedMemoryMergeStrategy(
|
|
1668
|
-
input.mergeStrategy ??
|
|
1803
|
+
input.mergeStrategy ?? sourceDeps.reasoning?.defaultMergeStrategy ?? "rrf"
|
|
1669
1804
|
) !== "rrf") {
|
|
1670
1805
|
warnings.push({
|
|
1671
1806
|
source: "memory",
|
|
@@ -1696,6 +1831,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1696
1831
|
const searchStartedAt = monotonicNow();
|
|
1697
1832
|
const query = input.query.trim();
|
|
1698
1833
|
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
1834
|
+
const sourceStates = [];
|
|
1699
1835
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
1700
1836
|
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(
|
|
1701
1837
|
input.mergeStrategy ?? deps.reasoning?.defaultMergeStrategy ?? "rrf"
|
|
@@ -1721,7 +1857,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1721
1857
|
sources,
|
|
1722
1858
|
results: [],
|
|
1723
1859
|
count: 0,
|
|
1724
|
-
warnings
|
|
1860
|
+
warnings,
|
|
1861
|
+
sourceStates
|
|
1725
1862
|
};
|
|
1726
1863
|
}
|
|
1727
1864
|
let memorySubs;
|
|
@@ -1732,6 +1869,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1732
1869
|
reasoningInfo.dateRange = { from: input.dateFrom, to: input.dateTo };
|
|
1733
1870
|
}
|
|
1734
1871
|
if (sources.includes("memory")) {
|
|
1872
|
+
let availability = "not_ready";
|
|
1735
1873
|
const hasRawProviders = typeof deps.searchRawMessagesAnn === "function" || typeof deps.searchRawMessagesHybrid === "function" || typeof deps.searchRawMessagesLexical === "function";
|
|
1736
1874
|
if (isRawMessageStorageAvailable() || hasRawProviders) {
|
|
1737
1875
|
try {
|
|
@@ -1746,6 +1884,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1746
1884
|
runtimeContext
|
|
1747
1885
|
);
|
|
1748
1886
|
memorySubs = memoryRun.results;
|
|
1887
|
+
availability = memoryRun.availability;
|
|
1749
1888
|
memoryTimings = memoryRun.timings;
|
|
1750
1889
|
memorySubs = {
|
|
1751
1890
|
semantic: dedupeChannelByParent(memorySubs.semantic, "memory-semantic"),
|
|
@@ -1770,11 +1909,12 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1770
1909
|
warnings.push({ source: "memory", code, message });
|
|
1771
1910
|
}
|
|
1772
1911
|
} catch (error) {
|
|
1912
|
+
availability = availability === "complete" || availability === "partial" ? "partial" : "failed";
|
|
1773
1913
|
logger.warn?.("[memory-store] memory source failed:", error);
|
|
1774
1914
|
warnings.push({
|
|
1775
1915
|
source: "memory",
|
|
1776
1916
|
code: "memory_search_failed",
|
|
1777
|
-
message: `Memory search failed: ${error.message ?? "Unknown error"}
|
|
1917
|
+
message: `Memory search failed: ${error.message ?? "Unknown error"}.`
|
|
1778
1918
|
});
|
|
1779
1919
|
}
|
|
1780
1920
|
} else {
|
|
@@ -1784,6 +1924,51 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1784
1924
|
message: "Raw memory storage is not available in this environment."
|
|
1785
1925
|
});
|
|
1786
1926
|
}
|
|
1927
|
+
const count = new Set(
|
|
1928
|
+
[
|
|
1929
|
+
...memorySubs?.semantic ?? [],
|
|
1930
|
+
...memorySubs?.lexical ?? [],
|
|
1931
|
+
...memorySubs?.hybrid ?? [],
|
|
1932
|
+
...memorySubs?.planner ?? [],
|
|
1933
|
+
...memorySubs?.entity ?? []
|
|
1934
|
+
].map((hit) => hit.id)
|
|
1935
|
+
).size;
|
|
1936
|
+
if (availability === "complete" && count === 0 && retrievalStatus?.lexicalReady === false && (!retrievalStatus.semanticReady || !deps.embedQuery)) {
|
|
1937
|
+
availability = "not_ready";
|
|
1938
|
+
}
|
|
1939
|
+
const memoryWarnings = warnings.filter((warning) => warning.source === "memory");
|
|
1940
|
+
sourceStates.push({
|
|
1941
|
+
source: "memory",
|
|
1942
|
+
status: availability,
|
|
1943
|
+
count,
|
|
1944
|
+
...memoryWarnings.length ? { message: memoryWarnings.map((warning) => warning.message).join("; ") } : {}
|
|
1945
|
+
});
|
|
1946
|
+
}
|
|
1947
|
+
const workspaceHits = [];
|
|
1948
|
+
if (sources.includes("workspace")) {
|
|
1949
|
+
if (!deps.searchWorkspace) {
|
|
1950
|
+
sourceStates.push({ source: "workspace", status: "not_configured", count: 0 });
|
|
1951
|
+
} else {
|
|
1952
|
+
try {
|
|
1953
|
+
const result = await deps.searchWorkspace({
|
|
1954
|
+
userId: input.userId,
|
|
1955
|
+
query,
|
|
1956
|
+
limit: candidateLimit,
|
|
1957
|
+
...runtimeContext ?? {}
|
|
1958
|
+
});
|
|
1959
|
+
workspaceHits.push(...result.hits.map((hit) => ({ ...hit, type: "workspace" })));
|
|
1960
|
+
sourceStates.push({
|
|
1961
|
+
source: "workspace",
|
|
1962
|
+
status: result.status,
|
|
1963
|
+
count: result.hits.length,
|
|
1964
|
+
...result.message ? { message: result.message } : {}
|
|
1965
|
+
});
|
|
1966
|
+
} catch (error) {
|
|
1967
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1968
|
+
sourceStates.push({ source: "workspace", status: "failed", count: 0, message });
|
|
1969
|
+
warnings.push({ source: "workspace", code: "workspace_search_failed", message });
|
|
1970
|
+
}
|
|
1971
|
+
}
|
|
1787
1972
|
}
|
|
1788
1973
|
const insightHits = [];
|
|
1789
1974
|
if (sources.includes("insights") && typeof deps.searchInsights === "function") {
|
|
@@ -1805,10 +1990,18 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1805
1990
|
id: hit.id,
|
|
1806
1991
|
content: hit.content,
|
|
1807
1992
|
similarity: hit.similarity,
|
|
1808
|
-
metadata: hit.metadata
|
|
1993
|
+
metadata: hit.metadata,
|
|
1994
|
+
provenance: hit.provenance
|
|
1809
1995
|
});
|
|
1810
1996
|
}
|
|
1997
|
+
sourceStates.push({ source: "insights", status: "complete", count: insightHits.length });
|
|
1811
1998
|
} catch (error) {
|
|
1999
|
+
sourceStates.push({
|
|
2000
|
+
source: "insights",
|
|
2001
|
+
status: "failed",
|
|
2002
|
+
count: 0,
|
|
2003
|
+
message: error instanceof Error ? error.message : String(error)
|
|
2004
|
+
});
|
|
1812
2005
|
logger.warn?.("[memory-store] insights source failed:", error);
|
|
1813
2006
|
warnings.push({
|
|
1814
2007
|
source: "insights",
|
|
@@ -1816,7 +2009,9 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1816
2009
|
message: error.message ?? "insights_search_failed"
|
|
1817
2010
|
});
|
|
1818
2011
|
}
|
|
1819
|
-
} else if (sources.includes("insights"))
|
|
2012
|
+
} else if (sources.includes("insights")) {
|
|
2013
|
+
sourceStates.push({ source: "insights", status: "not_configured", count: 0 });
|
|
2014
|
+
}
|
|
1820
2015
|
const knowledgeHits = [];
|
|
1821
2016
|
if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
|
|
1822
2017
|
try {
|
|
@@ -1835,7 +2030,14 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1835
2030
|
for (const hit of hits) {
|
|
1836
2031
|
knowledgeHits.push(toKnowledgeResult(hit));
|
|
1837
2032
|
}
|
|
2033
|
+
sourceStates.push({ source: "knowledge", status: "complete", count: knowledgeHits.length });
|
|
1838
2034
|
} catch (error) {
|
|
2035
|
+
sourceStates.push({
|
|
2036
|
+
source: "knowledge",
|
|
2037
|
+
status: "failed",
|
|
2038
|
+
count: 0,
|
|
2039
|
+
message: error instanceof Error ? error.message : String(error)
|
|
2040
|
+
});
|
|
1839
2041
|
logger.warn?.("[memory-store] knowledge source failed:", error);
|
|
1840
2042
|
warnings.push({
|
|
1841
2043
|
source: "knowledge",
|
|
@@ -1843,12 +2045,15 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1843
2045
|
message: error.message ?? "knowledge_search_failed"
|
|
1844
2046
|
});
|
|
1845
2047
|
}
|
|
1846
|
-
} else if (sources.includes("knowledge"))
|
|
2048
|
+
} else if (sources.includes("knowledge")) {
|
|
2049
|
+
sourceStates.push({ source: "knowledge", status: "not_configured", count: 0 });
|
|
2050
|
+
}
|
|
1847
2051
|
const fusionStartedAt = monotonicNow();
|
|
1848
2052
|
const merged = mergeAcrossSources({
|
|
1849
2053
|
memorySubs,
|
|
1850
2054
|
insightHits,
|
|
1851
2055
|
knowledgeHits,
|
|
2056
|
+
workspaceHits,
|
|
1852
2057
|
limit: candidateLimit,
|
|
1853
2058
|
strategy: mergeStrategy,
|
|
1854
2059
|
rrfK,
|
|
@@ -1875,7 +2080,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1875
2080
|
sources,
|
|
1876
2081
|
results: ranked,
|
|
1877
2082
|
count: ranked.length,
|
|
1878
|
-
warnings
|
|
2083
|
+
warnings,
|
|
2084
|
+
sourceStates
|
|
1879
2085
|
};
|
|
1880
2086
|
if (input.includeRetrievalDiagnostics) {
|
|
1881
2087
|
output.retrievalDiagnostics = {
|
|
@@ -1985,15 +2191,17 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1985
2191
|
evidence: [],
|
|
1986
2192
|
count: 0,
|
|
1987
2193
|
warnings: peerScope.warnings,
|
|
2194
|
+
sourceStates: [],
|
|
1988
2195
|
answer: ""
|
|
1989
2196
|
};
|
|
1990
2197
|
return empty;
|
|
1991
2198
|
}
|
|
1992
|
-
const tiers = input.tiers ? [...input.tiers] : wantsSynthesis ? ["summary", "raw", "insight", "knowledge"] : ["raw", "insight", "knowledge"];
|
|
2199
|
+
const tiers = input.tiers ? [...input.tiers] : wantsSynthesis ? ["summary", "raw", "insight", "knowledge", "workspace"] : ["raw", "insight", "knowledge", "workspace"];
|
|
1993
2200
|
const wantsSummaryTier = tiers.includes("summary");
|
|
1994
2201
|
const wantsRawTier = tiers.includes("raw");
|
|
1995
2202
|
const wantsInsightTier = tiers.includes("insight");
|
|
1996
2203
|
const wantsKnowledgeTier = tiers.includes("knowledge");
|
|
2204
|
+
const wantsWorkspaceTier = tiers.includes("workspace");
|
|
1997
2205
|
const warnings = [...peerScope.warnings];
|
|
1998
2206
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
1999
2207
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
@@ -2001,11 +2209,13 @@ function createUnifiedSearch(deps = {}) {
|
|
|
2001
2209
|
let sources = input.sources ? [...input.sources] : [];
|
|
2002
2210
|
let unifiedReasoning;
|
|
2003
2211
|
let retrievalDiagnostics;
|
|
2004
|
-
|
|
2212
|
+
let sourceStates = [];
|
|
2213
|
+
if (wantsRawTier || wantsInsightTier || wantsKnowledgeTier || wantsWorkspaceTier) {
|
|
2005
2214
|
const inferredSources = sources.length > 0 ? sources : [
|
|
2006
2215
|
...wantsRawTier ? ["memory"] : [],
|
|
2007
2216
|
...wantsInsightTier ? ["insights"] : [],
|
|
2008
|
-
...wantsKnowledgeTier ? ["knowledge"] : []
|
|
2217
|
+
...wantsKnowledgeTier ? ["knowledge"] : [],
|
|
2218
|
+
...wantsWorkspaceTier ? ["workspace"] : []
|
|
2009
2219
|
];
|
|
2010
2220
|
const unified = await searchUnifiedMemoryWithRuntime(
|
|
2011
2221
|
{
|
|
@@ -2018,6 +2228,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
2018
2228
|
sources = unified.sources;
|
|
2019
2229
|
unifiedReasoning = unified.reasoning;
|
|
2020
2230
|
retrievalDiagnostics = unified.retrievalDiagnostics;
|
|
2231
|
+
sourceStates = unified.sourceStates;
|
|
2021
2232
|
warnings.push(...unified.warnings);
|
|
2022
2233
|
}
|
|
2023
2234
|
let summaryHits = [];
|
|
@@ -2056,6 +2267,17 @@ function createUnifiedSearch(deps = {}) {
|
|
|
2056
2267
|
source: mapHitToEvidenceSource(hit),
|
|
2057
2268
|
snippet: hit.content,
|
|
2058
2269
|
score: hit.similarity,
|
|
2270
|
+
provenance: {
|
|
2271
|
+
citation: hit.provenance?.citation ?? null,
|
|
2272
|
+
origin: hit.provenance?.origin ?? null,
|
|
2273
|
+
businessReference: hit.provenance?.businessReference ?? null,
|
|
2274
|
+
version: hit.provenance?.version ?? hit.provenance?.citation?.version_id ?? null,
|
|
2275
|
+
contentHash: hit.provenance?.contentHash ?? hit.provenance?.citation?.content_hash ?? null,
|
|
2276
|
+
scope: hit.provenance?.scope ?? null,
|
|
2277
|
+
derivedFrom: hit.provenance?.derivedFrom ?? null,
|
|
2278
|
+
confirmation: hit.provenance?.confirmation ?? null,
|
|
2279
|
+
freshness: hit.provenance?.freshness ?? null
|
|
2280
|
+
},
|
|
2059
2281
|
timestamp: getCandidateTimestamp(hit.metadata),
|
|
2060
2282
|
metadata: hit.metadata
|
|
2061
2283
|
}));
|
|
@@ -2065,7 +2287,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
2065
2287
|
results: hits,
|
|
2066
2288
|
evidence,
|
|
2067
2289
|
count: hits.length,
|
|
2068
|
-
warnings
|
|
2290
|
+
warnings,
|
|
2291
|
+
sourceStates
|
|
2069
2292
|
};
|
|
2070
2293
|
if (unifiedReasoning) {
|
|
2071
2294
|
base.reasoning = unifiedReasoning;
|
|
@@ -2088,8 +2311,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
2088
2311
|
return base;
|
|
2089
2312
|
}
|
|
2090
2313
|
async function search(input, runtimeContext) {
|
|
2314
|
+
deps.signal?.throwIfAborted();
|
|
2315
|
+
if (input.sources !== void 0) normalizeUnifiedMemorySearchSources(input.sources);
|
|
2091
2316
|
const resolvedRuntime = resolveSearchRuntimeContext(input, runtimeContext);
|
|
2092
|
-
|
|
2317
|
+
const runSearch = boundedSearchOperation(() => searchWithResolvedRuntime(input, resolvedRuntime), {
|
|
2318
|
+
signal: deps.signal,
|
|
2319
|
+
sourceTimeoutMs: deps.searchTimeoutMs ?? 3e4
|
|
2320
|
+
});
|
|
2321
|
+
const result = await runSearch();
|
|
2322
|
+
deps.signal?.throwIfAborted();
|
|
2323
|
+
return result;
|
|
2093
2324
|
}
|
|
2094
2325
|
return {
|
|
2095
2326
|
search,
|
|
@@ -2204,6 +2435,7 @@ function selectSessionNeighborResults(results, options, limit) {
|
|
|
2204
2435
|
function mergeAcrossSources(input) {
|
|
2205
2436
|
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|
|
2206
2437
|
const lists = buildChannelLists(memorySubs, input.insightHits, input.knowledgeHits);
|
|
2438
|
+
if (input.workspaceHits.length) lists.push({ name: "workspace", hits: input.workspaceHits });
|
|
2207
2439
|
if (input.strategy !== "rrf") {
|
|
2208
2440
|
const seen = /* @__PURE__ */ new Map();
|
|
2209
2441
|
for (const hit of [
|
|
@@ -2212,7 +2444,8 @@ function mergeAcrossSources(input) {
|
|
|
2212
2444
|
...memorySubs.planner ?? [],
|
|
2213
2445
|
...memorySubs.hybrid ?? [],
|
|
2214
2446
|
...input.insightHits,
|
|
2215
|
-
...input.knowledgeHits
|
|
2447
|
+
...input.knowledgeHits,
|
|
2448
|
+
...input.workspaceHits
|
|
2216
2449
|
]) {
|
|
2217
2450
|
const key = `${hit.type}::${hit.id}`;
|
|
2218
2451
|
const existing = seen.get(key);
|
|
@@ -2223,7 +2456,10 @@ function mergeAcrossSources(input) {
|
|
|
2223
2456
|
const other = preferred === hit ? existing : hit;
|
|
2224
2457
|
seen.set(
|
|
2225
2458
|
key,
|
|
2226
|
-
|
|
2459
|
+
mergeSearchResultProvenance(
|
|
2460
|
+
preferred.type === "memory" && other.type === "memory" ? mergeMatchedEvidence(preferred, other) : preferred,
|
|
2461
|
+
other
|
|
2462
|
+
)
|
|
2227
2463
|
);
|
|
2228
2464
|
}
|
|
2229
2465
|
}
|
|
@@ -2239,7 +2475,8 @@ function mergeAcrossSources(input) {
|
|
|
2239
2475
|
...memorySubs.hybrid ?? [],
|
|
2240
2476
|
...memorySubs.entity ?? [],
|
|
2241
2477
|
...input.insightHits,
|
|
2242
|
-
...input.knowledgeHits
|
|
2478
|
+
...input.knowledgeHits,
|
|
2479
|
+
...input.workspaceHits
|
|
2243
2480
|
];
|
|
2244
2481
|
const merged = mergeUnifiedMemorySearchResultsRrf(
|
|
2245
2482
|
lists.length > 0 ? lists : [{ name: "memory-semantic", hits: all }],
|
|
@@ -2334,7 +2571,8 @@ async function applyReflectedPlan(deps, store, input, _logger) {
|
|
|
2334
2571
|
source: item.source ?? "raw",
|
|
2335
2572
|
snippet: item.snippet,
|
|
2336
2573
|
score: item.score,
|
|
2337
|
-
timestamp: item.timestamp
|
|
2574
|
+
timestamp: item.timestamp,
|
|
2575
|
+
provenance: item.provenance
|
|
2338
2576
|
}));
|
|
2339
2577
|
const records = evidence.map((item) => toMemoryEvidenceRecord(item, input.userId));
|
|
2340
2578
|
const applyInput = {
|
|
@@ -3079,9 +3317,9 @@ async function startMcpServer(options = {}) {
|
|
|
3079
3317
|
const searchSchema = {
|
|
3080
3318
|
userId: z.string(),
|
|
3081
3319
|
query: z.string().min(1),
|
|
3082
|
-
sources: z.array(z.enum(["memory", "insights", "knowledge"])).optional().describe("Default: ['memory','insights','knowledge']"),
|
|
3083
|
-
tiers: z.array(z.enum(["summary", "raw", "insight", "knowledge"])).optional().describe(
|
|
3084
|
-
"
|
|
3320
|
+
sources: z.array(z.enum(["memory", "insights", "knowledge", "workspace"])).min(1).optional().describe("Default: ['memory','insights','knowledge','workspace']"),
|
|
3321
|
+
tiers: z.array(z.enum(["summary", "raw", "insight", "knowledge", "workspace"])).optional().describe(
|
|
3322
|
+
"Evidence tiers. Default: ['raw','insight','knowledge','workspace']; synthesis additionally includes 'summary'."
|
|
3085
3323
|
),
|
|
3086
3324
|
reasoningStrategy: z.enum(["none", "rewrite", "iterative", "union"]).optional().describe(
|
|
3087
3325
|
"LLM reasoning strategy. `rewrite` rephrases the query before embedding; `iterative` runs a planner that searches, notes evidence, and searches again; `union` merges the planner's evidence with the baseline top-k hits (evidence first, deduped, capped at limit). Requires `unified.reasoning.{queryRewriter,iterativePlanner}` to be wired into the server."
|