@melandlabs/memory-store 1.3.8 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{config-9RzSCdXG.d.ts → config-Cnk2b7fH.d.ts} +25 -4
- package/dist/{evidence-selector-uOHsxwch.d.ts → evidence-selector-C_ZUQsio.d.ts} +34 -5
- package/dist/http.d.ts +2 -2
- package/dist/http.js +356 -112
- package/dist/index.d.ts +6 -6
- package/dist/index.js +349 -111
- package/dist/mcp.d.ts +2 -2
- package/dist/mcp.js +352 -114
- package/dist/search/derive.d.ts +2 -2
- package/dist/search/derive.js +1 -1
- package/dist/search/distill.d.ts +2 -2
- package/dist/search/evidence-selector.d.ts +2 -1
- package/dist/search/unified-search.d.ts +3 -3
- package/dist/search/unified-search.js +349 -111
- package/dist/server/cli-http.js +356 -112
- package/dist/server/cli-mcp.js +352 -114
- package/dist/server/cli-shared.d.ts +2 -2
- package/dist/storage/postgres-raw-message-factory.d.ts +2 -2
- package/dist/storage/raw-message-store.d.ts +2 -2
- package/dist/storage/sqlite-raw-message-store.d.ts +2 -2
- package/dist/storage/sqlite-vector-index.d.ts +2 -2
- package/dist/{unified-search-BBC3jkPJ.d.ts → unified-search-DfAYG3S9.d.ts} +3 -3
- package/package.json +6 -5
package/dist/server/cli-mcp.js
CHANGED
|
@@ -321,6 +321,46 @@ function resolveSearchRuntimeContext(input, runtimeContext, now = Date.now) {
|
|
|
321
321
|
};
|
|
322
322
|
}
|
|
323
323
|
|
|
324
|
+
// src/search/execution-budget.ts
|
|
325
|
+
function boundedSearchOperation(operation, budget) {
|
|
326
|
+
const timeout = budget.sourceTimeoutMs ?? 5e3;
|
|
327
|
+
if (!Number.isFinite(timeout) || timeout <= 0)
|
|
328
|
+
throw new TypeError("sourceTimeoutMs must be positive and finite");
|
|
329
|
+
return (...args) => new Promise((resolve, reject) => {
|
|
330
|
+
const signal = budget.signal;
|
|
331
|
+
if (signal?.aborted) {
|
|
332
|
+
reject(signal.reason);
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
const cleanup2 = () => {
|
|
336
|
+
clearTimeout(timer);
|
|
337
|
+
signal?.removeEventListener("abort", onAbort);
|
|
338
|
+
};
|
|
339
|
+
const onAbort = () => {
|
|
340
|
+
cleanup2();
|
|
341
|
+
reject(signal?.reason);
|
|
342
|
+
};
|
|
343
|
+
const timer = setTimeout(() => {
|
|
344
|
+
cleanup2();
|
|
345
|
+
reject(new Error(`Search operation exceeded ${timeout}ms waiting budget`));
|
|
346
|
+
}, timeout);
|
|
347
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
348
|
+
Promise.resolve().then(() => {
|
|
349
|
+
signal?.throwIfAborted();
|
|
350
|
+
return operation(...args);
|
|
351
|
+
}).then(
|
|
352
|
+
(value) => {
|
|
353
|
+
cleanup2();
|
|
354
|
+
resolve(value);
|
|
355
|
+
},
|
|
356
|
+
(error) => {
|
|
357
|
+
cleanup2();
|
|
358
|
+
reject(error);
|
|
359
|
+
}
|
|
360
|
+
);
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
|
|
324
364
|
// src/search/reranker.ts
|
|
325
365
|
async function applyReranker(reranker, query, results) {
|
|
326
366
|
if (!reranker || results.length === 0) {
|
|
@@ -467,16 +507,21 @@ function deriveLexicalKeywords(query) {
|
|
|
467
507
|
}
|
|
468
508
|
var DEFAULT_LIMIT = 10;
|
|
469
509
|
var DEFAULT_THRESHOLD = 0.7;
|
|
470
|
-
var DEFAULT_SOURCES = ["memory", "insights", "knowledge"];
|
|
510
|
+
var DEFAULT_SOURCES = ["memory", "insights", "knowledge", "workspace"];
|
|
471
511
|
var SOURCE_SET = new Set(DEFAULT_SOURCES);
|
|
472
512
|
function normalizeUnifiedMemorySearchSources(sources) {
|
|
473
|
-
if (
|
|
513
|
+
if (sources === void 0) {
|
|
474
514
|
return [...DEFAULT_SOURCES];
|
|
475
515
|
}
|
|
516
|
+
if (!Array.isArray(sources) || sources.length === 0) {
|
|
517
|
+
throw new TypeError("Search sources must be a non-empty array when specified");
|
|
518
|
+
}
|
|
476
519
|
const normalized = sources.filter((source) => typeof source === "string").map((source) => source.trim().toLowerCase()).filter(
|
|
477
520
|
(source) => SOURCE_SET.has(source)
|
|
478
521
|
);
|
|
479
|
-
|
|
522
|
+
if (normalized.length !== sources.length)
|
|
523
|
+
throw new TypeError("Search sources contain an unsupported source");
|
|
524
|
+
return Array.from(new Set(normalized));
|
|
480
525
|
}
|
|
481
526
|
function clampUnifiedMemorySearchLimit(limit) {
|
|
482
527
|
const parsed = typeof limit === "number" ? limit : Number(limit ?? DEFAULT_LIMIT);
|
|
@@ -518,6 +563,20 @@ function normalizeUnifiedMemoryReasoningStrategy(value, defaultStrategy = "none"
|
|
|
518
563
|
return defaultStrategy;
|
|
519
564
|
}
|
|
520
565
|
var DEFAULT_RRF_K = 60;
|
|
566
|
+
function mergeSearchResultProvenance(preferred, other) {
|
|
567
|
+
const first = preferred.provenance;
|
|
568
|
+
const second = other.provenance;
|
|
569
|
+
const firstVersion = first?.version ?? first?.citation?.version_id;
|
|
570
|
+
const secondVersion = second?.version ?? second?.citation?.version_id;
|
|
571
|
+
const firstHash = first?.contentHash ?? first?.citation?.content_hash;
|
|
572
|
+
const secondHash = second?.contentHash ?? second?.citation?.content_hash;
|
|
573
|
+
if (firstVersion != null && secondVersion != null && firstVersion !== secondVersion || firstHash != null && secondHash != null && firstHash !== secondHash || first?.citation && second?.citation && first.citation.id !== second.citation.id || preferred.content !== other.content && !(firstHash && firstHash === secondHash))
|
|
574
|
+
return preferred;
|
|
575
|
+
const metadata = { ...other.metadata, ...preferred.metadata };
|
|
576
|
+
if (!second) return { ...preferred, metadata };
|
|
577
|
+
const knownFirst = Object.fromEntries(Object.entries(first ?? {}).filter(([, value]) => value != null));
|
|
578
|
+
return { ...preferred, metadata, provenance: { ...second, ...knownFirst } };
|
|
579
|
+
}
|
|
521
580
|
function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K, weights = {}) {
|
|
522
581
|
if (!Array.isArray(lists) || lists.length === 0) {
|
|
523
582
|
return [];
|
|
@@ -531,6 +590,7 @@ function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K, wei
|
|
|
531
590
|
const scores = /* @__PURE__ */ new Map();
|
|
532
591
|
const order = [];
|
|
533
592
|
for (const list of lists) {
|
|
593
|
+
const seenInList = /* @__PURE__ */ new Set();
|
|
534
594
|
const channel = listNameToChannel(list.name);
|
|
535
595
|
const configuredWeight = channel ? effectiveWeights[channel] : void 0;
|
|
536
596
|
const weight = typeof configuredWeight === "number" && Number.isFinite(configuredWeight) && configuredWeight >= 0 ? configuredWeight : 1;
|
|
@@ -538,7 +598,8 @@ function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K, wei
|
|
|
538
598
|
for (let index = 0; index < list.hits.length; index += 1) {
|
|
539
599
|
const hit = list.hits[index];
|
|
540
600
|
const key = `${hit.type}::${hit.id}`;
|
|
541
|
-
const contribution = weight / (safeK +
|
|
601
|
+
const contribution = seenInList.has(key) ? 0 : weight / (safeK + seenInList.size + 1);
|
|
602
|
+
seenInList.add(key);
|
|
542
603
|
const existing = scores.get(key);
|
|
543
604
|
const rankedHit = hit.type === "memory" ? withMatchedEvidence(hit, list.name, index + 1) : hit;
|
|
544
605
|
if (existing) {
|
|
@@ -546,6 +607,7 @@ function mergeUnifiedMemorySearchResultsRrf(lists, limit, k = DEFAULT_RRF_K, wei
|
|
|
546
607
|
if (hit.type === "memory" && existing.hit.type === "memory") {
|
|
547
608
|
existing.hit = mergeMatchedEvidence(existing.hit, rankedHit);
|
|
548
609
|
}
|
|
610
|
+
existing.hit = mergeSearchResultProvenance(existing.hit, rankedHit);
|
|
549
611
|
} else {
|
|
550
612
|
scores.set(key, { hit: rankedHit, rrf: contribution });
|
|
551
613
|
order.push(key);
|
|
@@ -752,14 +814,15 @@ function buildSynthesisPrompt(input) {
|
|
|
752
814
|
grouped.set(item.source, bucket);
|
|
753
815
|
}
|
|
754
816
|
const sections = [];
|
|
755
|
-
for (const tier of DEFAULT_TIERS) {
|
|
817
|
+
for (const tier of [...DEFAULT_TIERS, "workspace"]) {
|
|
756
818
|
const items = grouped.get(tier) ?? [];
|
|
757
819
|
if (items.length === 0) {
|
|
758
820
|
continue;
|
|
759
821
|
}
|
|
760
822
|
const lines = items.map((item, index) => {
|
|
761
823
|
const factSource = typeof item.metadata?.source === "string" && item.metadata.source.length > 0 ? item.metadata.source : item.source ?? "unknown";
|
|
762
|
-
|
|
824
|
+
const provenance = item.provenance ? ` provenance=${JSON.stringify(item.provenance)}` : "";
|
|
825
|
+
return ` [${index + 1}] (${item.id}, source=${factSource}, score=${item.score.toFixed(4)}) ${item.snippet}${provenance}`;
|
|
763
826
|
});
|
|
764
827
|
sections.push(`## ${tier}
|
|
765
828
|
${lines.join("\n")}`);
|
|
@@ -850,7 +913,8 @@ async function synthesizeAnswer(input) {
|
|
|
850
913
|
score: item.score,
|
|
851
914
|
// Forward per-fact provenance (e.g. `metadata.source` from
|
|
852
915
|
// `opencontext add --source`) so the synthesis prompt can cite it.
|
|
853
|
-
metadata: item.metadata
|
|
916
|
+
metadata: item.metadata,
|
|
917
|
+
provenance: item.provenance
|
|
854
918
|
}));
|
|
855
919
|
const warnings = [];
|
|
856
920
|
const complete = deps.reasoning?.complete;
|
|
@@ -1071,7 +1135,7 @@ async function embedQueryVariant(embedQuery, input, query) {
|
|
|
1071
1135
|
authToken: input.authToken
|
|
1072
1136
|
});
|
|
1073
1137
|
}
|
|
1074
|
-
async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit, threshold, logger, peerPeers, warnings, runtimeContext) {
|
|
1138
|
+
async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit, threshold, logger, peerPeers, warnings, runtimeContext, execution) {
|
|
1075
1139
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds.map((botId) => ({ botId })) : [{}];
|
|
1076
1140
|
let semantic = [];
|
|
1077
1141
|
if (typeof deps.searchRawMessagesAnn === "function") {
|
|
@@ -1099,7 +1163,10 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
1099
1163
|
semantic = (await Promise.all(
|
|
1100
1164
|
filters.map((filter) => {
|
|
1101
1165
|
const botId = "botId" in filter ? filter.botId : void 0;
|
|
1102
|
-
return
|
|
1166
|
+
return boundedSearchOperation(
|
|
1167
|
+
searchRawMessagesWithChroma,
|
|
1168
|
+
deps
|
|
1169
|
+
)({
|
|
1103
1170
|
userId: input.userId,
|
|
1104
1171
|
queryEmbedding,
|
|
1105
1172
|
limit,
|
|
@@ -1108,19 +1175,21 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
1108
1175
|
});
|
|
1109
1176
|
})
|
|
1110
1177
|
)).flat().map(toMemoryResult);
|
|
1178
|
+
if (execution) execution.completed++;
|
|
1111
1179
|
logger.log?.("[memory-store] Raw message semantic search completed", {
|
|
1112
1180
|
backend: "chroma",
|
|
1113
1181
|
dimensions: queryEmbedding.length,
|
|
1114
1182
|
count: semantic.length
|
|
1115
1183
|
});
|
|
1116
1184
|
} catch (error) {
|
|
1185
|
+
if (execution) execution.failed++;
|
|
1117
1186
|
logger.warn?.(
|
|
1118
1187
|
"[memory-store] Chroma raw message search failed; falling back to database search:",
|
|
1119
1188
|
error
|
|
1120
1189
|
);
|
|
1121
1190
|
}
|
|
1122
1191
|
}
|
|
1123
|
-
if (semantic.length === 0 && runtimeContext !== void 0) {
|
|
1192
|
+
if (semantic.length === 0 && runtimeContext !== void 0 && !deps.searchRawMessagesAnn) {
|
|
1124
1193
|
warnApplicabilityNotEnforced(warnings);
|
|
1125
1194
|
return [];
|
|
1126
1195
|
}
|
|
@@ -1130,7 +1199,10 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
1130
1199
|
const manager2 = await getRawMessageManager2();
|
|
1131
1200
|
if (typeof manager2.searchMessagesSemantically === "function") {
|
|
1132
1201
|
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
1133
|
-
const results = await
|
|
1202
|
+
const results = await boundedSearchOperation(
|
|
1203
|
+
manager2.searchMessagesSemantically.bind(manager2),
|
|
1204
|
+
deps
|
|
1205
|
+
)({
|
|
1134
1206
|
userId: input.userId,
|
|
1135
1207
|
queryEmbedding,
|
|
1136
1208
|
limit,
|
|
@@ -1140,6 +1212,7 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
1140
1212
|
...peerPeers.length > 0 ? { peers: peerPeers } : {},
|
|
1141
1213
|
...factTypes ? { factTypes } : {}
|
|
1142
1214
|
});
|
|
1215
|
+
if (execution) execution.completed++;
|
|
1143
1216
|
semantic = results.map((r) => ({
|
|
1144
1217
|
type: "memory",
|
|
1145
1218
|
id: r.id,
|
|
@@ -1149,6 +1222,7 @@ async function runSemanticSearchForEmbedding(deps, input, queryEmbedding, limit,
|
|
|
1149
1222
|
}));
|
|
1150
1223
|
}
|
|
1151
1224
|
} catch (error) {
|
|
1225
|
+
if (execution) execution.failed++;
|
|
1152
1226
|
logger.warn?.("[memory-store] SQLite semantic search failed:", error);
|
|
1153
1227
|
}
|
|
1154
1228
|
}
|
|
@@ -1199,7 +1273,7 @@ function toIterativeRecallCandidate(hit) {
|
|
|
1199
1273
|
metadata: hit.metadata
|
|
1200
1274
|
};
|
|
1201
1275
|
}
|
|
1202
|
-
async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger, peerPeers, warnings, runtimeContext) {
|
|
1276
|
+
async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger, peerPeers, warnings, runtimeContext, execution) {
|
|
1203
1277
|
if (keywords.length === 0) {
|
|
1204
1278
|
return [];
|
|
1205
1279
|
}
|
|
@@ -1249,9 +1323,12 @@ async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger,
|
|
|
1249
1323
|
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
1250
1324
|
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
1251
1325
|
const includeDeprecated = input.includeDeprecated === true;
|
|
1252
|
-
|
|
1326
|
+
const results = (await Promise.all(
|
|
1253
1327
|
filters.map(
|
|
1254
|
-
(botId) =>
|
|
1328
|
+
(botId) => boundedSearchOperation(
|
|
1329
|
+
lexicalSearchRawMessages2,
|
|
1330
|
+
deps
|
|
1331
|
+
)({
|
|
1255
1332
|
userId: input.userId,
|
|
1256
1333
|
keywords,
|
|
1257
1334
|
limit: Math.ceil(limit / filters.length),
|
|
@@ -1266,7 +1343,10 @@ async function runLexicalSearchForKeywords(deps, input, keywords, limit, logger,
|
|
|
1266
1343
|
r
|
|
1267
1344
|
)
|
|
1268
1345
|
);
|
|
1346
|
+
if (execution) execution.completed++;
|
|
1347
|
+
return results;
|
|
1269
1348
|
} catch (error) {
|
|
1349
|
+
if (execution) execution.failed++;
|
|
1270
1350
|
logger.warn?.("[memory-store] SQLite lexical search failed:", error);
|
|
1271
1351
|
}
|
|
1272
1352
|
return [];
|
|
@@ -1313,7 +1393,21 @@ function mapHitToEvidenceSource(hit) {
|
|
|
1313
1393
|
}
|
|
1314
1394
|
return hit.type;
|
|
1315
1395
|
}
|
|
1316
|
-
function createUnifiedSearch(
|
|
1396
|
+
function createUnifiedSearch(inputDeps = {}) {
|
|
1397
|
+
const deps = {
|
|
1398
|
+
...inputDeps,
|
|
1399
|
+
embedQuery: inputDeps.embedQuery && boundedSearchOperation(inputDeps.embedQuery, inputDeps),
|
|
1400
|
+
searchRawMessagesAnn: inputDeps.searchRawMessagesAnn && boundedSearchOperation(inputDeps.searchRawMessagesAnn, inputDeps),
|
|
1401
|
+
searchRawMessagesHybrid: inputDeps.searchRawMessagesHybrid && boundedSearchOperation(inputDeps.searchRawMessagesHybrid, inputDeps),
|
|
1402
|
+
searchRawMessagesLexical: inputDeps.searchRawMessagesLexical && boundedSearchOperation(inputDeps.searchRawMessagesLexical, inputDeps),
|
|
1403
|
+
searchWorkspace: inputDeps.searchWorkspace && boundedSearchOperation(inputDeps.searchWorkspace, inputDeps),
|
|
1404
|
+
searchInsights: inputDeps.searchInsights && boundedSearchOperation(inputDeps.searchInsights, inputDeps),
|
|
1405
|
+
searchKnowledge: inputDeps.searchKnowledge && boundedSearchOperation(inputDeps.searchKnowledge, inputDeps),
|
|
1406
|
+
searchSummaries: inputDeps.searchSummaries && boundedSearchOperation(inputDeps.searchSummaries, inputDeps),
|
|
1407
|
+
getRawMessageRetrievalStatus: inputDeps.getRawMessageRetrievalStatus && boundedSearchOperation(inputDeps.getRawMessageRetrievalStatus, inputDeps),
|
|
1408
|
+
entitySearch: inputDeps.entitySearch && boundedSearchOperation(inputDeps.entitySearch, inputDeps),
|
|
1409
|
+
searchRawMessageNeighbors: inputDeps.searchRawMessageNeighbors && boundedSearchOperation(inputDeps.searchRawMessageNeighbors, inputDeps)
|
|
1410
|
+
};
|
|
1317
1411
|
const logger = deps.logger ?? console;
|
|
1318
1412
|
function resolveReasoningStrategy(input) {
|
|
1319
1413
|
const requested = normalizeUnifiedMemoryReasoningStrategy(input.reasoningStrategy);
|
|
@@ -1323,6 +1417,25 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1323
1417
|
return normalizeUnifiedMemoryReasoningStrategy(deps.reasoning?.defaultStrategy, "none");
|
|
1324
1418
|
}
|
|
1325
1419
|
async function runMemorySource(input, limit, threshold, warnings, reasoningStrategy = "none", reasoningInfo, peerPeers = [], runtimeContext) {
|
|
1420
|
+
const execution = { completed: 0, failed: 0 };
|
|
1421
|
+
const observe = (operation, preparation = false) => operation && (async (...args) => {
|
|
1422
|
+
try {
|
|
1423
|
+
const result = await operation(...args);
|
|
1424
|
+
if (!preparation) execution.completed++;
|
|
1425
|
+
return result;
|
|
1426
|
+
} catch (error) {
|
|
1427
|
+
execution.failed++;
|
|
1428
|
+
throw error;
|
|
1429
|
+
}
|
|
1430
|
+
});
|
|
1431
|
+
const sourceDeps = {
|
|
1432
|
+
...deps,
|
|
1433
|
+
embedQuery: observe(deps.embedQuery, true),
|
|
1434
|
+
searchRawMessagesAnn: observe(deps.searchRawMessagesAnn),
|
|
1435
|
+
searchRawMessagesHybrid: observe(deps.searchRawMessagesHybrid),
|
|
1436
|
+
searchRawMessagesLexical: observe(deps.searchRawMessagesLexical),
|
|
1437
|
+
entitySearch: observe(deps.entitySearch)
|
|
1438
|
+
};
|
|
1326
1439
|
const sourceStartedAt = monotonicNow();
|
|
1327
1440
|
const timings = {
|
|
1328
1441
|
semanticMs: 0,
|
|
@@ -1340,9 +1453,10 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1340
1453
|
};
|
|
1341
1454
|
const finish = (results) => ({
|
|
1342
1455
|
results,
|
|
1343
|
-
timings: { ...timings, memorySourceMs: monotonicNow() - sourceStartedAt }
|
|
1456
|
+
timings: { ...timings, memorySourceMs: monotonicNow() - sourceStartedAt },
|
|
1457
|
+
availability: execution.failed > 0 ? execution.completed > 0 ? "partial" : "failed" : execution.completed > 0 || Object.values(results).some((hits) => hits.length > 0) ? "complete" : "not_ready"
|
|
1344
1458
|
});
|
|
1345
|
-
if ((reasoningStrategy === "iterative" || reasoningStrategy === "union") && !
|
|
1459
|
+
if ((reasoningStrategy === "iterative" || reasoningStrategy === "union") && !sourceDeps.reasoning?.iterativePlanner) {
|
|
1346
1460
|
warnings.push({
|
|
1347
1461
|
source: "memory",
|
|
1348
1462
|
code: "memory_iterative_planner_not_configured",
|
|
@@ -1350,7 +1464,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1350
1464
|
});
|
|
1351
1465
|
if (reasoningInfo) reasoningInfo.degraded = true;
|
|
1352
1466
|
}
|
|
1353
|
-
if (reasoningStrategy === "rewrite" && !
|
|
1467
|
+
if (reasoningStrategy === "rewrite" && !sourceDeps.reasoning?.queryRewriter) {
|
|
1354
1468
|
warnings.push({
|
|
1355
1469
|
source: "memory",
|
|
1356
1470
|
code: "memory_query_rewrite_not_configured",
|
|
@@ -1359,8 +1473,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1359
1473
|
if (reasoningInfo) reasoningInfo.degraded = true;
|
|
1360
1474
|
}
|
|
1361
1475
|
let unionEvidence = null;
|
|
1362
|
-
if ((reasoningStrategy === "iterative" || reasoningStrategy === "union") &&
|
|
1363
|
-
const planner =
|
|
1476
|
+
if ((reasoningStrategy === "iterative" || reasoningStrategy === "union") && sourceDeps.reasoning?.iterativePlanner) {
|
|
1477
|
+
const planner = sourceDeps.reasoning.iterativePlanner;
|
|
1364
1478
|
const result = await planner.plan({
|
|
1365
1479
|
query: input.query,
|
|
1366
1480
|
...input.includeRetrievalDiagnostics ? { options: { collectDiagnostics: true } } : {},
|
|
@@ -1372,26 +1486,27 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1372
1486
|
const lexicalHits = await measure(
|
|
1373
1487
|
"lexicalMs",
|
|
1374
1488
|
() => runLexicalSearchForKeywords(
|
|
1375
|
-
|
|
1489
|
+
sourceDeps,
|
|
1376
1490
|
input,
|
|
1377
1491
|
keywords2,
|
|
1378
1492
|
limit,
|
|
1379
1493
|
logger,
|
|
1380
1494
|
peerPeers,
|
|
1381
1495
|
warnings,
|
|
1382
|
-
runtimeContext
|
|
1496
|
+
runtimeContext,
|
|
1497
|
+
execution
|
|
1383
1498
|
)
|
|
1384
1499
|
);
|
|
1385
1500
|
let semanticHits = [];
|
|
1386
|
-
if (typeof
|
|
1501
|
+
if (typeof sourceDeps.embedQuery === "function") {
|
|
1387
1502
|
try {
|
|
1388
1503
|
semanticHits = await measure("semanticMs", async () => {
|
|
1389
1504
|
const semanticQuery = keywords2.join(" ");
|
|
1390
|
-
const queryEmbedder =
|
|
1505
|
+
const queryEmbedder = sourceDeps.embedQuery;
|
|
1391
1506
|
if (typeof queryEmbedder !== "function") return [];
|
|
1392
1507
|
const queryEmbedding = await embedQueryVariant(queryEmbedder, input, semanticQuery);
|
|
1393
1508
|
return runSemanticSearchForEmbedding(
|
|
1394
|
-
|
|
1509
|
+
sourceDeps,
|
|
1395
1510
|
input,
|
|
1396
1511
|
queryEmbedding,
|
|
1397
1512
|
limit,
|
|
@@ -1399,7 +1514,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1399
1514
|
logger,
|
|
1400
1515
|
peerPeers,
|
|
1401
1516
|
warnings,
|
|
1402
|
-
runtimeContext
|
|
1517
|
+
runtimeContext,
|
|
1518
|
+
execution
|
|
1403
1519
|
);
|
|
1404
1520
|
});
|
|
1405
1521
|
} catch (error) {
|
|
@@ -1452,66 +1568,74 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1452
1568
|
}
|
|
1453
1569
|
unionEvidence = evidence;
|
|
1454
1570
|
}
|
|
1455
|
-
|
|
1571
|
+
const lexicalFallback = async () => {
|
|
1572
|
+
const lexical2 = await runLexicalSearchForKeywords(
|
|
1573
|
+
sourceDeps,
|
|
1574
|
+
input,
|
|
1575
|
+
deriveLexicalKeywords(input.query),
|
|
1576
|
+
limit,
|
|
1577
|
+
logger,
|
|
1578
|
+
peerPeers,
|
|
1579
|
+
warnings,
|
|
1580
|
+
runtimeContext,
|
|
1581
|
+
execution
|
|
1582
|
+
);
|
|
1583
|
+
return finish({ semantic: [], lexical: filterByDateRange(lexical2, input.dateFrom, input.dateTo) });
|
|
1584
|
+
};
|
|
1585
|
+
if (typeof sourceDeps.embedQuery !== "function") {
|
|
1456
1586
|
warnings.push({
|
|
1457
1587
|
source: "memory",
|
|
1458
1588
|
code: "memory_lexical_search_fallback",
|
|
1459
1589
|
message: "No embedding provider is configured; using lexical (BM25) search as fallback. Pass an `embedQuery` function via `unified.embedQuery` to enable semantic retrieval."
|
|
1460
1590
|
});
|
|
1461
|
-
|
|
1462
|
-
const lexical2 = filterByDateRange(
|
|
1463
|
-
await runLexicalSearchForKeywords(
|
|
1464
|
-
deps,
|
|
1465
|
-
input,
|
|
1466
|
-
keywords2,
|
|
1467
|
-
limit,
|
|
1468
|
-
logger,
|
|
1469
|
-
peerPeers,
|
|
1470
|
-
warnings,
|
|
1471
|
-
runtimeContext
|
|
1472
|
-
),
|
|
1473
|
-
input.dateFrom,
|
|
1474
|
-
input.dateTo
|
|
1475
|
-
);
|
|
1476
|
-
return finish({ semantic: [], lexical: lexical2 });
|
|
1591
|
+
return lexicalFallback();
|
|
1477
1592
|
}
|
|
1478
1593
|
let semantic = [];
|
|
1479
1594
|
let lexicalQueries = [input.query];
|
|
1480
|
-
const embedQuery =
|
|
1595
|
+
const embedQuery = sourceDeps.embedQuery;
|
|
1481
1596
|
const activeMergeStrategy = normalizeUnifiedMemoryMergeStrategy(
|
|
1482
|
-
input.mergeStrategy ??
|
|
1597
|
+
input.mergeStrategy ?? sourceDeps.reasoning?.defaultMergeStrategy ?? "rrf"
|
|
1483
1598
|
);
|
|
1484
|
-
if (reasoningStrategy === "none" && activeMergeStrategy === "rrf" && input.threshold === void 0 && typeof
|
|
1485
|
-
|
|
1486
|
-
const
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1599
|
+
if (reasoningStrategy === "none" && activeMergeStrategy === "rrf" && input.threshold === void 0 && typeof sourceDeps.searchRawMessagesHybrid === "function") {
|
|
1600
|
+
try {
|
|
1601
|
+
const hybrid = await measure("hybridMs", async () => {
|
|
1602
|
+
const queryEmbedding = await embedQueryVariant(embedQuery, input, input.query);
|
|
1603
|
+
const filters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
1604
|
+
return (await Promise.all(
|
|
1605
|
+
filters.map(
|
|
1606
|
+
(botId) => sourceDeps.searchRawMessagesHybrid?.({
|
|
1607
|
+
userId: input.userId,
|
|
1608
|
+
query: input.query,
|
|
1609
|
+
queryEmbedding,
|
|
1610
|
+
limit: Math.ceil(limit / filters.length),
|
|
1611
|
+
botId,
|
|
1612
|
+
...runtimeContext ?? {}
|
|
1613
|
+
})
|
|
1614
|
+
)
|
|
1615
|
+
)).flatMap((items) => items ?? []).filter(isRawMemorySemanticResult).map(toMemoryResult);
|
|
1616
|
+
});
|
|
1617
|
+
return finish({
|
|
1618
|
+
semantic: [],
|
|
1619
|
+
lexical: [],
|
|
1620
|
+
hybrid: filterByDateRange(hybrid, input.dateFrom, input.dateTo)
|
|
1621
|
+
});
|
|
1622
|
+
} catch (error) {
|
|
1623
|
+
warnings.push({
|
|
1624
|
+
source: "memory",
|
|
1625
|
+
code: "memory_hybrid_search_failed",
|
|
1626
|
+
message: error instanceof Error ? error.message : String(error)
|
|
1627
|
+
});
|
|
1628
|
+
return lexicalFallback();
|
|
1629
|
+
}
|
|
1506
1630
|
}
|
|
1507
|
-
if (reasoningStrategy === "rewrite" &&
|
|
1631
|
+
if (reasoningStrategy === "rewrite" && sourceDeps.reasoning?.queryRewriter) {
|
|
1508
1632
|
try {
|
|
1509
|
-
const variants = await
|
|
1633
|
+
const variants = await sourceDeps.reasoning.queryRewriter.rewrite({
|
|
1510
1634
|
query: input.query,
|
|
1511
1635
|
userId: input.userId,
|
|
1512
1636
|
authToken: input.authToken
|
|
1513
1637
|
});
|
|
1514
|
-
if (
|
|
1638
|
+
if (sourceDeps.reasoning.rewriteLexical === true) {
|
|
1515
1639
|
const seen = /* @__PURE__ */ new Set();
|
|
1516
1640
|
lexicalQueries = [input.query, ...variants].map((query) => query.trim()).filter((query) => {
|
|
1517
1641
|
const key = query.toLowerCase();
|
|
@@ -1522,7 +1646,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1522
1646
|
}
|
|
1523
1647
|
if (reasoningInfo) {
|
|
1524
1648
|
reasoningInfo.rewrittenQueries = variants;
|
|
1525
|
-
if (
|
|
1649
|
+
if (sourceDeps.reasoning.queryRewriter.lastDegraded?.()) {
|
|
1526
1650
|
reasoningInfo.degraded = true;
|
|
1527
1651
|
}
|
|
1528
1652
|
}
|
|
@@ -1533,7 +1657,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1533
1657
|
return Promise.all(
|
|
1534
1658
|
embeddings.map(
|
|
1535
1659
|
(embedding) => runSemanticSearchForEmbedding(
|
|
1536
|
-
|
|
1660
|
+
sourceDeps,
|
|
1537
1661
|
input,
|
|
1538
1662
|
embedding,
|
|
1539
1663
|
limit,
|
|
@@ -1541,12 +1665,13 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1541
1665
|
logger,
|
|
1542
1666
|
peerPeers,
|
|
1543
1667
|
warnings,
|
|
1544
|
-
runtimeContext
|
|
1668
|
+
runtimeContext,
|
|
1669
|
+
execution
|
|
1545
1670
|
)
|
|
1546
1671
|
)
|
|
1547
1672
|
);
|
|
1548
1673
|
});
|
|
1549
|
-
if (activeMergeStrategy === "rrf" &&
|
|
1674
|
+
if (activeMergeStrategy === "rrf" && sourceDeps.reasoning.rewriteSemanticMerge === "rrf" && lists.length > 1 && lists.some((hits) => hits.length > 0)) {
|
|
1550
1675
|
semantic = mergeUnifiedMemorySearchResultsRrf(
|
|
1551
1676
|
lists.map((hits, index) => ({
|
|
1552
1677
|
name: `memory-semantic-query:${index}`,
|
|
@@ -1569,24 +1694,33 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1569
1694
|
}
|
|
1570
1695
|
}
|
|
1571
1696
|
if (semantic.length === 0) {
|
|
1572
|
-
|
|
1573
|
-
|
|
1574
|
-
|
|
1575
|
-
|
|
1576
|
-
|
|
1577
|
-
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
|
|
1581
|
-
|
|
1582
|
-
|
|
1583
|
-
|
|
1584
|
-
|
|
1585
|
-
|
|
1697
|
+
try {
|
|
1698
|
+
semantic = await measure("semanticMs", async () => {
|
|
1699
|
+
const queryEmbedding = await embedQueryVariant(embedQuery, input, input.query);
|
|
1700
|
+
return runSemanticSearchForEmbedding(
|
|
1701
|
+
sourceDeps,
|
|
1702
|
+
input,
|
|
1703
|
+
queryEmbedding,
|
|
1704
|
+
limit,
|
|
1705
|
+
threshold,
|
|
1706
|
+
logger,
|
|
1707
|
+
peerPeers,
|
|
1708
|
+
warnings,
|
|
1709
|
+
runtimeContext,
|
|
1710
|
+
execution
|
|
1711
|
+
);
|
|
1712
|
+
});
|
|
1713
|
+
} catch (error) {
|
|
1714
|
+
warnings.push({
|
|
1715
|
+
source: "memory",
|
|
1716
|
+
code: "memory_semantic_search_failed",
|
|
1717
|
+
message: error instanceof Error ? error.message : String(error)
|
|
1718
|
+
});
|
|
1719
|
+
}
|
|
1586
1720
|
}
|
|
1587
1721
|
let lexical = [];
|
|
1588
1722
|
const keywords = deriveLexicalKeywords(input.query);
|
|
1589
|
-
if (activeMergeStrategy === "rrf" && lexicalQueries.length > 1 && typeof
|
|
1723
|
+
if (activeMergeStrategy === "rrf" && lexicalQueries.length > 1 && typeof sourceDeps.searchRawMessagesLexical === "function") {
|
|
1590
1724
|
const lists = await Promise.all(
|
|
1591
1725
|
lexicalQueries.map(async (query) => ({
|
|
1592
1726
|
name: "memory-bm25",
|
|
@@ -1594,14 +1728,15 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1594
1728
|
await measure(
|
|
1595
1729
|
"lexicalMs",
|
|
1596
1730
|
() => runLexicalSearchForKeywords(
|
|
1597
|
-
|
|
1731
|
+
sourceDeps,
|
|
1598
1732
|
input,
|
|
1599
1733
|
deriveLexicalKeywords(query),
|
|
1600
1734
|
limit,
|
|
1601
1735
|
logger,
|
|
1602
1736
|
peerPeers,
|
|
1603
1737
|
warnings,
|
|
1604
|
-
runtimeContext
|
|
1738
|
+
runtimeContext,
|
|
1739
|
+
execution
|
|
1605
1740
|
)
|
|
1606
1741
|
),
|
|
1607
1742
|
"memory-bm25"
|
|
@@ -1614,11 +1749,11 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1614
1749
|
}));
|
|
1615
1750
|
if (reasoningInfo) reasoningInfo.lexicalRewrittenQueries = lexicalQueries;
|
|
1616
1751
|
} else if (keywords.length > 0) {
|
|
1617
|
-
if (typeof
|
|
1752
|
+
if (typeof sourceDeps.searchRawMessagesLexical === "function") {
|
|
1618
1753
|
const lexicalStartedAt = monotonicNow();
|
|
1619
1754
|
try {
|
|
1620
1755
|
const lexFilters = input.botIds && input.botIds.length > 0 ? input.botIds : [void 0];
|
|
1621
|
-
const searchRawMessagesLexical =
|
|
1756
|
+
const searchRawMessagesLexical = sourceDeps.searchRawMessagesLexical;
|
|
1622
1757
|
const factTypes = input.factTypes?.length ? input.factTypes : void 0;
|
|
1623
1758
|
const includeDeprecated = input.includeDeprecated === true;
|
|
1624
1759
|
lexical = (await Promise.all(
|
|
@@ -1658,16 +1793,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1658
1793
|
}
|
|
1659
1794
|
}
|
|
1660
1795
|
let entity;
|
|
1661
|
-
if (typeof
|
|
1796
|
+
if (typeof sourceDeps.entitySearch === "function") {
|
|
1662
1797
|
entity = filterByDateRange(
|
|
1663
|
-
await runEntitySearchForQuery(
|
|
1798
|
+
await runEntitySearchForQuery(sourceDeps, input, limit, logger, peerPeers, runtimeContext),
|
|
1664
1799
|
input.dateFrom,
|
|
1665
1800
|
input.dateTo
|
|
1666
1801
|
);
|
|
1667
1802
|
if (entity.length === 0) {
|
|
1668
1803
|
entity = void 0;
|
|
1669
1804
|
} else if (normalizeUnifiedMemoryMergeStrategy(
|
|
1670
|
-
input.mergeStrategy ??
|
|
1805
|
+
input.mergeStrategy ?? sourceDeps.reasoning?.defaultMergeStrategy ?? "rrf"
|
|
1671
1806
|
) !== "rrf") {
|
|
1672
1807
|
warnings.push({
|
|
1673
1808
|
source: "memory",
|
|
@@ -1698,6 +1833,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1698
1833
|
const searchStartedAt = monotonicNow();
|
|
1699
1834
|
const query = input.query.trim();
|
|
1700
1835
|
const sources = normalizeUnifiedMemorySearchSources(input.sources);
|
|
1836
|
+
const sourceStates = [];
|
|
1701
1837
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
1702
1838
|
const mergeStrategy = normalizeUnifiedMemoryMergeStrategy(
|
|
1703
1839
|
input.mergeStrategy ?? deps.reasoning?.defaultMergeStrategy ?? "rrf"
|
|
@@ -1723,7 +1859,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1723
1859
|
sources,
|
|
1724
1860
|
results: [],
|
|
1725
1861
|
count: 0,
|
|
1726
|
-
warnings
|
|
1862
|
+
warnings,
|
|
1863
|
+
sourceStates
|
|
1727
1864
|
};
|
|
1728
1865
|
}
|
|
1729
1866
|
let memorySubs;
|
|
@@ -1734,6 +1871,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1734
1871
|
reasoningInfo.dateRange = { from: input.dateFrom, to: input.dateTo };
|
|
1735
1872
|
}
|
|
1736
1873
|
if (sources.includes("memory")) {
|
|
1874
|
+
let availability = "not_ready";
|
|
1737
1875
|
const hasRawProviders = typeof deps.searchRawMessagesAnn === "function" || typeof deps.searchRawMessagesHybrid === "function" || typeof deps.searchRawMessagesLexical === "function";
|
|
1738
1876
|
if (isRawMessageStorageAvailable() || hasRawProviders) {
|
|
1739
1877
|
try {
|
|
@@ -1748,6 +1886,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1748
1886
|
runtimeContext
|
|
1749
1887
|
);
|
|
1750
1888
|
memorySubs = memoryRun.results;
|
|
1889
|
+
availability = memoryRun.availability;
|
|
1751
1890
|
memoryTimings = memoryRun.timings;
|
|
1752
1891
|
memorySubs = {
|
|
1753
1892
|
semantic: dedupeChannelByParent(memorySubs.semantic, "memory-semantic"),
|
|
@@ -1772,11 +1911,12 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1772
1911
|
warnings.push({ source: "memory", code, message });
|
|
1773
1912
|
}
|
|
1774
1913
|
} catch (error) {
|
|
1914
|
+
availability = availability === "complete" || availability === "partial" ? "partial" : "failed";
|
|
1775
1915
|
logger.warn?.("[memory-store] memory source failed:", error);
|
|
1776
1916
|
warnings.push({
|
|
1777
1917
|
source: "memory",
|
|
1778
1918
|
code: "memory_search_failed",
|
|
1779
|
-
message: `Memory search failed: ${error.message ?? "Unknown error"}
|
|
1919
|
+
message: `Memory search failed: ${error.message ?? "Unknown error"}.`
|
|
1780
1920
|
});
|
|
1781
1921
|
}
|
|
1782
1922
|
} else {
|
|
@@ -1786,6 +1926,51 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1786
1926
|
message: "Raw memory storage is not available in this environment."
|
|
1787
1927
|
});
|
|
1788
1928
|
}
|
|
1929
|
+
const count = new Set(
|
|
1930
|
+
[
|
|
1931
|
+
...memorySubs?.semantic ?? [],
|
|
1932
|
+
...memorySubs?.lexical ?? [],
|
|
1933
|
+
...memorySubs?.hybrid ?? [],
|
|
1934
|
+
...memorySubs?.planner ?? [],
|
|
1935
|
+
...memorySubs?.entity ?? []
|
|
1936
|
+
].map((hit) => hit.id)
|
|
1937
|
+
).size;
|
|
1938
|
+
if (availability === "complete" && count === 0 && retrievalStatus?.lexicalReady === false && (!retrievalStatus.semanticReady || !deps.embedQuery)) {
|
|
1939
|
+
availability = "not_ready";
|
|
1940
|
+
}
|
|
1941
|
+
const memoryWarnings = warnings.filter((warning) => warning.source === "memory");
|
|
1942
|
+
sourceStates.push({
|
|
1943
|
+
source: "memory",
|
|
1944
|
+
status: availability,
|
|
1945
|
+
count,
|
|
1946
|
+
...memoryWarnings.length ? { message: memoryWarnings.map((warning) => warning.message).join("; ") } : {}
|
|
1947
|
+
});
|
|
1948
|
+
}
|
|
1949
|
+
const workspaceHits = [];
|
|
1950
|
+
if (sources.includes("workspace")) {
|
|
1951
|
+
if (!deps.searchWorkspace) {
|
|
1952
|
+
sourceStates.push({ source: "workspace", status: "not_configured", count: 0 });
|
|
1953
|
+
} else {
|
|
1954
|
+
try {
|
|
1955
|
+
const result = await deps.searchWorkspace({
|
|
1956
|
+
userId: input.userId,
|
|
1957
|
+
query,
|
|
1958
|
+
limit: candidateLimit,
|
|
1959
|
+
...runtimeContext ?? {}
|
|
1960
|
+
});
|
|
1961
|
+
workspaceHits.push(...result.hits.map((hit) => ({ ...hit, type: "workspace" })));
|
|
1962
|
+
sourceStates.push({
|
|
1963
|
+
source: "workspace",
|
|
1964
|
+
status: result.status,
|
|
1965
|
+
count: result.hits.length,
|
|
1966
|
+
...result.message ? { message: result.message } : {}
|
|
1967
|
+
});
|
|
1968
|
+
} catch (error) {
|
|
1969
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
1970
|
+
sourceStates.push({ source: "workspace", status: "failed", count: 0, message });
|
|
1971
|
+
warnings.push({ source: "workspace", code: "workspace_search_failed", message });
|
|
1972
|
+
}
|
|
1973
|
+
}
|
|
1789
1974
|
}
|
|
1790
1975
|
const insightHits = [];
|
|
1791
1976
|
if (sources.includes("insights") && typeof deps.searchInsights === "function") {
|
|
@@ -1807,10 +1992,18 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1807
1992
|
id: hit.id,
|
|
1808
1993
|
content: hit.content,
|
|
1809
1994
|
similarity: hit.similarity,
|
|
1810
|
-
metadata: hit.metadata
|
|
1995
|
+
metadata: hit.metadata,
|
|
1996
|
+
provenance: hit.provenance
|
|
1811
1997
|
});
|
|
1812
1998
|
}
|
|
1999
|
+
sourceStates.push({ source: "insights", status: "complete", count: insightHits.length });
|
|
1813
2000
|
} catch (error) {
|
|
2001
|
+
sourceStates.push({
|
|
2002
|
+
source: "insights",
|
|
2003
|
+
status: "failed",
|
|
2004
|
+
count: 0,
|
|
2005
|
+
message: error instanceof Error ? error.message : String(error)
|
|
2006
|
+
});
|
|
1814
2007
|
logger.warn?.("[memory-store] insights source failed:", error);
|
|
1815
2008
|
warnings.push({
|
|
1816
2009
|
source: "insights",
|
|
@@ -1818,7 +2011,9 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1818
2011
|
message: error.message ?? "insights_search_failed"
|
|
1819
2012
|
});
|
|
1820
2013
|
}
|
|
1821
|
-
} else if (sources.includes("insights"))
|
|
2014
|
+
} else if (sources.includes("insights")) {
|
|
2015
|
+
sourceStates.push({ source: "insights", status: "not_configured", count: 0 });
|
|
2016
|
+
}
|
|
1822
2017
|
const knowledgeHits = [];
|
|
1823
2018
|
if (sources.includes("knowledge") && typeof deps.searchKnowledge === "function") {
|
|
1824
2019
|
try {
|
|
@@ -1837,7 +2032,14 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1837
2032
|
for (const hit of hits) {
|
|
1838
2033
|
knowledgeHits.push(toKnowledgeResult(hit));
|
|
1839
2034
|
}
|
|
2035
|
+
sourceStates.push({ source: "knowledge", status: "complete", count: knowledgeHits.length });
|
|
1840
2036
|
} catch (error) {
|
|
2037
|
+
sourceStates.push({
|
|
2038
|
+
source: "knowledge",
|
|
2039
|
+
status: "failed",
|
|
2040
|
+
count: 0,
|
|
2041
|
+
message: error instanceof Error ? error.message : String(error)
|
|
2042
|
+
});
|
|
1841
2043
|
logger.warn?.("[memory-store] knowledge source failed:", error);
|
|
1842
2044
|
warnings.push({
|
|
1843
2045
|
source: "knowledge",
|
|
@@ -1845,12 +2047,15 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1845
2047
|
message: error.message ?? "knowledge_search_failed"
|
|
1846
2048
|
});
|
|
1847
2049
|
}
|
|
1848
|
-
} else if (sources.includes("knowledge"))
|
|
2050
|
+
} else if (sources.includes("knowledge")) {
|
|
2051
|
+
sourceStates.push({ source: "knowledge", status: "not_configured", count: 0 });
|
|
2052
|
+
}
|
|
1849
2053
|
const fusionStartedAt = monotonicNow();
|
|
1850
2054
|
const merged = mergeAcrossSources({
|
|
1851
2055
|
memorySubs,
|
|
1852
2056
|
insightHits,
|
|
1853
2057
|
knowledgeHits,
|
|
2058
|
+
workspaceHits,
|
|
1854
2059
|
limit: candidateLimit,
|
|
1855
2060
|
strategy: mergeStrategy,
|
|
1856
2061
|
rrfK,
|
|
@@ -1877,7 +2082,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1877
2082
|
sources,
|
|
1878
2083
|
results: ranked,
|
|
1879
2084
|
count: ranked.length,
|
|
1880
|
-
warnings
|
|
2085
|
+
warnings,
|
|
2086
|
+
sourceStates
|
|
1881
2087
|
};
|
|
1882
2088
|
if (input.includeRetrievalDiagnostics) {
|
|
1883
2089
|
output.retrievalDiagnostics = {
|
|
@@ -1987,15 +2193,17 @@ function createUnifiedSearch(deps = {}) {
|
|
|
1987
2193
|
evidence: [],
|
|
1988
2194
|
count: 0,
|
|
1989
2195
|
warnings: peerScope.warnings,
|
|
2196
|
+
sourceStates: [],
|
|
1990
2197
|
answer: ""
|
|
1991
2198
|
};
|
|
1992
2199
|
return empty;
|
|
1993
2200
|
}
|
|
1994
|
-
const tiers = input.tiers ? [...input.tiers] : wantsSynthesis ? ["summary", "raw", "insight", "knowledge"] : ["raw", "insight", "knowledge"];
|
|
2201
|
+
const tiers = input.tiers ? [...input.tiers] : wantsSynthesis ? ["summary", "raw", "insight", "knowledge", "workspace"] : ["raw", "insight", "knowledge", "workspace"];
|
|
1995
2202
|
const wantsSummaryTier = tiers.includes("summary");
|
|
1996
2203
|
const wantsRawTier = tiers.includes("raw");
|
|
1997
2204
|
const wantsInsightTier = tiers.includes("insight");
|
|
1998
2205
|
const wantsKnowledgeTier = tiers.includes("knowledge");
|
|
2206
|
+
const wantsWorkspaceTier = tiers.includes("workspace");
|
|
1999
2207
|
const warnings = [...peerScope.warnings];
|
|
2000
2208
|
const limit = clampUnifiedMemorySearchLimit(input.limit);
|
|
2001
2209
|
const threshold = clampUnifiedMemorySearchThreshold(input.threshold);
|
|
@@ -2003,11 +2211,13 @@ function createUnifiedSearch(deps = {}) {
|
|
|
2003
2211
|
let sources = input.sources ? [...input.sources] : [];
|
|
2004
2212
|
let unifiedReasoning;
|
|
2005
2213
|
let retrievalDiagnostics;
|
|
2006
|
-
|
|
2214
|
+
let sourceStates = [];
|
|
2215
|
+
if (wantsRawTier || wantsInsightTier || wantsKnowledgeTier || wantsWorkspaceTier) {
|
|
2007
2216
|
const inferredSources = sources.length > 0 ? sources : [
|
|
2008
2217
|
...wantsRawTier ? ["memory"] : [],
|
|
2009
2218
|
...wantsInsightTier ? ["insights"] : [],
|
|
2010
|
-
...wantsKnowledgeTier ? ["knowledge"] : []
|
|
2219
|
+
...wantsKnowledgeTier ? ["knowledge"] : [],
|
|
2220
|
+
...wantsWorkspaceTier ? ["workspace"] : []
|
|
2011
2221
|
];
|
|
2012
2222
|
const unified = await searchUnifiedMemoryWithRuntime(
|
|
2013
2223
|
{
|
|
@@ -2020,6 +2230,7 @@ function createUnifiedSearch(deps = {}) {
|
|
|
2020
2230
|
sources = unified.sources;
|
|
2021
2231
|
unifiedReasoning = unified.reasoning;
|
|
2022
2232
|
retrievalDiagnostics = unified.retrievalDiagnostics;
|
|
2233
|
+
sourceStates = unified.sourceStates;
|
|
2023
2234
|
warnings.push(...unified.warnings);
|
|
2024
2235
|
}
|
|
2025
2236
|
let summaryHits = [];
|
|
@@ -2058,6 +2269,17 @@ function createUnifiedSearch(deps = {}) {
|
|
|
2058
2269
|
source: mapHitToEvidenceSource(hit),
|
|
2059
2270
|
snippet: hit.content,
|
|
2060
2271
|
score: hit.similarity,
|
|
2272
|
+
provenance: {
|
|
2273
|
+
citation: hit.provenance?.citation ?? null,
|
|
2274
|
+
origin: hit.provenance?.origin ?? null,
|
|
2275
|
+
businessReference: hit.provenance?.businessReference ?? null,
|
|
2276
|
+
version: hit.provenance?.version ?? hit.provenance?.citation?.version_id ?? null,
|
|
2277
|
+
contentHash: hit.provenance?.contentHash ?? hit.provenance?.citation?.content_hash ?? null,
|
|
2278
|
+
scope: hit.provenance?.scope ?? null,
|
|
2279
|
+
derivedFrom: hit.provenance?.derivedFrom ?? null,
|
|
2280
|
+
confirmation: hit.provenance?.confirmation ?? null,
|
|
2281
|
+
freshness: hit.provenance?.freshness ?? null
|
|
2282
|
+
},
|
|
2061
2283
|
timestamp: getCandidateTimestamp(hit.metadata),
|
|
2062
2284
|
metadata: hit.metadata
|
|
2063
2285
|
}));
|
|
@@ -2067,7 +2289,8 @@ function createUnifiedSearch(deps = {}) {
|
|
|
2067
2289
|
results: hits,
|
|
2068
2290
|
evidence,
|
|
2069
2291
|
count: hits.length,
|
|
2070
|
-
warnings
|
|
2292
|
+
warnings,
|
|
2293
|
+
sourceStates
|
|
2071
2294
|
};
|
|
2072
2295
|
if (unifiedReasoning) {
|
|
2073
2296
|
base.reasoning = unifiedReasoning;
|
|
@@ -2090,8 +2313,16 @@ function createUnifiedSearch(deps = {}) {
|
|
|
2090
2313
|
return base;
|
|
2091
2314
|
}
|
|
2092
2315
|
async function search(input, runtimeContext) {
|
|
2316
|
+
deps.signal?.throwIfAborted();
|
|
2317
|
+
if (input.sources !== void 0) normalizeUnifiedMemorySearchSources(input.sources);
|
|
2093
2318
|
const resolvedRuntime = resolveSearchRuntimeContext(input, runtimeContext);
|
|
2094
|
-
|
|
2319
|
+
const runSearch = boundedSearchOperation(() => searchWithResolvedRuntime(input, resolvedRuntime), {
|
|
2320
|
+
signal: deps.signal,
|
|
2321
|
+
sourceTimeoutMs: deps.searchTimeoutMs ?? 3e4
|
|
2322
|
+
});
|
|
2323
|
+
const result = await runSearch();
|
|
2324
|
+
deps.signal?.throwIfAborted();
|
|
2325
|
+
return result;
|
|
2095
2326
|
}
|
|
2096
2327
|
return {
|
|
2097
2328
|
search,
|
|
@@ -2206,6 +2437,7 @@ function selectSessionNeighborResults(results, options, limit) {
|
|
|
2206
2437
|
function mergeAcrossSources(input) {
|
|
2207
2438
|
const memorySubs = input.memorySubs ?? { semantic: [], lexical: [] };
|
|
2208
2439
|
const lists = buildChannelLists(memorySubs, input.insightHits, input.knowledgeHits);
|
|
2440
|
+
if (input.workspaceHits.length) lists.push({ name: "workspace", hits: input.workspaceHits });
|
|
2209
2441
|
if (input.strategy !== "rrf") {
|
|
2210
2442
|
const seen = /* @__PURE__ */ new Map();
|
|
2211
2443
|
for (const hit of [
|
|
@@ -2214,7 +2446,8 @@ function mergeAcrossSources(input) {
|
|
|
2214
2446
|
...memorySubs.planner ?? [],
|
|
2215
2447
|
...memorySubs.hybrid ?? [],
|
|
2216
2448
|
...input.insightHits,
|
|
2217
|
-
...input.knowledgeHits
|
|
2449
|
+
...input.knowledgeHits,
|
|
2450
|
+
...input.workspaceHits
|
|
2218
2451
|
]) {
|
|
2219
2452
|
const key = `${hit.type}::${hit.id}`;
|
|
2220
2453
|
const existing = seen.get(key);
|
|
@@ -2225,7 +2458,10 @@ function mergeAcrossSources(input) {
|
|
|
2225
2458
|
const other = preferred === hit ? existing : hit;
|
|
2226
2459
|
seen.set(
|
|
2227
2460
|
key,
|
|
2228
|
-
|
|
2461
|
+
mergeSearchResultProvenance(
|
|
2462
|
+
preferred.type === "memory" && other.type === "memory" ? mergeMatchedEvidence(preferred, other) : preferred,
|
|
2463
|
+
other
|
|
2464
|
+
)
|
|
2229
2465
|
);
|
|
2230
2466
|
}
|
|
2231
2467
|
}
|
|
@@ -2241,7 +2477,8 @@ function mergeAcrossSources(input) {
|
|
|
2241
2477
|
...memorySubs.hybrid ?? [],
|
|
2242
2478
|
...memorySubs.entity ?? [],
|
|
2243
2479
|
...input.insightHits,
|
|
2244
|
-
...input.knowledgeHits
|
|
2480
|
+
...input.knowledgeHits,
|
|
2481
|
+
...input.workspaceHits
|
|
2245
2482
|
];
|
|
2246
2483
|
const merged = mergeUnifiedMemorySearchResultsRrf(
|
|
2247
2484
|
lists.length > 0 ? lists : [{ name: "memory-semantic", hits: all }],
|
|
@@ -2336,7 +2573,8 @@ async function applyReflectedPlan(deps, store, input, _logger) {
|
|
|
2336
2573
|
source: item.source ?? "raw",
|
|
2337
2574
|
snippet: item.snippet,
|
|
2338
2575
|
score: item.score,
|
|
2339
|
-
timestamp: item.timestamp
|
|
2576
|
+
timestamp: item.timestamp,
|
|
2577
|
+
provenance: item.provenance
|
|
2340
2578
|
}));
|
|
2341
2579
|
const records = evidence.map((item) => toMemoryEvidenceRecord(item, input.userId));
|
|
2342
2580
|
const applyInput = {
|
|
@@ -3081,9 +3319,9 @@ async function startMcpServer(options = {}) {
|
|
|
3081
3319
|
const searchSchema = {
|
|
3082
3320
|
userId: z.string(),
|
|
3083
3321
|
query: z.string().min(1),
|
|
3084
|
-
sources: z.array(z.enum(["memory", "insights", "knowledge"])).optional().describe("Default: ['memory','insights','knowledge']"),
|
|
3085
|
-
tiers: z.array(z.enum(["summary", "raw", "insight", "knowledge"])).optional().describe(
|
|
3086
|
-
"
|
|
3322
|
+
sources: z.array(z.enum(["memory", "insights", "knowledge", "workspace"])).min(1).optional().describe("Default: ['memory','insights','knowledge','workspace']"),
|
|
3323
|
+
tiers: z.array(z.enum(["summary", "raw", "insight", "knowledge", "workspace"])).optional().describe(
|
|
3324
|
+
"Evidence tiers. Default: ['raw','insight','knowledge','workspace']; synthesis additionally includes 'summary'."
|
|
3087
3325
|
),
|
|
3088
3326
|
reasoningStrategy: z.enum(["none", "rewrite", "iterative", "union"]).optional().describe(
|
|
3089
3327
|
"LLM reasoning strategy. `rewrite` rephrases the query before embedding; `iterative` runs a planner that searches, notes evidence, and searches again; `union` merges the planner's evidence with the baseline top-k hits (evidence first, deduped, capped at limit). Requires `unified.reasoning.{queryRewriter,iterativePlanner}` to be wired into the server."
|