@remnic/core 9.3.637 → 9.3.638
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +11 -11
- package/dist/access-http.d.ts +2 -2
- package/dist/access-http.js +8 -6
- package/dist/access-mcp.d.ts +2 -2
- package/dist/access-mcp.js +7 -5
- package/dist/{access-service-GVOE5U3C.d.ts → access-service-DvEvBv9_.d.ts} +2 -1
- package/dist/access-service.d.ts +2 -2
- package/dist/access-service.js +6 -4
- package/dist/bootstrap.d.ts +1 -1
- package/dist/causal-behavior.js +2 -2
- package/dist/causal-chain.js +2 -2
- package/dist/causal-consolidation.js +2 -2
- package/dist/causal-retrieval.js +2 -2
- package/dist/causal-trajectory-graph.js +1 -1
- package/dist/causal-trajectory.js +1 -1
- package/dist/{chunk-3XNMN242.js → chunk-2JIYTTQJ.js} +3 -3
- package/dist/{chunk-HZVIYZYN.js → chunk-7KRYJUTB.js} +6 -6
- package/dist/{chunk-2IT7WFYE.js → chunk-FWIROLS6.js} +2 -1
- package/dist/chunk-FWIROLS6.js.map +1 -0
- package/dist/{chunk-QJGTSCJN.js → chunk-I3YAA4BI.js} +8 -8
- package/dist/{chunk-A6D7A2FW.js → chunk-IHG27THY.js} +4 -4
- package/dist/{chunk-3EJ5OOZ3.js → chunk-KXTLEB4L.js} +2 -2
- package/dist/{chunk-XM6M6DHG.js → chunk-MNDFVT4F.js} +195 -6
- package/dist/chunk-MNDFVT4F.js.map +1 -0
- package/dist/{chunk-7V2SGZ3H.js → chunk-NE566K4E.js} +2 -2
- package/dist/{chunk-NFNQ7TWF.js → chunk-OSPVAP56.js} +3 -3
- package/dist/{chunk-UXFOGILU.js → chunk-QDAWVW75.js} +1 -1
- package/dist/{chunk-TQOU3VAT.js → chunk-SSOMTUCA.js} +1 -1
- package/dist/{chunk-TFICNXXU.js → chunk-TJRV4MWX.js} +990 -252
- package/dist/chunk-TJRV4MWX.js.map +1 -0
- package/dist/{chunk-BK35NOJL.js → chunk-U2YPUWRB.js} +3 -3
- package/dist/{chunk-EWC6W6AB.js → chunk-UXA5L2DZ.js} +2 -2
- package/dist/{chunk-XL7FK7PJ.js → chunk-Y56J7CXW.js} +33 -16
- package/dist/{chunk-XL7FK7PJ.js.map → chunk-Y56J7CXW.js.map} +1 -1
- package/dist/{cli-B-fHk59D.d.ts → cli-BJBKNWIS.d.ts} +2 -2
- package/dist/cli.d.ts +3 -3
- package/dist/cli.js +12 -11
- package/dist/compounding/engine.js +1 -1
- package/dist/explicit-capture.d.ts +1 -1
- package/dist/{graph-edge-decay-5ZOK7ZNO.js → graph-edge-decay-PUFNHOBS.js} +2 -2
- package/dist/graph-snapshot.js +2 -2
- package/dist/graph.d.ts +3 -0
- package/dist/graph.js +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.js +16 -16
- package/dist/mcp-memory-inspector-app.d.ts +2 -2
- package/dist/operator-toolkit.js +2 -2
- package/dist/{orchestrator-CoqytbK_.d.ts → orchestrator-ibTTeoUa.d.ts} +9 -0
- package/dist/orchestrator.d.ts +1 -1
- package/dist/orchestrator.js +9 -9
- package/dist/recall-state.d.ts +2 -0
- package/dist/recall-state.js +1 -1
- package/dist/resume-bundles.js +3 -3
- package/dist/schemas.d.ts +22 -22
- package/dist/summarizer.js +2 -2
- package/dist/transcript.js +2 -2
- package/dist/transfer/types.d.ts +12 -12
- package/package.json +1 -1
- package/src/access-service-namespace.test.ts +740 -0
- package/src/access-service.ts +222 -3
- package/src/graph.ts +39 -17
- package/src/orchestrator.ts +1286 -331
- package/src/recall-state.ts +3 -0
- package/dist/chunk-2IT7WFYE.js.map +0 -1
- package/dist/chunk-TFICNXXU.js.map +0 -1
- package/dist/chunk-XM6M6DHG.js.map +0 -1
- /package/dist/{chunk-3XNMN242.js.map → chunk-2JIYTTQJ.js.map} +0 -0
- /package/dist/{chunk-HZVIYZYN.js.map → chunk-7KRYJUTB.js.map} +0 -0
- /package/dist/{chunk-QJGTSCJN.js.map → chunk-I3YAA4BI.js.map} +0 -0
- /package/dist/{chunk-A6D7A2FW.js.map → chunk-IHG27THY.js.map} +0 -0
- /package/dist/{chunk-3EJ5OOZ3.js.map → chunk-KXTLEB4L.js.map} +0 -0
- /package/dist/{chunk-7V2SGZ3H.js.map → chunk-NE566K4E.js.map} +0 -0
- /package/dist/{chunk-NFNQ7TWF.js.map → chunk-OSPVAP56.js.map} +0 -0
- /package/dist/{chunk-UXFOGILU.js.map → chunk-QDAWVW75.js.map} +0 -0
- /package/dist/{chunk-TQOU3VAT.js.map → chunk-SSOMTUCA.js.map} +0 -0
- /package/dist/{chunk-BK35NOJL.js.map → chunk-U2YPUWRB.js.map} +0 -0
- /package/dist/{chunk-EWC6W6AB.js.map → chunk-UXA5L2DZ.js.map} +0 -0
- /package/dist/{graph-edge-decay-5ZOK7ZNO.js.map → graph-edge-decay-PUFNHOBS.js.map} +0 -0
package/src/orchestrator.ts
CHANGED
|
@@ -153,6 +153,7 @@ import {
|
|
|
153
153
|
import { CODEX_THREAD_KEY_PREFIX } from "./codex-thread-key.js";
|
|
154
154
|
import { isDisagreementPrompt } from "./signal.js";
|
|
155
155
|
import { lintWorkspaceFiles, rotateMarkdownFileToArchive } from "./hygiene.js";
|
|
156
|
+
import { isPathInsideStorageRoot } from "./storage-paths.js";
|
|
156
157
|
import { EmbeddingFallback } from "./embedding-fallback.js";
|
|
157
158
|
import {
|
|
158
159
|
decideSemanticDedup,
|
|
@@ -341,7 +342,11 @@ import {
|
|
|
341
342
|
dedupeBehaviorSignalsByMemoryAndHash,
|
|
342
343
|
} from "./behavior-signals.js";
|
|
343
344
|
import { ProfilingCollector } from "./profiling.js";
|
|
344
|
-
import {
|
|
345
|
+
import {
|
|
346
|
+
keyring,
|
|
347
|
+
secureStoreDir,
|
|
348
|
+
SecureStoreLockedError,
|
|
349
|
+
} from "./secure-store/index.js";
|
|
345
350
|
import type {
|
|
346
351
|
AccessTrackingEntry,
|
|
347
352
|
BehaviorLoopPolicyState,
|
|
@@ -710,6 +715,52 @@ async function raceRecallAbort<T>(
|
|
|
710
715
|
}
|
|
711
716
|
}
|
|
712
717
|
|
|
718
|
+
function qmdCollectionPathParts(resultPath: string): {
|
|
719
|
+
collection: string;
|
|
720
|
+
relativePath: string;
|
|
721
|
+
} | null {
|
|
722
|
+
if (!resultPath || path.isAbsolute(resultPath)) return null;
|
|
723
|
+
const normalized = resultPath.replace(/\\/g, "/").replace(/^\/+/, "");
|
|
724
|
+
const slashIndex = normalized.indexOf("/");
|
|
725
|
+
if (slashIndex <= 0 || slashIndex >= normalized.length - 1) return null;
|
|
726
|
+
const collection = normalized.slice(0, slashIndex);
|
|
727
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(collection)) return null;
|
|
728
|
+
return {
|
|
729
|
+
collection,
|
|
730
|
+
relativePath: normalized.slice(slashIndex + 1),
|
|
731
|
+
};
|
|
732
|
+
}
|
|
733
|
+
|
|
734
|
+
function qmdResultPathCandidates(
|
|
735
|
+
storageDir: string,
|
|
736
|
+
resultPath: string,
|
|
737
|
+
): string[] {
|
|
738
|
+
const candidates = new Set<string>();
|
|
739
|
+
const storageRoot = path.resolve(storageDir);
|
|
740
|
+
const addCandidate = (candidate: string) => {
|
|
741
|
+
const resolved = path.resolve(candidate);
|
|
742
|
+
if (isPathInsideStorageRoot(storageRoot, resolved)) {
|
|
743
|
+
candidates.add(resolved);
|
|
744
|
+
}
|
|
745
|
+
};
|
|
746
|
+
const addRelativeCandidates = (relativePath: string) => {
|
|
747
|
+
const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "");
|
|
748
|
+
if (!normalized) return;
|
|
749
|
+
addCandidate(path.join(storageRoot, normalized));
|
|
750
|
+
if (/^\d{4}-\d{2}-\d{2}\//.test(normalized)) {
|
|
751
|
+
addCandidate(path.join(storageRoot, "facts", normalized));
|
|
752
|
+
}
|
|
753
|
+
};
|
|
754
|
+
|
|
755
|
+
if (path.isAbsolute(resultPath)) {
|
|
756
|
+
addCandidate(resultPath);
|
|
757
|
+
} else {
|
|
758
|
+
addRelativeCandidates(resultPath);
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
return [...candidates];
|
|
762
|
+
}
|
|
763
|
+
|
|
713
764
|
/** Maximum age (ms) before a compaction-reset signal file is considered stale and removed. */
|
|
714
765
|
const COMPACTION_SIGNAL_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
715
766
|
const DEFAULT_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS = 10_000;
|
|
@@ -5939,6 +5990,7 @@ export class Orchestrator {
|
|
|
5939
5990
|
memoryResults: QmdSearchResult[];
|
|
5940
5991
|
recallNamespaces: string[];
|
|
5941
5992
|
recallResultLimit: number;
|
|
5993
|
+
deadlineAtMs?: number | null;
|
|
5942
5994
|
/** Issue #681 — when true, bypass graphTraversalConfidenceFloor. */
|
|
5943
5995
|
includeLowConfidence?: boolean;
|
|
5944
5996
|
}): Promise<{
|
|
@@ -5947,16 +5999,90 @@ export class Orchestrator {
|
|
|
5947
5999
|
expandedPaths: GraphRecallExpandedEntry[];
|
|
5948
6000
|
seedResults: QmdSearchResult[];
|
|
5949
6001
|
}> {
|
|
6002
|
+
const deadlineExpired = (): boolean =>
|
|
6003
|
+
typeof options.deadlineAtMs === "number" &&
|
|
6004
|
+
Date.now() >= options.deadlineAtMs;
|
|
5950
6005
|
const byNamespace = new Map<string, QmdSearchResult[]>();
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
|
|
6006
|
+
const addResultForNamespace = (
|
|
6007
|
+
namespace: string,
|
|
6008
|
+
result: QmdSearchResult,
|
|
6009
|
+
): void => {
|
|
6010
|
+
const existing = byNamespace.get(namespace);
|
|
5955
6011
|
if (existing) {
|
|
5956
6012
|
existing.push(result);
|
|
5957
6013
|
} else {
|
|
5958
|
-
byNamespace.set(
|
|
6014
|
+
byNamespace.set(namespace, [result]);
|
|
6015
|
+
}
|
|
6016
|
+
};
|
|
6017
|
+
const resolvedAmbiguousSeeds = new Map<
|
|
6018
|
+
string,
|
|
6019
|
+
{ namespace: string; result: QmdSearchResult } | null
|
|
6020
|
+
>();
|
|
6021
|
+
const resolveAmbiguousSeedOwner = async (
|
|
6022
|
+
result: QmdSearchResult,
|
|
6023
|
+
parts: { collection: string; relativePath: string } | null,
|
|
6024
|
+
): Promise<{ namespace: string; result: QmdSearchResult } | null> => {
|
|
6025
|
+
const cached = resolvedAmbiguousSeeds.get(result.path);
|
|
6026
|
+
if (cached !== undefined) return cached;
|
|
6027
|
+
if (deadlineExpired()) {
|
|
6028
|
+
resolvedAmbiguousSeeds.set(result.path, null);
|
|
6029
|
+
return null;
|
|
6030
|
+
}
|
|
6031
|
+
|
|
6032
|
+
let resolvedPath = result.path;
|
|
6033
|
+
let resolvedResult = result;
|
|
6034
|
+
if (parts) {
|
|
6035
|
+
const resolvedCold = await this.resolveColdQmdResultForRecall(
|
|
6036
|
+
result,
|
|
6037
|
+
this.storage,
|
|
6038
|
+
options.recallNamespaces,
|
|
6039
|
+
);
|
|
6040
|
+
if (!resolvedCold || deadlineExpired()) {
|
|
6041
|
+
resolvedAmbiguousSeeds.set(result.path, null);
|
|
6042
|
+
return null;
|
|
6043
|
+
}
|
|
6044
|
+
resolvedPath = resolvedCold.result.path;
|
|
6045
|
+
resolvedResult = resolvedCold.result;
|
|
6046
|
+
}
|
|
6047
|
+
|
|
6048
|
+
if (!path.isAbsolute(resolvedPath)) {
|
|
6049
|
+
resolvedAmbiguousSeeds.set(result.path, null);
|
|
6050
|
+
return null;
|
|
5959
6051
|
}
|
|
6052
|
+
const ownerStorage = await this.storageForAbsoluteQmdResultPath(
|
|
6053
|
+
resolvedPath,
|
|
6054
|
+
this.storage,
|
|
6055
|
+
options.recallNamespaces,
|
|
6056
|
+
);
|
|
6057
|
+
const ownerNamespace = ownerStorage?.namespace ?? null;
|
|
6058
|
+
const resolved =
|
|
6059
|
+
ownerNamespace && options.recallNamespaces.includes(ownerNamespace)
|
|
6060
|
+
? { namespace: ownerNamespace, result: resolvedResult }
|
|
6061
|
+
: null;
|
|
6062
|
+
resolvedAmbiguousSeeds.set(result.path, resolved);
|
|
6063
|
+
return resolved;
|
|
6064
|
+
};
|
|
6065
|
+
const coldCollection = this.config.qmdColdCollection ?? "openclaw-engram-cold";
|
|
6066
|
+
for (const result of options.memoryResults) {
|
|
6067
|
+
if (deadlineExpired()) break;
|
|
6068
|
+
const parts = qmdCollectionPathParts(result.path);
|
|
6069
|
+
if (parts?.collection === coldCollection) {
|
|
6070
|
+
const resolved = await resolveAmbiguousSeedOwner(result, parts);
|
|
6071
|
+
if (resolved) {
|
|
6072
|
+
addResultForNamespace(resolved.namespace, resolved.result);
|
|
6073
|
+
}
|
|
6074
|
+
continue;
|
|
6075
|
+
}
|
|
6076
|
+
if (path.isAbsolute(result.path)) {
|
|
6077
|
+
const resolved = await resolveAmbiguousSeedOwner(result, null);
|
|
6078
|
+
if (resolved) {
|
|
6079
|
+
addResultForNamespace(resolved.namespace, resolved.result);
|
|
6080
|
+
}
|
|
6081
|
+
continue;
|
|
6082
|
+
}
|
|
6083
|
+
const ns = this.namespaceFromPath(result.path);
|
|
6084
|
+
if (!options.recallNamespaces.includes(ns)) continue;
|
|
6085
|
+
addResultForNamespace(ns, result);
|
|
5960
6086
|
}
|
|
5961
6087
|
|
|
5962
6088
|
const perNamespaceSeedCap = Math.max(3, options.recallResultLimit);
|
|
@@ -5967,15 +6093,31 @@ export class Orchestrator {
|
|
|
5967
6093
|
const expandedResults: QmdSearchResult[] = [];
|
|
5968
6094
|
|
|
5969
6095
|
for (const [namespace, nsResults] of byNamespace.entries()) {
|
|
6096
|
+
if (deadlineExpired()) break;
|
|
5970
6097
|
const storage = await this.storageRouter.storageFor(namespace);
|
|
5971
6098
|
const seedCandidates = nsResults.slice(0, perNamespaceSeedCap);
|
|
5972
6099
|
seedResults.push(...seedCandidates);
|
|
5973
|
-
const seedRelativePaths =
|
|
5974
|
-
.
|
|
5975
|
-
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
|
|
6100
|
+
const seedRelativePaths =
|
|
6101
|
+
typeof options.deadlineAtMs === "number"
|
|
6102
|
+
? await this.graphSeedPathsWithinDeadline(
|
|
6103
|
+
storage,
|
|
6104
|
+
seedCandidates,
|
|
6105
|
+
options.deadlineAtMs,
|
|
6106
|
+
[namespace],
|
|
6107
|
+
)
|
|
6108
|
+
: (
|
|
6109
|
+
await Promise.all(
|
|
6110
|
+
seedCandidates.map((result) =>
|
|
6111
|
+
this.graphSeedPathRelativeToStorage(storage, result, [
|
|
6112
|
+
namespace,
|
|
6113
|
+
]),
|
|
6114
|
+
),
|
|
6115
|
+
)
|
|
6116
|
+
).filter(
|
|
6117
|
+
(value): value is string =>
|
|
6118
|
+
typeof value === "string" && value.length > 0,
|
|
6119
|
+
);
|
|
6120
|
+
if (deadlineExpired()) break;
|
|
5979
6121
|
if (seedRelativePaths.length === 0) continue;
|
|
5980
6122
|
|
|
5981
6123
|
const seedRecallScore = seedCandidates.reduce(
|
|
@@ -5989,14 +6131,22 @@ export class Orchestrator {
|
|
|
5989
6131
|
const expanded = await this.graphIndexFor(storage).spreadingActivation(
|
|
5990
6132
|
seedRelativePaths,
|
|
5991
6133
|
this.config.maxGraphTraversalSteps,
|
|
5992
|
-
|
|
6134
|
+
{
|
|
6135
|
+
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
6136
|
+
...(typeof options.deadlineAtMs === "number"
|
|
6137
|
+
? { deadlineAtMs: options.deadlineAtMs }
|
|
6138
|
+
: {}),
|
|
6139
|
+
},
|
|
5993
6140
|
);
|
|
5994
6141
|
if (expanded.length === 0) continue;
|
|
6142
|
+
if (deadlineExpired()) break;
|
|
5995
6143
|
|
|
5996
6144
|
for (const candidate of expanded.slice(0, perNamespaceExpandedCap)) {
|
|
6145
|
+
if (deadlineExpired()) break;
|
|
5997
6146
|
if (seedSet.has(candidate.path)) continue;
|
|
5998
6147
|
const memoryPath = path.resolve(storage.dir, candidate.path);
|
|
5999
6148
|
const memory = await storage.readMemoryByPath(memoryPath);
|
|
6149
|
+
if (deadlineExpired()) break;
|
|
6000
6150
|
if (!memory) continue;
|
|
6001
6151
|
if (isArtifactMemoryPath(memory.path)) continue;
|
|
6002
6152
|
if (memory.frontmatter.status && memory.frontmatter.status !== "active")
|
|
@@ -8737,17 +8887,18 @@ export class Orchestrator {
|
|
|
8737
8887
|
// several seconds on large box directories due to sequential I/O). We kick it
|
|
8738
8888
|
// off here so it overlaps with QMD and other concurrent work rather than
|
|
8739
8889
|
// running sequentially in phase-2 and blocking assembly.
|
|
8740
|
-
const recentBoxesPromise
|
|
8890
|
+
const recentBoxesPromise = observeEnrichmentPromise(
|
|
8741
8891
|
this.isRecallSectionEnabled(
|
|
8742
8892
|
"memory-boxes",
|
|
8743
8893
|
this.config.memoryBoxesEnabled === true,
|
|
8744
8894
|
) &&
|
|
8745
|
-
|
|
8746
|
-
|
|
8895
|
+
this.config.memoryBoxesEnabled &&
|
|
8896
|
+
this.config.boxRecallDays > 0
|
|
8747
8897
|
? this.boxBuilderFor(profileStorage)
|
|
8748
8898
|
.readRecentBoxes(this.config.boxRecallDays)
|
|
8749
8899
|
.catch(() => [] as BoxFrontmatter[])
|
|
8750
|
-
: Promise.resolve([] as BoxFrontmatter[])
|
|
8900
|
+
: Promise.resolve([] as BoxFrontmatter[]),
|
|
8901
|
+
);
|
|
8751
8902
|
|
|
8752
8903
|
// --- Wait for core sections first, then bounded enrichment ---
|
|
8753
8904
|
this.profiler.startSpan("phase-1-parallel", profileTraceId);
|
|
@@ -8931,6 +9082,70 @@ export class Orchestrator {
|
|
|
8931
9082
|
return finalizeEnrichmentOutcome(outcome);
|
|
8932
9083
|
};
|
|
8933
9084
|
|
|
9085
|
+
const remainingEnrichmentAssemblyMs = (): number | null =>
|
|
9086
|
+
enrichmentAssemblyDeadlineAtMs === null
|
|
9087
|
+
? null
|
|
9088
|
+
: Math.max(0, enrichmentAssemblyDeadlineAtMs - Date.now());
|
|
9089
|
+
|
|
9090
|
+
const awaitAssemblyStep = async <T>(
|
|
9091
|
+
name: string,
|
|
9092
|
+
task: () => Promise<T>,
|
|
9093
|
+
fallback: T,
|
|
9094
|
+
): Promise<T> => {
|
|
9095
|
+
if (options.abortSignal?.aborted) {
|
|
9096
|
+
log.debug(
|
|
9097
|
+
`recall phase-1 assembly [${name}]: skipped after abort at +${Date.now() - phase1Start}ms`,
|
|
9098
|
+
);
|
|
9099
|
+
return fallback;
|
|
9100
|
+
}
|
|
9101
|
+
|
|
9102
|
+
const timeoutMs = remainingEnrichmentAssemblyMs();
|
|
9103
|
+
if (timeoutMs === 0) {
|
|
9104
|
+
log.debug(
|
|
9105
|
+
`recall phase-1 assembly [${name}]: skipped after shared ${enrichmentSectionDeadlineMs}ms budget expired ` +
|
|
9106
|
+
`at +${Date.now() - phase1Start}ms`,
|
|
9107
|
+
);
|
|
9108
|
+
return fallback;
|
|
9109
|
+
}
|
|
9110
|
+
|
|
9111
|
+
let timeoutHandle: NodeJS.Timeout | undefined;
|
|
9112
|
+
try {
|
|
9113
|
+
const result = await (timeoutMs !== null
|
|
9114
|
+
? Promise.race<T | { status: "timed_out" }>([
|
|
9115
|
+
task(),
|
|
9116
|
+
new Promise<{ status: "timed_out" }>((resolve) => {
|
|
9117
|
+
timeoutHandle = setTimeout(
|
|
9118
|
+
() => resolve({ status: "timed_out" }),
|
|
9119
|
+
timeoutMs,
|
|
9120
|
+
);
|
|
9121
|
+
}),
|
|
9122
|
+
])
|
|
9123
|
+
: task());
|
|
9124
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
9125
|
+
if (
|
|
9126
|
+
typeof result === "object" &&
|
|
9127
|
+
result !== null &&
|
|
9128
|
+
"status" in result &&
|
|
9129
|
+
(result as { status?: unknown }).status === "timed_out"
|
|
9130
|
+
) {
|
|
9131
|
+
log.debug(
|
|
9132
|
+
`recall phase-1 assembly [${name}]: timed out within shared ${enrichmentSectionDeadlineMs}ms budget ` +
|
|
9133
|
+
`at +${Date.now() - phase1Start}ms`,
|
|
9134
|
+
);
|
|
9135
|
+
return fallback;
|
|
9136
|
+
}
|
|
9137
|
+
return result as T;
|
|
9138
|
+
} catch (err) {
|
|
9139
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
9140
|
+
log.warn(
|
|
9141
|
+
`recall phase-1 assembly [${name}] failed open: ${
|
|
9142
|
+
err instanceof Error ? err.message : String(err)
|
|
9143
|
+
}`,
|
|
9144
|
+
);
|
|
9145
|
+
return fallback;
|
|
9146
|
+
}
|
|
9147
|
+
};
|
|
9148
|
+
|
|
8934
9149
|
// --- Phase 2: Assemble sections in correct order ---
|
|
8935
9150
|
this.profiler.startSpan("assembly", profileTraceId);
|
|
8936
9151
|
|
|
@@ -9263,8 +9478,11 @@ export class Orchestrator {
|
|
|
9263
9478
|
// 1d. Memory Boxes (topic continuity windows, v8.0 Phase 2A)
|
|
9264
9479
|
// recentBoxesPromise was kicked off before phase-1 so it ran concurrently.
|
|
9265
9480
|
{
|
|
9266
|
-
const recentBoxes = await
|
|
9267
|
-
|
|
9481
|
+
const recentBoxes = await awaitEnrichmentSection(
|
|
9482
|
+
"memory-boxes",
|
|
9483
|
+
recentBoxesPromise,
|
|
9484
|
+
);
|
|
9485
|
+
if (recentBoxes && recentBoxes.length > 0) {
|
|
9268
9486
|
const boxLines = recentBoxes.slice(0, 5).map((b: BoxFrontmatter) => {
|
|
9269
9487
|
const sealedDate = b.sealedAt
|
|
9270
9488
|
? b.sealedAt.slice(0, 16).replace("T", " ")
|
|
@@ -9418,6 +9636,8 @@ export class Orchestrator {
|
|
|
9418
9636
|
}
|
|
9419
9637
|
|
|
9420
9638
|
// 2. QMD results — post-process and format
|
|
9639
|
+
const qmdWasSettledBeforeAssemblyWait =
|
|
9640
|
+
qmdPromise.getSettledOutcome() !== undefined;
|
|
9421
9641
|
const qmdResult = await awaitEnrichmentSection("qmd", qmdPromise);
|
|
9422
9642
|
if (qmdResult) {
|
|
9423
9643
|
const t0 = Date.now();
|
|
@@ -9480,56 +9700,82 @@ export class Orchestrator {
|
|
|
9480
9700
|
graphExpandedResultPaths.clear();
|
|
9481
9701
|
} else {
|
|
9482
9702
|
try {
|
|
9483
|
-
const
|
|
9484
|
-
|
|
9485
|
-
|
|
9486
|
-
|
|
9487
|
-
|
|
9488
|
-
|
|
9489
|
-
|
|
9490
|
-
|
|
9491
|
-
|
|
9492
|
-
|
|
9493
|
-
|
|
9494
|
-
graphSnapshotStatus = "completed";
|
|
9495
|
-
graphDecisionStatus = "completed";
|
|
9496
|
-
graphDecisionReason = graphShadowEvalEnabled
|
|
9497
|
-
? "graph shadow evaluation completed without altering injected context"
|
|
9498
|
-
: "graph expansion merged into recall ranking";
|
|
9499
|
-
graphSnapshotReason = graphDecisionReason;
|
|
9500
|
-
graphSnapshotSeedPaths = seedPaths;
|
|
9501
|
-
graphSnapshotExpandedPaths = expandedPaths;
|
|
9502
|
-
graphSnapshotSeedResults = this.buildGraphRecallRankedResults(
|
|
9503
|
-
seedResults,
|
|
9504
|
-
() => ["baseline"],
|
|
9505
|
-
);
|
|
9506
|
-
graphExpandedResultPaths.clear();
|
|
9507
|
-
expandedPaths.forEach((entry) =>
|
|
9508
|
-
graphExpandedResultPaths.add(entry.path),
|
|
9703
|
+
const graphExpansion = await awaitAssemblyStep(
|
|
9704
|
+
"graph-expansion",
|
|
9705
|
+
() =>
|
|
9706
|
+
this.expandResultsViaGraph({
|
|
9707
|
+
memoryResults,
|
|
9708
|
+
recallNamespaces,
|
|
9709
|
+
recallResultLimit,
|
|
9710
|
+
deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
|
|
9711
|
+
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
9712
|
+
}),
|
|
9713
|
+
null as Awaited<ReturnType<typeof this.expandResultsViaGraph>> | null,
|
|
9509
9714
|
);
|
|
9510
|
-
|
|
9511
|
-
|
|
9512
|
-
|
|
9513
|
-
|
|
9514
|
-
|
|
9515
|
-
|
|
9715
|
+
if (!graphExpansion) {
|
|
9716
|
+
graphSnapshotStatus = "aborted";
|
|
9717
|
+
graphDecisionStatus = "aborted";
|
|
9718
|
+
graphDecisionReason = options.abortSignal?.aborted
|
|
9719
|
+
? "graph expansion skipped because recall assembly was aborted"
|
|
9720
|
+
: "graph expansion skipped because shared post-retrieval assembly budget expired";
|
|
9721
|
+
graphSnapshotReason = graphDecisionReason;
|
|
9722
|
+
graphSnapshotSeedPaths = baselineMemoryResults
|
|
9723
|
+
.slice(0, Math.max(1, recallResultLimit))
|
|
9724
|
+
.map((result) => result.path);
|
|
9725
|
+
graphSnapshotSeedResults = this.buildGraphRecallRankedResults(
|
|
9516
9726
|
baselineMemoryResults,
|
|
9727
|
+
() => ["baseline"],
|
|
9728
|
+
);
|
|
9729
|
+
graphSnapshotExpandedPaths = [];
|
|
9730
|
+
graphExpandedResultPaths.clear();
|
|
9731
|
+
memoryResults = baselineMemoryResults;
|
|
9732
|
+
} else {
|
|
9733
|
+
const {
|
|
9517
9734
|
merged,
|
|
9518
|
-
|
|
9735
|
+
seedPaths,
|
|
9736
|
+
expandedPaths,
|
|
9737
|
+
seedResults = baselineMemoryResults,
|
|
9738
|
+
} = graphExpansion;
|
|
9739
|
+
graphSnapshotStatus = "completed";
|
|
9740
|
+
graphDecisionStatus = "completed";
|
|
9741
|
+
graphDecisionReason = graphShadowEvalEnabled
|
|
9742
|
+
? "graph shadow evaluation completed without altering injected context"
|
|
9743
|
+
: "graph expansion merged into recall ranking";
|
|
9744
|
+
graphSnapshotReason = graphDecisionReason;
|
|
9745
|
+
graphSnapshotSeedPaths = seedPaths;
|
|
9746
|
+
graphSnapshotExpandedPaths = expandedPaths;
|
|
9747
|
+
graphSnapshotSeedResults = this.buildGraphRecallRankedResults(
|
|
9748
|
+
seedResults,
|
|
9749
|
+
() => ["baseline"],
|
|
9519
9750
|
);
|
|
9520
|
-
|
|
9521
|
-
|
|
9522
|
-
|
|
9523
|
-
|
|
9524
|
-
|
|
9525
|
-
|
|
9526
|
-
|
|
9527
|
-
|
|
9528
|
-
|
|
9529
|
-
|
|
9530
|
-
|
|
9531
|
-
|
|
9532
|
-
|
|
9751
|
+
graphExpandedResultPaths.clear();
|
|
9752
|
+
expandedPaths.forEach((entry) =>
|
|
9753
|
+
graphExpandedResultPaths.add(entry.path),
|
|
9754
|
+
);
|
|
9755
|
+
memoryResults = graphShadowEvalEnabled
|
|
9756
|
+
? baselineMemoryResults
|
|
9757
|
+
: merged;
|
|
9758
|
+
|
|
9759
|
+
if (graphShadowEvalEnabled) {
|
|
9760
|
+
const comparison = summarizeGraphShadowComparison(
|
|
9761
|
+
baselineMemoryResults,
|
|
9762
|
+
merged,
|
|
9763
|
+
recallResultLimit,
|
|
9764
|
+
);
|
|
9765
|
+
graphSnapshotShadowComparison = comparison;
|
|
9766
|
+
recordRecallSectionMetric({
|
|
9767
|
+
section: "graphShadow",
|
|
9768
|
+
priority: "enrichment",
|
|
9769
|
+
durationMs: Date.now() - t0,
|
|
9770
|
+
deadlineMs: enrichmentSectionDeadlineMs,
|
|
9771
|
+
source: "fresh",
|
|
9772
|
+
success: true,
|
|
9773
|
+
timing:
|
|
9774
|
+
`on b=${comparison.baselineCount} g=${comparison.graphCount} ` +
|
|
9775
|
+
`ov=${comparison.overlapCount} (${comparison.overlapRatio.toFixed(2)}) ` +
|
|
9776
|
+
`avgDelta=${comparison.averageOverlapDelta.toFixed(3)}`,
|
|
9777
|
+
});
|
|
9778
|
+
}
|
|
9533
9779
|
}
|
|
9534
9780
|
} catch (err) {
|
|
9535
9781
|
graphSnapshotStatus = "aborted";
|
|
@@ -9551,13 +9797,41 @@ export class Orchestrator {
|
|
|
9551
9797
|
}
|
|
9552
9798
|
}
|
|
9553
9799
|
|
|
9554
|
-
// Apply
|
|
9555
|
-
|
|
9800
|
+
// Apply mandatory recall safety filters before deadline-bound scoring
|
|
9801
|
+
// enrichment. If scoring times out, we must fall back to this filtered
|
|
9802
|
+
// list rather than raw QMD hits; boostSearchResults also removes
|
|
9803
|
+
// forgotten, lifecycle-filtered, superseded/as_of-invalid, and
|
|
9804
|
+
// dream/procedural memories.
|
|
9805
|
+
const qmdBoostInput = await this.filterSearchResultsForRecall(
|
|
9556
9806
|
memoryResults,
|
|
9557
|
-
recallNamespaces,
|
|
9558
|
-
retrievalQuery,
|
|
9559
9807
|
undefined,
|
|
9560
|
-
{
|
|
9808
|
+
{
|
|
9809
|
+
asOfMs,
|
|
9810
|
+
// If QMD had already settled before the ordered assembly reached it,
|
|
9811
|
+
// do not let unrelated slow enrichment turn those known results into
|
|
9812
|
+
// unchecked misses. QMD that only settles during its own wait remains
|
|
9813
|
+
// bounded by the shared post-retrieval assembly deadline.
|
|
9814
|
+
deadlineAtMs: qmdWasSettledBeforeAssemblyWait
|
|
9815
|
+
? null
|
|
9816
|
+
: enrichmentAssemblyDeadlineAtMs,
|
|
9817
|
+
abortSignal: options.abortSignal,
|
|
9818
|
+
dropUnresolved: true,
|
|
9819
|
+
recallNamespaces,
|
|
9820
|
+
},
|
|
9821
|
+
);
|
|
9822
|
+
|
|
9823
|
+
// Apply recency and access count boosting
|
|
9824
|
+
memoryResults = await awaitAssemblyStep(
|
|
9825
|
+
"qmd-boost",
|
|
9826
|
+
() =>
|
|
9827
|
+
this.boostSearchResults(
|
|
9828
|
+
qmdBoostInput.results,
|
|
9829
|
+
recallNamespaces,
|
|
9830
|
+
retrievalQuery,
|
|
9831
|
+
qmdBoostInput.memoryByPath,
|
|
9832
|
+
{ asOfMs },
|
|
9833
|
+
),
|
|
9834
|
+
qmdBoostInput.results,
|
|
9561
9835
|
);
|
|
9562
9836
|
|
|
9563
9837
|
// Optional LLM reranking (default off). Fail-open if rerank fails/slow.
|
|
@@ -9739,41 +10013,47 @@ export class Orchestrator {
|
|
|
9739
10013
|
// low-relevance results from polluting context.
|
|
9740
10014
|
const queryAwarePrefilter = await queryAwarePrefilterPromise;
|
|
9741
10015
|
if (queryAwarePrefilter.candidatePaths?.size !== 0) {
|
|
9742
|
-
const
|
|
9743
|
-
|
|
9744
|
-
|
|
9745
|
-
|
|
9746
|
-
|
|
9747
|
-
|
|
9748
|
-
|
|
9749
|
-
|
|
9750
|
-
|
|
9751
|
-
|
|
9752
|
-
|
|
9753
|
-
|
|
9754
|
-
|
|
9755
|
-
|
|
9756
|
-
|
|
10016
|
+
const scoped = await awaitAssemblyStep(
|
|
10017
|
+
"embedding-fallback",
|
|
10018
|
+
async () => {
|
|
10019
|
+
const embeddingResults = await this.searchEmbeddingFallback(
|
|
10020
|
+
retrievalQuery,
|
|
10021
|
+
embeddingFetchLimit,
|
|
10022
|
+
);
|
|
10023
|
+
const prefilteredEmbeddingResults = applyQueryAwareCandidateFilter(
|
|
10024
|
+
embeddingResults,
|
|
10025
|
+
queryAwarePrefilter.candidatePaths,
|
|
10026
|
+
);
|
|
10027
|
+
const scopedCandidates = filterRecallCandidates(
|
|
10028
|
+
prefilteredEmbeddingResults,
|
|
10029
|
+
{
|
|
10030
|
+
namespacesEnabled: this.config.namespacesEnabled,
|
|
10031
|
+
recallNamespaces,
|
|
10032
|
+
resolveNamespace: (p) => this.namespaceFromPath(p),
|
|
10033
|
+
limit: embeddingFetchLimit,
|
|
10034
|
+
},
|
|
10035
|
+
);
|
|
10036
|
+
const boostedScoped = await this.boostSearchResults(
|
|
10037
|
+
scopedCandidates,
|
|
10038
|
+
recallNamespaces,
|
|
10039
|
+
retrievalQuery,
|
|
10040
|
+
undefined,
|
|
10041
|
+
{ asOfMs },
|
|
10042
|
+
);
|
|
10043
|
+
// MMR runs on the pre-truncation pool so diverse candidates just
|
|
10044
|
+
// below the cutoff can be promoted into the injected set.
|
|
10045
|
+
xrayBranchPoolSize.hot_embedding = Math.max(
|
|
10046
|
+
xrayBranchPoolSize.hot_embedding,
|
|
10047
|
+
boostedScoped.length,
|
|
10048
|
+
);
|
|
10049
|
+
return this.diversifyAndLimitRecallResults(
|
|
10050
|
+
"memories",
|
|
10051
|
+
boostedScoped,
|
|
10052
|
+
recallResultLimit,
|
|
10053
|
+
retrievalQuery,
|
|
10054
|
+
);
|
|
9757
10055
|
},
|
|
9758
|
-
|
|
9759
|
-
const boostedScoped = await this.boostSearchResults(
|
|
9760
|
-
scopedCandidates,
|
|
9761
|
-
recallNamespaces,
|
|
9762
|
-
retrievalQuery,
|
|
9763
|
-
undefined,
|
|
9764
|
-
{ asOfMs },
|
|
9765
|
-
);
|
|
9766
|
-
// MMR runs on the pre-truncation pool so diverse candidates just
|
|
9767
|
-
// below the cutoff can be promoted into the injected set.
|
|
9768
|
-
xrayBranchPoolSize.hot_embedding = Math.max(
|
|
9769
|
-
xrayBranchPoolSize.hot_embedding,
|
|
9770
|
-
boostedScoped.length,
|
|
9771
|
-
);
|
|
9772
|
-
const scoped = this.diversifyAndLimitRecallResults(
|
|
9773
|
-
"memories",
|
|
9774
|
-
boostedScoped,
|
|
9775
|
-
recallResultLimit,
|
|
9776
|
-
retrievalQuery,
|
|
10056
|
+
[] as QmdSearchResult[],
|
|
9777
10057
|
);
|
|
9778
10058
|
if (scoped.length > 0) {
|
|
9779
10059
|
if (shouldPersistGraphSnapshot) {
|
|
@@ -9811,6 +10091,7 @@ export class Orchestrator {
|
|
|
9811
10091
|
queryAwarePrefilter,
|
|
9812
10092
|
abortSignal: options.abortSignal,
|
|
9813
10093
|
xrayPoolSizeSink: xrayColdPoolSink,
|
|
10094
|
+
deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
|
|
9814
10095
|
asOfMs,
|
|
9815
10096
|
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
9816
10097
|
});
|
|
@@ -9886,42 +10167,48 @@ export class Orchestrator {
|
|
|
9886
10167
|
// Fallback: embeddings first, then recency-only.
|
|
9887
10168
|
const queryAwarePrefilter = await queryAwarePrefilterPromise;
|
|
9888
10169
|
if (queryAwarePrefilter.candidatePaths?.size !== 0) {
|
|
9889
|
-
|
|
9890
|
-
|
|
9891
|
-
|
|
9892
|
-
|
|
9893
|
-
|
|
9894
|
-
|
|
9895
|
-
|
|
9896
|
-
|
|
9897
|
-
|
|
9898
|
-
|
|
9899
|
-
|
|
9900
|
-
|
|
9901
|
-
|
|
9902
|
-
|
|
9903
|
-
|
|
9904
|
-
|
|
9905
|
-
|
|
9906
|
-
|
|
9907
|
-
|
|
9908
|
-
|
|
9909
|
-
|
|
9910
|
-
|
|
9911
|
-
|
|
9912
|
-
|
|
9913
|
-
|
|
9914
|
-
|
|
9915
|
-
|
|
9916
|
-
|
|
9917
|
-
|
|
9918
|
-
|
|
9919
|
-
|
|
9920
|
-
|
|
9921
|
-
|
|
9922
|
-
|
|
9923
|
-
|
|
9924
|
-
|
|
10170
|
+
const scoped = await awaitAssemblyStep(
|
|
10171
|
+
"embedding-fallback",
|
|
10172
|
+
async () => {
|
|
10173
|
+
const embeddingResults = await this.searchEmbeddingFallback(
|
|
10174
|
+
retrievalQuery,
|
|
10175
|
+
embeddingFetchLimit,
|
|
10176
|
+
);
|
|
10177
|
+
const prefilteredEmbeddingResults = applyQueryAwareCandidateFilter(
|
|
10178
|
+
embeddingResults,
|
|
10179
|
+
queryAwarePrefilter.candidatePaths,
|
|
10180
|
+
);
|
|
10181
|
+
const scopedCandidates = filterRecallCandidates(
|
|
10182
|
+
prefilteredEmbeddingResults,
|
|
10183
|
+
{
|
|
10184
|
+
namespacesEnabled: this.config.namespacesEnabled,
|
|
10185
|
+
recallNamespaces,
|
|
10186
|
+
resolveNamespace: (p) => this.namespaceFromPath(p),
|
|
10187
|
+
limit: embeddingFetchLimit,
|
|
10188
|
+
},
|
|
10189
|
+
);
|
|
10190
|
+
const boostedScoped = await this.boostSearchResults(
|
|
10191
|
+
scopedCandidates,
|
|
10192
|
+
recallNamespaces,
|
|
10193
|
+
retrievalQuery,
|
|
10194
|
+
undefined,
|
|
10195
|
+
{ asOfMs },
|
|
10196
|
+
);
|
|
10197
|
+
// MMR runs on the pre-truncation pool so diverse candidates just
|
|
10198
|
+
// below the cutoff can be promoted into the injected set.
|
|
10199
|
+
xrayBranchPoolSize.hot_embedding = Math.max(
|
|
10200
|
+
xrayBranchPoolSize.hot_embedding,
|
|
10201
|
+
boostedScoped.length,
|
|
10202
|
+
);
|
|
10203
|
+
return this.diversifyAndLimitRecallResults(
|
|
10204
|
+
"memories",
|
|
10205
|
+
boostedScoped,
|
|
10206
|
+
recallResultLimit,
|
|
10207
|
+
retrievalQuery,
|
|
10208
|
+
);
|
|
10209
|
+
},
|
|
10210
|
+
[] as QmdSearchResult[],
|
|
10211
|
+
);
|
|
9925
10212
|
if (scoped.length > 0) {
|
|
9926
10213
|
if (shouldPersistGraphSnapshot) {
|
|
9927
10214
|
graphSnapshotFinalResults = this.buildGraphRecallRankedResults(
|
|
@@ -9950,8 +10237,11 @@ export class Orchestrator {
|
|
|
9950
10237
|
xrayRecalledResults = scoped;
|
|
9951
10238
|
impressionRecorded = true;
|
|
9952
10239
|
} else {
|
|
9953
|
-
const memories =
|
|
9954
|
-
|
|
10240
|
+
const memories = await awaitAssemblyStep(
|
|
10241
|
+
"recent-memory-read",
|
|
10242
|
+
() => this.readAllMemoriesForNamespaces(recallNamespaces),
|
|
10243
|
+
[] as MemoryFile[],
|
|
10244
|
+
);
|
|
9955
10245
|
if (memories.length > 0) {
|
|
9956
10246
|
// Filter out non-active memories. Delegate to
|
|
9957
10247
|
// shouldFilterSupersededFromRecall for superseded-status logic so
|
|
@@ -10023,6 +10313,7 @@ export class Orchestrator {
|
|
|
10023
10313
|
queryAwarePrefilter,
|
|
10024
10314
|
abortSignal: options.abortSignal,
|
|
10025
10315
|
xrayPoolSizeSink: xrayColdPoolSink,
|
|
10316
|
+
deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
|
|
10026
10317
|
asOfMs,
|
|
10027
10318
|
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
10028
10319
|
});
|
|
@@ -10049,44 +10340,50 @@ export class Orchestrator {
|
|
|
10049
10340
|
impressionRecorded = true;
|
|
10050
10341
|
}
|
|
10051
10342
|
} else {
|
|
10052
|
-
const
|
|
10053
|
-
|
|
10054
|
-
|
|
10055
|
-
|
|
10056
|
-
|
|
10057
|
-
|
|
10058
|
-
|
|
10059
|
-
|
|
10060
|
-
|
|
10061
|
-
|
|
10062
|
-
|
|
10063
|
-
|
|
10064
|
-
|
|
10065
|
-
|
|
10066
|
-
|
|
10067
|
-
|
|
10068
|
-
|
|
10069
|
-
|
|
10070
|
-
|
|
10071
|
-
|
|
10072
|
-
|
|
10073
|
-
|
|
10074
|
-
|
|
10075
|
-
|
|
10076
|
-
|
|
10077
|
-
|
|
10078
|
-
|
|
10079
|
-
|
|
10080
|
-
|
|
10081
|
-
|
|
10082
|
-
|
|
10083
|
-
|
|
10084
|
-
|
|
10085
|
-
|
|
10086
|
-
|
|
10087
|
-
|
|
10088
|
-
|
|
10089
|
-
|
|
10343
|
+
const recent = await awaitAssemblyStep(
|
|
10344
|
+
"recent-memory-scan",
|
|
10345
|
+
async () => {
|
|
10346
|
+
const recentSorted = queryAwareScopedMemories.sort(
|
|
10347
|
+
(a, b) =>
|
|
10348
|
+
new Date(b.frontmatter.updated).getTime() -
|
|
10349
|
+
new Date(a.frontmatter.updated).getTime(),
|
|
10350
|
+
);
|
|
10351
|
+
const preloadedMap = new Map<string, MemoryFile>(
|
|
10352
|
+
queryAwareScopedMemories
|
|
10353
|
+
.filter((m) => m.path)
|
|
10354
|
+
.map((m) => [m.path, m]),
|
|
10355
|
+
);
|
|
10356
|
+
const recentAsResults: QmdSearchResult[] = recentSorted.map(
|
|
10357
|
+
(m, i) => ({
|
|
10358
|
+
docid: m.frontmatter.id,
|
|
10359
|
+
path: m.path,
|
|
10360
|
+
snippet: m.content,
|
|
10361
|
+
score: 1.0 - i / Math.max(recentSorted.length, 1),
|
|
10362
|
+
}),
|
|
10363
|
+
);
|
|
10364
|
+
const boostedRecent = (
|
|
10365
|
+
await this.boostSearchResults(
|
|
10366
|
+
recentAsResults,
|
|
10367
|
+
recallNamespaces,
|
|
10368
|
+
retrievalQuery,
|
|
10369
|
+
preloadedMap,
|
|
10370
|
+
{ asOfMs },
|
|
10371
|
+
)
|
|
10372
|
+
).sort((a, b) => b.score - a.score);
|
|
10373
|
+
// MMR runs on the pre-truncation pool so diverse candidates just
|
|
10374
|
+
// below the cutoff can be promoted into the injected set.
|
|
10375
|
+
xrayBranchPoolSize.recent_scan = Math.max(
|
|
10376
|
+
xrayBranchPoolSize.recent_scan,
|
|
10377
|
+
boostedRecent.length,
|
|
10378
|
+
);
|
|
10379
|
+
return this.diversifyAndLimitRecallResults(
|
|
10380
|
+
"memories",
|
|
10381
|
+
boostedRecent,
|
|
10382
|
+
recallResultLimit,
|
|
10383
|
+
retrievalQuery,
|
|
10384
|
+
);
|
|
10385
|
+
},
|
|
10386
|
+
[] as QmdSearchResult[],
|
|
10090
10387
|
);
|
|
10091
10388
|
|
|
10092
10389
|
if (recent.length > 0) {
|
|
@@ -10125,6 +10422,7 @@ export class Orchestrator {
|
|
|
10125
10422
|
queryAwarePrefilter,
|
|
10126
10423
|
abortSignal: options.abortSignal,
|
|
10127
10424
|
xrayPoolSizeSink: xrayColdPoolSink,
|
|
10425
|
+
deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
|
|
10128
10426
|
asOfMs,
|
|
10129
10427
|
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
10130
10428
|
});
|
|
@@ -10168,6 +10466,7 @@ export class Orchestrator {
|
|
|
10168
10466
|
queryAwarePrefilter,
|
|
10169
10467
|
abortSignal: options.abortSignal,
|
|
10170
10468
|
xrayPoolSizeSink: xrayColdPoolSink,
|
|
10469
|
+
deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
|
|
10171
10470
|
asOfMs,
|
|
10172
10471
|
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
10173
10472
|
});
|
|
@@ -10547,6 +10846,7 @@ export class Orchestrator {
|
|
|
10547
10846
|
query: retrievalQuery,
|
|
10548
10847
|
memoryIds: recalledMemoryIds,
|
|
10549
10848
|
namespace: selfNamespace,
|
|
10849
|
+
recallNamespaces,
|
|
10550
10850
|
traceId,
|
|
10551
10851
|
plannerMode: recallMode,
|
|
10552
10852
|
requestedMode,
|
|
@@ -15482,36 +15782,307 @@ export class Orchestrator {
|
|
|
15482
15782
|
*
|
|
15483
15783
|
* Callers must pass the full candidate pool (post-rerank, pre-slice).
|
|
15484
15784
|
*/
|
|
15485
|
-
private
|
|
15486
|
-
|
|
15487
|
-
|
|
15488
|
-
|
|
15489
|
-
|
|
15490
|
-
|
|
15491
|
-
|
|
15492
|
-
|
|
15493
|
-
|
|
15494
|
-
|
|
15495
|
-
|
|
15496
|
-
|
|
15497
|
-
|
|
15785
|
+
private qmdCollectionNamespaceFromPrefix(collectionPrefix: string): string | null {
|
|
15786
|
+
const baseCollection = this.config.qmdCollection;
|
|
15787
|
+
if (collectionPrefix === baseCollection) return this.config.defaultNamespace;
|
|
15788
|
+
const namespaceSuffix = collectionPrefix.startsWith(`${baseCollection}--`)
|
|
15789
|
+
? collectionPrefix.slice(baseCollection.length + 2)
|
|
15790
|
+
: "";
|
|
15791
|
+
if (!namespaceSuffix) return null;
|
|
15792
|
+
|
|
15793
|
+
const decoded = namespaceIdentityFromToken(namespaceSuffix);
|
|
15794
|
+
if (decoded !== null) return decoded || this.config.defaultNamespace;
|
|
15795
|
+
if (namespaceSuffix.startsWith("ns--")) {
|
|
15796
|
+
const legacyNamespace = namespaceSuffix.slice("ns--".length).trim();
|
|
15797
|
+
return legacyNamespace || null;
|
|
15798
|
+
}
|
|
15799
|
+
return null;
|
|
15800
|
+
}
|
|
15498
15801
|
|
|
15499
|
-
|
|
15500
|
-
|
|
15501
|
-
|
|
15502
|
-
|
|
15503
|
-
|
|
15504
|
-
|
|
15505
|
-
|
|
15506
|
-
|
|
15802
|
+
private async readQmdResultMemory(
|
|
15803
|
+
resultPath: string,
|
|
15804
|
+
fallbackStorage: StorageManager,
|
|
15805
|
+
recallNamespaces: readonly string[] = [],
|
|
15806
|
+
): Promise<MemoryFile | null> {
|
|
15807
|
+
const parts = qmdCollectionPathParts(resultPath);
|
|
15808
|
+
const storageDirFor = (storage: StorageManager): string | null =>
|
|
15809
|
+
typeof (storage as { dir?: unknown }).dir === "string" &&
|
|
15810
|
+
(storage as { dir?: string }).dir
|
|
15811
|
+
? (storage as { dir: string }).dir
|
|
15812
|
+
: null;
|
|
15813
|
+
const fallbackStorageDir = storageDirFor(fallbackStorage);
|
|
15814
|
+
const coldCollection = this.config.qmdColdCollection ?? "openclaw-engram-cold";
|
|
15815
|
+
if (parts && parts.collection === coldCollection) {
|
|
15816
|
+
const storages: StorageManager[] = [];
|
|
15817
|
+
const seenStorageDirs = new Set<string>();
|
|
15818
|
+
const addStorage = (storage: StorageManager): void => {
|
|
15819
|
+
const storageDir = storageDirFor(storage);
|
|
15820
|
+
const storageKey = storageDir
|
|
15821
|
+
? path.resolve(storageDir)
|
|
15822
|
+
: `storage-without-dir-${storages.length}`;
|
|
15823
|
+
if (seenStorageDirs.has(storageKey)) return;
|
|
15824
|
+
seenStorageDirs.add(storageKey);
|
|
15825
|
+
storages.push(storage);
|
|
15826
|
+
};
|
|
15827
|
+
|
|
15828
|
+
const fallbackNamespace =
|
|
15829
|
+
fallbackStorageDir !== null
|
|
15830
|
+
? this.namespaceFromStorageDir(fallbackStorageDir)
|
|
15831
|
+
: this.config.defaultNamespace;
|
|
15832
|
+
if (
|
|
15833
|
+
recallNamespaces.length === 0 ||
|
|
15834
|
+
!this.config.namespacesEnabled ||
|
|
15835
|
+
recallNamespaces.includes(fallbackNamespace)
|
|
15836
|
+
) {
|
|
15837
|
+
addStorage(fallbackStorage);
|
|
15507
15838
|
}
|
|
15508
|
-
}
|
|
15509
15839
|
|
|
15510
|
-
|
|
15511
|
-
|
|
15512
|
-
|
|
15513
|
-
|
|
15514
|
-
|
|
15840
|
+
if (recallNamespaces.length > 0) {
|
|
15841
|
+
for (const namespace of recallNamespaces) {
|
|
15842
|
+
try {
|
|
15843
|
+
addStorage(await this.storageRouter.storageFor(namespace));
|
|
15844
|
+
} catch (err) {
|
|
15845
|
+
log.debug("qmd cold result namespace storage lookup skipped", {
|
|
15846
|
+
path: resultPath,
|
|
15847
|
+
namespace,
|
|
15848
|
+
error: (err as Error).message,
|
|
15849
|
+
});
|
|
15850
|
+
}
|
|
15851
|
+
}
|
|
15852
|
+
}
|
|
15853
|
+
|
|
15854
|
+
for (const storage of storages) {
|
|
15855
|
+
const storageDir = storageDirFor(storage);
|
|
15856
|
+
if (!storageDir) {
|
|
15857
|
+
const memory = await storage.readMemoryByPath(resultPath);
|
|
15858
|
+
if (memory) return memory;
|
|
15859
|
+
continue;
|
|
15860
|
+
}
|
|
15861
|
+
try {
|
|
15862
|
+
const coldRoot = path.join(storageDir, "cold");
|
|
15863
|
+
for (const candidate of qmdResultPathCandidates(
|
|
15864
|
+
coldRoot,
|
|
15865
|
+
parts.relativePath,
|
|
15866
|
+
)) {
|
|
15867
|
+
const memory = await storage.readMemoryByPath(candidate);
|
|
15868
|
+
if (memory) return memory;
|
|
15869
|
+
}
|
|
15870
|
+
} catch (err) {
|
|
15871
|
+
if (err instanceof SecureStoreLockedError) throw err;
|
|
15872
|
+
log.debug("qmd cold result path lookup failed open", {
|
|
15873
|
+
path: resultPath,
|
|
15874
|
+
collection: coldCollection,
|
|
15875
|
+
error: (err as Error).message,
|
|
15876
|
+
});
|
|
15877
|
+
}
|
|
15878
|
+
}
|
|
15879
|
+
return null;
|
|
15880
|
+
}
|
|
15881
|
+
const collectionNamespace = parts
|
|
15882
|
+
? this.qmdCollectionNamespaceFromPrefix(parts.collection)
|
|
15883
|
+
: null;
|
|
15884
|
+
|
|
15885
|
+
if (parts && collectionNamespace) {
|
|
15886
|
+
try {
|
|
15887
|
+
const collectionStorage =
|
|
15888
|
+
await this.storageRouter.storageFor(collectionNamespace);
|
|
15889
|
+
for (const candidate of qmdResultPathCandidates(
|
|
15890
|
+
collectionStorage.dir,
|
|
15891
|
+
parts.relativePath,
|
|
15892
|
+
)) {
|
|
15893
|
+
const memory = await collectionStorage.readMemoryByPath(candidate);
|
|
15894
|
+
if (memory) return memory;
|
|
15895
|
+
}
|
|
15896
|
+
return null;
|
|
15897
|
+
} catch (err) {
|
|
15898
|
+
if (err instanceof SecureStoreLockedError) throw err;
|
|
15899
|
+
log.debug("qmd result namespace path lookup failed open", {
|
|
15900
|
+
path: resultPath,
|
|
15901
|
+
namespace: collectionNamespace,
|
|
15902
|
+
error: (err as Error).message,
|
|
15903
|
+
});
|
|
15904
|
+
return null;
|
|
15905
|
+
}
|
|
15906
|
+
}
|
|
15907
|
+
|
|
15908
|
+
if (path.isAbsolute(resultPath)) {
|
|
15909
|
+
if (!fallbackStorageDir) {
|
|
15910
|
+
return await fallbackStorage.readMemoryByPath(resultPath);
|
|
15911
|
+
}
|
|
15912
|
+
const ownerStorage = await this.storageForAbsoluteQmdResultPath(
|
|
15913
|
+
resultPath,
|
|
15914
|
+
fallbackStorage,
|
|
15915
|
+
recallNamespaces,
|
|
15916
|
+
);
|
|
15917
|
+
if (!ownerStorage) return null;
|
|
15918
|
+
for (const candidate of qmdResultPathCandidates(
|
|
15919
|
+
ownerStorage.dir,
|
|
15920
|
+
resultPath,
|
|
15921
|
+
)) {
|
|
15922
|
+
const memory = await ownerStorage.storage.readMemoryByPath(candidate);
|
|
15923
|
+
if (memory) return memory;
|
|
15924
|
+
}
|
|
15925
|
+
return null;
|
|
15926
|
+
}
|
|
15927
|
+
|
|
15928
|
+
if (!fallbackStorageDir) {
|
|
15929
|
+
return await fallbackStorage.readMemoryByPath(resultPath);
|
|
15930
|
+
}
|
|
15931
|
+
for (const candidate of qmdResultPathCandidates(
|
|
15932
|
+
fallbackStorageDir,
|
|
15933
|
+
resultPath,
|
|
15934
|
+
)) {
|
|
15935
|
+
const memory = await fallbackStorage.readMemoryByPath(candidate);
|
|
15936
|
+
if (memory) return memory;
|
|
15937
|
+
}
|
|
15938
|
+
return null;
|
|
15939
|
+
}
|
|
15940
|
+
|
|
15941
|
+
private async resolveColdQmdResultForRecall(
|
|
15942
|
+
result: QmdSearchResult,
|
|
15943
|
+
fallbackStorage: StorageManager,
|
|
15944
|
+
recallNamespaces: readonly string[] = [],
|
|
15945
|
+
): Promise<{ namespace: string; result: QmdSearchResult } | null> {
|
|
15946
|
+
const memory = await this.readQmdResultMemory(
|
|
15947
|
+
result.path,
|
|
15948
|
+
fallbackStorage,
|
|
15949
|
+
recallNamespaces,
|
|
15950
|
+
);
|
|
15951
|
+
if (!memory) return null;
|
|
15952
|
+
|
|
15953
|
+
let ownerNamespace: string | null = null;
|
|
15954
|
+
if (path.isAbsolute(memory.path)) {
|
|
15955
|
+
const ownerStorage = await this.storageForAbsoluteQmdResultPath(
|
|
15956
|
+
memory.path,
|
|
15957
|
+
fallbackStorage,
|
|
15958
|
+
recallNamespaces,
|
|
15959
|
+
);
|
|
15960
|
+
ownerNamespace = ownerStorage?.namespace ?? null;
|
|
15961
|
+
if (!ownerNamespace && this.config.namespacesEnabled) return null;
|
|
15962
|
+
}
|
|
15963
|
+
ownerNamespace ??= this.namespaceFromPath(memory.path);
|
|
15964
|
+
if (
|
|
15965
|
+
recallNamespaces.length > 0 &&
|
|
15966
|
+
!recallNamespaces.includes(ownerNamespace)
|
|
15967
|
+
) {
|
|
15968
|
+
return null;
|
|
15969
|
+
}
|
|
15970
|
+
|
|
15971
|
+
return {
|
|
15972
|
+
namespace: ownerNamespace,
|
|
15973
|
+
result: {
|
|
15974
|
+
...result,
|
|
15975
|
+
docid: result.docid || memory.frontmatter.id,
|
|
15976
|
+
path: memory.path,
|
|
15977
|
+
snippet: result.snippet || memory.content.slice(0, 400),
|
|
15978
|
+
},
|
|
15979
|
+
};
|
|
15980
|
+
}
|
|
15981
|
+
|
|
15982
|
+
private async storageForAbsoluteQmdResultPath(
|
|
15983
|
+
resultPath: string,
|
|
15984
|
+
fallbackStorage: StorageManager,
|
|
15985
|
+
recallNamespaces: readonly string[] = [],
|
|
15986
|
+
): Promise<{ storage: StorageManager; dir: string; namespace: string } | null> {
|
|
15987
|
+
const resolvedPath = path.resolve(resultPath);
|
|
15988
|
+
const memoryRoot = path.resolve(this.config.memoryDir);
|
|
15989
|
+
const namespacesRoot = path.join(memoryRoot, "namespaces");
|
|
15990
|
+
const fallbackStorageDir =
|
|
15991
|
+
typeof (fallbackStorage as { dir?: unknown }).dir === "string" &&
|
|
15992
|
+
(fallbackStorage as { dir?: string }).dir
|
|
15993
|
+
? (fallbackStorage as { dir: string }).dir
|
|
15994
|
+
: null;
|
|
15995
|
+
const matches: Array<{ storage: StorageManager; dir: string; namespace: string }> = [];
|
|
15996
|
+
const seenDirs = new Set<string>();
|
|
15997
|
+
|
|
15998
|
+
const maybeAddStorage = (storage: StorageManager, namespace: string) => {
|
|
15999
|
+
const storageDir =
|
|
16000
|
+
typeof (storage as { dir?: unknown }).dir === "string" &&
|
|
16001
|
+
(storage as { dir?: string }).dir
|
|
16002
|
+
? (storage as { dir: string }).dir
|
|
16003
|
+
: null;
|
|
16004
|
+
if (!storageDir) return;
|
|
16005
|
+
const candidateRoot = path.resolve(storageDir);
|
|
16006
|
+
if (seenDirs.has(candidateRoot)) return;
|
|
16007
|
+
if (!isPathInsideStorageRoot(candidateRoot, resolvedPath)) return;
|
|
16008
|
+
if (
|
|
16009
|
+
candidateRoot === memoryRoot &&
|
|
16010
|
+
isPathInsideStorageRoot(namespacesRoot, resolvedPath)
|
|
16011
|
+
) {
|
|
16012
|
+
return;
|
|
16013
|
+
}
|
|
16014
|
+
seenDirs.add(candidateRoot);
|
|
16015
|
+
matches.push({ storage, dir: candidateRoot, namespace });
|
|
16016
|
+
};
|
|
16017
|
+
|
|
16018
|
+
const fallbackNamespace =
|
|
16019
|
+
fallbackStorageDir !== null
|
|
16020
|
+
? this.namespaceFromStorageDir(fallbackStorageDir)
|
|
16021
|
+
: this.config.defaultNamespace;
|
|
16022
|
+
maybeAddStorage(fallbackStorage, fallbackNamespace);
|
|
16023
|
+
|
|
16024
|
+
const candidateNamespaces = new Set<string>();
|
|
16025
|
+
candidateNamespaces.add(this.config.defaultNamespace);
|
|
16026
|
+
candidateNamespaces.add(this.config.sharedNamespace);
|
|
16027
|
+
for (const ns of recallNamespaces) {
|
|
16028
|
+
candidateNamespaces.add(ns);
|
|
16029
|
+
}
|
|
16030
|
+
if (isPathInsideStorageRoot(namespacesRoot, resolvedPath)) {
|
|
16031
|
+
const relativeToNamespaces = path.relative(namespacesRoot, resolvedPath);
|
|
16032
|
+
const [namespaceSegment] = relativeToNamespaces.split(/[\\/]/);
|
|
16033
|
+
if (namespaceSegment) {
|
|
16034
|
+
candidateNamespaces.add(
|
|
16035
|
+
namespaceIdentityFromToken(namespaceSegment) ?? namespaceSegment,
|
|
16036
|
+
);
|
|
16037
|
+
}
|
|
16038
|
+
}
|
|
16039
|
+
for (const policy of this.config.namespacePolicies ?? []) {
|
|
16040
|
+
candidateNamespaces.add(policy.name);
|
|
16041
|
+
}
|
|
16042
|
+
|
|
16043
|
+
for (const ns of candidateNamespaces) {
|
|
16044
|
+
if (!ns) continue;
|
|
16045
|
+
try {
|
|
16046
|
+
maybeAddStorage(await this.storageRouter.storageFor(ns), ns);
|
|
16047
|
+
} catch {
|
|
16048
|
+
continue;
|
|
16049
|
+
}
|
|
16050
|
+
}
|
|
16051
|
+
|
|
16052
|
+
matches.sort((a, b) => b.dir.length - a.dir.length);
|
|
16053
|
+
return matches[0] ?? null;
|
|
16054
|
+
}
|
|
16055
|
+
|
|
16056
|
+
private async applyMemoryWorthRerank(
|
|
16057
|
+
results: QmdSearchResult[],
|
|
16058
|
+
namespaces: string[],
|
|
16059
|
+
): Promise<QmdSearchResult[]> {
|
|
16060
|
+
// Build the counter lookup. We union frontmatter counters across every
|
|
16061
|
+
// namespace the recall spans — the recall path itself already
|
|
16062
|
+
// aggregates candidates from multiple namespaces, so we must do the
|
|
16063
|
+
// same when looking up counters. Per-namespace results are cached with
|
|
16064
|
+
// a short TTL so interactive recall doesn't trigger a full
|
|
16065
|
+
// `readAllMemories` scan per query (addresses codex P2 on PR 4).
|
|
16066
|
+
const counters = new Map<string, MemoryWorthCounters>();
|
|
16067
|
+
const seenNamespaces = new Set<string>();
|
|
16068
|
+
const nowMs = Date.now();
|
|
16069
|
+
|
|
16070
|
+
// Evict all expired entries on every call so long-running processes
|
|
16071
|
+
// touching a high-cardinality namespace set (coding/project overlays,
|
|
16072
|
+
// per-branch) don't grow the cache unboundedly. Without this, an entry
|
|
16073
|
+
// for a namespace that's never looked up again would pin its full
|
|
16074
|
+
// counter map forever.
|
|
16075
|
+
for (const [key, entry] of this.memoryWorthCounterCache) {
|
|
16076
|
+
if (nowMs - entry.at >= Orchestrator.MEMORY_WORTH_CACHE_TTL_MS) {
|
|
16077
|
+
this.memoryWorthCounterCache.delete(key);
|
|
16078
|
+
}
|
|
16079
|
+
}
|
|
16080
|
+
|
|
16081
|
+
for (const ns of namespaces) {
|
|
16082
|
+
if (seenNamespaces.has(ns)) continue;
|
|
16083
|
+
seenNamespaces.add(ns);
|
|
16084
|
+
try {
|
|
16085
|
+
const cached = this.memoryWorthCounterCache.get(ns);
|
|
15515
16086
|
let nsMap: ReadonlyMap<string, MemoryWorthCounters> | undefined;
|
|
15516
16087
|
if (
|
|
15517
16088
|
cached &&
|
|
@@ -15557,7 +16128,7 @@ export class Orchestrator {
|
|
|
15557
16128
|
if (reader) {
|
|
15558
16129
|
for (const r of missing) {
|
|
15559
16130
|
try {
|
|
15560
|
-
const memory = await
|
|
16131
|
+
const memory = await this.readQmdResultMemory(r.path, reader, namespaces);
|
|
15561
16132
|
if (!memory) continue;
|
|
15562
16133
|
const fm = memory.frontmatter;
|
|
15563
16134
|
if (fm.mw_success === undefined && fm.mw_fail === undefined) continue;
|
|
@@ -15973,6 +16544,7 @@ export class Orchestrator {
|
|
|
15973
16544
|
* Unset by default so existing call sites are unaffected.
|
|
15974
16545
|
*/
|
|
15975
16546
|
xrayPoolSizeSink?: { size: number };
|
|
16547
|
+
deadlineAtMs?: number | null;
|
|
15976
16548
|
/** Issue #681 — when true, bypass graphTraversalConfidenceFloor. */
|
|
15977
16549
|
includeLowConfidence?: boolean;
|
|
15978
16550
|
}): Promise<QmdSearchResult[]> {
|
|
@@ -15980,6 +16552,50 @@ export class Orchestrator {
|
|
|
15980
16552
|
if (options.xrayPoolSizeSink) options.xrayPoolSizeSink.size = 0;
|
|
15981
16553
|
return [];
|
|
15982
16554
|
}
|
|
16555
|
+
const deadlineRemainingMs = (): number | null =>
|
|
16556
|
+
typeof options.deadlineAtMs === "number"
|
|
16557
|
+
? Math.max(0, options.deadlineAtMs - Date.now())
|
|
16558
|
+
: null;
|
|
16559
|
+
const runColdStepWithinDeadline = async <T>(
|
|
16560
|
+
label: string,
|
|
16561
|
+
fallback: T,
|
|
16562
|
+
task: () => Promise<T>,
|
|
16563
|
+
): Promise<T> => {
|
|
16564
|
+
throwIfRecallAborted(options.abortSignal);
|
|
16565
|
+
const remainingMs = deadlineRemainingMs();
|
|
16566
|
+
if (remainingMs === 0) {
|
|
16567
|
+
log.debug(`cold-tier recall ${label} skipped: shared assembly deadline expired`);
|
|
16568
|
+
return fallback;
|
|
16569
|
+
}
|
|
16570
|
+
if (remainingMs === null) return task();
|
|
16571
|
+
|
|
16572
|
+
let timeoutHandle: NodeJS.Timeout | undefined;
|
|
16573
|
+
let timedOut = false;
|
|
16574
|
+
const taskPromise = task().catch((err) => {
|
|
16575
|
+
if (timedOut) {
|
|
16576
|
+
log.debug(`cold-tier recall ${label} failed after deadline: ${err}`);
|
|
16577
|
+
return fallback;
|
|
16578
|
+
}
|
|
16579
|
+
throw err;
|
|
16580
|
+
});
|
|
16581
|
+
|
|
16582
|
+
try {
|
|
16583
|
+
return await Promise.race<T>([
|
|
16584
|
+
taskPromise,
|
|
16585
|
+
new Promise<T>((resolve) => {
|
|
16586
|
+
timeoutHandle = setTimeout(() => {
|
|
16587
|
+
timedOut = true;
|
|
16588
|
+
log.debug(
|
|
16589
|
+
`cold-tier recall ${label} skipped: shared assembly deadline expired`,
|
|
16590
|
+
);
|
|
16591
|
+
resolve(fallback);
|
|
16592
|
+
}, remainingMs);
|
|
16593
|
+
}),
|
|
16594
|
+
]);
|
|
16595
|
+
} finally {
|
|
16596
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
16597
|
+
}
|
|
16598
|
+
};
|
|
15983
16599
|
|
|
15984
16600
|
const coldQmdEnabled = this.config.qmdColdTierEnabled === true;
|
|
15985
16601
|
const coldCollection =
|
|
@@ -15999,19 +16615,24 @@ export class Orchestrator {
|
|
|
15999
16615
|
false,
|
|
16000
16616
|
0,
|
|
16001
16617
|
);
|
|
16002
|
-
longTerm = await
|
|
16003
|
-
|
|
16004
|
-
|
|
16005
|
-
|
|
16006
|
-
|
|
16007
|
-
|
|
16008
|
-
|
|
16009
|
-
|
|
16010
|
-
|
|
16011
|
-
|
|
16012
|
-
|
|
16013
|
-
|
|
16014
|
-
|
|
16618
|
+
longTerm = await runColdStepWithinDeadline(
|
|
16619
|
+
"qmd lookup",
|
|
16620
|
+
[],
|
|
16621
|
+
() =>
|
|
16622
|
+
this.fetchQmdMemoryResultsWithArtifactTopUp(
|
|
16623
|
+
options.prompt,
|
|
16624
|
+
coldFetchLimit,
|
|
16625
|
+
coldHybridLimit,
|
|
16626
|
+
{
|
|
16627
|
+
namespacesEnabled: this.config.namespacesEnabled,
|
|
16628
|
+
recallNamespaces: options.recallNamespaces,
|
|
16629
|
+
resolveNamespace: (p) => this.namespaceFromPath(p),
|
|
16630
|
+
collection: coldCollection,
|
|
16631
|
+
queryAwarePrefilter: options.queryAwarePrefilter,
|
|
16632
|
+
searchOptions: this.buildConfiguredQmdSearchOptions(options.prompt),
|
|
16633
|
+
abortSignal: options.abortSignal,
|
|
16634
|
+
},
|
|
16635
|
+
),
|
|
16015
16636
|
);
|
|
16016
16637
|
if (longTerm.length > 0) {
|
|
16017
16638
|
log.debug(
|
|
@@ -16021,12 +16642,17 @@ export class Orchestrator {
|
|
|
16021
16642
|
}
|
|
16022
16643
|
}
|
|
16023
16644
|
if (longTerm.length === 0) {
|
|
16024
|
-
longTerm = await
|
|
16025
|
-
|
|
16026
|
-
|
|
16027
|
-
|
|
16028
|
-
|
|
16029
|
-
|
|
16645
|
+
longTerm = await runColdStepWithinDeadline(
|
|
16646
|
+
"archive scan",
|
|
16647
|
+
[],
|
|
16648
|
+
() =>
|
|
16649
|
+
this.searchLongTermArchiveFallback(
|
|
16650
|
+
options.prompt,
|
|
16651
|
+
options.recallNamespaces,
|
|
16652
|
+
options.recallResultLimit,
|
|
16653
|
+
options.queryAwarePrefilter,
|
|
16654
|
+
options.abortSignal,
|
|
16655
|
+
),
|
|
16030
16656
|
);
|
|
16031
16657
|
if (longTerm.length > 0) {
|
|
16032
16658
|
log.debug("cold-tier recall source=archive-scan");
|
|
@@ -16036,9 +16662,57 @@ export class Orchestrator {
|
|
|
16036
16662
|
|
|
16037
16663
|
let results = longTerm;
|
|
16038
16664
|
if (this.config.namespacesEnabled) {
|
|
16039
|
-
|
|
16040
|
-
|
|
16041
|
-
)
|
|
16665
|
+
const recallRoots: string[] = [];
|
|
16666
|
+
const seenRecallRoots = new Set<string>();
|
|
16667
|
+
for (const namespace of options.recallNamespaces) {
|
|
16668
|
+
try {
|
|
16669
|
+
const storage = await this.storageRouter.storageFor(namespace);
|
|
16670
|
+
const storageDir =
|
|
16671
|
+
typeof (storage as { dir?: unknown }).dir === "string" &&
|
|
16672
|
+
(storage as { dir?: string }).dir
|
|
16673
|
+
? (storage as { dir: string }).dir
|
|
16674
|
+
: null;
|
|
16675
|
+
if (!storageDir) continue;
|
|
16676
|
+
const recallRoot = path.resolve(storageDir);
|
|
16677
|
+
if (seenRecallRoots.has(recallRoot)) continue;
|
|
16678
|
+
seenRecallRoots.add(recallRoot);
|
|
16679
|
+
recallRoots.push(recallRoot);
|
|
16680
|
+
} catch (err) {
|
|
16681
|
+
log.debug("cold-tier recall namespace root lookup skipped", {
|
|
16682
|
+
namespace,
|
|
16683
|
+
error: (err as Error).message,
|
|
16684
|
+
});
|
|
16685
|
+
}
|
|
16686
|
+
}
|
|
16687
|
+
const scopedResults: QmdSearchResult[] = [];
|
|
16688
|
+
for (const result of results) {
|
|
16689
|
+
if (options.abortSignal?.aborted || deadlineRemainingMs() === 0) break;
|
|
16690
|
+
const parts = qmdCollectionPathParts(result.path);
|
|
16691
|
+
if (parts?.collection === coldCollection) {
|
|
16692
|
+
const resolvedCold = await this.resolveColdQmdResultForRecall(
|
|
16693
|
+
result,
|
|
16694
|
+
this.storage,
|
|
16695
|
+
options.recallNamespaces,
|
|
16696
|
+
);
|
|
16697
|
+
if (resolvedCold) scopedResults.push(resolvedCold.result);
|
|
16698
|
+
continue;
|
|
16699
|
+
}
|
|
16700
|
+
if (path.isAbsolute(result.path)) {
|
|
16701
|
+
const resolvedPath = path.resolve(result.path);
|
|
16702
|
+
if (
|
|
16703
|
+
recallRoots.some((recallRoot) =>
|
|
16704
|
+
isPathInsideStorageRoot(recallRoot, resolvedPath),
|
|
16705
|
+
)
|
|
16706
|
+
) {
|
|
16707
|
+
scopedResults.push(result);
|
|
16708
|
+
}
|
|
16709
|
+
continue;
|
|
16710
|
+
}
|
|
16711
|
+
if (options.recallNamespaces.includes(this.namespaceFromPath(result.path))) {
|
|
16712
|
+
scopedResults.push(result);
|
|
16713
|
+
}
|
|
16714
|
+
}
|
|
16715
|
+
results = scopedResults;
|
|
16042
16716
|
}
|
|
16043
16717
|
// Artifact isolation contract: generic recall paths must exclude artifacts.
|
|
16044
16718
|
results = results.filter((r) => !isArtifactMemoryPath(r.path));
|
|
@@ -16059,18 +16733,75 @@ export class Orchestrator {
|
|
|
16059
16733
|
memoryResults: results,
|
|
16060
16734
|
recallNamespaces: options.recallNamespaces,
|
|
16061
16735
|
recallResultLimit: options.recallResultLimit,
|
|
16736
|
+
deadlineAtMs: options.deadlineAtMs,
|
|
16062
16737
|
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
16063
16738
|
});
|
|
16064
16739
|
results = merged;
|
|
16065
16740
|
}
|
|
16066
16741
|
|
|
16067
|
-
|
|
16742
|
+
const boostInput = await this.filterSearchResultsForRecall(
|
|
16068
16743
|
results,
|
|
16069
|
-
options.recallNamespaces,
|
|
16070
|
-
options.prompt,
|
|
16071
16744
|
undefined,
|
|
16072
|
-
{
|
|
16745
|
+
{
|
|
16746
|
+
allowLifecycleFiltered: true,
|
|
16747
|
+
asOfMs: options.asOfMs,
|
|
16748
|
+
deadlineAtMs: options.deadlineAtMs,
|
|
16749
|
+
abortSignal: options.abortSignal,
|
|
16750
|
+
dropUnresolved: true,
|
|
16751
|
+
recallNamespaces: options.recallNamespaces,
|
|
16752
|
+
},
|
|
16073
16753
|
);
|
|
16754
|
+
results = boostInput.results;
|
|
16755
|
+
const boostTimeoutMs =
|
|
16756
|
+
typeof options.deadlineAtMs === "number"
|
|
16757
|
+
? Math.max(0, options.deadlineAtMs - Date.now())
|
|
16758
|
+
: null;
|
|
16759
|
+
if (boostTimeoutMs !== 0) {
|
|
16760
|
+
let timeoutHandle: NodeJS.Timeout | undefined;
|
|
16761
|
+
try {
|
|
16762
|
+
const boosted = await (boostTimeoutMs !== null
|
|
16763
|
+
? Promise.race<QmdSearchResult[] | { status: "timed_out" }>([
|
|
16764
|
+
this.boostSearchResults(
|
|
16765
|
+
boostInput.results,
|
|
16766
|
+
options.recallNamespaces,
|
|
16767
|
+
options.prompt,
|
|
16768
|
+
boostInput.memoryByPath,
|
|
16769
|
+
{ allowLifecycleFiltered: true, asOfMs: options.asOfMs },
|
|
16770
|
+
),
|
|
16771
|
+
new Promise<{ status: "timed_out" }>((resolve) => {
|
|
16772
|
+
timeoutHandle = setTimeout(
|
|
16773
|
+
() => resolve({ status: "timed_out" }),
|
|
16774
|
+
boostTimeoutMs,
|
|
16775
|
+
);
|
|
16776
|
+
}),
|
|
16777
|
+
])
|
|
16778
|
+
: this.boostSearchResults(
|
|
16779
|
+
boostInput.results,
|
|
16780
|
+
options.recallNamespaces,
|
|
16781
|
+
options.prompt,
|
|
16782
|
+
boostInput.memoryByPath,
|
|
16783
|
+
{ allowLifecycleFiltered: true, asOfMs: options.asOfMs },
|
|
16784
|
+
));
|
|
16785
|
+
if (
|
|
16786
|
+
typeof boosted === "object" &&
|
|
16787
|
+
boosted !== null &&
|
|
16788
|
+
"status" in boosted &&
|
|
16789
|
+
boosted.status === "timed_out"
|
|
16790
|
+
) {
|
|
16791
|
+
log.debug("cold-tier recall boost skipped: shared assembly deadline expired");
|
|
16792
|
+
} else if (Array.isArray(boosted)) {
|
|
16793
|
+
results = boosted;
|
|
16794
|
+
} else {
|
|
16795
|
+
results = boostInput.results;
|
|
16796
|
+
}
|
|
16797
|
+
} catch (err) {
|
|
16798
|
+
log.debug(`cold-tier recall boost failed open: ${err}`);
|
|
16799
|
+
} finally {
|
|
16800
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
16801
|
+
}
|
|
16802
|
+
} else {
|
|
16803
|
+
log.debug("cold-tier recall boost skipped: shared assembly deadline already expired");
|
|
16804
|
+
}
|
|
16074
16805
|
|
|
16075
16806
|
if (this.config.rerankEnabled && this.config.rerankProvider === "local") {
|
|
16076
16807
|
const ranked = await rerankLocalOrNoop({
|
|
@@ -16218,104 +16949,133 @@ export class Orchestrator {
|
|
|
16218
16949
|
log.debug(`flushed ${entries.length} access tracking entries`);
|
|
16219
16950
|
}
|
|
16220
16951
|
|
|
16221
|
-
|
|
16222
|
-
* Apply recency, access count, and importance boosting to QMD search results.
|
|
16223
|
-
* Returns re-ranked results.
|
|
16224
|
-
*/
|
|
16225
|
-
private async boostSearchResults(
|
|
16952
|
+
private async loadSearchResultMemoryMap(
|
|
16226
16953
|
results: QmdSearchResult[],
|
|
16227
|
-
_recallNamespaces: string[],
|
|
16228
|
-
prompt?: string,
|
|
16229
16954
|
preloadedMemoryMap?: Map<string, MemoryFile>,
|
|
16230
16955
|
options?: {
|
|
16231
|
-
|
|
16232
|
-
|
|
16233
|
-
|
|
16234
|
-
* Historical recall point in ms-since-epoch (issue #680). When
|
|
16235
|
-
* set, drops candidates that were not authoritative at this
|
|
16236
|
-
* instant per `temporal-validity.isValidAsOf`. Caller is
|
|
16237
|
-
* responsible for parsing/validating the user-supplied ISO
|
|
16238
|
-
* string at the input boundary (CLI / HTTP / MCP).
|
|
16239
|
-
*/
|
|
16240
|
-
asOfMs?: number;
|
|
16956
|
+
deadlineAtMs?: number | null;
|
|
16957
|
+
abortSignal?: AbortSignal;
|
|
16958
|
+
recallNamespaces?: readonly string[];
|
|
16241
16959
|
},
|
|
16242
|
-
): Promise<
|
|
16243
|
-
|
|
16244
|
-
|
|
16245
|
-
|
|
16246
|
-
|
|
16247
|
-
|
|
16960
|
+
): Promise<{
|
|
16961
|
+
memoryByPath: Map<string, MemoryFile>;
|
|
16962
|
+
checkedPaths: Set<string>;
|
|
16963
|
+
unreadablePaths: Set<string>;
|
|
16964
|
+
completed: boolean;
|
|
16965
|
+
}> {
|
|
16248
16966
|
const memoryByPath: Map<string, MemoryFile> = preloadedMemoryMap
|
|
16249
16967
|
? new Map(preloadedMemoryMap)
|
|
16250
16968
|
: new Map();
|
|
16969
|
+
const checkedPaths = new Set<string>();
|
|
16970
|
+
const unreadablePaths = new Set<string>();
|
|
16251
16971
|
|
|
16252
|
-
|
|
16253
|
-
|
|
16254
|
-
|
|
16255
|
-
)
|
|
16256
|
-
|
|
16257
|
-
|
|
16258
|
-
|
|
16259
|
-
|
|
16260
|
-
|
|
16972
|
+
const markChecked = (result: QmdSearchResult): void => {
|
|
16973
|
+
if (result.path) checkedPaths.add(result.path);
|
|
16974
|
+
};
|
|
16975
|
+
const markUnreadable = (result: QmdSearchResult, err: unknown): void => {
|
|
16976
|
+
if (!result.path) return;
|
|
16977
|
+
checkedPaths.add(result.path);
|
|
16978
|
+
unreadablePaths.add(result.path);
|
|
16979
|
+
log.warn("recall safety filter dropped unreadable secure-store candidate", {
|
|
16980
|
+
path: result.path,
|
|
16981
|
+
error: (err as Error).message,
|
|
16982
|
+
});
|
|
16983
|
+
};
|
|
16984
|
+
const deadlineExpired = (): boolean =>
|
|
16985
|
+
typeof options?.deadlineAtMs === "number" &&
|
|
16986
|
+
Date.now() >= options.deadlineAtMs;
|
|
16987
|
+
|
|
16988
|
+
if (options?.deadlineAtMs == null) {
|
|
16989
|
+
const batchSize = options?.abortSignal ? 16 : results.length;
|
|
16990
|
+
for (let offset = 0; offset < results.length; offset += batchSize) {
|
|
16991
|
+
if (options?.abortSignal?.aborted) {
|
|
16992
|
+
return {
|
|
16993
|
+
memoryByPath,
|
|
16994
|
+
checkedPaths,
|
|
16995
|
+
unreadablePaths,
|
|
16996
|
+
completed: false,
|
|
16997
|
+
};
|
|
16998
|
+
}
|
|
16999
|
+
await Promise.all(
|
|
17000
|
+
results.slice(offset, offset + batchSize).map(async (r) => {
|
|
17001
|
+
if (!r.path) return;
|
|
17002
|
+
if (memoryByPath.has(r.path)) {
|
|
17003
|
+
markChecked(r);
|
|
17004
|
+
return;
|
|
17005
|
+
}
|
|
17006
|
+
try {
|
|
17007
|
+
const mem = await this.readQmdResultMemory(
|
|
17008
|
+
r.path,
|
|
17009
|
+
this.storage,
|
|
17010
|
+
options?.recallNamespaces,
|
|
17011
|
+
);
|
|
17012
|
+
markChecked(r);
|
|
17013
|
+
if (mem) memoryByPath.set(r.path, mem);
|
|
17014
|
+
} catch (err) {
|
|
17015
|
+
if (err instanceof SecureStoreLockedError) {
|
|
17016
|
+
markUnreadable(r, err);
|
|
17017
|
+
return;
|
|
17018
|
+
}
|
|
17019
|
+
throw err;
|
|
17020
|
+
}
|
|
17021
|
+
}),
|
|
17022
|
+
);
|
|
16261
17023
|
}
|
|
16262
|
-
promptTags = extractTagsFromPrompt(prompt);
|
|
16263
|
-
}
|
|
16264
17024
|
|
|
16265
|
-
|
|
16266
|
-
|
|
16267
|
-
Promise.all(
|
|
16268
|
-
results.map(async (r) => {
|
|
16269
|
-
if (!r.path || memoryByPath.has(r.path)) return;
|
|
16270
|
-
const mem = await this.storage.readMemoryByPath(r.path);
|
|
16271
|
-
if (mem) memoryByPath.set(r.path, mem);
|
|
16272
|
-
}),
|
|
16273
|
-
),
|
|
16274
|
-
temporalFromDate !== null
|
|
16275
|
-
? queryByDateRangeAsync(this.config.memoryDir, temporalFromDate)
|
|
16276
|
-
: Promise.resolve<Set<string> | null>(null),
|
|
16277
|
-
promptTags.length > 0
|
|
16278
|
-
? queryByTagsAsync(this.config.memoryDir, promptTags)
|
|
16279
|
-
: Promise.resolve<Set<string> | null>(null),
|
|
16280
|
-
]);
|
|
16281
|
-
|
|
16282
|
-
const queryIntent =
|
|
16283
|
-
this.config.intentRoutingEnabled && prompt
|
|
16284
|
-
? inferIntentFromText(prompt)
|
|
16285
|
-
: null;
|
|
17025
|
+
return { memoryByPath, checkedPaths, unreadablePaths, completed: true };
|
|
17026
|
+
}
|
|
16286
17027
|
|
|
16287
|
-
|
|
16288
|
-
|
|
16289
|
-
|
|
16290
|
-
|
|
16291
|
-
|
|
16292
|
-
const maxCandidates = this.config.queryAwareIndexingMaxCandidates;
|
|
16293
|
-
const capSet = (s: Set<string> | null): Set<string> | null => {
|
|
16294
|
-
if (!s) return null;
|
|
16295
|
-
// Intersect with result paths first so out-of-scope paths don't exhaust the budget
|
|
16296
|
-
const scoped = new Set(Array.from(s).filter((p) => resultPaths.has(p)));
|
|
16297
|
-
if (maxCandidates === 0 || scoped.size <= maxCandidates)
|
|
16298
|
-
return scoped;
|
|
16299
|
-
return new Set(Array.from(scoped).slice(0, maxCandidates));
|
|
16300
|
-
};
|
|
16301
|
-
if (temporalFromDate !== null) {
|
|
16302
|
-
temporalCandidates = capSet(rawTemporal);
|
|
17028
|
+
for (const result of results) {
|
|
17029
|
+
if (!result.path) continue;
|
|
17030
|
+
if (memoryByPath.has(result.path)) {
|
|
17031
|
+
markChecked(result);
|
|
17032
|
+
continue;
|
|
16303
17033
|
}
|
|
16304
|
-
if (
|
|
16305
|
-
|
|
17034
|
+
if (options?.abortSignal?.aborted || deadlineExpired()) {
|
|
17035
|
+
return { memoryByPath, checkedPaths, unreadablePaths, completed: false };
|
|
17036
|
+
}
|
|
17037
|
+
try {
|
|
17038
|
+
const mem = await this.readQmdResultMemory(
|
|
17039
|
+
result.path,
|
|
17040
|
+
this.storage,
|
|
17041
|
+
options?.recallNamespaces,
|
|
17042
|
+
);
|
|
17043
|
+
markChecked(result);
|
|
17044
|
+
if (mem) memoryByPath.set(result.path, mem);
|
|
17045
|
+
} catch (err) {
|
|
17046
|
+
if (err instanceof SecureStoreLockedError) {
|
|
17047
|
+
markUnreadable(result, err);
|
|
17048
|
+
continue;
|
|
17049
|
+
}
|
|
17050
|
+
throw err;
|
|
16306
17051
|
}
|
|
16307
17052
|
}
|
|
16308
17053
|
|
|
17054
|
+
return { memoryByPath, checkedPaths, unreadablePaths, completed: true };
|
|
17055
|
+
}
|
|
17056
|
+
|
|
17057
|
+
private filterSearchResultsByRecallSafety(
|
|
17058
|
+
results: QmdSearchResult[],
|
|
17059
|
+
memoryByPath: Map<string, MemoryFile>,
|
|
17060
|
+
options?: {
|
|
17061
|
+
allowLifecycleFiltered?: boolean;
|
|
17062
|
+
allowDedicatedSurface?: boolean;
|
|
17063
|
+
asOfMs?: number;
|
|
17064
|
+
blockedPaths?: Set<string>;
|
|
17065
|
+
},
|
|
17066
|
+
): QmdSearchResult[] {
|
|
16309
17067
|
let lifecycleFilteredCount = 0;
|
|
16310
17068
|
let temporalSupersededFilteredCount = 0;
|
|
16311
17069
|
let dedicatedSurfaceFilteredCount = 0;
|
|
16312
17070
|
let forgottenFilteredCount = 0;
|
|
16313
|
-
|
|
16314
|
-
const
|
|
17071
|
+
let blockedPathFilteredCount = 0;
|
|
17072
|
+
const filtered: QmdSearchResult[] = [];
|
|
16315
17073
|
for (const r of results) {
|
|
17074
|
+
if (r.path && options?.blockedPaths?.has(r.path)) {
|
|
17075
|
+
blockedPathFilteredCount += 1;
|
|
17076
|
+
continue;
|
|
17077
|
+
}
|
|
16316
17078
|
const memory = memoryByPath.get(r.path);
|
|
16317
|
-
let score = r.score;
|
|
16318
|
-
|
|
16319
17079
|
if (memory) {
|
|
16320
17080
|
if (memory.frontmatter.status === "forgotten") {
|
|
16321
17081
|
forgottenFilteredCount += 1;
|
|
@@ -16377,7 +17137,178 @@ export class Orchestrator {
|
|
|
16377
17137
|
dedicatedSurfaceFilteredCount += 1;
|
|
16378
17138
|
continue;
|
|
16379
17139
|
}
|
|
17140
|
+
}
|
|
17141
|
+
filtered.push(r);
|
|
17142
|
+
}
|
|
17143
|
+
if (lifecycleFilteredCount > 0) {
|
|
17144
|
+
log.debug(
|
|
17145
|
+
`lifecycle retrieval filter removed ${lifecycleFilteredCount} stale/archived candidates`,
|
|
17146
|
+
);
|
|
17147
|
+
}
|
|
17148
|
+
if (temporalSupersededFilteredCount > 0) {
|
|
17149
|
+
log.debug(
|
|
17150
|
+
`temporal supersession filter removed ${temporalSupersededFilteredCount} superseded candidates`,
|
|
17151
|
+
);
|
|
17152
|
+
}
|
|
17153
|
+
if (dedicatedSurfaceFilteredCount > 0) {
|
|
17154
|
+
log.debug(
|
|
17155
|
+
`dedicated surface filter removed ${dedicatedSurfaceFilteredCount} dream/procedural candidates from generic recall`,
|
|
17156
|
+
);
|
|
17157
|
+
}
|
|
17158
|
+
if (forgottenFilteredCount > 0) {
|
|
17159
|
+
log.debug(
|
|
17160
|
+
`forgotten status filter removed ${forgottenFilteredCount} candidates from recall`,
|
|
17161
|
+
);
|
|
17162
|
+
}
|
|
17163
|
+
if (blockedPathFilteredCount > 0) {
|
|
17164
|
+
log.debug(
|
|
17165
|
+
`unreadable-path filter removed ${blockedPathFilteredCount} candidates from recall`,
|
|
17166
|
+
);
|
|
17167
|
+
}
|
|
17168
|
+
return filtered;
|
|
17169
|
+
}
|
|
17170
|
+
|
|
17171
|
+
private async filterSearchResultsForRecall(
|
|
17172
|
+
results: QmdSearchResult[],
|
|
17173
|
+
preloadedMemoryMap?: Map<string, MemoryFile>,
|
|
17174
|
+
options?: {
|
|
17175
|
+
allowLifecycleFiltered?: boolean;
|
|
17176
|
+
allowDedicatedSurface?: boolean;
|
|
17177
|
+
asOfMs?: number;
|
|
17178
|
+
deadlineAtMs?: number | null;
|
|
17179
|
+
abortSignal?: AbortSignal;
|
|
17180
|
+
dropUnresolved?: boolean;
|
|
17181
|
+
recallNamespaces?: readonly string[];
|
|
17182
|
+
},
|
|
17183
|
+
): Promise<{ results: QmdSearchResult[]; memoryByPath: Map<string, MemoryFile> }> {
|
|
17184
|
+
if (results.length === 0) {
|
|
17185
|
+
return {
|
|
17186
|
+
results,
|
|
17187
|
+
memoryByPath: preloadedMemoryMap ? new Map(preloadedMemoryMap) : new Map(),
|
|
17188
|
+
};
|
|
17189
|
+
}
|
|
17190
|
+
const loaded = await this.loadSearchResultMemoryMap(
|
|
17191
|
+
results,
|
|
17192
|
+
preloadedMemoryMap,
|
|
17193
|
+
options,
|
|
17194
|
+
);
|
|
17195
|
+
const candidateResults = loaded.completed
|
|
17196
|
+
? results
|
|
17197
|
+
: results.filter((result) => !result.path || loaded.checkedPaths.has(result.path));
|
|
17198
|
+
if (!loaded.completed) {
|
|
17199
|
+
log.debug(
|
|
17200
|
+
`recall safety filter stopped before validating all candidates (${candidateResults.length}/${results.length} eligible)`,
|
|
17201
|
+
);
|
|
17202
|
+
}
|
|
17203
|
+
const blockedPaths = new Set(loaded.unreadablePaths);
|
|
17204
|
+
if (options?.dropUnresolved === true) {
|
|
17205
|
+
for (const resultPath of loaded.checkedPaths) {
|
|
17206
|
+
if (!loaded.memoryByPath.has(resultPath)) {
|
|
17207
|
+
blockedPaths.add(resultPath);
|
|
17208
|
+
}
|
|
17209
|
+
}
|
|
17210
|
+
}
|
|
17211
|
+
return {
|
|
17212
|
+
results: this.filterSearchResultsByRecallSafety(
|
|
17213
|
+
candidateResults,
|
|
17214
|
+
loaded.memoryByPath,
|
|
17215
|
+
{ ...options, blockedPaths },
|
|
17216
|
+
),
|
|
17217
|
+
memoryByPath: loaded.memoryByPath,
|
|
17218
|
+
};
|
|
17219
|
+
}
|
|
16380
17220
|
|
|
17221
|
+
/**
|
|
17222
|
+
* Apply recency, access count, and importance boosting to QMD search results.
|
|
17223
|
+
* Returns re-ranked results.
|
|
17224
|
+
*/
|
|
17225
|
+
private async boostSearchResults(
|
|
17226
|
+
results: QmdSearchResult[],
|
|
17227
|
+
_recallNamespaces: string[],
|
|
17228
|
+
prompt?: string,
|
|
17229
|
+
preloadedMemoryMap?: Map<string, MemoryFile>,
|
|
17230
|
+
options?: {
|
|
17231
|
+
allowLifecycleFiltered?: boolean;
|
|
17232
|
+
allowDedicatedSurface?: boolean;
|
|
17233
|
+
/**
|
|
17234
|
+
* Historical recall point in ms-since-epoch (issue #680). When
|
|
17235
|
+
* set, drops candidates that were not authoritative at this
|
|
17236
|
+
* instant per `temporal-validity.isValidAsOf`. Caller is
|
|
17237
|
+
* responsible for parsing/validating the user-supplied ISO
|
|
17238
|
+
* string at the input boundary (CLI / HTTP / MCP).
|
|
17239
|
+
*/
|
|
17240
|
+
asOfMs?: number;
|
|
17241
|
+
},
|
|
17242
|
+
): Promise<QmdSearchResult[]> {
|
|
17243
|
+
if (results.length === 0) return results;
|
|
17244
|
+
|
|
17245
|
+
const safety = await this.filterSearchResultsForRecall(
|
|
17246
|
+
results,
|
|
17247
|
+
preloadedMemoryMap,
|
|
17248
|
+
{ ...options, recallNamespaces: _recallNamespaces },
|
|
17249
|
+
);
|
|
17250
|
+
const safeResults = safety.results;
|
|
17251
|
+
const memoryByPath = safety.memoryByPath;
|
|
17252
|
+
if (safeResults.length === 0) return safeResults;
|
|
17253
|
+
|
|
17254
|
+
const now = Date.now();
|
|
17255
|
+
|
|
17256
|
+
// Determine temporal/tag query params before index I/O (pure computation).
|
|
17257
|
+
const resultPaths = new Set(
|
|
17258
|
+
safeResults.map((r) => r.path).filter(Boolean) as string[],
|
|
17259
|
+
);
|
|
17260
|
+
let temporalFromDate: string | null = null;
|
|
17261
|
+
let promptTags: string[] = [];
|
|
17262
|
+
if (this.config.queryAwareIndexingEnabled && prompt) {
|
|
17263
|
+
if (isTemporalQuery(prompt)) {
|
|
17264
|
+
temporalFromDate = recencyWindowFromPrompt(prompt, now);
|
|
17265
|
+
}
|
|
17266
|
+
promptTags = extractTagsFromPrompt(prompt);
|
|
17267
|
+
}
|
|
17268
|
+
|
|
17269
|
+
const [rawTemporal, rawTags] = await Promise.all([
|
|
17270
|
+
temporalFromDate !== null
|
|
17271
|
+
? queryByDateRangeAsync(this.config.memoryDir, temporalFromDate)
|
|
17272
|
+
: Promise.resolve<Set<string> | null>(null),
|
|
17273
|
+
promptTags.length > 0
|
|
17274
|
+
? queryByTagsAsync(this.config.memoryDir, promptTags)
|
|
17275
|
+
: Promise.resolve<Set<string> | null>(null),
|
|
17276
|
+
]);
|
|
17277
|
+
|
|
17278
|
+
const queryIntent =
|
|
17279
|
+
this.config.intentRoutingEnabled && prompt
|
|
17280
|
+
? inferIntentFromText(prompt)
|
|
17281
|
+
: null;
|
|
17282
|
+
|
|
17283
|
+
// v8.1: Temporal + Tag prefilter candidate set
|
|
17284
|
+
// Scope to result paths first so cross-namespace paths don't consume the cap.
|
|
17285
|
+
let temporalCandidates: Set<string> | null = null;
|
|
17286
|
+
let tagCandidates: Set<string> | null = null;
|
|
17287
|
+
if (this.config.queryAwareIndexingEnabled && prompt) {
|
|
17288
|
+
const maxCandidates = this.config.queryAwareIndexingMaxCandidates;
|
|
17289
|
+
const capSet = (s: Set<string> | null): Set<string> | null => {
|
|
17290
|
+
if (!s) return null;
|
|
17291
|
+
// Intersect with result paths first so out-of-scope paths don't exhaust the budget
|
|
17292
|
+
const scoped = new Set(Array.from(s).filter((p) => resultPaths.has(p)));
|
|
17293
|
+
if (maxCandidates === 0 || scoped.size <= maxCandidates)
|
|
17294
|
+
return scoped;
|
|
17295
|
+
return new Set(Array.from(scoped).slice(0, maxCandidates));
|
|
17296
|
+
};
|
|
17297
|
+
if (temporalFromDate !== null) {
|
|
17298
|
+
temporalCandidates = capSet(rawTemporal);
|
|
17299
|
+
}
|
|
17300
|
+
if (promptTags.length > 0) {
|
|
17301
|
+
tagCandidates = capSet(rawTags);
|
|
17302
|
+
}
|
|
17303
|
+
}
|
|
17304
|
+
|
|
17305
|
+
const boosted: QmdSearchResult[] = [];
|
|
17306
|
+
const recencyWeight = this.effectiveRecencyWeight();
|
|
17307
|
+
for (const r of safeResults) {
|
|
17308
|
+
const memory = memoryByPath.get(r.path);
|
|
17309
|
+
let score = r.score;
|
|
17310
|
+
|
|
17311
|
+
if (memory) {
|
|
16381
17312
|
// Recency boost: exponential decay over 7 days
|
|
16382
17313
|
if (recencyWeight > 0) {
|
|
16383
17314
|
const createdAt = new Date(memory.frontmatter.created).getTime();
|
|
@@ -16522,26 +17453,6 @@ export class Orchestrator {
|
|
|
16522
17453
|
|
|
16523
17454
|
boosted.push({ ...r, score });
|
|
16524
17455
|
}
|
|
16525
|
-
if (lifecycleFilteredCount > 0) {
|
|
16526
|
-
log.debug(
|
|
16527
|
-
`lifecycle retrieval filter removed ${lifecycleFilteredCount} stale/archived candidates`,
|
|
16528
|
-
);
|
|
16529
|
-
}
|
|
16530
|
-
if (temporalSupersededFilteredCount > 0) {
|
|
16531
|
-
log.debug(
|
|
16532
|
-
`temporal supersession filter removed ${temporalSupersededFilteredCount} superseded candidates`,
|
|
16533
|
-
);
|
|
16534
|
-
}
|
|
16535
|
-
if (dedicatedSurfaceFilteredCount > 0) {
|
|
16536
|
-
log.debug(
|
|
16537
|
-
`dedicated surface filter removed ${dedicatedSurfaceFilteredCount} dream/procedural candidates from generic recall`,
|
|
16538
|
-
);
|
|
16539
|
-
}
|
|
16540
|
-
if (forgottenFilteredCount > 0) {
|
|
16541
|
-
log.debug(
|
|
16542
|
-
`forgotten status filter removed ${forgottenFilteredCount} candidates from recall`,
|
|
16543
|
-
);
|
|
16544
|
-
}
|
|
16545
17456
|
|
|
16546
17457
|
// Re-sort by boosted score
|
|
16547
17458
|
return boosted.sort((a, b) => b.score - a.score);
|
|
@@ -16769,8 +17680,52 @@ export class Orchestrator {
|
|
|
16769
17680
|
}));
|
|
16770
17681
|
}
|
|
16771
17682
|
|
|
17683
|
+
private async graphSeedPathRelativeToStorage(
|
|
17684
|
+
storage: StorageManager,
|
|
17685
|
+
result: QmdSearchResult,
|
|
17686
|
+
recallNamespaces: readonly string[] = [],
|
|
17687
|
+
): Promise<string | null> {
|
|
17688
|
+
const parts = qmdCollectionPathParts(result.path);
|
|
17689
|
+
if (parts) {
|
|
17690
|
+
const memory = await this.readQmdResultMemory(
|
|
17691
|
+
result.path,
|
|
17692
|
+
storage,
|
|
17693
|
+
recallNamespaces,
|
|
17694
|
+
);
|
|
17695
|
+
return memory
|
|
17696
|
+
? graphPathRelativeToStorage(storage.dir, memory.path)
|
|
17697
|
+
: null;
|
|
17698
|
+
}
|
|
17699
|
+
return graphPathRelativeToStorage(storage.dir, result.path);
|
|
17700
|
+
}
|
|
17701
|
+
|
|
17702
|
+
private async graphSeedPathsWithinDeadline(
|
|
17703
|
+
storage: StorageManager,
|
|
17704
|
+
results: QmdSearchResult[],
|
|
17705
|
+
deadlineAtMs: number,
|
|
17706
|
+
recallNamespaces: readonly string[] = [],
|
|
17707
|
+
): Promise<string[]> {
|
|
17708
|
+
const resolved: string[] = [];
|
|
17709
|
+
for (const result of results) {
|
|
17710
|
+
if (Date.now() >= deadlineAtMs) break;
|
|
17711
|
+
const seedPath = await this.graphSeedPathRelativeToStorage(
|
|
17712
|
+
storage,
|
|
17713
|
+
result,
|
|
17714
|
+
recallNamespaces,
|
|
17715
|
+
);
|
|
17716
|
+
if (Date.now() >= deadlineAtMs) break;
|
|
17717
|
+
if (seedPath) resolved.push(seedPath);
|
|
17718
|
+
}
|
|
17719
|
+
return resolved;
|
|
17720
|
+
}
|
|
17721
|
+
|
|
16772
17722
|
private namespaceFromPath(p: string): string {
|
|
16773
17723
|
if (!this.config.namespacesEnabled) return this.config.defaultNamespace;
|
|
17724
|
+
const parts = qmdCollectionPathParts(p);
|
|
17725
|
+
const collectionNamespace = parts
|
|
17726
|
+
? this.qmdCollectionNamespaceFromPrefix(parts.collection)
|
|
17727
|
+
: null;
|
|
17728
|
+
if (collectionNamespace) return collectionNamespace;
|
|
16774
17729
|
const m = p.match(/[\\/]+namespaces[\\/]+([^\\/]+)(?:[\\/]|$)/);
|
|
16775
17730
|
if (!m?.[1]) return this.config.defaultNamespace;
|
|
16776
17731
|
return namespaceIdentityFromToken(m[1]) ?? m[1];
|