@remnic/core 9.3.636 → 9.3.638

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/dist/access-cli.js +12 -12
  2. package/dist/access-http.d.ts +2 -2
  3. package/dist/access-http.js +8 -6
  4. package/dist/access-mcp.d.ts +2 -2
  5. package/dist/access-mcp.js +7 -5
  6. package/dist/{access-service-GVOE5U3C.d.ts → access-service-DvEvBv9_.d.ts} +2 -1
  7. package/dist/access-service.d.ts +2 -2
  8. package/dist/access-service.js +6 -4
  9. package/dist/active-recall.js +1 -1
  10. package/dist/bootstrap.d.ts +1 -1
  11. package/dist/causal-behavior.js +2 -2
  12. package/dist/causal-chain.js +2 -2
  13. package/dist/causal-consolidation.js +2 -2
  14. package/dist/causal-retrieval.js +2 -2
  15. package/dist/causal-trajectory-graph.js +1 -1
  16. package/dist/causal-trajectory.js +1 -1
  17. package/dist/{chunk-VTH5SKQW.js → chunk-2JIYTTQJ.js} +4 -4
  18. package/dist/{chunk-HZVIYZYN.js → chunk-7KRYJUTB.js} +6 -6
  19. package/dist/{chunk-7H5WCPBS.js → chunk-A3HKTPPV.js} +21 -6
  20. package/dist/chunk-A3HKTPPV.js.map +1 -0
  21. package/dist/{chunk-2IT7WFYE.js → chunk-FWIROLS6.js} +2 -1
  22. package/dist/chunk-FWIROLS6.js.map +1 -0
  23. package/dist/{chunk-JYFO32V4.js → chunk-I3YAA4BI.js} +9 -9
  24. package/dist/{chunk-A6D7A2FW.js → chunk-IHG27THY.js} +4 -4
  25. package/dist/{chunk-W25I7G6U.js → chunk-KXTLEB4L.js} +3 -3
  26. package/dist/{chunk-XM6M6DHG.js → chunk-MNDFVT4F.js} +195 -6
  27. package/dist/chunk-MNDFVT4F.js.map +1 -0
  28. package/dist/{chunk-7V2SGZ3H.js → chunk-NE566K4E.js} +2 -2
  29. package/dist/{chunk-NFNQ7TWF.js → chunk-OSPVAP56.js} +3 -3
  30. package/dist/{chunk-UXFOGILU.js → chunk-QDAWVW75.js} +1 -1
  31. package/dist/{chunk-TQOU3VAT.js → chunk-SSOMTUCA.js} +1 -1
  32. package/dist/{chunk-TFICNXXU.js → chunk-TJRV4MWX.js} +990 -252
  33. package/dist/chunk-TJRV4MWX.js.map +1 -0
  34. package/dist/{chunk-BK35NOJL.js → chunk-U2YPUWRB.js} +3 -3
  35. package/dist/{chunk-EWC6W6AB.js → chunk-UXA5L2DZ.js} +2 -2
  36. package/dist/{chunk-XL7FK7PJ.js → chunk-Y56J7CXW.js} +33 -16
  37. package/dist/{chunk-XL7FK7PJ.js.map → chunk-Y56J7CXW.js.map} +1 -1
  38. package/dist/{cli-B-fHk59D.d.ts → cli-BJBKNWIS.d.ts} +2 -2
  39. package/dist/cli.d.ts +3 -3
  40. package/dist/cli.js +13 -12
  41. package/dist/compounding/engine.js +1 -1
  42. package/dist/config.js +1 -1
  43. package/dist/explicit-capture.d.ts +1 -1
  44. package/dist/{graph-edge-decay-5ZOK7ZNO.js → graph-edge-decay-PUFNHOBS.js} +2 -2
  45. package/dist/graph-snapshot.js +2 -2
  46. package/dist/graph.d.ts +3 -0
  47. package/dist/graph.js +1 -1
  48. package/dist/index.d.ts +4 -4
  49. package/dist/index.js +17 -17
  50. package/dist/mcp-memory-inspector-app.d.ts +2 -2
  51. package/dist/operator-toolkit.js +3 -3
  52. package/dist/{orchestrator-CoqytbK_.d.ts → orchestrator-ibTTeoUa.d.ts} +9 -0
  53. package/dist/orchestrator.d.ts +1 -1
  54. package/dist/orchestrator.js +9 -9
  55. package/dist/recall-state.d.ts +2 -0
  56. package/dist/recall-state.js +1 -1
  57. package/dist/resume-bundles.js +4 -4
  58. package/dist/summarizer.js +2 -2
  59. package/dist/transcript.js +2 -2
  60. package/package.json +1 -1
  61. package/src/access-service-namespace.test.ts +740 -0
  62. package/src/access-service.ts +222 -3
  63. package/src/config.test.ts +118 -0
  64. package/src/config.ts +45 -10
  65. package/src/graph.ts +39 -17
  66. package/src/orchestrator.ts +1286 -331
  67. package/src/recall-state.ts +3 -0
  68. package/dist/chunk-2IT7WFYE.js.map +0 -1
  69. package/dist/chunk-7H5WCPBS.js.map +0 -1
  70. package/dist/chunk-TFICNXXU.js.map +0 -1
  71. package/dist/chunk-XM6M6DHG.js.map +0 -1
  72. /package/dist/{chunk-VTH5SKQW.js.map → chunk-2JIYTTQJ.js.map} +0 -0
  73. /package/dist/{chunk-HZVIYZYN.js.map → chunk-7KRYJUTB.js.map} +0 -0
  74. /package/dist/{chunk-JYFO32V4.js.map → chunk-I3YAA4BI.js.map} +0 -0
  75. /package/dist/{chunk-A6D7A2FW.js.map → chunk-IHG27THY.js.map} +0 -0
  76. /package/dist/{chunk-W25I7G6U.js.map → chunk-KXTLEB4L.js.map} +0 -0
  77. /package/dist/{chunk-7V2SGZ3H.js.map → chunk-NE566K4E.js.map} +0 -0
  78. /package/dist/{chunk-NFNQ7TWF.js.map → chunk-OSPVAP56.js.map} +0 -0
  79. /package/dist/{chunk-UXFOGILU.js.map → chunk-QDAWVW75.js.map} +0 -0
  80. /package/dist/{chunk-TQOU3VAT.js.map → chunk-SSOMTUCA.js.map} +0 -0
  81. /package/dist/{chunk-BK35NOJL.js.map → chunk-U2YPUWRB.js.map} +0 -0
  82. /package/dist/{chunk-EWC6W6AB.js.map → chunk-UXA5L2DZ.js.map} +0 -0
  83. /package/dist/{graph-edge-decay-5ZOK7ZNO.js.map → graph-edge-decay-PUFNHOBS.js.map} +0 -0
@@ -153,6 +153,7 @@ import {
153
153
  import { CODEX_THREAD_KEY_PREFIX } from "./codex-thread-key.js";
154
154
  import { isDisagreementPrompt } from "./signal.js";
155
155
  import { lintWorkspaceFiles, rotateMarkdownFileToArchive } from "./hygiene.js";
156
+ import { isPathInsideStorageRoot } from "./storage-paths.js";
156
157
  import { EmbeddingFallback } from "./embedding-fallback.js";
157
158
  import {
158
159
  decideSemanticDedup,
@@ -341,7 +342,11 @@ import {
341
342
  dedupeBehaviorSignalsByMemoryAndHash,
342
343
  } from "./behavior-signals.js";
343
344
  import { ProfilingCollector } from "./profiling.js";
344
- import { keyring, secureStoreDir } from "./secure-store/index.js";
345
+ import {
346
+ keyring,
347
+ secureStoreDir,
348
+ SecureStoreLockedError,
349
+ } from "./secure-store/index.js";
345
350
  import type {
346
351
  AccessTrackingEntry,
347
352
  BehaviorLoopPolicyState,
@@ -710,6 +715,52 @@ async function raceRecallAbort<T>(
710
715
  }
711
716
  }
712
717
 
718
+ function qmdCollectionPathParts(resultPath: string): {
719
+ collection: string;
720
+ relativePath: string;
721
+ } | null {
722
+ if (!resultPath || path.isAbsolute(resultPath)) return null;
723
+ const normalized = resultPath.replace(/\\/g, "/").replace(/^\/+/, "");
724
+ const slashIndex = normalized.indexOf("/");
725
+ if (slashIndex <= 0 || slashIndex >= normalized.length - 1) return null;
726
+ const collection = normalized.slice(0, slashIndex);
727
+ if (/^\d{4}-\d{2}-\d{2}$/.test(collection)) return null;
728
+ return {
729
+ collection,
730
+ relativePath: normalized.slice(slashIndex + 1),
731
+ };
732
+ }
733
+
734
+ function qmdResultPathCandidates(
735
+ storageDir: string,
736
+ resultPath: string,
737
+ ): string[] {
738
+ const candidates = new Set<string>();
739
+ const storageRoot = path.resolve(storageDir);
740
+ const addCandidate = (candidate: string) => {
741
+ const resolved = path.resolve(candidate);
742
+ if (isPathInsideStorageRoot(storageRoot, resolved)) {
743
+ candidates.add(resolved);
744
+ }
745
+ };
746
+ const addRelativeCandidates = (relativePath: string) => {
747
+ const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "");
748
+ if (!normalized) return;
749
+ addCandidate(path.join(storageRoot, normalized));
750
+ if (/^\d{4}-\d{2}-\d{2}\//.test(normalized)) {
751
+ addCandidate(path.join(storageRoot, "facts", normalized));
752
+ }
753
+ };
754
+
755
+ if (path.isAbsolute(resultPath)) {
756
+ addCandidate(resultPath);
757
+ } else {
758
+ addRelativeCandidates(resultPath);
759
+ }
760
+
761
+ return [...candidates];
762
+ }
763
+
713
764
  /** Maximum age (ms) before a compaction-reset signal file is considered stale and removed. */
714
765
  const COMPACTION_SIGNAL_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
715
766
  const DEFAULT_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS = 10_000;
@@ -5939,6 +5990,7 @@ export class Orchestrator {
5939
5990
  memoryResults: QmdSearchResult[];
5940
5991
  recallNamespaces: string[];
5941
5992
  recallResultLimit: number;
5993
+ deadlineAtMs?: number | null;
5942
5994
  /** Issue #681 — when true, bypass graphTraversalConfidenceFloor. */
5943
5995
  includeLowConfidence?: boolean;
5944
5996
  }): Promise<{
@@ -5947,16 +5999,90 @@ export class Orchestrator {
5947
5999
  expandedPaths: GraphRecallExpandedEntry[];
5948
6000
  seedResults: QmdSearchResult[];
5949
6001
  }> {
6002
+ const deadlineExpired = (): boolean =>
6003
+ typeof options.deadlineAtMs === "number" &&
6004
+ Date.now() >= options.deadlineAtMs;
5950
6005
  const byNamespace = new Map<string, QmdSearchResult[]>();
5951
- for (const result of options.memoryResults) {
5952
- const ns = this.namespaceFromPath(result.path);
5953
- if (!options.recallNamespaces.includes(ns)) continue;
5954
- const existing = byNamespace.get(ns);
6006
+ const addResultForNamespace = (
6007
+ namespace: string,
6008
+ result: QmdSearchResult,
6009
+ ): void => {
6010
+ const existing = byNamespace.get(namespace);
5955
6011
  if (existing) {
5956
6012
  existing.push(result);
5957
6013
  } else {
5958
- byNamespace.set(ns, [result]);
6014
+ byNamespace.set(namespace, [result]);
6015
+ }
6016
+ };
6017
+ const resolvedAmbiguousSeeds = new Map<
6018
+ string,
6019
+ { namespace: string; result: QmdSearchResult } | null
6020
+ >();
6021
+ const resolveAmbiguousSeedOwner = async (
6022
+ result: QmdSearchResult,
6023
+ parts: { collection: string; relativePath: string } | null,
6024
+ ): Promise<{ namespace: string; result: QmdSearchResult } | null> => {
6025
+ const cached = resolvedAmbiguousSeeds.get(result.path);
6026
+ if (cached !== undefined) return cached;
6027
+ if (deadlineExpired()) {
6028
+ resolvedAmbiguousSeeds.set(result.path, null);
6029
+ return null;
6030
+ }
6031
+
6032
+ let resolvedPath = result.path;
6033
+ let resolvedResult = result;
6034
+ if (parts) {
6035
+ const resolvedCold = await this.resolveColdQmdResultForRecall(
6036
+ result,
6037
+ this.storage,
6038
+ options.recallNamespaces,
6039
+ );
6040
+ if (!resolvedCold || deadlineExpired()) {
6041
+ resolvedAmbiguousSeeds.set(result.path, null);
6042
+ return null;
6043
+ }
6044
+ resolvedPath = resolvedCold.result.path;
6045
+ resolvedResult = resolvedCold.result;
6046
+ }
6047
+
6048
+ if (!path.isAbsolute(resolvedPath)) {
6049
+ resolvedAmbiguousSeeds.set(result.path, null);
6050
+ return null;
5959
6051
  }
6052
+ const ownerStorage = await this.storageForAbsoluteQmdResultPath(
6053
+ resolvedPath,
6054
+ this.storage,
6055
+ options.recallNamespaces,
6056
+ );
6057
+ const ownerNamespace = ownerStorage?.namespace ?? null;
6058
+ const resolved =
6059
+ ownerNamespace && options.recallNamespaces.includes(ownerNamespace)
6060
+ ? { namespace: ownerNamespace, result: resolvedResult }
6061
+ : null;
6062
+ resolvedAmbiguousSeeds.set(result.path, resolved);
6063
+ return resolved;
6064
+ };
6065
+ const coldCollection = this.config.qmdColdCollection ?? "openclaw-engram-cold";
6066
+ for (const result of options.memoryResults) {
6067
+ if (deadlineExpired()) break;
6068
+ const parts = qmdCollectionPathParts(result.path);
6069
+ if (parts?.collection === coldCollection) {
6070
+ const resolved = await resolveAmbiguousSeedOwner(result, parts);
6071
+ if (resolved) {
6072
+ addResultForNamespace(resolved.namespace, resolved.result);
6073
+ }
6074
+ continue;
6075
+ }
6076
+ if (path.isAbsolute(result.path)) {
6077
+ const resolved = await resolveAmbiguousSeedOwner(result, null);
6078
+ if (resolved) {
6079
+ addResultForNamespace(resolved.namespace, resolved.result);
6080
+ }
6081
+ continue;
6082
+ }
6083
+ const ns = this.namespaceFromPath(result.path);
6084
+ if (!options.recallNamespaces.includes(ns)) continue;
6085
+ addResultForNamespace(ns, result);
5960
6086
  }
5961
6087
 
5962
6088
  const perNamespaceSeedCap = Math.max(3, options.recallResultLimit);
@@ -5967,15 +6093,31 @@ export class Orchestrator {
5967
6093
  const expandedResults: QmdSearchResult[] = [];
5968
6094
 
5969
6095
  for (const [namespace, nsResults] of byNamespace.entries()) {
6096
+ if (deadlineExpired()) break;
5970
6097
  const storage = await this.storageRouter.storageFor(namespace);
5971
6098
  const seedCandidates = nsResults.slice(0, perNamespaceSeedCap);
5972
6099
  seedResults.push(...seedCandidates);
5973
- const seedRelativePaths = seedCandidates
5974
- .map((result) => graphPathRelativeToStorage(storage.dir, result.path))
5975
- .filter(
5976
- (value): value is string =>
5977
- typeof value === "string" && value.length > 0,
5978
- );
6100
+ const seedRelativePaths =
6101
+ typeof options.deadlineAtMs === "number"
6102
+ ? await this.graphSeedPathsWithinDeadline(
6103
+ storage,
6104
+ seedCandidates,
6105
+ options.deadlineAtMs,
6106
+ [namespace],
6107
+ )
6108
+ : (
6109
+ await Promise.all(
6110
+ seedCandidates.map((result) =>
6111
+ this.graphSeedPathRelativeToStorage(storage, result, [
6112
+ namespace,
6113
+ ]),
6114
+ ),
6115
+ )
6116
+ ).filter(
6117
+ (value): value is string =>
6118
+ typeof value === "string" && value.length > 0,
6119
+ );
6120
+ if (deadlineExpired()) break;
5979
6121
  if (seedRelativePaths.length === 0) continue;
5980
6122
 
5981
6123
  const seedRecallScore = seedCandidates.reduce(
@@ -5989,14 +6131,22 @@ export class Orchestrator {
5989
6131
  const expanded = await this.graphIndexFor(storage).spreadingActivation(
5990
6132
  seedRelativePaths,
5991
6133
  this.config.maxGraphTraversalSteps,
5992
- options.includeLowConfidence === true ? { includeLowConfidence: true } : undefined,
6134
+ {
6135
+ ...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
6136
+ ...(typeof options.deadlineAtMs === "number"
6137
+ ? { deadlineAtMs: options.deadlineAtMs }
6138
+ : {}),
6139
+ },
5993
6140
  );
5994
6141
  if (expanded.length === 0) continue;
6142
+ if (deadlineExpired()) break;
5995
6143
 
5996
6144
  for (const candidate of expanded.slice(0, perNamespaceExpandedCap)) {
6145
+ if (deadlineExpired()) break;
5997
6146
  if (seedSet.has(candidate.path)) continue;
5998
6147
  const memoryPath = path.resolve(storage.dir, candidate.path);
5999
6148
  const memory = await storage.readMemoryByPath(memoryPath);
6149
+ if (deadlineExpired()) break;
6000
6150
  if (!memory) continue;
6001
6151
  if (isArtifactMemoryPath(memory.path)) continue;
6002
6152
  if (memory.frontmatter.status && memory.frontmatter.status !== "active")
@@ -8737,17 +8887,18 @@ export class Orchestrator {
8737
8887
  // several seconds on large box directories due to sequential I/O). We kick it
8738
8888
  // off here so it overlaps with QMD and other concurrent work rather than
8739
8889
  // running sequentially in phase-2 and blocking assembly.
8740
- const recentBoxesPromise: Promise<BoxFrontmatter[]> =
8890
+ const recentBoxesPromise = observeEnrichmentPromise(
8741
8891
  this.isRecallSectionEnabled(
8742
8892
  "memory-boxes",
8743
8893
  this.config.memoryBoxesEnabled === true,
8744
8894
  ) &&
8745
- this.config.memoryBoxesEnabled &&
8746
- this.config.boxRecallDays > 0
8895
+ this.config.memoryBoxesEnabled &&
8896
+ this.config.boxRecallDays > 0
8747
8897
  ? this.boxBuilderFor(profileStorage)
8748
8898
  .readRecentBoxes(this.config.boxRecallDays)
8749
8899
  .catch(() => [] as BoxFrontmatter[])
8750
- : Promise.resolve([] as BoxFrontmatter[]);
8900
+ : Promise.resolve([] as BoxFrontmatter[]),
8901
+ );
8751
8902
 
8752
8903
  // --- Wait for core sections first, then bounded enrichment ---
8753
8904
  this.profiler.startSpan("phase-1-parallel", profileTraceId);
@@ -8931,6 +9082,70 @@ export class Orchestrator {
8931
9082
  return finalizeEnrichmentOutcome(outcome);
8932
9083
  };
8933
9084
 
9085
+ const remainingEnrichmentAssemblyMs = (): number | null =>
9086
+ enrichmentAssemblyDeadlineAtMs === null
9087
+ ? null
9088
+ : Math.max(0, enrichmentAssemblyDeadlineAtMs - Date.now());
9089
+
9090
+ const awaitAssemblyStep = async <T>(
9091
+ name: string,
9092
+ task: () => Promise<T>,
9093
+ fallback: T,
9094
+ ): Promise<T> => {
9095
+ if (options.abortSignal?.aborted) {
9096
+ log.debug(
9097
+ `recall phase-1 assembly [${name}]: skipped after abort at +${Date.now() - phase1Start}ms`,
9098
+ );
9099
+ return fallback;
9100
+ }
9101
+
9102
+ const timeoutMs = remainingEnrichmentAssemblyMs();
9103
+ if (timeoutMs === 0) {
9104
+ log.debug(
9105
+ `recall phase-1 assembly [${name}]: skipped after shared ${enrichmentSectionDeadlineMs}ms budget expired ` +
9106
+ `at +${Date.now() - phase1Start}ms`,
9107
+ );
9108
+ return fallback;
9109
+ }
9110
+
9111
+ let timeoutHandle: NodeJS.Timeout | undefined;
9112
+ try {
9113
+ const result = await (timeoutMs !== null
9114
+ ? Promise.race<T | { status: "timed_out" }>([
9115
+ task(),
9116
+ new Promise<{ status: "timed_out" }>((resolve) => {
9117
+ timeoutHandle = setTimeout(
9118
+ () => resolve({ status: "timed_out" }),
9119
+ timeoutMs,
9120
+ );
9121
+ }),
9122
+ ])
9123
+ : task());
9124
+ if (timeoutHandle) clearTimeout(timeoutHandle);
9125
+ if (
9126
+ typeof result === "object" &&
9127
+ result !== null &&
9128
+ "status" in result &&
9129
+ (result as { status?: unknown }).status === "timed_out"
9130
+ ) {
9131
+ log.debug(
9132
+ `recall phase-1 assembly [${name}]: timed out within shared ${enrichmentSectionDeadlineMs}ms budget ` +
9133
+ `at +${Date.now() - phase1Start}ms`,
9134
+ );
9135
+ return fallback;
9136
+ }
9137
+ return result as T;
9138
+ } catch (err) {
9139
+ if (timeoutHandle) clearTimeout(timeoutHandle);
9140
+ log.warn(
9141
+ `recall phase-1 assembly [${name}] failed open: ${
9142
+ err instanceof Error ? err.message : String(err)
9143
+ }`,
9144
+ );
9145
+ return fallback;
9146
+ }
9147
+ };
9148
+
8934
9149
  // --- Phase 2: Assemble sections in correct order ---
8935
9150
  this.profiler.startSpan("assembly", profileTraceId);
8936
9151
 
@@ -9263,8 +9478,11 @@ export class Orchestrator {
9263
9478
  // 1d. Memory Boxes (topic continuity windows, v8.0 Phase 2A)
9264
9479
  // recentBoxesPromise was kicked off before phase-1 so it ran concurrently.
9265
9480
  {
9266
- const recentBoxes = await recentBoxesPromise;
9267
- if (recentBoxes.length > 0) {
9481
+ const recentBoxes = await awaitEnrichmentSection(
9482
+ "memory-boxes",
9483
+ recentBoxesPromise,
9484
+ );
9485
+ if (recentBoxes && recentBoxes.length > 0) {
9268
9486
  const boxLines = recentBoxes.slice(0, 5).map((b: BoxFrontmatter) => {
9269
9487
  const sealedDate = b.sealedAt
9270
9488
  ? b.sealedAt.slice(0, 16).replace("T", " ")
@@ -9418,6 +9636,8 @@ export class Orchestrator {
9418
9636
  }
9419
9637
 
9420
9638
  // 2. QMD results — post-process and format
9639
+ const qmdWasSettledBeforeAssemblyWait =
9640
+ qmdPromise.getSettledOutcome() !== undefined;
9421
9641
  const qmdResult = await awaitEnrichmentSection("qmd", qmdPromise);
9422
9642
  if (qmdResult) {
9423
9643
  const t0 = Date.now();
@@ -9480,56 +9700,82 @@ export class Orchestrator {
9480
9700
  graphExpandedResultPaths.clear();
9481
9701
  } else {
9482
9702
  try {
9483
- const {
9484
- merged,
9485
- seedPaths,
9486
- expandedPaths,
9487
- seedResults = baselineMemoryResults,
9488
- } = await this.expandResultsViaGraph({
9489
- memoryResults,
9490
- recallNamespaces,
9491
- recallResultLimit,
9492
- ...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
9493
- });
9494
- graphSnapshotStatus = "completed";
9495
- graphDecisionStatus = "completed";
9496
- graphDecisionReason = graphShadowEvalEnabled
9497
- ? "graph shadow evaluation completed without altering injected context"
9498
- : "graph expansion merged into recall ranking";
9499
- graphSnapshotReason = graphDecisionReason;
9500
- graphSnapshotSeedPaths = seedPaths;
9501
- graphSnapshotExpandedPaths = expandedPaths;
9502
- graphSnapshotSeedResults = this.buildGraphRecallRankedResults(
9503
- seedResults,
9504
- () => ["baseline"],
9505
- );
9506
- graphExpandedResultPaths.clear();
9507
- expandedPaths.forEach((entry) =>
9508
- graphExpandedResultPaths.add(entry.path),
9703
+ const graphExpansion = await awaitAssemblyStep(
9704
+ "graph-expansion",
9705
+ () =>
9706
+ this.expandResultsViaGraph({
9707
+ memoryResults,
9708
+ recallNamespaces,
9709
+ recallResultLimit,
9710
+ deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
9711
+ ...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
9712
+ }),
9713
+ null as Awaited<ReturnType<typeof this.expandResultsViaGraph>> | null,
9509
9714
  );
9510
- memoryResults = graphShadowEvalEnabled
9511
- ? baselineMemoryResults
9512
- : merged;
9513
-
9514
- if (graphShadowEvalEnabled) {
9515
- const comparison = summarizeGraphShadowComparison(
9715
+ if (!graphExpansion) {
9716
+ graphSnapshotStatus = "aborted";
9717
+ graphDecisionStatus = "aborted";
9718
+ graphDecisionReason = options.abortSignal?.aborted
9719
+ ? "graph expansion skipped because recall assembly was aborted"
9720
+ : "graph expansion skipped because shared post-retrieval assembly budget expired";
9721
+ graphSnapshotReason = graphDecisionReason;
9722
+ graphSnapshotSeedPaths = baselineMemoryResults
9723
+ .slice(0, Math.max(1, recallResultLimit))
9724
+ .map((result) => result.path);
9725
+ graphSnapshotSeedResults = this.buildGraphRecallRankedResults(
9516
9726
  baselineMemoryResults,
9727
+ () => ["baseline"],
9728
+ );
9729
+ graphSnapshotExpandedPaths = [];
9730
+ graphExpandedResultPaths.clear();
9731
+ memoryResults = baselineMemoryResults;
9732
+ } else {
9733
+ const {
9517
9734
  merged,
9518
- recallResultLimit,
9735
+ seedPaths,
9736
+ expandedPaths,
9737
+ seedResults = baselineMemoryResults,
9738
+ } = graphExpansion;
9739
+ graphSnapshotStatus = "completed";
9740
+ graphDecisionStatus = "completed";
9741
+ graphDecisionReason = graphShadowEvalEnabled
9742
+ ? "graph shadow evaluation completed without altering injected context"
9743
+ : "graph expansion merged into recall ranking";
9744
+ graphSnapshotReason = graphDecisionReason;
9745
+ graphSnapshotSeedPaths = seedPaths;
9746
+ graphSnapshotExpandedPaths = expandedPaths;
9747
+ graphSnapshotSeedResults = this.buildGraphRecallRankedResults(
9748
+ seedResults,
9749
+ () => ["baseline"],
9519
9750
  );
9520
- graphSnapshotShadowComparison = comparison;
9521
- recordRecallSectionMetric({
9522
- section: "graphShadow",
9523
- priority: "enrichment",
9524
- durationMs: Date.now() - t0,
9525
- deadlineMs: enrichmentSectionDeadlineMs,
9526
- source: "fresh",
9527
- success: true,
9528
- timing:
9529
- `on b=${comparison.baselineCount} g=${comparison.graphCount} ` +
9530
- `ov=${comparison.overlapCount} (${comparison.overlapRatio.toFixed(2)}) ` +
9531
- `avgDelta=${comparison.averageOverlapDelta.toFixed(3)}`,
9532
- });
9751
+ graphExpandedResultPaths.clear();
9752
+ expandedPaths.forEach((entry) =>
9753
+ graphExpandedResultPaths.add(entry.path),
9754
+ );
9755
+ memoryResults = graphShadowEvalEnabled
9756
+ ? baselineMemoryResults
9757
+ : merged;
9758
+
9759
+ if (graphShadowEvalEnabled) {
9760
+ const comparison = summarizeGraphShadowComparison(
9761
+ baselineMemoryResults,
9762
+ merged,
9763
+ recallResultLimit,
9764
+ );
9765
+ graphSnapshotShadowComparison = comparison;
9766
+ recordRecallSectionMetric({
9767
+ section: "graphShadow",
9768
+ priority: "enrichment",
9769
+ durationMs: Date.now() - t0,
9770
+ deadlineMs: enrichmentSectionDeadlineMs,
9771
+ source: "fresh",
9772
+ success: true,
9773
+ timing:
9774
+ `on b=${comparison.baselineCount} g=${comparison.graphCount} ` +
9775
+ `ov=${comparison.overlapCount} (${comparison.overlapRatio.toFixed(2)}) ` +
9776
+ `avgDelta=${comparison.averageOverlapDelta.toFixed(3)}`,
9777
+ });
9778
+ }
9533
9779
  }
9534
9780
  } catch (err) {
9535
9781
  graphSnapshotStatus = "aborted";
@@ -9551,13 +9797,41 @@ export class Orchestrator {
9551
9797
  }
9552
9798
  }
9553
9799
 
9554
- // Apply recency and access count boosting
9555
- memoryResults = await this.boostSearchResults(
9800
+ // Apply mandatory recall safety filters before deadline-bound scoring
9801
+ // enrichment. If scoring times out, we must fall back to this filtered
9802
+ // list rather than raw QMD hits; boostSearchResults also removes
9803
+ // forgotten, lifecycle-filtered, superseded/as_of-invalid, and
9804
+ // dream/procedural memories.
9805
+ const qmdBoostInput = await this.filterSearchResultsForRecall(
9556
9806
  memoryResults,
9557
- recallNamespaces,
9558
- retrievalQuery,
9559
9807
  undefined,
9560
- { asOfMs },
9808
+ {
9809
+ asOfMs,
9810
+ // If QMD had already settled before the ordered assembly reached it,
9811
+ // do not let unrelated slow enrichment turn those known results into
9812
+ // unchecked misses. QMD that only settles during its own wait remains
9813
+ // bounded by the shared post-retrieval assembly deadline.
9814
+ deadlineAtMs: qmdWasSettledBeforeAssemblyWait
9815
+ ? null
9816
+ : enrichmentAssemblyDeadlineAtMs,
9817
+ abortSignal: options.abortSignal,
9818
+ dropUnresolved: true,
9819
+ recallNamespaces,
9820
+ },
9821
+ );
9822
+
9823
+ // Apply recency and access count boosting
9824
+ memoryResults = await awaitAssemblyStep(
9825
+ "qmd-boost",
9826
+ () =>
9827
+ this.boostSearchResults(
9828
+ qmdBoostInput.results,
9829
+ recallNamespaces,
9830
+ retrievalQuery,
9831
+ qmdBoostInput.memoryByPath,
9832
+ { asOfMs },
9833
+ ),
9834
+ qmdBoostInput.results,
9561
9835
  );
9562
9836
 
9563
9837
  // Optional LLM reranking (default off). Fail-open if rerank fails/slow.
@@ -9739,41 +10013,47 @@ export class Orchestrator {
9739
10013
  // low-relevance results from polluting context.
9740
10014
  const queryAwarePrefilter = await queryAwarePrefilterPromise;
9741
10015
  if (queryAwarePrefilter.candidatePaths?.size !== 0) {
9742
- const embeddingResults = await this.searchEmbeddingFallback(
9743
- retrievalQuery,
9744
- embeddingFetchLimit,
9745
- );
9746
- const prefilteredEmbeddingResults = applyQueryAwareCandidateFilter(
9747
- embeddingResults,
9748
- queryAwarePrefilter.candidatePaths,
9749
- );
9750
- const scopedCandidates = filterRecallCandidates(
9751
- prefilteredEmbeddingResults,
9752
- {
9753
- namespacesEnabled: this.config.namespacesEnabled,
9754
- recallNamespaces,
9755
- resolveNamespace: (p) => this.namespaceFromPath(p),
9756
- limit: embeddingFetchLimit,
10016
+ const scoped = await awaitAssemblyStep(
10017
+ "embedding-fallback",
10018
+ async () => {
10019
+ const embeddingResults = await this.searchEmbeddingFallback(
10020
+ retrievalQuery,
10021
+ embeddingFetchLimit,
10022
+ );
10023
+ const prefilteredEmbeddingResults = applyQueryAwareCandidateFilter(
10024
+ embeddingResults,
10025
+ queryAwarePrefilter.candidatePaths,
10026
+ );
10027
+ const scopedCandidates = filterRecallCandidates(
10028
+ prefilteredEmbeddingResults,
10029
+ {
10030
+ namespacesEnabled: this.config.namespacesEnabled,
10031
+ recallNamespaces,
10032
+ resolveNamespace: (p) => this.namespaceFromPath(p),
10033
+ limit: embeddingFetchLimit,
10034
+ },
10035
+ );
10036
+ const boostedScoped = await this.boostSearchResults(
10037
+ scopedCandidates,
10038
+ recallNamespaces,
10039
+ retrievalQuery,
10040
+ undefined,
10041
+ { asOfMs },
10042
+ );
10043
+ // MMR runs on the pre-truncation pool so diverse candidates just
10044
+ // below the cutoff can be promoted into the injected set.
10045
+ xrayBranchPoolSize.hot_embedding = Math.max(
10046
+ xrayBranchPoolSize.hot_embedding,
10047
+ boostedScoped.length,
10048
+ );
10049
+ return this.diversifyAndLimitRecallResults(
10050
+ "memories",
10051
+ boostedScoped,
10052
+ recallResultLimit,
10053
+ retrievalQuery,
10054
+ );
9757
10055
  },
9758
- );
9759
- const boostedScoped = await this.boostSearchResults(
9760
- scopedCandidates,
9761
- recallNamespaces,
9762
- retrievalQuery,
9763
- undefined,
9764
- { asOfMs },
9765
- );
9766
- // MMR runs on the pre-truncation pool so diverse candidates just
9767
- // below the cutoff can be promoted into the injected set.
9768
- xrayBranchPoolSize.hot_embedding = Math.max(
9769
- xrayBranchPoolSize.hot_embedding,
9770
- boostedScoped.length,
9771
- );
9772
- const scoped = this.diversifyAndLimitRecallResults(
9773
- "memories",
9774
- boostedScoped,
9775
- recallResultLimit,
9776
- retrievalQuery,
10056
+ [] as QmdSearchResult[],
9777
10057
  );
9778
10058
  if (scoped.length > 0) {
9779
10059
  if (shouldPersistGraphSnapshot) {
@@ -9811,6 +10091,7 @@ export class Orchestrator {
9811
10091
  queryAwarePrefilter,
9812
10092
  abortSignal: options.abortSignal,
9813
10093
  xrayPoolSizeSink: xrayColdPoolSink,
10094
+ deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
9814
10095
  asOfMs,
9815
10096
  ...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
9816
10097
  });
@@ -9886,42 +10167,48 @@ export class Orchestrator {
9886
10167
  // Fallback: embeddings first, then recency-only.
9887
10168
  const queryAwarePrefilter = await queryAwarePrefilterPromise;
9888
10169
  if (queryAwarePrefilter.candidatePaths?.size !== 0) {
9889
- const embeddingResults = await this.searchEmbeddingFallback(
9890
- retrievalQuery,
9891
- embeddingFetchLimit,
9892
- );
9893
- const prefilteredEmbeddingResults = applyQueryAwareCandidateFilter(
9894
- embeddingResults,
9895
- queryAwarePrefilter.candidatePaths,
9896
- );
9897
- const scopedCandidates = filterRecallCandidates(
9898
- prefilteredEmbeddingResults,
9899
- {
9900
- namespacesEnabled: this.config.namespacesEnabled,
9901
- recallNamespaces,
9902
- resolveNamespace: (p) => this.namespaceFromPath(p),
9903
- limit: embeddingFetchLimit,
9904
- },
9905
- );
9906
- const boostedScoped = await this.boostSearchResults(
9907
- scopedCandidates,
9908
- recallNamespaces,
9909
- retrievalQuery,
9910
- undefined,
9911
- { asOfMs },
9912
- );
9913
- // MMR runs on the pre-truncation pool so diverse candidates just
9914
- // below the cutoff can be promoted into the injected set.
9915
- xrayBranchPoolSize.hot_embedding = Math.max(
9916
- xrayBranchPoolSize.hot_embedding,
9917
- boostedScoped.length,
9918
- );
9919
- const scoped = this.diversifyAndLimitRecallResults(
9920
- "memories",
9921
- boostedScoped,
9922
- recallResultLimit,
9923
- retrievalQuery,
9924
- );
10170
+ const scoped = await awaitAssemblyStep(
10171
+ "embedding-fallback",
10172
+ async () => {
10173
+ const embeddingResults = await this.searchEmbeddingFallback(
10174
+ retrievalQuery,
10175
+ embeddingFetchLimit,
10176
+ );
10177
+ const prefilteredEmbeddingResults = applyQueryAwareCandidateFilter(
10178
+ embeddingResults,
10179
+ queryAwarePrefilter.candidatePaths,
10180
+ );
10181
+ const scopedCandidates = filterRecallCandidates(
10182
+ prefilteredEmbeddingResults,
10183
+ {
10184
+ namespacesEnabled: this.config.namespacesEnabled,
10185
+ recallNamespaces,
10186
+ resolveNamespace: (p) => this.namespaceFromPath(p),
10187
+ limit: embeddingFetchLimit,
10188
+ },
10189
+ );
10190
+ const boostedScoped = await this.boostSearchResults(
10191
+ scopedCandidates,
10192
+ recallNamespaces,
10193
+ retrievalQuery,
10194
+ undefined,
10195
+ { asOfMs },
10196
+ );
10197
+ // MMR runs on the pre-truncation pool so diverse candidates just
10198
+ // below the cutoff can be promoted into the injected set.
10199
+ xrayBranchPoolSize.hot_embedding = Math.max(
10200
+ xrayBranchPoolSize.hot_embedding,
10201
+ boostedScoped.length,
10202
+ );
10203
+ return this.diversifyAndLimitRecallResults(
10204
+ "memories",
10205
+ boostedScoped,
10206
+ recallResultLimit,
10207
+ retrievalQuery,
10208
+ );
10209
+ },
10210
+ [] as QmdSearchResult[],
10211
+ );
9925
10212
  if (scoped.length > 0) {
9926
10213
  if (shouldPersistGraphSnapshot) {
9927
10214
  graphSnapshotFinalResults = this.buildGraphRecallRankedResults(
@@ -9950,8 +10237,11 @@ export class Orchestrator {
9950
10237
  xrayRecalledResults = scoped;
9951
10238
  impressionRecorded = true;
9952
10239
  } else {
9953
- const memories =
9954
- await this.readAllMemoriesForNamespaces(recallNamespaces);
10240
+ const memories = await awaitAssemblyStep(
10241
+ "recent-memory-read",
10242
+ () => this.readAllMemoriesForNamespaces(recallNamespaces),
10243
+ [] as MemoryFile[],
10244
+ );
9955
10245
  if (memories.length > 0) {
9956
10246
  // Filter out non-active memories. Delegate to
9957
10247
  // shouldFilterSupersededFromRecall for superseded-status logic so
@@ -10023,6 +10313,7 @@ export class Orchestrator {
10023
10313
  queryAwarePrefilter,
10024
10314
  abortSignal: options.abortSignal,
10025
10315
  xrayPoolSizeSink: xrayColdPoolSink,
10316
+ deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
10026
10317
  asOfMs,
10027
10318
  ...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
10028
10319
  });
@@ -10049,44 +10340,50 @@ export class Orchestrator {
10049
10340
  impressionRecorded = true;
10050
10341
  }
10051
10342
  } else {
10052
- const recentSorted = queryAwareScopedMemories.sort(
10053
- (a, b) =>
10054
- new Date(b.frontmatter.updated).getTime() -
10055
- new Date(a.frontmatter.updated).getTime(),
10056
- );
10057
- const preloadedMap = new Map<string, MemoryFile>(
10058
- queryAwareScopedMemories
10059
- .filter((m) => m.path)
10060
- .map((m) => [m.path, m]),
10061
- );
10062
- const recentAsResults: QmdSearchResult[] = recentSorted.map(
10063
- (m, i) => ({
10064
- docid: m.frontmatter.id,
10065
- path: m.path,
10066
- snippet: m.content,
10067
- score: 1.0 - i / Math.max(recentSorted.length, 1),
10068
- }),
10069
- );
10070
- const boostedRecent = (
10071
- await this.boostSearchResults(
10072
- recentAsResults,
10073
- recallNamespaces,
10074
- retrievalQuery,
10075
- preloadedMap,
10076
- { asOfMs },
10077
- )
10078
- ).sort((a, b) => b.score - a.score);
10079
- // MMR runs on the pre-truncation pool so diverse candidates just
10080
- // below the cutoff can be promoted into the injected set.
10081
- xrayBranchPoolSize.recent_scan = Math.max(
10082
- xrayBranchPoolSize.recent_scan,
10083
- boostedRecent.length,
10084
- );
10085
- const recent = this.diversifyAndLimitRecallResults(
10086
- "memories",
10087
- boostedRecent,
10088
- recallResultLimit,
10089
- retrievalQuery,
10343
+ const recent = await awaitAssemblyStep(
10344
+ "recent-memory-scan",
10345
+ async () => {
10346
+ const recentSorted = queryAwareScopedMemories.sort(
10347
+ (a, b) =>
10348
+ new Date(b.frontmatter.updated).getTime() -
10349
+ new Date(a.frontmatter.updated).getTime(),
10350
+ );
10351
+ const preloadedMap = new Map<string, MemoryFile>(
10352
+ queryAwareScopedMemories
10353
+ .filter((m) => m.path)
10354
+ .map((m) => [m.path, m]),
10355
+ );
10356
+ const recentAsResults: QmdSearchResult[] = recentSorted.map(
10357
+ (m, i) => ({
10358
+ docid: m.frontmatter.id,
10359
+ path: m.path,
10360
+ snippet: m.content,
10361
+ score: 1.0 - i / Math.max(recentSorted.length, 1),
10362
+ }),
10363
+ );
10364
+ const boostedRecent = (
10365
+ await this.boostSearchResults(
10366
+ recentAsResults,
10367
+ recallNamespaces,
10368
+ retrievalQuery,
10369
+ preloadedMap,
10370
+ { asOfMs },
10371
+ )
10372
+ ).sort((a, b) => b.score - a.score);
10373
+ // MMR runs on the pre-truncation pool so diverse candidates just
10374
+ // below the cutoff can be promoted into the injected set.
10375
+ xrayBranchPoolSize.recent_scan = Math.max(
10376
+ xrayBranchPoolSize.recent_scan,
10377
+ boostedRecent.length,
10378
+ );
10379
+ return this.diversifyAndLimitRecallResults(
10380
+ "memories",
10381
+ boostedRecent,
10382
+ recallResultLimit,
10383
+ retrievalQuery,
10384
+ );
10385
+ },
10386
+ [] as QmdSearchResult[],
10090
10387
  );
10091
10388
 
10092
10389
  if (recent.length > 0) {
@@ -10125,6 +10422,7 @@ export class Orchestrator {
10125
10422
  queryAwarePrefilter,
10126
10423
  abortSignal: options.abortSignal,
10127
10424
  xrayPoolSizeSink: xrayColdPoolSink,
10425
+ deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
10128
10426
  asOfMs,
10129
10427
  ...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
10130
10428
  });
@@ -10168,6 +10466,7 @@ export class Orchestrator {
10168
10466
  queryAwarePrefilter,
10169
10467
  abortSignal: options.abortSignal,
10170
10468
  xrayPoolSizeSink: xrayColdPoolSink,
10469
+ deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
10171
10470
  asOfMs,
10172
10471
  ...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
10173
10472
  });
@@ -10547,6 +10846,7 @@ export class Orchestrator {
10547
10846
  query: retrievalQuery,
10548
10847
  memoryIds: recalledMemoryIds,
10549
10848
  namespace: selfNamespace,
10849
+ recallNamespaces,
10550
10850
  traceId,
10551
10851
  plannerMode: recallMode,
10552
10852
  requestedMode,
@@ -15482,36 +15782,307 @@ export class Orchestrator {
15482
15782
  *
15483
15783
  * Callers must pass the full candidate pool (post-rerank, pre-slice).
15484
15784
  */
15485
- private async applyMemoryWorthRerank(
15486
- results: QmdSearchResult[],
15487
- namespaces: string[],
15488
- ): Promise<QmdSearchResult[]> {
15489
- // Build the counter lookup. We union frontmatter counters across every
15490
- // namespace the recall spans — the recall path itself already
15491
- // aggregates candidates from multiple namespaces, so we must do the
15492
- // same when looking up counters. Per-namespace results are cached with
15493
- // a short TTL so interactive recall doesn't trigger a full
15494
- // `readAllMemories` scan per query (addresses codex P2 on PR 4).
15495
- const counters = new Map<string, MemoryWorthCounters>();
15496
- const seenNamespaces = new Set<string>();
15497
- const nowMs = Date.now();
15785
+ private qmdCollectionNamespaceFromPrefix(collectionPrefix: string): string | null {
15786
+ const baseCollection = this.config.qmdCollection;
15787
+ if (collectionPrefix === baseCollection) return this.config.defaultNamespace;
15788
+ const namespaceSuffix = collectionPrefix.startsWith(`${baseCollection}--`)
15789
+ ? collectionPrefix.slice(baseCollection.length + 2)
15790
+ : "";
15791
+ if (!namespaceSuffix) return null;
15792
+
15793
+ const decoded = namespaceIdentityFromToken(namespaceSuffix);
15794
+ if (decoded !== null) return decoded || this.config.defaultNamespace;
15795
+ if (namespaceSuffix.startsWith("ns--")) {
15796
+ const legacyNamespace = namespaceSuffix.slice("ns--".length).trim();
15797
+ return legacyNamespace || null;
15798
+ }
15799
+ return null;
15800
+ }
15498
15801
 
15499
- // Evict all expired entries on every call so long-running processes
15500
- // touching a high-cardinality namespace set (coding/project overlays,
15501
- // per-branch) don't grow the cache unboundedly. Without this, an entry
15502
- // for a namespace that's never looked up again would pin its full
15503
- // counter map forever.
15504
- for (const [key, entry] of this.memoryWorthCounterCache) {
15505
- if (nowMs - entry.at >= Orchestrator.MEMORY_WORTH_CACHE_TTL_MS) {
15506
- this.memoryWorthCounterCache.delete(key);
15802
+ private async readQmdResultMemory(
15803
+ resultPath: string,
15804
+ fallbackStorage: StorageManager,
15805
+ recallNamespaces: readonly string[] = [],
15806
+ ): Promise<MemoryFile | null> {
15807
+ const parts = qmdCollectionPathParts(resultPath);
15808
+ const storageDirFor = (storage: StorageManager): string | null =>
15809
+ typeof (storage as { dir?: unknown }).dir === "string" &&
15810
+ (storage as { dir?: string }).dir
15811
+ ? (storage as { dir: string }).dir
15812
+ : null;
15813
+ const fallbackStorageDir = storageDirFor(fallbackStorage);
15814
+ const coldCollection = this.config.qmdColdCollection ?? "openclaw-engram-cold";
15815
+ if (parts && parts.collection === coldCollection) {
15816
+ const storages: StorageManager[] = [];
15817
+ const seenStorageDirs = new Set<string>();
15818
+ const addStorage = (storage: StorageManager): void => {
15819
+ const storageDir = storageDirFor(storage);
15820
+ const storageKey = storageDir
15821
+ ? path.resolve(storageDir)
15822
+ : `storage-without-dir-${storages.length}`;
15823
+ if (seenStorageDirs.has(storageKey)) return;
15824
+ seenStorageDirs.add(storageKey);
15825
+ storages.push(storage);
15826
+ };
15827
+
15828
+ const fallbackNamespace =
15829
+ fallbackStorageDir !== null
15830
+ ? this.namespaceFromStorageDir(fallbackStorageDir)
15831
+ : this.config.defaultNamespace;
15832
+ if (
15833
+ recallNamespaces.length === 0 ||
15834
+ !this.config.namespacesEnabled ||
15835
+ recallNamespaces.includes(fallbackNamespace)
15836
+ ) {
15837
+ addStorage(fallbackStorage);
15507
15838
  }
15508
- }
15509
15839
 
15510
- for (const ns of namespaces) {
15511
- if (seenNamespaces.has(ns)) continue;
15512
- seenNamespaces.add(ns);
15513
- try {
15514
- const cached = this.memoryWorthCounterCache.get(ns);
15840
+ if (recallNamespaces.length > 0) {
15841
+ for (const namespace of recallNamespaces) {
15842
+ try {
15843
+ addStorage(await this.storageRouter.storageFor(namespace));
15844
+ } catch (err) {
15845
+ log.debug("qmd cold result namespace storage lookup skipped", {
15846
+ path: resultPath,
15847
+ namespace,
15848
+ error: (err as Error).message,
15849
+ });
15850
+ }
15851
+ }
15852
+ }
15853
+
15854
+ for (const storage of storages) {
15855
+ const storageDir = storageDirFor(storage);
15856
+ if (!storageDir) {
15857
+ const memory = await storage.readMemoryByPath(resultPath);
15858
+ if (memory) return memory;
15859
+ continue;
15860
+ }
15861
+ try {
15862
+ const coldRoot = path.join(storageDir, "cold");
15863
+ for (const candidate of qmdResultPathCandidates(
15864
+ coldRoot,
15865
+ parts.relativePath,
15866
+ )) {
15867
+ const memory = await storage.readMemoryByPath(candidate);
15868
+ if (memory) return memory;
15869
+ }
15870
+ } catch (err) {
15871
+ if (err instanceof SecureStoreLockedError) throw err;
15872
+ log.debug("qmd cold result path lookup failed open", {
15873
+ path: resultPath,
15874
+ collection: coldCollection,
15875
+ error: (err as Error).message,
15876
+ });
15877
+ }
15878
+ }
15879
+ return null;
15880
+ }
15881
+ const collectionNamespace = parts
15882
+ ? this.qmdCollectionNamespaceFromPrefix(parts.collection)
15883
+ : null;
15884
+
15885
+ if (parts && collectionNamespace) {
15886
+ try {
15887
+ const collectionStorage =
15888
+ await this.storageRouter.storageFor(collectionNamespace);
15889
+ for (const candidate of qmdResultPathCandidates(
15890
+ collectionStorage.dir,
15891
+ parts.relativePath,
15892
+ )) {
15893
+ const memory = await collectionStorage.readMemoryByPath(candidate);
15894
+ if (memory) return memory;
15895
+ }
15896
+ return null;
15897
+ } catch (err) {
15898
+ if (err instanceof SecureStoreLockedError) throw err;
15899
+ log.debug("qmd result namespace path lookup failed open", {
15900
+ path: resultPath,
15901
+ namespace: collectionNamespace,
15902
+ error: (err as Error).message,
15903
+ });
15904
+ return null;
15905
+ }
15906
+ }
15907
+
15908
+ if (path.isAbsolute(resultPath)) {
15909
+ if (!fallbackStorageDir) {
15910
+ return await fallbackStorage.readMemoryByPath(resultPath);
15911
+ }
15912
+ const ownerStorage = await this.storageForAbsoluteQmdResultPath(
15913
+ resultPath,
15914
+ fallbackStorage,
15915
+ recallNamespaces,
15916
+ );
15917
+ if (!ownerStorage) return null;
15918
+ for (const candidate of qmdResultPathCandidates(
15919
+ ownerStorage.dir,
15920
+ resultPath,
15921
+ )) {
15922
+ const memory = await ownerStorage.storage.readMemoryByPath(candidate);
15923
+ if (memory) return memory;
15924
+ }
15925
+ return null;
15926
+ }
15927
+
15928
+ if (!fallbackStorageDir) {
15929
+ return await fallbackStorage.readMemoryByPath(resultPath);
15930
+ }
15931
+ for (const candidate of qmdResultPathCandidates(
15932
+ fallbackStorageDir,
15933
+ resultPath,
15934
+ )) {
15935
+ const memory = await fallbackStorage.readMemoryByPath(candidate);
15936
+ if (memory) return memory;
15937
+ }
15938
+ return null;
15939
+ }
15940
+
15941
+ private async resolveColdQmdResultForRecall(
15942
+ result: QmdSearchResult,
15943
+ fallbackStorage: StorageManager,
15944
+ recallNamespaces: readonly string[] = [],
15945
+ ): Promise<{ namespace: string; result: QmdSearchResult } | null> {
15946
+ const memory = await this.readQmdResultMemory(
15947
+ result.path,
15948
+ fallbackStorage,
15949
+ recallNamespaces,
15950
+ );
15951
+ if (!memory) return null;
15952
+
15953
+ let ownerNamespace: string | null = null;
15954
+ if (path.isAbsolute(memory.path)) {
15955
+ const ownerStorage = await this.storageForAbsoluteQmdResultPath(
15956
+ memory.path,
15957
+ fallbackStorage,
15958
+ recallNamespaces,
15959
+ );
15960
+ ownerNamespace = ownerStorage?.namespace ?? null;
15961
+ if (!ownerNamespace && this.config.namespacesEnabled) return null;
15962
+ }
15963
+ ownerNamespace ??= this.namespaceFromPath(memory.path);
15964
+ if (
15965
+ recallNamespaces.length > 0 &&
15966
+ !recallNamespaces.includes(ownerNamespace)
15967
+ ) {
15968
+ return null;
15969
+ }
15970
+
15971
+ return {
15972
+ namespace: ownerNamespace,
15973
+ result: {
15974
+ ...result,
15975
+ docid: result.docid || memory.frontmatter.id,
15976
+ path: memory.path,
15977
+ snippet: result.snippet || memory.content.slice(0, 400),
15978
+ },
15979
+ };
15980
+ }
15981
+
15982
+ private async storageForAbsoluteQmdResultPath(
15983
+ resultPath: string,
15984
+ fallbackStorage: StorageManager,
15985
+ recallNamespaces: readonly string[] = [],
15986
+ ): Promise<{ storage: StorageManager; dir: string; namespace: string } | null> {
15987
+ const resolvedPath = path.resolve(resultPath);
15988
+ const memoryRoot = path.resolve(this.config.memoryDir);
15989
+ const namespacesRoot = path.join(memoryRoot, "namespaces");
15990
+ const fallbackStorageDir =
15991
+ typeof (fallbackStorage as { dir?: unknown }).dir === "string" &&
15992
+ (fallbackStorage as { dir?: string }).dir
15993
+ ? (fallbackStorage as { dir: string }).dir
15994
+ : null;
15995
+ const matches: Array<{ storage: StorageManager; dir: string; namespace: string }> = [];
15996
+ const seenDirs = new Set<string>();
15997
+
15998
+ const maybeAddStorage = (storage: StorageManager, namespace: string) => {
15999
+ const storageDir =
16000
+ typeof (storage as { dir?: unknown }).dir === "string" &&
16001
+ (storage as { dir?: string }).dir
16002
+ ? (storage as { dir: string }).dir
16003
+ : null;
16004
+ if (!storageDir) return;
16005
+ const candidateRoot = path.resolve(storageDir);
16006
+ if (seenDirs.has(candidateRoot)) return;
16007
+ if (!isPathInsideStorageRoot(candidateRoot, resolvedPath)) return;
16008
+ if (
16009
+ candidateRoot === memoryRoot &&
16010
+ isPathInsideStorageRoot(namespacesRoot, resolvedPath)
16011
+ ) {
16012
+ return;
16013
+ }
16014
+ seenDirs.add(candidateRoot);
16015
+ matches.push({ storage, dir: candidateRoot, namespace });
16016
+ };
16017
+
16018
+ const fallbackNamespace =
16019
+ fallbackStorageDir !== null
16020
+ ? this.namespaceFromStorageDir(fallbackStorageDir)
16021
+ : this.config.defaultNamespace;
16022
+ maybeAddStorage(fallbackStorage, fallbackNamespace);
16023
+
16024
+ const candidateNamespaces = new Set<string>();
16025
+ candidateNamespaces.add(this.config.defaultNamespace);
16026
+ candidateNamespaces.add(this.config.sharedNamespace);
16027
+ for (const ns of recallNamespaces) {
16028
+ candidateNamespaces.add(ns);
16029
+ }
16030
+ if (isPathInsideStorageRoot(namespacesRoot, resolvedPath)) {
16031
+ const relativeToNamespaces = path.relative(namespacesRoot, resolvedPath);
16032
+ const [namespaceSegment] = relativeToNamespaces.split(/[\\/]/);
16033
+ if (namespaceSegment) {
16034
+ candidateNamespaces.add(
16035
+ namespaceIdentityFromToken(namespaceSegment) ?? namespaceSegment,
16036
+ );
16037
+ }
16038
+ }
16039
+ for (const policy of this.config.namespacePolicies ?? []) {
16040
+ candidateNamespaces.add(policy.name);
16041
+ }
16042
+
16043
+ for (const ns of candidateNamespaces) {
16044
+ if (!ns) continue;
16045
+ try {
16046
+ maybeAddStorage(await this.storageRouter.storageFor(ns), ns);
16047
+ } catch {
16048
+ continue;
16049
+ }
16050
+ }
16051
+
16052
+ matches.sort((a, b) => b.dir.length - a.dir.length);
16053
+ return matches[0] ?? null;
16054
+ }
16055
+
16056
+ private async applyMemoryWorthRerank(
16057
+ results: QmdSearchResult[],
16058
+ namespaces: string[],
16059
+ ): Promise<QmdSearchResult[]> {
16060
+ // Build the counter lookup. We union frontmatter counters across every
16061
+ // namespace the recall spans — the recall path itself already
16062
+ // aggregates candidates from multiple namespaces, so we must do the
16063
+ // same when looking up counters. Per-namespace results are cached with
16064
+ // a short TTL so interactive recall doesn't trigger a full
16065
+ // `readAllMemories` scan per query (addresses codex P2 on PR 4).
16066
+ const counters = new Map<string, MemoryWorthCounters>();
16067
+ const seenNamespaces = new Set<string>();
16068
+ const nowMs = Date.now();
16069
+
16070
+ // Evict all expired entries on every call so long-running processes
16071
+ // touching a high-cardinality namespace set (coding/project overlays,
16072
+ // per-branch) don't grow the cache unboundedly. Without this, an entry
16073
+ // for a namespace that's never looked up again would pin its full
16074
+ // counter map forever.
16075
+ for (const [key, entry] of this.memoryWorthCounterCache) {
16076
+ if (nowMs - entry.at >= Orchestrator.MEMORY_WORTH_CACHE_TTL_MS) {
16077
+ this.memoryWorthCounterCache.delete(key);
16078
+ }
16079
+ }
16080
+
16081
+ for (const ns of namespaces) {
16082
+ if (seenNamespaces.has(ns)) continue;
16083
+ seenNamespaces.add(ns);
16084
+ try {
16085
+ const cached = this.memoryWorthCounterCache.get(ns);
15515
16086
  let nsMap: ReadonlyMap<string, MemoryWorthCounters> | undefined;
15516
16087
  if (
15517
16088
  cached &&
@@ -15557,7 +16128,7 @@ export class Orchestrator {
15557
16128
  if (reader) {
15558
16129
  for (const r of missing) {
15559
16130
  try {
15560
- const memory = await reader.readMemoryByPath(r.path);
16131
+ const memory = await this.readQmdResultMemory(r.path, reader, namespaces);
15561
16132
  if (!memory) continue;
15562
16133
  const fm = memory.frontmatter;
15563
16134
  if (fm.mw_success === undefined && fm.mw_fail === undefined) continue;
@@ -15973,6 +16544,7 @@ export class Orchestrator {
15973
16544
  * Unset by default so existing call sites are unaffected.
15974
16545
  */
15975
16546
  xrayPoolSizeSink?: { size: number };
16547
+ deadlineAtMs?: number | null;
15976
16548
  /** Issue #681 — when true, bypass graphTraversalConfidenceFloor. */
15977
16549
  includeLowConfidence?: boolean;
15978
16550
  }): Promise<QmdSearchResult[]> {
@@ -15980,6 +16552,50 @@ export class Orchestrator {
15980
16552
  if (options.xrayPoolSizeSink) options.xrayPoolSizeSink.size = 0;
15981
16553
  return [];
15982
16554
  }
16555
+ const deadlineRemainingMs = (): number | null =>
16556
+ typeof options.deadlineAtMs === "number"
16557
+ ? Math.max(0, options.deadlineAtMs - Date.now())
16558
+ : null;
16559
+ const runColdStepWithinDeadline = async <T>(
16560
+ label: string,
16561
+ fallback: T,
16562
+ task: () => Promise<T>,
16563
+ ): Promise<T> => {
16564
+ throwIfRecallAborted(options.abortSignal);
16565
+ const remainingMs = deadlineRemainingMs();
16566
+ if (remainingMs === 0) {
16567
+ log.debug(`cold-tier recall ${label} skipped: shared assembly deadline expired`);
16568
+ return fallback;
16569
+ }
16570
+ if (remainingMs === null) return task();
16571
+
16572
+ let timeoutHandle: NodeJS.Timeout | undefined;
16573
+ let timedOut = false;
16574
+ const taskPromise = task().catch((err) => {
16575
+ if (timedOut) {
16576
+ log.debug(`cold-tier recall ${label} failed after deadline: ${err}`);
16577
+ return fallback;
16578
+ }
16579
+ throw err;
16580
+ });
16581
+
16582
+ try {
16583
+ return await Promise.race<T>([
16584
+ taskPromise,
16585
+ new Promise<T>((resolve) => {
16586
+ timeoutHandle = setTimeout(() => {
16587
+ timedOut = true;
16588
+ log.debug(
16589
+ `cold-tier recall ${label} skipped: shared assembly deadline expired`,
16590
+ );
16591
+ resolve(fallback);
16592
+ }, remainingMs);
16593
+ }),
16594
+ ]);
16595
+ } finally {
16596
+ if (timeoutHandle) clearTimeout(timeoutHandle);
16597
+ }
16598
+ };
15983
16599
 
15984
16600
  const coldQmdEnabled = this.config.qmdColdTierEnabled === true;
15985
16601
  const coldCollection =
@@ -15999,19 +16615,24 @@ export class Orchestrator {
15999
16615
  false,
16000
16616
  0,
16001
16617
  );
16002
- longTerm = await this.fetchQmdMemoryResultsWithArtifactTopUp(
16003
- options.prompt,
16004
- coldFetchLimit,
16005
- coldHybridLimit,
16006
- {
16007
- namespacesEnabled: this.config.namespacesEnabled,
16008
- recallNamespaces: options.recallNamespaces,
16009
- resolveNamespace: (p) => this.namespaceFromPath(p),
16010
- collection: coldCollection,
16011
- queryAwarePrefilter: options.queryAwarePrefilter,
16012
- searchOptions: this.buildConfiguredQmdSearchOptions(options.prompt),
16013
- abortSignal: options.abortSignal,
16014
- },
16618
+ longTerm = await runColdStepWithinDeadline(
16619
+ "qmd lookup",
16620
+ [],
16621
+ () =>
16622
+ this.fetchQmdMemoryResultsWithArtifactTopUp(
16623
+ options.prompt,
16624
+ coldFetchLimit,
16625
+ coldHybridLimit,
16626
+ {
16627
+ namespacesEnabled: this.config.namespacesEnabled,
16628
+ recallNamespaces: options.recallNamespaces,
16629
+ resolveNamespace: (p) => this.namespaceFromPath(p),
16630
+ collection: coldCollection,
16631
+ queryAwarePrefilter: options.queryAwarePrefilter,
16632
+ searchOptions: this.buildConfiguredQmdSearchOptions(options.prompt),
16633
+ abortSignal: options.abortSignal,
16634
+ },
16635
+ ),
16015
16636
  );
16016
16637
  if (longTerm.length > 0) {
16017
16638
  log.debug(
@@ -16021,12 +16642,17 @@ export class Orchestrator {
16021
16642
  }
16022
16643
  }
16023
16644
  if (longTerm.length === 0) {
16024
- longTerm = await this.searchLongTermArchiveFallback(
16025
- options.prompt,
16026
- options.recallNamespaces,
16027
- options.recallResultLimit,
16028
- options.queryAwarePrefilter,
16029
- options.abortSignal,
16645
+ longTerm = await runColdStepWithinDeadline(
16646
+ "archive scan",
16647
+ [],
16648
+ () =>
16649
+ this.searchLongTermArchiveFallback(
16650
+ options.prompt,
16651
+ options.recallNamespaces,
16652
+ options.recallResultLimit,
16653
+ options.queryAwarePrefilter,
16654
+ options.abortSignal,
16655
+ ),
16030
16656
  );
16031
16657
  if (longTerm.length > 0) {
16032
16658
  log.debug("cold-tier recall source=archive-scan");
@@ -16036,9 +16662,57 @@ export class Orchestrator {
16036
16662
 
16037
16663
  let results = longTerm;
16038
16664
  if (this.config.namespacesEnabled) {
16039
- results = results.filter((r) =>
16040
- options.recallNamespaces.includes(this.namespaceFromPath(r.path)),
16041
- );
16665
+ const recallRoots: string[] = [];
16666
+ const seenRecallRoots = new Set<string>();
16667
+ for (const namespace of options.recallNamespaces) {
16668
+ try {
16669
+ const storage = await this.storageRouter.storageFor(namespace);
16670
+ const storageDir =
16671
+ typeof (storage as { dir?: unknown }).dir === "string" &&
16672
+ (storage as { dir?: string }).dir
16673
+ ? (storage as { dir: string }).dir
16674
+ : null;
16675
+ if (!storageDir) continue;
16676
+ const recallRoot = path.resolve(storageDir);
16677
+ if (seenRecallRoots.has(recallRoot)) continue;
16678
+ seenRecallRoots.add(recallRoot);
16679
+ recallRoots.push(recallRoot);
16680
+ } catch (err) {
16681
+ log.debug("cold-tier recall namespace root lookup skipped", {
16682
+ namespace,
16683
+ error: (err as Error).message,
16684
+ });
16685
+ }
16686
+ }
16687
+ const scopedResults: QmdSearchResult[] = [];
16688
+ for (const result of results) {
16689
+ if (options.abortSignal?.aborted || deadlineRemainingMs() === 0) break;
16690
+ const parts = qmdCollectionPathParts(result.path);
16691
+ if (parts?.collection === coldCollection) {
16692
+ const resolvedCold = await this.resolveColdQmdResultForRecall(
16693
+ result,
16694
+ this.storage,
16695
+ options.recallNamespaces,
16696
+ );
16697
+ if (resolvedCold) scopedResults.push(resolvedCold.result);
16698
+ continue;
16699
+ }
16700
+ if (path.isAbsolute(result.path)) {
16701
+ const resolvedPath = path.resolve(result.path);
16702
+ if (
16703
+ recallRoots.some((recallRoot) =>
16704
+ isPathInsideStorageRoot(recallRoot, resolvedPath),
16705
+ )
16706
+ ) {
16707
+ scopedResults.push(result);
16708
+ }
16709
+ continue;
16710
+ }
16711
+ if (options.recallNamespaces.includes(this.namespaceFromPath(result.path))) {
16712
+ scopedResults.push(result);
16713
+ }
16714
+ }
16715
+ results = scopedResults;
16042
16716
  }
16043
16717
  // Artifact isolation contract: generic recall paths must exclude artifacts.
16044
16718
  results = results.filter((r) => !isArtifactMemoryPath(r.path));
@@ -16059,18 +16733,75 @@ export class Orchestrator {
16059
16733
  memoryResults: results,
16060
16734
  recallNamespaces: options.recallNamespaces,
16061
16735
  recallResultLimit: options.recallResultLimit,
16736
+ deadlineAtMs: options.deadlineAtMs,
16062
16737
  ...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
16063
16738
  });
16064
16739
  results = merged;
16065
16740
  }
16066
16741
 
16067
- results = await this.boostSearchResults(
16742
+ const boostInput = await this.filterSearchResultsForRecall(
16068
16743
  results,
16069
- options.recallNamespaces,
16070
- options.prompt,
16071
16744
  undefined,
16072
- { allowLifecycleFiltered: true, asOfMs: options.asOfMs },
16745
+ {
16746
+ allowLifecycleFiltered: true,
16747
+ asOfMs: options.asOfMs,
16748
+ deadlineAtMs: options.deadlineAtMs,
16749
+ abortSignal: options.abortSignal,
16750
+ dropUnresolved: true,
16751
+ recallNamespaces: options.recallNamespaces,
16752
+ },
16073
16753
  );
16754
+ results = boostInput.results;
16755
+ const boostTimeoutMs =
16756
+ typeof options.deadlineAtMs === "number"
16757
+ ? Math.max(0, options.deadlineAtMs - Date.now())
16758
+ : null;
16759
+ if (boostTimeoutMs !== 0) {
16760
+ let timeoutHandle: NodeJS.Timeout | undefined;
16761
+ try {
16762
+ const boosted = await (boostTimeoutMs !== null
16763
+ ? Promise.race<QmdSearchResult[] | { status: "timed_out" }>([
16764
+ this.boostSearchResults(
16765
+ boostInput.results,
16766
+ options.recallNamespaces,
16767
+ options.prompt,
16768
+ boostInput.memoryByPath,
16769
+ { allowLifecycleFiltered: true, asOfMs: options.asOfMs },
16770
+ ),
16771
+ new Promise<{ status: "timed_out" }>((resolve) => {
16772
+ timeoutHandle = setTimeout(
16773
+ () => resolve({ status: "timed_out" }),
16774
+ boostTimeoutMs,
16775
+ );
16776
+ }),
16777
+ ])
16778
+ : this.boostSearchResults(
16779
+ boostInput.results,
16780
+ options.recallNamespaces,
16781
+ options.prompt,
16782
+ boostInput.memoryByPath,
16783
+ { allowLifecycleFiltered: true, asOfMs: options.asOfMs },
16784
+ ));
16785
+ if (
16786
+ typeof boosted === "object" &&
16787
+ boosted !== null &&
16788
+ "status" in boosted &&
16789
+ boosted.status === "timed_out"
16790
+ ) {
16791
+ log.debug("cold-tier recall boost skipped: shared assembly deadline expired");
16792
+ } else if (Array.isArray(boosted)) {
16793
+ results = boosted;
16794
+ } else {
16795
+ results = boostInput.results;
16796
+ }
16797
+ } catch (err) {
16798
+ log.debug(`cold-tier recall boost failed open: ${err}`);
16799
+ } finally {
16800
+ if (timeoutHandle) clearTimeout(timeoutHandle);
16801
+ }
16802
+ } else {
16803
+ log.debug("cold-tier recall boost skipped: shared assembly deadline already expired");
16804
+ }
16074
16805
 
16075
16806
  if (this.config.rerankEnabled && this.config.rerankProvider === "local") {
16076
16807
  const ranked = await rerankLocalOrNoop({
@@ -16218,104 +16949,133 @@ export class Orchestrator {
16218
16949
  log.debug(`flushed ${entries.length} access tracking entries`);
16219
16950
  }
16220
16951
 
16221
- /**
16222
- * Apply recency, access count, and importance boosting to QMD search results.
16223
- * Returns re-ranked results.
16224
- */
16225
- private async boostSearchResults(
16952
+ private async loadSearchResultMemoryMap(
16226
16953
  results: QmdSearchResult[],
16227
- _recallNamespaces: string[],
16228
- prompt?: string,
16229
16954
  preloadedMemoryMap?: Map<string, MemoryFile>,
16230
16955
  options?: {
16231
- allowLifecycleFiltered?: boolean;
16232
- allowDedicatedSurface?: boolean;
16233
- /**
16234
- * Historical recall point in ms-since-epoch (issue #680). When
16235
- * set, drops candidates that were not authoritative at this
16236
- * instant per `temporal-validity.isValidAsOf`. Caller is
16237
- * responsible for parsing/validating the user-supplied ISO
16238
- * string at the input boundary (CLI / HTTP / MCP).
16239
- */
16240
- asOfMs?: number;
16956
+ deadlineAtMs?: number | null;
16957
+ abortSignal?: AbortSignal;
16958
+ recallNamespaces?: readonly string[];
16241
16959
  },
16242
- ): Promise<QmdSearchResult[]> {
16243
- if (results.length === 0) return results;
16244
-
16245
- const now = Date.now();
16246
- // Seed with any pre-loaded memories (e.g. from the recency fallback path)
16247
- // to avoid redundant disk reads for files already in memory.
16960
+ ): Promise<{
16961
+ memoryByPath: Map<string, MemoryFile>;
16962
+ checkedPaths: Set<string>;
16963
+ unreadablePaths: Set<string>;
16964
+ completed: boolean;
16965
+ }> {
16248
16966
  const memoryByPath: Map<string, MemoryFile> = preloadedMemoryMap
16249
16967
  ? new Map(preloadedMemoryMap)
16250
16968
  : new Map();
16969
+ const checkedPaths = new Set<string>();
16970
+ const unreadablePaths = new Set<string>();
16251
16971
 
16252
- // Determine temporal/tag query params before I/O (pure computation).
16253
- const resultPaths = new Set(
16254
- results.map((r) => r.path).filter(Boolean) as string[],
16255
- );
16256
- let temporalFromDate: string | null = null;
16257
- let promptTags: string[] = [];
16258
- if (this.config.queryAwareIndexingEnabled && prompt) {
16259
- if (isTemporalQuery(prompt)) {
16260
- temporalFromDate = recencyWindowFromPrompt(prompt, now);
16972
+ const markChecked = (result: QmdSearchResult): void => {
16973
+ if (result.path) checkedPaths.add(result.path);
16974
+ };
16975
+ const markUnreadable = (result: QmdSearchResult, err: unknown): void => {
16976
+ if (!result.path) return;
16977
+ checkedPaths.add(result.path);
16978
+ unreadablePaths.add(result.path);
16979
+ log.warn("recall safety filter dropped unreadable secure-store candidate", {
16980
+ path: result.path,
16981
+ error: (err as Error).message,
16982
+ });
16983
+ };
16984
+ const deadlineExpired = (): boolean =>
16985
+ typeof options?.deadlineAtMs === "number" &&
16986
+ Date.now() >= options.deadlineAtMs;
16987
+
16988
+ if (options?.deadlineAtMs == null) {
16989
+ const batchSize = options?.abortSignal ? 16 : results.length;
16990
+ for (let offset = 0; offset < results.length; offset += batchSize) {
16991
+ if (options?.abortSignal?.aborted) {
16992
+ return {
16993
+ memoryByPath,
16994
+ checkedPaths,
16995
+ unreadablePaths,
16996
+ completed: false,
16997
+ };
16998
+ }
16999
+ await Promise.all(
17000
+ results.slice(offset, offset + batchSize).map(async (r) => {
17001
+ if (!r.path) return;
17002
+ if (memoryByPath.has(r.path)) {
17003
+ markChecked(r);
17004
+ return;
17005
+ }
17006
+ try {
17007
+ const mem = await this.readQmdResultMemory(
17008
+ r.path,
17009
+ this.storage,
17010
+ options?.recallNamespaces,
17011
+ );
17012
+ markChecked(r);
17013
+ if (mem) memoryByPath.set(r.path, mem);
17014
+ } catch (err) {
17015
+ if (err instanceof SecureStoreLockedError) {
17016
+ markUnreadable(r, err);
17017
+ return;
17018
+ }
17019
+ throw err;
17020
+ }
17021
+ }),
17022
+ );
16261
17023
  }
16262
- promptTags = extractTagsFromPrompt(prompt);
16263
- }
16264
17024
 
16265
- // Run all file I/O in parallel: memory files not yet preloaded + index files.
16266
- const [, rawTemporal, rawTags] = await Promise.all([
16267
- Promise.all(
16268
- results.map(async (r) => {
16269
- if (!r.path || memoryByPath.has(r.path)) return;
16270
- const mem = await this.storage.readMemoryByPath(r.path);
16271
- if (mem) memoryByPath.set(r.path, mem);
16272
- }),
16273
- ),
16274
- temporalFromDate !== null
16275
- ? queryByDateRangeAsync(this.config.memoryDir, temporalFromDate)
16276
- : Promise.resolve<Set<string> | null>(null),
16277
- promptTags.length > 0
16278
- ? queryByTagsAsync(this.config.memoryDir, promptTags)
16279
- : Promise.resolve<Set<string> | null>(null),
16280
- ]);
16281
-
16282
- const queryIntent =
16283
- this.config.intentRoutingEnabled && prompt
16284
- ? inferIntentFromText(prompt)
16285
- : null;
17025
+ return { memoryByPath, checkedPaths, unreadablePaths, completed: true };
17026
+ }
16286
17027
 
16287
- // v8.1: Temporal + Tag prefilter candidate set
16288
- // Scope to result paths first so cross-namespace paths don't consume the cap.
16289
- let temporalCandidates: Set<string> | null = null;
16290
- let tagCandidates: Set<string> | null = null;
16291
- if (this.config.queryAwareIndexingEnabled && prompt) {
16292
- const maxCandidates = this.config.queryAwareIndexingMaxCandidates;
16293
- const capSet = (s: Set<string> | null): Set<string> | null => {
16294
- if (!s) return null;
16295
- // Intersect with result paths first so out-of-scope paths don't exhaust the budget
16296
- const scoped = new Set(Array.from(s).filter((p) => resultPaths.has(p)));
16297
- if (maxCandidates === 0 || scoped.size <= maxCandidates)
16298
- return scoped;
16299
- return new Set(Array.from(scoped).slice(0, maxCandidates));
16300
- };
16301
- if (temporalFromDate !== null) {
16302
- temporalCandidates = capSet(rawTemporal);
17028
+ for (const result of results) {
17029
+ if (!result.path) continue;
17030
+ if (memoryByPath.has(result.path)) {
17031
+ markChecked(result);
17032
+ continue;
16303
17033
  }
16304
- if (promptTags.length > 0) {
16305
- tagCandidates = capSet(rawTags);
17034
+ if (options?.abortSignal?.aborted || deadlineExpired()) {
17035
+ return { memoryByPath, checkedPaths, unreadablePaths, completed: false };
17036
+ }
17037
+ try {
17038
+ const mem = await this.readQmdResultMemory(
17039
+ result.path,
17040
+ this.storage,
17041
+ options?.recallNamespaces,
17042
+ );
17043
+ markChecked(result);
17044
+ if (mem) memoryByPath.set(result.path, mem);
17045
+ } catch (err) {
17046
+ if (err instanceof SecureStoreLockedError) {
17047
+ markUnreadable(result, err);
17048
+ continue;
17049
+ }
17050
+ throw err;
16306
17051
  }
16307
17052
  }
16308
17053
 
17054
+ return { memoryByPath, checkedPaths, unreadablePaths, completed: true };
17055
+ }
17056
+
17057
+ private filterSearchResultsByRecallSafety(
17058
+ results: QmdSearchResult[],
17059
+ memoryByPath: Map<string, MemoryFile>,
17060
+ options?: {
17061
+ allowLifecycleFiltered?: boolean;
17062
+ allowDedicatedSurface?: boolean;
17063
+ asOfMs?: number;
17064
+ blockedPaths?: Set<string>;
17065
+ },
17066
+ ): QmdSearchResult[] {
16309
17067
  let lifecycleFilteredCount = 0;
16310
17068
  let temporalSupersededFilteredCount = 0;
16311
17069
  let dedicatedSurfaceFilteredCount = 0;
16312
17070
  let forgottenFilteredCount = 0;
16313
- const boosted: QmdSearchResult[] = [];
16314
- const recencyWeight = this.effectiveRecencyWeight();
17071
+ let blockedPathFilteredCount = 0;
17072
+ const filtered: QmdSearchResult[] = [];
16315
17073
  for (const r of results) {
17074
+ if (r.path && options?.blockedPaths?.has(r.path)) {
17075
+ blockedPathFilteredCount += 1;
17076
+ continue;
17077
+ }
16316
17078
  const memory = memoryByPath.get(r.path);
16317
- let score = r.score;
16318
-
16319
17079
  if (memory) {
16320
17080
  if (memory.frontmatter.status === "forgotten") {
16321
17081
  forgottenFilteredCount += 1;
@@ -16377,7 +17137,178 @@ export class Orchestrator {
16377
17137
  dedicatedSurfaceFilteredCount += 1;
16378
17138
  continue;
16379
17139
  }
17140
+ }
17141
+ filtered.push(r);
17142
+ }
17143
+ if (lifecycleFilteredCount > 0) {
17144
+ log.debug(
17145
+ `lifecycle retrieval filter removed ${lifecycleFilteredCount} stale/archived candidates`,
17146
+ );
17147
+ }
17148
+ if (temporalSupersededFilteredCount > 0) {
17149
+ log.debug(
17150
+ `temporal supersession filter removed ${temporalSupersededFilteredCount} superseded candidates`,
17151
+ );
17152
+ }
17153
+ if (dedicatedSurfaceFilteredCount > 0) {
17154
+ log.debug(
17155
+ `dedicated surface filter removed ${dedicatedSurfaceFilteredCount} dream/procedural candidates from generic recall`,
17156
+ );
17157
+ }
17158
+ if (forgottenFilteredCount > 0) {
17159
+ log.debug(
17160
+ `forgotten status filter removed ${forgottenFilteredCount} candidates from recall`,
17161
+ );
17162
+ }
17163
+ if (blockedPathFilteredCount > 0) {
17164
+ log.debug(
17165
+ `unreadable-path filter removed ${blockedPathFilteredCount} candidates from recall`,
17166
+ );
17167
+ }
17168
+ return filtered;
17169
+ }
17170
+
17171
+ private async filterSearchResultsForRecall(
17172
+ results: QmdSearchResult[],
17173
+ preloadedMemoryMap?: Map<string, MemoryFile>,
17174
+ options?: {
17175
+ allowLifecycleFiltered?: boolean;
17176
+ allowDedicatedSurface?: boolean;
17177
+ asOfMs?: number;
17178
+ deadlineAtMs?: number | null;
17179
+ abortSignal?: AbortSignal;
17180
+ dropUnresolved?: boolean;
17181
+ recallNamespaces?: readonly string[];
17182
+ },
17183
+ ): Promise<{ results: QmdSearchResult[]; memoryByPath: Map<string, MemoryFile> }> {
17184
+ if (results.length === 0) {
17185
+ return {
17186
+ results,
17187
+ memoryByPath: preloadedMemoryMap ? new Map(preloadedMemoryMap) : new Map(),
17188
+ };
17189
+ }
17190
+ const loaded = await this.loadSearchResultMemoryMap(
17191
+ results,
17192
+ preloadedMemoryMap,
17193
+ options,
17194
+ );
17195
+ const candidateResults = loaded.completed
17196
+ ? results
17197
+ : results.filter((result) => !result.path || loaded.checkedPaths.has(result.path));
17198
+ if (!loaded.completed) {
17199
+ log.debug(
17200
+ `recall safety filter stopped before validating all candidates (${candidateResults.length}/${results.length} eligible)`,
17201
+ );
17202
+ }
17203
+ const blockedPaths = new Set(loaded.unreadablePaths);
17204
+ if (options?.dropUnresolved === true) {
17205
+ for (const resultPath of loaded.checkedPaths) {
17206
+ if (!loaded.memoryByPath.has(resultPath)) {
17207
+ blockedPaths.add(resultPath);
17208
+ }
17209
+ }
17210
+ }
17211
+ return {
17212
+ results: this.filterSearchResultsByRecallSafety(
17213
+ candidateResults,
17214
+ loaded.memoryByPath,
17215
+ { ...options, blockedPaths },
17216
+ ),
17217
+ memoryByPath: loaded.memoryByPath,
17218
+ };
17219
+ }
16380
17220
 
17221
+ /**
17222
+ * Apply recency, access count, and importance boosting to QMD search results.
17223
+ * Returns re-ranked results.
17224
+ */
17225
+ private async boostSearchResults(
17226
+ results: QmdSearchResult[],
17227
+ _recallNamespaces: string[],
17228
+ prompt?: string,
17229
+ preloadedMemoryMap?: Map<string, MemoryFile>,
17230
+ options?: {
17231
+ allowLifecycleFiltered?: boolean;
17232
+ allowDedicatedSurface?: boolean;
17233
+ /**
17234
+ * Historical recall point in ms-since-epoch (issue #680). When
17235
+ * set, drops candidates that were not authoritative at this
17236
+ * instant per `temporal-validity.isValidAsOf`. Caller is
17237
+ * responsible for parsing/validating the user-supplied ISO
17238
+ * string at the input boundary (CLI / HTTP / MCP).
17239
+ */
17240
+ asOfMs?: number;
17241
+ },
17242
+ ): Promise<QmdSearchResult[]> {
17243
+ if (results.length === 0) return results;
17244
+
17245
+ const safety = await this.filterSearchResultsForRecall(
17246
+ results,
17247
+ preloadedMemoryMap,
17248
+ { ...options, recallNamespaces: _recallNamespaces },
17249
+ );
17250
+ const safeResults = safety.results;
17251
+ const memoryByPath = safety.memoryByPath;
17252
+ if (safeResults.length === 0) return safeResults;
17253
+
17254
+ const now = Date.now();
17255
+
17256
+ // Determine temporal/tag query params before index I/O (pure computation).
17257
+ const resultPaths = new Set(
17258
+ safeResults.map((r) => r.path).filter(Boolean) as string[],
17259
+ );
17260
+ let temporalFromDate: string | null = null;
17261
+ let promptTags: string[] = [];
17262
+ if (this.config.queryAwareIndexingEnabled && prompt) {
17263
+ if (isTemporalQuery(prompt)) {
17264
+ temporalFromDate = recencyWindowFromPrompt(prompt, now);
17265
+ }
17266
+ promptTags = extractTagsFromPrompt(prompt);
17267
+ }
17268
+
17269
+ const [rawTemporal, rawTags] = await Promise.all([
17270
+ temporalFromDate !== null
17271
+ ? queryByDateRangeAsync(this.config.memoryDir, temporalFromDate)
17272
+ : Promise.resolve<Set<string> | null>(null),
17273
+ promptTags.length > 0
17274
+ ? queryByTagsAsync(this.config.memoryDir, promptTags)
17275
+ : Promise.resolve<Set<string> | null>(null),
17276
+ ]);
17277
+
17278
+ const queryIntent =
17279
+ this.config.intentRoutingEnabled && prompt
17280
+ ? inferIntentFromText(prompt)
17281
+ : null;
17282
+
17283
+ // v8.1: Temporal + Tag prefilter candidate set
17284
+ // Scope to result paths first so cross-namespace paths don't consume the cap.
17285
+ let temporalCandidates: Set<string> | null = null;
17286
+ let tagCandidates: Set<string> | null = null;
17287
+ if (this.config.queryAwareIndexingEnabled && prompt) {
17288
+ const maxCandidates = this.config.queryAwareIndexingMaxCandidates;
17289
+ const capSet = (s: Set<string> | null): Set<string> | null => {
17290
+ if (!s) return null;
17291
+ // Intersect with result paths first so out-of-scope paths don't exhaust the budget
17292
+ const scoped = new Set(Array.from(s).filter((p) => resultPaths.has(p)));
17293
+ if (maxCandidates === 0 || scoped.size <= maxCandidates)
17294
+ return scoped;
17295
+ return new Set(Array.from(scoped).slice(0, maxCandidates));
17296
+ };
17297
+ if (temporalFromDate !== null) {
17298
+ temporalCandidates = capSet(rawTemporal);
17299
+ }
17300
+ if (promptTags.length > 0) {
17301
+ tagCandidates = capSet(rawTags);
17302
+ }
17303
+ }
17304
+
17305
+ const boosted: QmdSearchResult[] = [];
17306
+ const recencyWeight = this.effectiveRecencyWeight();
17307
+ for (const r of safeResults) {
17308
+ const memory = memoryByPath.get(r.path);
17309
+ let score = r.score;
17310
+
17311
+ if (memory) {
16381
17312
  // Recency boost: exponential decay over 7 days
16382
17313
  if (recencyWeight > 0) {
16383
17314
  const createdAt = new Date(memory.frontmatter.created).getTime();
@@ -16522,26 +17453,6 @@ export class Orchestrator {
16522
17453
 
16523
17454
  boosted.push({ ...r, score });
16524
17455
  }
16525
- if (lifecycleFilteredCount > 0) {
16526
- log.debug(
16527
- `lifecycle retrieval filter removed ${lifecycleFilteredCount} stale/archived candidates`,
16528
- );
16529
- }
16530
- if (temporalSupersededFilteredCount > 0) {
16531
- log.debug(
16532
- `temporal supersession filter removed ${temporalSupersededFilteredCount} superseded candidates`,
16533
- );
16534
- }
16535
- if (dedicatedSurfaceFilteredCount > 0) {
16536
- log.debug(
16537
- `dedicated surface filter removed ${dedicatedSurfaceFilteredCount} dream/procedural candidates from generic recall`,
16538
- );
16539
- }
16540
- if (forgottenFilteredCount > 0) {
16541
- log.debug(
16542
- `forgotten status filter removed ${forgottenFilteredCount} candidates from recall`,
16543
- );
16544
- }
16545
17456
 
16546
17457
  // Re-sort by boosted score
16547
17458
  return boosted.sort((a, b) => b.score - a.score);
@@ -16769,8 +17680,52 @@ export class Orchestrator {
16769
17680
  }));
16770
17681
  }
16771
17682
 
17683
+ private async graphSeedPathRelativeToStorage(
17684
+ storage: StorageManager,
17685
+ result: QmdSearchResult,
17686
+ recallNamespaces: readonly string[] = [],
17687
+ ): Promise<string | null> {
17688
+ const parts = qmdCollectionPathParts(result.path);
17689
+ if (parts) {
17690
+ const memory = await this.readQmdResultMemory(
17691
+ result.path,
17692
+ storage,
17693
+ recallNamespaces,
17694
+ );
17695
+ return memory
17696
+ ? graphPathRelativeToStorage(storage.dir, memory.path)
17697
+ : null;
17698
+ }
17699
+ return graphPathRelativeToStorage(storage.dir, result.path);
17700
+ }
17701
+
17702
+ private async graphSeedPathsWithinDeadline(
17703
+ storage: StorageManager,
17704
+ results: QmdSearchResult[],
17705
+ deadlineAtMs: number,
17706
+ recallNamespaces: readonly string[] = [],
17707
+ ): Promise<string[]> {
17708
+ const resolved: string[] = [];
17709
+ for (const result of results) {
17710
+ if (Date.now() >= deadlineAtMs) break;
17711
+ const seedPath = await this.graphSeedPathRelativeToStorage(
17712
+ storage,
17713
+ result,
17714
+ recallNamespaces,
17715
+ );
17716
+ if (Date.now() >= deadlineAtMs) break;
17717
+ if (seedPath) resolved.push(seedPath);
17718
+ }
17719
+ return resolved;
17720
+ }
17721
+
16772
17722
  private namespaceFromPath(p: string): string {
16773
17723
  if (!this.config.namespacesEnabled) return this.config.defaultNamespace;
17724
+ const parts = qmdCollectionPathParts(p);
17725
+ const collectionNamespace = parts
17726
+ ? this.qmdCollectionNamespaceFromPrefix(parts.collection)
17727
+ : null;
17728
+ if (collectionNamespace) return collectionNamespace;
16774
17729
  const m = p.match(/[\\/]+namespaces[\\/]+([^\\/]+)(?:[\\/]|$)/);
16775
17730
  if (!m?.[1]) return this.config.defaultNamespace;
16776
17731
  return namespaceIdentityFromToken(m[1]) ?? m[1];