@remnic/core 9.3.637 → 9.3.639
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-cli.js +29 -29
- package/dist/access-http.d.ts +56 -5
- package/dist/access-http.js +15 -13
- package/dist/access-mcp.d.ts +4 -4
- package/dist/access-mcp.js +14 -12
- package/dist/access-schema.d.ts +3 -0
- package/dist/access-schema.js +1 -1
- package/dist/{access-service-GVOE5U3C.d.ts → access-service-jyYute38.d.ts} +4 -2
- package/dist/access-service.d.ts +4 -4
- package/dist/access-service.js +12 -10
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/active-recall.js +1 -1
- package/dist/{auto-sync-RFADEHIQ.js → auto-sync-54QQHOG5.js} +4 -4
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +3 -3
- package/dist/briefing.d.ts +1 -1
- package/dist/briefing.js +3 -3
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer.d.ts +1 -1
- package/dist/calibration.d.ts +1 -1
- package/dist/calibration.js +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-behavior.js +2 -2
- package/dist/causal-chain.js +2 -2
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-consolidation.js +7 -7
- package/dist/causal-retrieval.js +2 -2
- package/dist/causal-trajectory-graph.js +1 -1
- package/dist/causal-trajectory.js +1 -1
- package/dist/{chunk-TFICNXXU.js → chunk-2BHNQRE5.js} +1153 -294
- package/dist/chunk-2BHNQRE5.js.map +1 -0
- package/dist/{chunk-3RDYU3JS.js → chunk-3AZDLWDW.js} +3 -3
- package/dist/{chunk-FPNQF475.js → chunk-3F7RMHC4.js} +1 -1
- package/dist/chunk-3F7RMHC4.js.map +1 -0
- package/dist/{chunk-3EJ5OOZ3.js → chunk-3SF7QNKF.js} +3 -3
- package/dist/{chunk-NFNQ7TWF.js → chunk-4VTVCWVB.js} +20 -12
- package/dist/chunk-4VTVCWVB.js.map +1 -0
- package/dist/{chunk-XM6M6DHG.js → chunk-52XI77OQ.js} +204 -13
- package/dist/chunk-52XI77OQ.js.map +1 -0
- package/dist/{chunk-6BDVBBBY.js → chunk-666A3MOW.js} +88 -30
- package/dist/chunk-666A3MOW.js.map +1 -0
- package/dist/{chunk-242XFZ36.js → chunk-7OH7KPF5.js} +2 -2
- package/dist/{chunk-HZVIYZYN.js → chunk-7R37SOT4.js} +6 -6
- package/dist/{chunk-7A2QKUUA.js → chunk-7XRSLMUD.js} +2 -2
- package/dist/{chunk-723OMPUI.js → chunk-A3BS64GV.js} +7 -1
- package/dist/chunk-A3BS64GV.js.map +1 -0
- package/dist/{chunk-JCLECECB.js → chunk-A3UPMOJZ.js} +2 -2
- package/dist/{chunk-YCX45GOQ.js → chunk-AFH2JXSF.js} +2 -2
- package/dist/{chunk-BK35NOJL.js → chunk-BB6FNMMB.js} +52 -6
- package/dist/chunk-BB6FNMMB.js.map +1 -0
- package/dist/{chunk-3XNMN242.js → chunk-BTY5XQG3.js} +7 -7
- package/dist/{chunk-QHXW3LZV.js → chunk-C3EJ3UIS.js} +3 -3
- package/dist/{chunk-A3HKTPPV.js → chunk-C63WC454.js} +12 -1
- package/dist/chunk-C63WC454.js.map +1 -0
- package/dist/{chunk-4S3N6HFG.js → chunk-C6XO6ONP.js} +2 -2
- package/dist/{chunk-WD2W4234.js → chunk-CDHRUNBX.js} +11 -2
- package/dist/chunk-CDHRUNBX.js.map +1 -0
- package/dist/{chunk-YQNADJCT.js → chunk-CMZLMIZI.js} +2 -2
- package/dist/{chunk-KVDUDYEN.js → chunk-DDRNDPX4.js} +2 -2
- package/dist/{chunk-PW6GURU3.js → chunk-EKIQBGAY.js} +2 -2
- package/dist/{chunk-QJGTSCJN.js → chunk-FGRV3X3Y.js} +19 -19
- package/dist/{chunk-2IT7WFYE.js → chunk-FWIROLS6.js} +2 -1
- package/dist/chunk-FWIROLS6.js.map +1 -0
- package/dist/{chunk-X7EJF46S.js → chunk-H4RWKZE2.js} +2 -2
- package/dist/{chunk-A6D7A2FW.js → chunk-IHG27THY.js} +4 -4
- package/dist/{chunk-PUEAEQSN.js → chunk-ISMZIMD3.js} +2 -2
- package/dist/{chunk-LEG7XWS2.js → chunk-IXSTQGYV.js} +2 -2
- package/dist/{chunk-CMN5AWAZ.js → chunk-J2KFC47S.js} +2 -2
- package/dist/{chunk-K47C6M2C.js → chunk-M75TBFKQ.js} +2 -2
- package/dist/{chunk-7V2SGZ3H.js → chunk-NE566K4E.js} +2 -2
- package/dist/{chunk-YROCXMCK.js → chunk-RJGPH2Q3.js} +2 -2
- package/dist/{chunk-UXFOGILU.js → chunk-SBHQBC37.js} +2 -2
- package/dist/{chunk-QYGIQ5NM.js → chunk-SHHT4JTC.js} +2 -2
- package/dist/{chunk-TQOU3VAT.js → chunk-SSOMTUCA.js} +1 -1
- package/dist/{chunk-NRST7W5Q.js → chunk-UI2UP7XX.js} +2 -2
- package/dist/{chunk-C4KKM62E.js → chunk-UU5LHD6J.js} +2 -2
- package/dist/{chunk-EWC6W6AB.js → chunk-UXA5L2DZ.js} +2 -2
- package/dist/{chunk-XL7FK7PJ.js → chunk-Y56J7CXW.js} +33 -16
- package/dist/{chunk-XL7FK7PJ.js.map → chunk-Y56J7CXW.js.map} +1 -1
- package/dist/{cli-B-fHk59D.d.ts → cli-A4AIhv9e.d.ts} +3 -3
- package/dist/cli.d.ts +5 -5
- package/dist/cli.js +28 -27
- package/dist/compounding/engine.d.ts +1 -1
- package/dist/compounding/engine.js +3 -3
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.js +1 -1
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.js +3 -3
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/connectors/index.js +3 -3
- package/dist/consolidation-provenance-check.d.ts +1 -1
- package/dist/consolidation-undo.d.ts +1 -1
- package/dist/contradiction/index.d.ts +1 -1
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer.d.ts +1 -1
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-retrieval.d.ts +1 -1
- package/dist/entity-retrieval.js +3 -3
- package/dist/entity-schema.d.ts +1 -1
- package/dist/explicit-capture.d.ts +3 -3
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction-judge.js +2 -2
- package/dist/extraction.d.ts +1 -1
- package/dist/extraction.js +2 -2
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/fallback-llm.js +1 -1
- package/dist/{graph-edge-decay-5ZOK7ZNO.js → graph-edge-decay-PUFNHOBS.js} +2 -2
- package/dist/graph-snapshot.js +2 -2
- package/dist/graph.d.ts +3 -0
- package/dist/graph.js +1 -1
- package/dist/identity-continuity.d.ts +1 -1
- package/dist/importance.d.ts +1 -1
- package/dist/index.d.ts +9 -9
- package/dist/index.js +40 -40
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +1 -1
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/maintenance/memory-governance-cron.d.ts +8 -0
- package/dist/maintenance/memory-governance-cron.js +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/maintenance/memory-governance.js +3 -3
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
- package/dist/maintenance/rebuild-memory-projection.js +4 -4
- package/dist/mcp-memory-inspector-app.d.ts +4 -4
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +1 -1
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +1 -1
- package/dist/namespaces/migrate.js +4 -4
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/storage.d.ts +1 -1
- package/dist/namespaces/storage.js +3 -3
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/operator-toolkit.d.ts +1 -1
- package/dist/operator-toolkit.js +8 -8
- package/dist/{orchestrator-CoqytbK_.d.ts → orchestrator-BEDTiuEO.d.ts} +20 -5
- package/dist/orchestrator.d.ts +3 -3
- package/dist/orchestrator.js +24 -24
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +1 -1
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-explain-renderer.js +3 -3
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-planner-llm.js +1 -1
- package/dist/recall-state.d.ts +3 -1
- package/dist/recall-state.js +1 -1
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-cli.js +4 -4
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray-renderer.js +3 -3
- package/dist/recall-xray.d.ts +1 -1
- package/dist/recall-xray.js +2 -2
- package/dist/resolve-auth-token.d.ts +1 -1
- package/dist/resume-bundles.js +4 -4
- package/dist/retrieval-agents.d.ts +1 -1
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/index.d.ts +1 -1
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/{semantic-consolidation-BPs6BURk.d.ts → semantic-consolidation-Byut1ufK.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-consolidation.js +4 -4
- package/dist/semantic-rule-promotion.js +3 -3
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/semantic-rule-verifier.js +3 -3
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +1 -1
- package/dist/signal.d.ts +1 -1
- package/dist/storage.d.ts +1 -1
- package/dist/storage.js +2 -2
- package/dist/summarizer.d.ts +1 -1
- package/dist/summarizer.js +3 -3
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/temporal-supersession.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +1 -1
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/transcript.js +2 -2
- package/dist/{types-CpMPD8xl.d.ts → types-BzA1FKbg.d.ts} +4 -0
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -1
- package/dist/utility-runtime.d.ts +1 -1
- package/dist/verified-recall.js +3 -3
- package/package.json +1 -1
- package/src/access-http.test.ts +110 -2
- package/src/access-http.ts +117 -1
- package/src/access-mcp.ts +15 -6
- package/src/access-schema.ts +14 -0
- package/src/access-service-namespace.test.ts +740 -0
- package/src/access-service.ts +226 -4
- package/src/config.ts +14 -0
- package/src/fallback-llm.ts +16 -0
- package/src/graph.ts +39 -17
- package/src/index.ts +9 -1
- package/src/maintenance/memory-governance-cron.ts +113 -30
- package/src/orchestrator.ts +1456 -336
- package/src/recall-state.ts +3 -0
- package/src/types.ts +4 -0
- package/dist/chunk-2IT7WFYE.js.map +0 -1
- package/dist/chunk-6BDVBBBY.js.map +0 -1
- package/dist/chunk-723OMPUI.js.map +0 -1
- package/dist/chunk-A3HKTPPV.js.map +0 -1
- package/dist/chunk-BK35NOJL.js.map +0 -1
- package/dist/chunk-FPNQF475.js.map +0 -1
- package/dist/chunk-NFNQ7TWF.js.map +0 -1
- package/dist/chunk-TFICNXXU.js.map +0 -1
- package/dist/chunk-WD2W4234.js.map +0 -1
- package/dist/chunk-XM6M6DHG.js.map +0 -1
- /package/dist/{auto-sync-RFADEHIQ.js.map → auto-sync-54QQHOG5.js.map} +0 -0
- /package/dist/{chunk-3RDYU3JS.js.map → chunk-3AZDLWDW.js.map} +0 -0
- /package/dist/{chunk-3EJ5OOZ3.js.map → chunk-3SF7QNKF.js.map} +0 -0
- /package/dist/{chunk-242XFZ36.js.map → chunk-7OH7KPF5.js.map} +0 -0
- /package/dist/{chunk-HZVIYZYN.js.map → chunk-7R37SOT4.js.map} +0 -0
- /package/dist/{chunk-7A2QKUUA.js.map → chunk-7XRSLMUD.js.map} +0 -0
- /package/dist/{chunk-JCLECECB.js.map → chunk-A3UPMOJZ.js.map} +0 -0
- /package/dist/{chunk-YCX45GOQ.js.map → chunk-AFH2JXSF.js.map} +0 -0
- /package/dist/{chunk-3XNMN242.js.map → chunk-BTY5XQG3.js.map} +0 -0
- /package/dist/{chunk-QHXW3LZV.js.map → chunk-C3EJ3UIS.js.map} +0 -0
- /package/dist/{chunk-4S3N6HFG.js.map → chunk-C6XO6ONP.js.map} +0 -0
- /package/dist/{chunk-YQNADJCT.js.map → chunk-CMZLMIZI.js.map} +0 -0
- /package/dist/{chunk-KVDUDYEN.js.map → chunk-DDRNDPX4.js.map} +0 -0
- /package/dist/{chunk-PW6GURU3.js.map → chunk-EKIQBGAY.js.map} +0 -0
- /package/dist/{chunk-QJGTSCJN.js.map → chunk-FGRV3X3Y.js.map} +0 -0
- /package/dist/{chunk-X7EJF46S.js.map → chunk-H4RWKZE2.js.map} +0 -0
- /package/dist/{chunk-A6D7A2FW.js.map → chunk-IHG27THY.js.map} +0 -0
- /package/dist/{chunk-PUEAEQSN.js.map → chunk-ISMZIMD3.js.map} +0 -0
- /package/dist/{chunk-LEG7XWS2.js.map → chunk-IXSTQGYV.js.map} +0 -0
- /package/dist/{chunk-CMN5AWAZ.js.map → chunk-J2KFC47S.js.map} +0 -0
- /package/dist/{chunk-K47C6M2C.js.map → chunk-M75TBFKQ.js.map} +0 -0
- /package/dist/{chunk-7V2SGZ3H.js.map → chunk-NE566K4E.js.map} +0 -0
- /package/dist/{chunk-YROCXMCK.js.map → chunk-RJGPH2Q3.js.map} +0 -0
- /package/dist/{chunk-UXFOGILU.js.map → chunk-SBHQBC37.js.map} +0 -0
- /package/dist/{chunk-QYGIQ5NM.js.map → chunk-SHHT4JTC.js.map} +0 -0
- /package/dist/{chunk-TQOU3VAT.js.map → chunk-SSOMTUCA.js.map} +0 -0
- /package/dist/{chunk-NRST7W5Q.js.map → chunk-UI2UP7XX.js.map} +0 -0
- /package/dist/{chunk-C4KKM62E.js.map → chunk-UU5LHD6J.js.map} +0 -0
- /package/dist/{chunk-EWC6W6AB.js.map → chunk-UXA5L2DZ.js.map} +0 -0
- /package/dist/{graph-edge-decay-5ZOK7ZNO.js.map → graph-edge-decay-PUFNHOBS.js.map} +0 -0
package/src/orchestrator.ts
CHANGED
|
@@ -78,6 +78,7 @@ import { LocalLlmClient } from "./local-llm.js";
|
|
|
78
78
|
import {
|
|
79
79
|
FallbackLlmClient,
|
|
80
80
|
fallbackLlmRuntimeContextFromConfig,
|
|
81
|
+
gatewayTaskChainOptions,
|
|
81
82
|
} from "./fallback-llm.js";
|
|
82
83
|
import {
|
|
83
84
|
ensureDaySummaryCron,
|
|
@@ -153,6 +154,7 @@ import {
|
|
|
153
154
|
import { CODEX_THREAD_KEY_PREFIX } from "./codex-thread-key.js";
|
|
154
155
|
import { isDisagreementPrompt } from "./signal.js";
|
|
155
156
|
import { lintWorkspaceFiles, rotateMarkdownFileToArchive } from "./hygiene.js";
|
|
157
|
+
import { isPathInsideStorageRoot } from "./storage-paths.js";
|
|
156
158
|
import { EmbeddingFallback } from "./embedding-fallback.js";
|
|
157
159
|
import {
|
|
158
160
|
decideSemanticDedup,
|
|
@@ -341,7 +343,11 @@ import {
|
|
|
341
343
|
dedupeBehaviorSignalsByMemoryAndHash,
|
|
342
344
|
} from "./behavior-signals.js";
|
|
343
345
|
import { ProfilingCollector } from "./profiling.js";
|
|
344
|
-
import {
|
|
346
|
+
import {
|
|
347
|
+
keyring,
|
|
348
|
+
secureStoreDir,
|
|
349
|
+
SecureStoreLockedError,
|
|
350
|
+
} from "./secure-store/index.js";
|
|
345
351
|
import type {
|
|
346
352
|
AccessTrackingEntry,
|
|
347
353
|
BehaviorLoopPolicyState,
|
|
@@ -710,10 +716,155 @@ async function raceRecallAbort<T>(
|
|
|
710
716
|
}
|
|
711
717
|
}
|
|
712
718
|
|
|
719
|
+
function qmdCollectionPathParts(resultPath: string): {
|
|
720
|
+
collection: string;
|
|
721
|
+
relativePath: string;
|
|
722
|
+
} | null {
|
|
723
|
+
if (!resultPath || path.isAbsolute(resultPath)) return null;
|
|
724
|
+
const normalized = resultPath.replace(/\\/g, "/").replace(/^\/+/, "");
|
|
725
|
+
const slashIndex = normalized.indexOf("/");
|
|
726
|
+
if (slashIndex <= 0 || slashIndex >= normalized.length - 1) return null;
|
|
727
|
+
const collection = normalized.slice(0, slashIndex);
|
|
728
|
+
if (/^\d{4}-\d{2}-\d{2}$/.test(collection)) return null;
|
|
729
|
+
return {
|
|
730
|
+
collection,
|
|
731
|
+
relativePath: normalized.slice(slashIndex + 1),
|
|
732
|
+
};
|
|
733
|
+
}
|
|
734
|
+
|
|
735
|
+
function qmdResultPathCandidates(
|
|
736
|
+
storageDir: string,
|
|
737
|
+
resultPath: string,
|
|
738
|
+
): string[] {
|
|
739
|
+
const candidates = new Set<string>();
|
|
740
|
+
const storageRoot = path.resolve(storageDir);
|
|
741
|
+
const addCandidate = (candidate: string) => {
|
|
742
|
+
const resolved = path.resolve(candidate);
|
|
743
|
+
if (isPathInsideStorageRoot(storageRoot, resolved)) {
|
|
744
|
+
candidates.add(resolved);
|
|
745
|
+
}
|
|
746
|
+
};
|
|
747
|
+
const addRelativeCandidates = (relativePath: string) => {
|
|
748
|
+
const normalized = relativePath.replace(/\\/g, "/").replace(/^\/+/, "");
|
|
749
|
+
if (!normalized) return;
|
|
750
|
+
addCandidate(path.join(storageRoot, normalized));
|
|
751
|
+
if (/^\d{4}-\d{2}-\d{2}\//.test(normalized)) {
|
|
752
|
+
addCandidate(path.join(storageRoot, "facts", normalized));
|
|
753
|
+
}
|
|
754
|
+
};
|
|
755
|
+
|
|
756
|
+
if (path.isAbsolute(resultPath)) {
|
|
757
|
+
addCandidate(resultPath);
|
|
758
|
+
} else {
|
|
759
|
+
addRelativeCandidates(resultPath);
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
return [...candidates];
|
|
763
|
+
}
|
|
764
|
+
|
|
713
765
|
/** Maximum age (ms) before a compaction-reset signal file is considered stale and removed. */
|
|
714
766
|
const COMPACTION_SIGNAL_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
715
767
|
const DEFAULT_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS = 10_000;
|
|
716
768
|
|
|
769
|
+
type DaySummaryGatherOptions = {
|
|
770
|
+
timeZone?: string;
|
|
771
|
+
now?: Date;
|
|
772
|
+
};
|
|
773
|
+
|
|
774
|
+
function normalizeIanaTimeZone(value: unknown): string | undefined {
|
|
775
|
+
if (typeof value !== "string") return undefined;
|
|
776
|
+
const trimmed = value.trim();
|
|
777
|
+
if (!trimmed) return undefined;
|
|
778
|
+
try {
|
|
779
|
+
Intl.DateTimeFormat(undefined, { timeZone: trimmed });
|
|
780
|
+
return trimmed;
|
|
781
|
+
} catch {
|
|
782
|
+
return undefined;
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
function formatDateInTimeZone(date: Date, timeZone: string): string {
|
|
787
|
+
const parts = new Intl.DateTimeFormat("en-US", {
|
|
788
|
+
timeZone,
|
|
789
|
+
year: "numeric",
|
|
790
|
+
month: "2-digit",
|
|
791
|
+
day: "2-digit",
|
|
792
|
+
}).formatToParts(date);
|
|
793
|
+
const get = (type: string): string => parts.find((part) => part.type === type)?.value ?? "";
|
|
794
|
+
return `${get("year")}-${get("month")}-${get("day")}`;
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
function utcDateKey(date: Date): string {
|
|
798
|
+
return date.toISOString().slice(0, 10);
|
|
799
|
+
}
|
|
800
|
+
|
|
801
|
+
function utcDateKeysAround(date: Date): string[] {
|
|
802
|
+
const dayMs = 86_400_000;
|
|
803
|
+
const keys = [
|
|
804
|
+
utcDateKey(new Date(date.getTime() - dayMs)),
|
|
805
|
+
utcDateKey(date),
|
|
806
|
+
utcDateKey(new Date(date.getTime() + dayMs)),
|
|
807
|
+
];
|
|
808
|
+
return keys.filter((value, index, array) => array.indexOf(value) === index);
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
function utcDateKeysForLocalDay(date: Date, timeZone: string): string[] {
|
|
812
|
+
const targetLocalDate = formatDateInTimeZone(date, timeZone);
|
|
813
|
+
const keys = new Set<string>();
|
|
814
|
+
const hourMs = 3_600_000;
|
|
815
|
+
const scanStart = date.getTime() - 48 * hourMs;
|
|
816
|
+
const scanEnd = date.getTime() + 48 * hourMs;
|
|
817
|
+
for (let ms = scanStart; ms <= scanEnd; ms += hourMs) {
|
|
818
|
+
const candidate = new Date(ms);
|
|
819
|
+
if (formatDateInTimeZone(candidate, timeZone) === targetLocalDate) {
|
|
820
|
+
keys.add(utcDateKey(candidate));
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
return keys.size > 0 ? [...keys].sort() : utcDateKeysAround(date);
|
|
824
|
+
}
|
|
825
|
+
|
|
826
|
+
function parseFiniteDate(value: unknown): Date | null {
|
|
827
|
+
if (typeof value !== "string" || value.trim().length === 0) return null;
|
|
828
|
+
const parsed = new Date(value);
|
|
829
|
+
return Number.isFinite(parsed.getTime()) ? parsed : null;
|
|
830
|
+
}
|
|
831
|
+
|
|
832
|
+
function filterHourlySummaryMarkdownForLocalDay(
|
|
833
|
+
raw: string,
|
|
834
|
+
utcDate: string,
|
|
835
|
+
timeZone: string,
|
|
836
|
+
targetLocalDate: string,
|
|
837
|
+
): string | null {
|
|
838
|
+
const hourHeaderPattern = /^## ([01]\d|2[0-3]):00[ \t]*$/gm;
|
|
839
|
+
const matches = Array.from(raw.matchAll(hourHeaderPattern));
|
|
840
|
+
if (matches.length === 0) return null;
|
|
841
|
+
|
|
842
|
+
const firstSectionStart = matches[0]?.index ?? 0;
|
|
843
|
+
const preamble = raw.slice(0, firstSectionStart).trim();
|
|
844
|
+
const sections: string[] = [];
|
|
845
|
+
for (let index = 0; index < matches.length; index += 1) {
|
|
846
|
+
const match = matches[index];
|
|
847
|
+
const hour = match[1];
|
|
848
|
+
if (!hour) continue;
|
|
849
|
+
const sectionTimestamp = parseFiniteDate(`${utcDate}T${hour}:00:00.000Z`);
|
|
850
|
+
if (
|
|
851
|
+
!sectionTimestamp ||
|
|
852
|
+
formatDateInTimeZone(sectionTimestamp, timeZone) !== targetLocalDate
|
|
853
|
+
) {
|
|
854
|
+
continue;
|
|
855
|
+
}
|
|
856
|
+
const sectionStart = match.index ?? 0;
|
|
857
|
+
const sectionEnd = matches[index + 1]?.index ?? raw.length;
|
|
858
|
+
const section = raw.slice(sectionStart, sectionEnd).trim();
|
|
859
|
+
if (section.length > 0) sections.push(section);
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
if (sections.length === 0) return null;
|
|
863
|
+
return [preamble, ...sections]
|
|
864
|
+
.filter((section) => section.length > 0)
|
|
865
|
+
.join("\n\n");
|
|
866
|
+
}
|
|
867
|
+
|
|
717
868
|
type SearchCollectionState = "present" | "missing" | "unknown" | "skipped";
|
|
718
869
|
|
|
719
870
|
function qmdStartupCollectionCheckTimeoutMs(): number {
|
|
@@ -2243,10 +2394,12 @@ export class Orchestrator {
|
|
|
2243
2394
|
? await this._fastGatewayLlm.chatCompletion(messages, {
|
|
2244
2395
|
maxTokens: targetTokens * 2,
|
|
2245
2396
|
timeoutMs: this.config.localLlmFastTimeoutMs,
|
|
2246
|
-
|
|
2247
|
-
|
|
2248
|
-
|
|
2249
|
-
|
|
2397
|
+
// LCM is latency-sensitive. A configured fast persona is the
|
|
2398
|
+
// explicit fast-tier route; otherwise use taskModelChain so LCM
|
|
2399
|
+
// avoids falling to an expensive gateway default. Issue #1473.
|
|
2400
|
+
...(this.config.fastGatewayAgentId
|
|
2401
|
+
? { agentId: this.config.fastGatewayAgentId }
|
|
2402
|
+
: gatewayTaskChainOptions(this.config)),
|
|
2250
2403
|
})
|
|
2251
2404
|
: await this.localLlm.chatCompletion(messages, {
|
|
2252
2405
|
maxTokens: targetTokens * 2,
|
|
@@ -3155,8 +3308,10 @@ export class Orchestrator {
|
|
|
3155
3308
|
}
|
|
3156
3309
|
|
|
3157
3310
|
/**
|
|
3158
|
-
* Auto-register the engram-day-summary cron job in OpenClaw
|
|
3311
|
+
* Auto-register the engram-day-summary cron job in OpenClaw.
|
|
3312
|
+
* Reconciles model and timezone on every startup so config changes propagate.
|
|
3159
3313
|
* Fire-and-forget — never blocks init or crashes on failure.
|
|
3314
|
+
* Issues #1474, #1475.
|
|
3160
3315
|
*/
|
|
3161
3316
|
private async autoRegisterDaySummaryCron(): Promise<void> {
|
|
3162
3317
|
const home = resolveHomeDir();
|
|
@@ -3169,15 +3324,57 @@ export class Orchestrator {
|
|
|
3169
3324
|
);
|
|
3170
3325
|
return;
|
|
3171
3326
|
}
|
|
3172
|
-
|
|
3173
|
-
|
|
3327
|
+
|
|
3328
|
+
// Resolve an OpenClaw cron-routing model only in gateway mode. In plugin
|
|
3329
|
+
// mode, summaryModel is a direct-client model id for Remnic's own LLM
|
|
3330
|
+
// calls and may be unroutable as an OpenClaw agentTurn model.
|
|
3331
|
+
const rawSummaryModel = this.config.summaryModel;
|
|
3332
|
+
const taskPrimary = this.config.taskModelChain?.primary;
|
|
3333
|
+
const isGateway = this.config.modelSource === "gateway";
|
|
3334
|
+
const model = isGateway ? (rawSummaryModel || taskPrimary || undefined) : undefined;
|
|
3335
|
+
// Attach task-chain fallbacks only when the model matches the task-chain
|
|
3336
|
+
// primary. If summaryModel is a distinct override, its fallbacks would
|
|
3337
|
+
// be unrelated to the task chain. Also append gateway default models as
|
|
3338
|
+
// tail fallbacks (de-duped) so a task-chain outage doesn't stop the cron
|
|
3339
|
+
// before reaching the gateway default chain. Mirrors hourly cron pattern.
|
|
3340
|
+
const fallbacks: string[] = [];
|
|
3341
|
+
if (model && taskPrimary && model === taskPrimary) {
|
|
3342
|
+
const seen = new Set<string>(model ? [model] : []);
|
|
3343
|
+
const addUnique = (value: string | undefined) => {
|
|
3344
|
+
if (typeof value !== "string") return;
|
|
3345
|
+
const trimmed = value.trim();
|
|
3346
|
+
if (trimmed.length > 0 && !seen.has(trimmed)) {
|
|
3347
|
+
seen.add(trimmed);
|
|
3348
|
+
fallbacks.push(trimmed);
|
|
3349
|
+
}
|
|
3350
|
+
};
|
|
3351
|
+
for (const fb of this.config.taskModelChain?.fallbacks ?? []) addUnique(fb);
|
|
3352
|
+
const gwDefaults = this.config.gatewayConfig?.agents?.defaults?.model;
|
|
3353
|
+
addUnique(gwDefaults?.primary);
|
|
3354
|
+
if (Array.isArray(gwDefaults?.fallbacks)) {
|
|
3355
|
+
for (const fb of gwDefaults.fallbacks) addUnique(fb);
|
|
3356
|
+
}
|
|
3357
|
+
}
|
|
3358
|
+
|
|
3359
|
+
// Resolve timezone: configurable override, then server default
|
|
3360
|
+
const timezone = this.config.daySummaryTimezone
|
|
3361
|
+
|| Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
3362
|
+
|
|
3363
|
+
const result = await ensureDaySummaryCron(jobsPath, {
|
|
3364
|
+
timezone,
|
|
3365
|
+
...(model ? { model } : {}),
|
|
3366
|
+
...(fallbacks.length > 0 ? { fallbacks } : {}),
|
|
3174
3367
|
});
|
|
3175
|
-
if (
|
|
3368
|
+
if (result.created) {
|
|
3176
3369
|
log.info(
|
|
3177
|
-
`day-summary cron auto-registered (${
|
|
3370
|
+
`day-summary cron auto-registered (${result.jobId}, 23:47 ${timezone}${model ? `, model: ${model}` : ""})`,
|
|
3371
|
+
);
|
|
3372
|
+
} else if (result.updated) {
|
|
3373
|
+
log.info(
|
|
3374
|
+
`day-summary cron reconciled (${result.jobId}, timezone: ${timezone}${model ? `, model: ${model}` : ""})`,
|
|
3178
3375
|
);
|
|
3179
3376
|
} else {
|
|
3180
|
-
log.debug("day-summary cron already
|
|
3377
|
+
log.debug("day-summary cron already up to date");
|
|
3181
3378
|
}
|
|
3182
3379
|
} catch (err) {
|
|
3183
3380
|
log.debug(`day-summary cron auto-register error: ${err}`);
|
|
@@ -4098,8 +4295,9 @@ export class Orchestrator {
|
|
|
4098
4295
|
*/
|
|
4099
4296
|
async generateDaySummaryAuto(
|
|
4100
4297
|
namespace?: string,
|
|
4298
|
+
options: DaySummaryGatherOptions = {},
|
|
4101
4299
|
): Promise<DaySummaryResult | null> {
|
|
4102
|
-
const gathered = await this.gatherTodayFacts(namespace);
|
|
4300
|
+
const gathered = await this.gatherTodayFacts(namespace, options);
|
|
4103
4301
|
if (!gathered || !gathered.trim()) {
|
|
4104
4302
|
log.warn("generateDaySummaryAuto: no facts found for today, skipping");
|
|
4105
4303
|
return null;
|
|
@@ -4111,24 +4309,27 @@ export class Orchestrator {
|
|
|
4111
4309
|
* Read today's facts and hourly summaries from storage, returning them
|
|
4112
4310
|
* as a formatted string suitable for generateDaySummary().
|
|
4113
4311
|
*/
|
|
4114
|
-
async gatherTodayFacts(
|
|
4312
|
+
async gatherTodayFacts(
|
|
4313
|
+
namespace?: string,
|
|
4314
|
+
options: DaySummaryGatherOptions = {},
|
|
4315
|
+
): Promise<string> {
|
|
4115
4316
|
const ns =
|
|
4116
4317
|
namespace && namespace.length > 0
|
|
4117
4318
|
? namespace
|
|
4118
4319
|
: this.config.defaultNamespace;
|
|
4119
4320
|
const storage = await this.storageRouter.storageFor(ns);
|
|
4120
|
-
|
|
4121
|
-
|
|
4122
|
-
|
|
4123
|
-
|
|
4124
|
-
const now =
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
4130
|
-
|
|
4131
|
-
);
|
|
4321
|
+
const configuredTimeZone = normalizeIanaTimeZone(options.timeZone)
|
|
4322
|
+
?? normalizeIanaTimeZone(this.config.daySummaryTimezone);
|
|
4323
|
+
const timeZone =
|
|
4324
|
+
configuredTimeZone ?? Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
4325
|
+
const now = options.now instanceof Date && Number.isFinite(options.now.getTime())
|
|
4326
|
+
? options.now
|
|
4327
|
+
: new Date();
|
|
4328
|
+
const targetLocalDate = formatDateInTimeZone(now, timeZone);
|
|
4329
|
+
// Facts are stored under UTC date directories, while the summary target is
|
|
4330
|
+
// a local calendar day. Scan the UTC-date envelope that overlaps the local
|
|
4331
|
+
// day, then filter parseable fact timestamps to that configured local day.
|
|
4332
|
+
const datesToScan = utcDateKeysForLocalDay(now, timeZone);
|
|
4132
4333
|
const factsBaseDir = path.join(storage.dir, "facts");
|
|
4133
4334
|
const MAX_CHARS = 100_000;
|
|
4134
4335
|
|
|
@@ -4155,13 +4356,21 @@ export class Orchestrator {
|
|
|
4155
4356
|
.slice(colonIdx + 1)
|
|
4156
4357
|
.trim();
|
|
4157
4358
|
}
|
|
4359
|
+
const created = fm.created || "unknown";
|
|
4360
|
+
const createdAt = parseFiniteDate(created);
|
|
4361
|
+
if (
|
|
4362
|
+
createdAt &&
|
|
4363
|
+
formatDateInTimeZone(createdAt, timeZone) !== targetLocalDate
|
|
4364
|
+
) {
|
|
4365
|
+
continue;
|
|
4366
|
+
}
|
|
4158
4367
|
facts.push({
|
|
4159
4368
|
path: fullPath,
|
|
4160
4369
|
frontmatter: {
|
|
4161
4370
|
id: fm.id || path.basename(entry.name, ".md"),
|
|
4162
4371
|
category: (fm.category as any) || "fact",
|
|
4163
|
-
created
|
|
4164
|
-
updated: fm.updated ||
|
|
4372
|
+
created,
|
|
4373
|
+
updated: fm.updated || created,
|
|
4165
4374
|
source: fm.source || "unknown",
|
|
4166
4375
|
confidence: parseFloat(fm.confidence || "0.8"),
|
|
4167
4376
|
confidenceTier: (fm.confidenceTier as any) || "implied",
|
|
@@ -4179,9 +4388,10 @@ export class Orchestrator {
|
|
|
4179
4388
|
}
|
|
4180
4389
|
|
|
4181
4390
|
// Sort facts by created timestamp (most recent last) so truncation keeps newest
|
|
4182
|
-
facts.sort((a, b) =>
|
|
4183
|
-
a.frontmatter.created
|
|
4184
|
-
|
|
4391
|
+
facts.sort((a, b) => {
|
|
4392
|
+
if (a.frontmatter.created === b.frontmatter.created) return 0;
|
|
4393
|
+
return a.frontmatter.created < b.frontmatter.created ? -1 : 1;
|
|
4394
|
+
});
|
|
4185
4395
|
|
|
4186
4396
|
// --- Read hourly summaries for the scanned dates ---
|
|
4187
4397
|
const hourlySummaries: string[] = [];
|
|
@@ -4194,8 +4404,14 @@ export class Orchestrator {
|
|
|
4194
4404
|
const summaryFile = path.join(hourlyBaseDir, sk.name, `${date}.md`);
|
|
4195
4405
|
try {
|
|
4196
4406
|
const raw = await readFile(summaryFile, "utf-8");
|
|
4197
|
-
|
|
4198
|
-
|
|
4407
|
+
const filtered = filterHourlySummaryMarkdownForLocalDay(
|
|
4408
|
+
raw,
|
|
4409
|
+
date,
|
|
4410
|
+
timeZone,
|
|
4411
|
+
targetLocalDate,
|
|
4412
|
+
);
|
|
4413
|
+
if (filtered) {
|
|
4414
|
+
hourlySummaries.push(filtered);
|
|
4199
4415
|
}
|
|
4200
4416
|
} catch {
|
|
4201
4417
|
// No summary file for this session/date
|
|
@@ -4233,7 +4449,7 @@ export class Orchestrator {
|
|
|
4233
4449
|
}
|
|
4234
4450
|
|
|
4235
4451
|
log.info(
|
|
4236
|
-
`gatherTodayFacts: collected ${facts.length} facts, ${hourlySummaries.length} hourly summaries (${formatted.length} chars)`,
|
|
4452
|
+
`gatherTodayFacts: collected ${facts.length} facts, ${hourlySummaries.length} hourly summaries for ${targetLocalDate} (${timeZone}, ${formatted.length} chars)`,
|
|
4237
4453
|
);
|
|
4238
4454
|
|
|
4239
4455
|
return formatted;
|
|
@@ -5939,6 +6155,7 @@ export class Orchestrator {
|
|
|
5939
6155
|
memoryResults: QmdSearchResult[];
|
|
5940
6156
|
recallNamespaces: string[];
|
|
5941
6157
|
recallResultLimit: number;
|
|
6158
|
+
deadlineAtMs?: number | null;
|
|
5942
6159
|
/** Issue #681 — when true, bypass graphTraversalConfidenceFloor. */
|
|
5943
6160
|
includeLowConfidence?: boolean;
|
|
5944
6161
|
}): Promise<{
|
|
@@ -5947,16 +6164,90 @@ export class Orchestrator {
|
|
|
5947
6164
|
expandedPaths: GraphRecallExpandedEntry[];
|
|
5948
6165
|
seedResults: QmdSearchResult[];
|
|
5949
6166
|
}> {
|
|
6167
|
+
const deadlineExpired = (): boolean =>
|
|
6168
|
+
typeof options.deadlineAtMs === "number" &&
|
|
6169
|
+
Date.now() >= options.deadlineAtMs;
|
|
5950
6170
|
const byNamespace = new Map<string, QmdSearchResult[]>();
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
|
|
6171
|
+
const addResultForNamespace = (
|
|
6172
|
+
namespace: string,
|
|
6173
|
+
result: QmdSearchResult,
|
|
6174
|
+
): void => {
|
|
6175
|
+
const existing = byNamespace.get(namespace);
|
|
5955
6176
|
if (existing) {
|
|
5956
6177
|
existing.push(result);
|
|
5957
6178
|
} else {
|
|
5958
|
-
byNamespace.set(
|
|
6179
|
+
byNamespace.set(namespace, [result]);
|
|
6180
|
+
}
|
|
6181
|
+
};
|
|
6182
|
+
const resolvedAmbiguousSeeds = new Map<
|
|
6183
|
+
string,
|
|
6184
|
+
{ namespace: string; result: QmdSearchResult } | null
|
|
6185
|
+
>();
|
|
6186
|
+
const resolveAmbiguousSeedOwner = async (
|
|
6187
|
+
result: QmdSearchResult,
|
|
6188
|
+
parts: { collection: string; relativePath: string } | null,
|
|
6189
|
+
): Promise<{ namespace: string; result: QmdSearchResult } | null> => {
|
|
6190
|
+
const cached = resolvedAmbiguousSeeds.get(result.path);
|
|
6191
|
+
if (cached !== undefined) return cached;
|
|
6192
|
+
if (deadlineExpired()) {
|
|
6193
|
+
resolvedAmbiguousSeeds.set(result.path, null);
|
|
6194
|
+
return null;
|
|
6195
|
+
}
|
|
6196
|
+
|
|
6197
|
+
let resolvedPath = result.path;
|
|
6198
|
+
let resolvedResult = result;
|
|
6199
|
+
if (parts) {
|
|
6200
|
+
const resolvedCold = await this.resolveColdQmdResultForRecall(
|
|
6201
|
+
result,
|
|
6202
|
+
this.storage,
|
|
6203
|
+
options.recallNamespaces,
|
|
6204
|
+
);
|
|
6205
|
+
if (!resolvedCold || deadlineExpired()) {
|
|
6206
|
+
resolvedAmbiguousSeeds.set(result.path, null);
|
|
6207
|
+
return null;
|
|
6208
|
+
}
|
|
6209
|
+
resolvedPath = resolvedCold.result.path;
|
|
6210
|
+
resolvedResult = resolvedCold.result;
|
|
6211
|
+
}
|
|
6212
|
+
|
|
6213
|
+
if (!path.isAbsolute(resolvedPath)) {
|
|
6214
|
+
resolvedAmbiguousSeeds.set(result.path, null);
|
|
6215
|
+
return null;
|
|
6216
|
+
}
|
|
6217
|
+
const ownerStorage = await this.storageForAbsoluteQmdResultPath(
|
|
6218
|
+
resolvedPath,
|
|
6219
|
+
this.storage,
|
|
6220
|
+
options.recallNamespaces,
|
|
6221
|
+
);
|
|
6222
|
+
const ownerNamespace = ownerStorage?.namespace ?? null;
|
|
6223
|
+
const resolved =
|
|
6224
|
+
ownerNamespace && options.recallNamespaces.includes(ownerNamespace)
|
|
6225
|
+
? { namespace: ownerNamespace, result: resolvedResult }
|
|
6226
|
+
: null;
|
|
6227
|
+
resolvedAmbiguousSeeds.set(result.path, resolved);
|
|
6228
|
+
return resolved;
|
|
6229
|
+
};
|
|
6230
|
+
const coldCollection = this.config.qmdColdCollection ?? "openclaw-engram-cold";
|
|
6231
|
+
for (const result of options.memoryResults) {
|
|
6232
|
+
if (deadlineExpired()) break;
|
|
6233
|
+
const parts = qmdCollectionPathParts(result.path);
|
|
6234
|
+
if (parts?.collection === coldCollection) {
|
|
6235
|
+
const resolved = await resolveAmbiguousSeedOwner(result, parts);
|
|
6236
|
+
if (resolved) {
|
|
6237
|
+
addResultForNamespace(resolved.namespace, resolved.result);
|
|
6238
|
+
}
|
|
6239
|
+
continue;
|
|
6240
|
+
}
|
|
6241
|
+
if (path.isAbsolute(result.path)) {
|
|
6242
|
+
const resolved = await resolveAmbiguousSeedOwner(result, null);
|
|
6243
|
+
if (resolved) {
|
|
6244
|
+
addResultForNamespace(resolved.namespace, resolved.result);
|
|
6245
|
+
}
|
|
6246
|
+
continue;
|
|
5959
6247
|
}
|
|
6248
|
+
const ns = this.namespaceFromPath(result.path);
|
|
6249
|
+
if (!options.recallNamespaces.includes(ns)) continue;
|
|
6250
|
+
addResultForNamespace(ns, result);
|
|
5960
6251
|
}
|
|
5961
6252
|
|
|
5962
6253
|
const perNamespaceSeedCap = Math.max(3, options.recallResultLimit);
|
|
@@ -5967,15 +6258,31 @@ export class Orchestrator {
|
|
|
5967
6258
|
const expandedResults: QmdSearchResult[] = [];
|
|
5968
6259
|
|
|
5969
6260
|
for (const [namespace, nsResults] of byNamespace.entries()) {
|
|
6261
|
+
if (deadlineExpired()) break;
|
|
5970
6262
|
const storage = await this.storageRouter.storageFor(namespace);
|
|
5971
6263
|
const seedCandidates = nsResults.slice(0, perNamespaceSeedCap);
|
|
5972
6264
|
seedResults.push(...seedCandidates);
|
|
5973
|
-
const seedRelativePaths =
|
|
5974
|
-
.
|
|
5975
|
-
|
|
5976
|
-
|
|
5977
|
-
|
|
5978
|
-
|
|
6265
|
+
const seedRelativePaths =
|
|
6266
|
+
typeof options.deadlineAtMs === "number"
|
|
6267
|
+
? await this.graphSeedPathsWithinDeadline(
|
|
6268
|
+
storage,
|
|
6269
|
+
seedCandidates,
|
|
6270
|
+
options.deadlineAtMs,
|
|
6271
|
+
[namespace],
|
|
6272
|
+
)
|
|
6273
|
+
: (
|
|
6274
|
+
await Promise.all(
|
|
6275
|
+
seedCandidates.map((result) =>
|
|
6276
|
+
this.graphSeedPathRelativeToStorage(storage, result, [
|
|
6277
|
+
namespace,
|
|
6278
|
+
]),
|
|
6279
|
+
),
|
|
6280
|
+
)
|
|
6281
|
+
).filter(
|
|
6282
|
+
(value): value is string =>
|
|
6283
|
+
typeof value === "string" && value.length > 0,
|
|
6284
|
+
);
|
|
6285
|
+
if (deadlineExpired()) break;
|
|
5979
6286
|
if (seedRelativePaths.length === 0) continue;
|
|
5980
6287
|
|
|
5981
6288
|
const seedRecallScore = seedCandidates.reduce(
|
|
@@ -5989,14 +6296,22 @@ export class Orchestrator {
|
|
|
5989
6296
|
const expanded = await this.graphIndexFor(storage).spreadingActivation(
|
|
5990
6297
|
seedRelativePaths,
|
|
5991
6298
|
this.config.maxGraphTraversalSteps,
|
|
5992
|
-
|
|
6299
|
+
{
|
|
6300
|
+
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
6301
|
+
...(typeof options.deadlineAtMs === "number"
|
|
6302
|
+
? { deadlineAtMs: options.deadlineAtMs }
|
|
6303
|
+
: {}),
|
|
6304
|
+
},
|
|
5993
6305
|
);
|
|
5994
6306
|
if (expanded.length === 0) continue;
|
|
6307
|
+
if (deadlineExpired()) break;
|
|
5995
6308
|
|
|
5996
6309
|
for (const candidate of expanded.slice(0, perNamespaceExpandedCap)) {
|
|
6310
|
+
if (deadlineExpired()) break;
|
|
5997
6311
|
if (seedSet.has(candidate.path)) continue;
|
|
5998
6312
|
const memoryPath = path.resolve(storage.dir, candidate.path);
|
|
5999
6313
|
const memory = await storage.readMemoryByPath(memoryPath);
|
|
6314
|
+
if (deadlineExpired()) break;
|
|
6000
6315
|
if (!memory) continue;
|
|
6001
6316
|
if (isArtifactMemoryPath(memory.path)) continue;
|
|
6002
6317
|
if (memory.frontmatter.status && memory.frontmatter.status !== "active")
|
|
@@ -8737,17 +9052,18 @@ export class Orchestrator {
|
|
|
8737
9052
|
// several seconds on large box directories due to sequential I/O). We kick it
|
|
8738
9053
|
// off here so it overlaps with QMD and other concurrent work rather than
|
|
8739
9054
|
// running sequentially in phase-2 and blocking assembly.
|
|
8740
|
-
const recentBoxesPromise
|
|
9055
|
+
const recentBoxesPromise = observeEnrichmentPromise(
|
|
8741
9056
|
this.isRecallSectionEnabled(
|
|
8742
9057
|
"memory-boxes",
|
|
8743
9058
|
this.config.memoryBoxesEnabled === true,
|
|
8744
9059
|
) &&
|
|
8745
|
-
|
|
8746
|
-
|
|
9060
|
+
this.config.memoryBoxesEnabled &&
|
|
9061
|
+
this.config.boxRecallDays > 0
|
|
8747
9062
|
? this.boxBuilderFor(profileStorage)
|
|
8748
9063
|
.readRecentBoxes(this.config.boxRecallDays)
|
|
8749
9064
|
.catch(() => [] as BoxFrontmatter[])
|
|
8750
|
-
: Promise.resolve([] as BoxFrontmatter[])
|
|
9065
|
+
: Promise.resolve([] as BoxFrontmatter[]),
|
|
9066
|
+
);
|
|
8751
9067
|
|
|
8752
9068
|
// --- Wait for core sections first, then bounded enrichment ---
|
|
8753
9069
|
this.profiler.startSpan("phase-1-parallel", profileTraceId);
|
|
@@ -8931,6 +9247,70 @@ export class Orchestrator {
|
|
|
8931
9247
|
return finalizeEnrichmentOutcome(outcome);
|
|
8932
9248
|
};
|
|
8933
9249
|
|
|
9250
|
+
const remainingEnrichmentAssemblyMs = (): number | null =>
|
|
9251
|
+
enrichmentAssemblyDeadlineAtMs === null
|
|
9252
|
+
? null
|
|
9253
|
+
: Math.max(0, enrichmentAssemblyDeadlineAtMs - Date.now());
|
|
9254
|
+
|
|
9255
|
+
const awaitAssemblyStep = async <T>(
|
|
9256
|
+
name: string,
|
|
9257
|
+
task: () => Promise<T>,
|
|
9258
|
+
fallback: T,
|
|
9259
|
+
): Promise<T> => {
|
|
9260
|
+
if (options.abortSignal?.aborted) {
|
|
9261
|
+
log.debug(
|
|
9262
|
+
`recall phase-1 assembly [${name}]: skipped after abort at +${Date.now() - phase1Start}ms`,
|
|
9263
|
+
);
|
|
9264
|
+
return fallback;
|
|
9265
|
+
}
|
|
9266
|
+
|
|
9267
|
+
const timeoutMs = remainingEnrichmentAssemblyMs();
|
|
9268
|
+
if (timeoutMs === 0) {
|
|
9269
|
+
log.debug(
|
|
9270
|
+
`recall phase-1 assembly [${name}]: skipped after shared ${enrichmentSectionDeadlineMs}ms budget expired ` +
|
|
9271
|
+
`at +${Date.now() - phase1Start}ms`,
|
|
9272
|
+
);
|
|
9273
|
+
return fallback;
|
|
9274
|
+
}
|
|
9275
|
+
|
|
9276
|
+
let timeoutHandle: NodeJS.Timeout | undefined;
|
|
9277
|
+
try {
|
|
9278
|
+
const result = await (timeoutMs !== null
|
|
9279
|
+
? Promise.race<T | { status: "timed_out" }>([
|
|
9280
|
+
task(),
|
|
9281
|
+
new Promise<{ status: "timed_out" }>((resolve) => {
|
|
9282
|
+
timeoutHandle = setTimeout(
|
|
9283
|
+
() => resolve({ status: "timed_out" }),
|
|
9284
|
+
timeoutMs,
|
|
9285
|
+
);
|
|
9286
|
+
}),
|
|
9287
|
+
])
|
|
9288
|
+
: task());
|
|
9289
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
9290
|
+
if (
|
|
9291
|
+
typeof result === "object" &&
|
|
9292
|
+
result !== null &&
|
|
9293
|
+
"status" in result &&
|
|
9294
|
+
(result as { status?: unknown }).status === "timed_out"
|
|
9295
|
+
) {
|
|
9296
|
+
log.debug(
|
|
9297
|
+
`recall phase-1 assembly [${name}]: timed out within shared ${enrichmentSectionDeadlineMs}ms budget ` +
|
|
9298
|
+
`at +${Date.now() - phase1Start}ms`,
|
|
9299
|
+
);
|
|
9300
|
+
return fallback;
|
|
9301
|
+
}
|
|
9302
|
+
return result as T;
|
|
9303
|
+
} catch (err) {
|
|
9304
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
9305
|
+
log.warn(
|
|
9306
|
+
`recall phase-1 assembly [${name}] failed open: ${
|
|
9307
|
+
err instanceof Error ? err.message : String(err)
|
|
9308
|
+
}`,
|
|
9309
|
+
);
|
|
9310
|
+
return fallback;
|
|
9311
|
+
}
|
|
9312
|
+
};
|
|
9313
|
+
|
|
8934
9314
|
// --- Phase 2: Assemble sections in correct order ---
|
|
8935
9315
|
this.profiler.startSpan("assembly", profileTraceId);
|
|
8936
9316
|
|
|
@@ -9263,8 +9643,11 @@ export class Orchestrator {
|
|
|
9263
9643
|
// 1d. Memory Boxes (topic continuity windows, v8.0 Phase 2A)
|
|
9264
9644
|
// recentBoxesPromise was kicked off before phase-1 so it ran concurrently.
|
|
9265
9645
|
{
|
|
9266
|
-
const recentBoxes = await
|
|
9267
|
-
|
|
9646
|
+
const recentBoxes = await awaitEnrichmentSection(
|
|
9647
|
+
"memory-boxes",
|
|
9648
|
+
recentBoxesPromise,
|
|
9649
|
+
);
|
|
9650
|
+
if (recentBoxes && recentBoxes.length > 0) {
|
|
9268
9651
|
const boxLines = recentBoxes.slice(0, 5).map((b: BoxFrontmatter) => {
|
|
9269
9652
|
const sealedDate = b.sealedAt
|
|
9270
9653
|
? b.sealedAt.slice(0, 16).replace("T", " ")
|
|
@@ -9418,6 +9801,8 @@ export class Orchestrator {
|
|
|
9418
9801
|
}
|
|
9419
9802
|
|
|
9420
9803
|
// 2. QMD results — post-process and format
|
|
9804
|
+
const qmdWasSettledBeforeAssemblyWait =
|
|
9805
|
+
qmdPromise.getSettledOutcome() !== undefined;
|
|
9421
9806
|
const qmdResult = await awaitEnrichmentSection("qmd", qmdPromise);
|
|
9422
9807
|
if (qmdResult) {
|
|
9423
9808
|
const t0 = Date.now();
|
|
@@ -9480,56 +9865,82 @@ export class Orchestrator {
|
|
|
9480
9865
|
graphExpandedResultPaths.clear();
|
|
9481
9866
|
} else {
|
|
9482
9867
|
try {
|
|
9483
|
-
const
|
|
9484
|
-
|
|
9485
|
-
|
|
9486
|
-
|
|
9487
|
-
|
|
9488
|
-
|
|
9489
|
-
|
|
9490
|
-
|
|
9491
|
-
|
|
9492
|
-
|
|
9493
|
-
|
|
9494
|
-
graphSnapshotStatus = "completed";
|
|
9495
|
-
graphDecisionStatus = "completed";
|
|
9496
|
-
graphDecisionReason = graphShadowEvalEnabled
|
|
9497
|
-
? "graph shadow evaluation completed without altering injected context"
|
|
9498
|
-
: "graph expansion merged into recall ranking";
|
|
9499
|
-
graphSnapshotReason = graphDecisionReason;
|
|
9500
|
-
graphSnapshotSeedPaths = seedPaths;
|
|
9501
|
-
graphSnapshotExpandedPaths = expandedPaths;
|
|
9502
|
-
graphSnapshotSeedResults = this.buildGraphRecallRankedResults(
|
|
9503
|
-
seedResults,
|
|
9504
|
-
() => ["baseline"],
|
|
9868
|
+
const graphExpansion = await awaitAssemblyStep(
|
|
9869
|
+
"graph-expansion",
|
|
9870
|
+
() =>
|
|
9871
|
+
this.expandResultsViaGraph({
|
|
9872
|
+
memoryResults,
|
|
9873
|
+
recallNamespaces,
|
|
9874
|
+
recallResultLimit,
|
|
9875
|
+
deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
|
|
9876
|
+
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
9877
|
+
}),
|
|
9878
|
+
null as Awaited<ReturnType<typeof this.expandResultsViaGraph>> | null,
|
|
9505
9879
|
);
|
|
9506
|
-
|
|
9507
|
-
|
|
9508
|
-
|
|
9509
|
-
|
|
9510
|
-
|
|
9511
|
-
|
|
9512
|
-
|
|
9513
|
-
|
|
9514
|
-
|
|
9515
|
-
|
|
9880
|
+
if (!graphExpansion) {
|
|
9881
|
+
graphSnapshotStatus = "aborted";
|
|
9882
|
+
graphDecisionStatus = "aborted";
|
|
9883
|
+
graphDecisionReason = options.abortSignal?.aborted
|
|
9884
|
+
? "graph expansion skipped because recall assembly was aborted"
|
|
9885
|
+
: "graph expansion skipped because shared post-retrieval assembly budget expired";
|
|
9886
|
+
graphSnapshotReason = graphDecisionReason;
|
|
9887
|
+
graphSnapshotSeedPaths = baselineMemoryResults
|
|
9888
|
+
.slice(0, Math.max(1, recallResultLimit))
|
|
9889
|
+
.map((result) => result.path);
|
|
9890
|
+
graphSnapshotSeedResults = this.buildGraphRecallRankedResults(
|
|
9516
9891
|
baselineMemoryResults,
|
|
9892
|
+
() => ["baseline"],
|
|
9893
|
+
);
|
|
9894
|
+
graphSnapshotExpandedPaths = [];
|
|
9895
|
+
graphExpandedResultPaths.clear();
|
|
9896
|
+
memoryResults = baselineMemoryResults;
|
|
9897
|
+
} else {
|
|
9898
|
+
const {
|
|
9517
9899
|
merged,
|
|
9518
|
-
|
|
9900
|
+
seedPaths,
|
|
9901
|
+
expandedPaths,
|
|
9902
|
+
seedResults = baselineMemoryResults,
|
|
9903
|
+
} = graphExpansion;
|
|
9904
|
+
graphSnapshotStatus = "completed";
|
|
9905
|
+
graphDecisionStatus = "completed";
|
|
9906
|
+
graphDecisionReason = graphShadowEvalEnabled
|
|
9907
|
+
? "graph shadow evaluation completed without altering injected context"
|
|
9908
|
+
: "graph expansion merged into recall ranking";
|
|
9909
|
+
graphSnapshotReason = graphDecisionReason;
|
|
9910
|
+
graphSnapshotSeedPaths = seedPaths;
|
|
9911
|
+
graphSnapshotExpandedPaths = expandedPaths;
|
|
9912
|
+
graphSnapshotSeedResults = this.buildGraphRecallRankedResults(
|
|
9913
|
+
seedResults,
|
|
9914
|
+
() => ["baseline"],
|
|
9519
9915
|
);
|
|
9520
|
-
|
|
9521
|
-
|
|
9522
|
-
|
|
9523
|
-
|
|
9524
|
-
|
|
9525
|
-
|
|
9526
|
-
|
|
9527
|
-
|
|
9528
|
-
|
|
9529
|
-
|
|
9530
|
-
|
|
9531
|
-
|
|
9532
|
-
|
|
9916
|
+
graphExpandedResultPaths.clear();
|
|
9917
|
+
expandedPaths.forEach((entry) =>
|
|
9918
|
+
graphExpandedResultPaths.add(entry.path),
|
|
9919
|
+
);
|
|
9920
|
+
memoryResults = graphShadowEvalEnabled
|
|
9921
|
+
? baselineMemoryResults
|
|
9922
|
+
: merged;
|
|
9923
|
+
|
|
9924
|
+
if (graphShadowEvalEnabled) {
|
|
9925
|
+
const comparison = summarizeGraphShadowComparison(
|
|
9926
|
+
baselineMemoryResults,
|
|
9927
|
+
merged,
|
|
9928
|
+
recallResultLimit,
|
|
9929
|
+
);
|
|
9930
|
+
graphSnapshotShadowComparison = comparison;
|
|
9931
|
+
recordRecallSectionMetric({
|
|
9932
|
+
section: "graphShadow",
|
|
9933
|
+
priority: "enrichment",
|
|
9934
|
+
durationMs: Date.now() - t0,
|
|
9935
|
+
deadlineMs: enrichmentSectionDeadlineMs,
|
|
9936
|
+
source: "fresh",
|
|
9937
|
+
success: true,
|
|
9938
|
+
timing:
|
|
9939
|
+
`on b=${comparison.baselineCount} g=${comparison.graphCount} ` +
|
|
9940
|
+
`ov=${comparison.overlapCount} (${comparison.overlapRatio.toFixed(2)}) ` +
|
|
9941
|
+
`avgDelta=${comparison.averageOverlapDelta.toFixed(3)}`,
|
|
9942
|
+
});
|
|
9943
|
+
}
|
|
9533
9944
|
}
|
|
9534
9945
|
} catch (err) {
|
|
9535
9946
|
graphSnapshotStatus = "aborted";
|
|
@@ -9551,13 +9962,41 @@ export class Orchestrator {
|
|
|
9551
9962
|
}
|
|
9552
9963
|
}
|
|
9553
9964
|
|
|
9554
|
-
// Apply
|
|
9555
|
-
|
|
9965
|
+
// Apply mandatory recall safety filters before deadline-bound scoring
|
|
9966
|
+
// enrichment. If scoring times out, we must fall back to this filtered
|
|
9967
|
+
// list rather than raw QMD hits; boostSearchResults also removes
|
|
9968
|
+
// forgotten, lifecycle-filtered, superseded/as_of-invalid, and
|
|
9969
|
+
// dream/procedural memories.
|
|
9970
|
+
const qmdBoostInput = await this.filterSearchResultsForRecall(
|
|
9556
9971
|
memoryResults,
|
|
9557
|
-
recallNamespaces,
|
|
9558
|
-
retrievalQuery,
|
|
9559
9972
|
undefined,
|
|
9560
|
-
{
|
|
9973
|
+
{
|
|
9974
|
+
asOfMs,
|
|
9975
|
+
// If QMD had already settled before the ordered assembly reached it,
|
|
9976
|
+
// do not let unrelated slow enrichment turn those known results into
|
|
9977
|
+
// unchecked misses. QMD that only settles during its own wait remains
|
|
9978
|
+
// bounded by the shared post-retrieval assembly deadline.
|
|
9979
|
+
deadlineAtMs: qmdWasSettledBeforeAssemblyWait
|
|
9980
|
+
? null
|
|
9981
|
+
: enrichmentAssemblyDeadlineAtMs,
|
|
9982
|
+
abortSignal: options.abortSignal,
|
|
9983
|
+
dropUnresolved: true,
|
|
9984
|
+
recallNamespaces,
|
|
9985
|
+
},
|
|
9986
|
+
);
|
|
9987
|
+
|
|
9988
|
+
// Apply recency and access count boosting
|
|
9989
|
+
memoryResults = await awaitAssemblyStep(
|
|
9990
|
+
"qmd-boost",
|
|
9991
|
+
() =>
|
|
9992
|
+
this.boostSearchResults(
|
|
9993
|
+
qmdBoostInput.results,
|
|
9994
|
+
recallNamespaces,
|
|
9995
|
+
retrievalQuery,
|
|
9996
|
+
qmdBoostInput.memoryByPath,
|
|
9997
|
+
{ asOfMs },
|
|
9998
|
+
),
|
|
9999
|
+
qmdBoostInput.results,
|
|
9561
10000
|
);
|
|
9562
10001
|
|
|
9563
10002
|
// Optional LLM reranking (default off). Fail-open if rerank fails/slow.
|
|
@@ -9739,41 +10178,47 @@ export class Orchestrator {
|
|
|
9739
10178
|
// low-relevance results from polluting context.
|
|
9740
10179
|
const queryAwarePrefilter = await queryAwarePrefilterPromise;
|
|
9741
10180
|
if (queryAwarePrefilter.candidatePaths?.size !== 0) {
|
|
9742
|
-
const
|
|
9743
|
-
|
|
9744
|
-
|
|
9745
|
-
|
|
9746
|
-
|
|
9747
|
-
|
|
9748
|
-
|
|
9749
|
-
|
|
9750
|
-
|
|
9751
|
-
|
|
9752
|
-
|
|
9753
|
-
|
|
9754
|
-
|
|
9755
|
-
|
|
9756
|
-
|
|
10181
|
+
const scoped = await awaitAssemblyStep(
|
|
10182
|
+
"embedding-fallback",
|
|
10183
|
+
async () => {
|
|
10184
|
+
const embeddingResults = await this.searchEmbeddingFallback(
|
|
10185
|
+
retrievalQuery,
|
|
10186
|
+
embeddingFetchLimit,
|
|
10187
|
+
);
|
|
10188
|
+
const prefilteredEmbeddingResults = applyQueryAwareCandidateFilter(
|
|
10189
|
+
embeddingResults,
|
|
10190
|
+
queryAwarePrefilter.candidatePaths,
|
|
10191
|
+
);
|
|
10192
|
+
const scopedCandidates = filterRecallCandidates(
|
|
10193
|
+
prefilteredEmbeddingResults,
|
|
10194
|
+
{
|
|
10195
|
+
namespacesEnabled: this.config.namespacesEnabled,
|
|
10196
|
+
recallNamespaces,
|
|
10197
|
+
resolveNamespace: (p) => this.namespaceFromPath(p),
|
|
10198
|
+
limit: embeddingFetchLimit,
|
|
10199
|
+
},
|
|
10200
|
+
);
|
|
10201
|
+
const boostedScoped = await this.boostSearchResults(
|
|
10202
|
+
scopedCandidates,
|
|
10203
|
+
recallNamespaces,
|
|
10204
|
+
retrievalQuery,
|
|
10205
|
+
undefined,
|
|
10206
|
+
{ asOfMs },
|
|
10207
|
+
);
|
|
10208
|
+
// MMR runs on the pre-truncation pool so diverse candidates just
|
|
10209
|
+
// below the cutoff can be promoted into the injected set.
|
|
10210
|
+
xrayBranchPoolSize.hot_embedding = Math.max(
|
|
10211
|
+
xrayBranchPoolSize.hot_embedding,
|
|
10212
|
+
boostedScoped.length,
|
|
10213
|
+
);
|
|
10214
|
+
return this.diversifyAndLimitRecallResults(
|
|
10215
|
+
"memories",
|
|
10216
|
+
boostedScoped,
|
|
10217
|
+
recallResultLimit,
|
|
10218
|
+
retrievalQuery,
|
|
10219
|
+
);
|
|
9757
10220
|
},
|
|
9758
|
-
|
|
9759
|
-
const boostedScoped = await this.boostSearchResults(
|
|
9760
|
-
scopedCandidates,
|
|
9761
|
-
recallNamespaces,
|
|
9762
|
-
retrievalQuery,
|
|
9763
|
-
undefined,
|
|
9764
|
-
{ asOfMs },
|
|
9765
|
-
);
|
|
9766
|
-
// MMR runs on the pre-truncation pool so diverse candidates just
|
|
9767
|
-
// below the cutoff can be promoted into the injected set.
|
|
9768
|
-
xrayBranchPoolSize.hot_embedding = Math.max(
|
|
9769
|
-
xrayBranchPoolSize.hot_embedding,
|
|
9770
|
-
boostedScoped.length,
|
|
9771
|
-
);
|
|
9772
|
-
const scoped = this.diversifyAndLimitRecallResults(
|
|
9773
|
-
"memories",
|
|
9774
|
-
boostedScoped,
|
|
9775
|
-
recallResultLimit,
|
|
9776
|
-
retrievalQuery,
|
|
10221
|
+
[] as QmdSearchResult[],
|
|
9777
10222
|
);
|
|
9778
10223
|
if (scoped.length > 0) {
|
|
9779
10224
|
if (shouldPersistGraphSnapshot) {
|
|
@@ -9811,6 +10256,7 @@ export class Orchestrator {
|
|
|
9811
10256
|
queryAwarePrefilter,
|
|
9812
10257
|
abortSignal: options.abortSignal,
|
|
9813
10258
|
xrayPoolSizeSink: xrayColdPoolSink,
|
|
10259
|
+
deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
|
|
9814
10260
|
asOfMs,
|
|
9815
10261
|
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
9816
10262
|
});
|
|
@@ -9886,42 +10332,48 @@ export class Orchestrator {
|
|
|
9886
10332
|
// Fallback: embeddings first, then recency-only.
|
|
9887
10333
|
const queryAwarePrefilter = await queryAwarePrefilterPromise;
|
|
9888
10334
|
if (queryAwarePrefilter.candidatePaths?.size !== 0) {
|
|
9889
|
-
|
|
9890
|
-
|
|
9891
|
-
|
|
9892
|
-
|
|
9893
|
-
|
|
9894
|
-
|
|
9895
|
-
|
|
9896
|
-
|
|
9897
|
-
|
|
9898
|
-
|
|
9899
|
-
|
|
9900
|
-
|
|
9901
|
-
|
|
9902
|
-
|
|
9903
|
-
|
|
9904
|
-
|
|
9905
|
-
|
|
9906
|
-
|
|
9907
|
-
|
|
9908
|
-
|
|
9909
|
-
|
|
9910
|
-
|
|
9911
|
-
|
|
9912
|
-
|
|
9913
|
-
|
|
9914
|
-
|
|
9915
|
-
|
|
9916
|
-
|
|
9917
|
-
|
|
9918
|
-
|
|
9919
|
-
|
|
9920
|
-
|
|
9921
|
-
|
|
9922
|
-
|
|
9923
|
-
|
|
9924
|
-
|
|
10335
|
+
const scoped = await awaitAssemblyStep(
|
|
10336
|
+
"embedding-fallback",
|
|
10337
|
+
async () => {
|
|
10338
|
+
const embeddingResults = await this.searchEmbeddingFallback(
|
|
10339
|
+
retrievalQuery,
|
|
10340
|
+
embeddingFetchLimit,
|
|
10341
|
+
);
|
|
10342
|
+
const prefilteredEmbeddingResults = applyQueryAwareCandidateFilter(
|
|
10343
|
+
embeddingResults,
|
|
10344
|
+
queryAwarePrefilter.candidatePaths,
|
|
10345
|
+
);
|
|
10346
|
+
const scopedCandidates = filterRecallCandidates(
|
|
10347
|
+
prefilteredEmbeddingResults,
|
|
10348
|
+
{
|
|
10349
|
+
namespacesEnabled: this.config.namespacesEnabled,
|
|
10350
|
+
recallNamespaces,
|
|
10351
|
+
resolveNamespace: (p) => this.namespaceFromPath(p),
|
|
10352
|
+
limit: embeddingFetchLimit,
|
|
10353
|
+
},
|
|
10354
|
+
);
|
|
10355
|
+
const boostedScoped = await this.boostSearchResults(
|
|
10356
|
+
scopedCandidates,
|
|
10357
|
+
recallNamespaces,
|
|
10358
|
+
retrievalQuery,
|
|
10359
|
+
undefined,
|
|
10360
|
+
{ asOfMs },
|
|
10361
|
+
);
|
|
10362
|
+
// MMR runs on the pre-truncation pool so diverse candidates just
|
|
10363
|
+
// below the cutoff can be promoted into the injected set.
|
|
10364
|
+
xrayBranchPoolSize.hot_embedding = Math.max(
|
|
10365
|
+
xrayBranchPoolSize.hot_embedding,
|
|
10366
|
+
boostedScoped.length,
|
|
10367
|
+
);
|
|
10368
|
+
return this.diversifyAndLimitRecallResults(
|
|
10369
|
+
"memories",
|
|
10370
|
+
boostedScoped,
|
|
10371
|
+
recallResultLimit,
|
|
10372
|
+
retrievalQuery,
|
|
10373
|
+
);
|
|
10374
|
+
},
|
|
10375
|
+
[] as QmdSearchResult[],
|
|
10376
|
+
);
|
|
9925
10377
|
if (scoped.length > 0) {
|
|
9926
10378
|
if (shouldPersistGraphSnapshot) {
|
|
9927
10379
|
graphSnapshotFinalResults = this.buildGraphRecallRankedResults(
|
|
@@ -9950,8 +10402,11 @@ export class Orchestrator {
|
|
|
9950
10402
|
xrayRecalledResults = scoped;
|
|
9951
10403
|
impressionRecorded = true;
|
|
9952
10404
|
} else {
|
|
9953
|
-
const memories =
|
|
9954
|
-
|
|
10405
|
+
const memories = await awaitAssemblyStep(
|
|
10406
|
+
"recent-memory-read",
|
|
10407
|
+
() => this.readAllMemoriesForNamespaces(recallNamespaces),
|
|
10408
|
+
[] as MemoryFile[],
|
|
10409
|
+
);
|
|
9955
10410
|
if (memories.length > 0) {
|
|
9956
10411
|
// Filter out non-active memories. Delegate to
|
|
9957
10412
|
// shouldFilterSupersededFromRecall for superseded-status logic so
|
|
@@ -10023,6 +10478,7 @@ export class Orchestrator {
|
|
|
10023
10478
|
queryAwarePrefilter,
|
|
10024
10479
|
abortSignal: options.abortSignal,
|
|
10025
10480
|
xrayPoolSizeSink: xrayColdPoolSink,
|
|
10481
|
+
deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
|
|
10026
10482
|
asOfMs,
|
|
10027
10483
|
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
10028
10484
|
});
|
|
@@ -10049,44 +10505,50 @@ export class Orchestrator {
|
|
|
10049
10505
|
impressionRecorded = true;
|
|
10050
10506
|
}
|
|
10051
10507
|
} else {
|
|
10052
|
-
const
|
|
10053
|
-
|
|
10054
|
-
|
|
10055
|
-
|
|
10056
|
-
|
|
10057
|
-
|
|
10058
|
-
|
|
10059
|
-
|
|
10060
|
-
|
|
10061
|
-
|
|
10062
|
-
|
|
10063
|
-
|
|
10064
|
-
|
|
10065
|
-
|
|
10066
|
-
|
|
10067
|
-
|
|
10068
|
-
|
|
10069
|
-
|
|
10070
|
-
|
|
10071
|
-
|
|
10072
|
-
|
|
10073
|
-
|
|
10074
|
-
|
|
10075
|
-
|
|
10076
|
-
|
|
10077
|
-
|
|
10078
|
-
|
|
10079
|
-
|
|
10080
|
-
|
|
10081
|
-
|
|
10082
|
-
|
|
10083
|
-
|
|
10084
|
-
|
|
10085
|
-
|
|
10086
|
-
|
|
10087
|
-
|
|
10088
|
-
|
|
10089
|
-
|
|
10508
|
+
const recent = await awaitAssemblyStep(
|
|
10509
|
+
"recent-memory-scan",
|
|
10510
|
+
async () => {
|
|
10511
|
+
const recentSorted = queryAwareScopedMemories.sort(
|
|
10512
|
+
(a, b) =>
|
|
10513
|
+
new Date(b.frontmatter.updated).getTime() -
|
|
10514
|
+
new Date(a.frontmatter.updated).getTime(),
|
|
10515
|
+
);
|
|
10516
|
+
const preloadedMap = new Map<string, MemoryFile>(
|
|
10517
|
+
queryAwareScopedMemories
|
|
10518
|
+
.filter((m) => m.path)
|
|
10519
|
+
.map((m) => [m.path, m]),
|
|
10520
|
+
);
|
|
10521
|
+
const recentAsResults: QmdSearchResult[] = recentSorted.map(
|
|
10522
|
+
(m, i) => ({
|
|
10523
|
+
docid: m.frontmatter.id,
|
|
10524
|
+
path: m.path,
|
|
10525
|
+
snippet: m.content,
|
|
10526
|
+
score: 1.0 - i / Math.max(recentSorted.length, 1),
|
|
10527
|
+
}),
|
|
10528
|
+
);
|
|
10529
|
+
const boostedRecent = (
|
|
10530
|
+
await this.boostSearchResults(
|
|
10531
|
+
recentAsResults,
|
|
10532
|
+
recallNamespaces,
|
|
10533
|
+
retrievalQuery,
|
|
10534
|
+
preloadedMap,
|
|
10535
|
+
{ asOfMs },
|
|
10536
|
+
)
|
|
10537
|
+
).sort((a, b) => b.score - a.score);
|
|
10538
|
+
// MMR runs on the pre-truncation pool so diverse candidates just
|
|
10539
|
+
// below the cutoff can be promoted into the injected set.
|
|
10540
|
+
xrayBranchPoolSize.recent_scan = Math.max(
|
|
10541
|
+
xrayBranchPoolSize.recent_scan,
|
|
10542
|
+
boostedRecent.length,
|
|
10543
|
+
);
|
|
10544
|
+
return this.diversifyAndLimitRecallResults(
|
|
10545
|
+
"memories",
|
|
10546
|
+
boostedRecent,
|
|
10547
|
+
recallResultLimit,
|
|
10548
|
+
retrievalQuery,
|
|
10549
|
+
);
|
|
10550
|
+
},
|
|
10551
|
+
[] as QmdSearchResult[],
|
|
10090
10552
|
);
|
|
10091
10553
|
|
|
10092
10554
|
if (recent.length > 0) {
|
|
@@ -10125,6 +10587,7 @@ export class Orchestrator {
|
|
|
10125
10587
|
queryAwarePrefilter,
|
|
10126
10588
|
abortSignal: options.abortSignal,
|
|
10127
10589
|
xrayPoolSizeSink: xrayColdPoolSink,
|
|
10590
|
+
deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
|
|
10128
10591
|
asOfMs,
|
|
10129
10592
|
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
10130
10593
|
});
|
|
@@ -10168,6 +10631,7 @@ export class Orchestrator {
|
|
|
10168
10631
|
queryAwarePrefilter,
|
|
10169
10632
|
abortSignal: options.abortSignal,
|
|
10170
10633
|
xrayPoolSizeSink: xrayColdPoolSink,
|
|
10634
|
+
deadlineAtMs: enrichmentAssemblyDeadlineAtMs,
|
|
10171
10635
|
asOfMs,
|
|
10172
10636
|
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
10173
10637
|
});
|
|
@@ -10547,6 +11011,7 @@ export class Orchestrator {
|
|
|
10547
11011
|
query: retrievalQuery,
|
|
10548
11012
|
memoryIds: recalledMemoryIds,
|
|
10549
11013
|
namespace: selfNamespace,
|
|
11014
|
+
recallNamespaces,
|
|
10550
11015
|
traceId,
|
|
10551
11016
|
plannerMode: recallMode,
|
|
10552
11017
|
requestedMode,
|
|
@@ -15482,6 +15947,277 @@ export class Orchestrator {
|
|
|
15482
15947
|
*
|
|
15483
15948
|
* Callers must pass the full candidate pool (post-rerank, pre-slice).
|
|
15484
15949
|
*/
|
|
15950
|
+
private qmdCollectionNamespaceFromPrefix(collectionPrefix: string): string | null {
|
|
15951
|
+
const baseCollection = this.config.qmdCollection;
|
|
15952
|
+
if (collectionPrefix === baseCollection) return this.config.defaultNamespace;
|
|
15953
|
+
const namespaceSuffix = collectionPrefix.startsWith(`${baseCollection}--`)
|
|
15954
|
+
? collectionPrefix.slice(baseCollection.length + 2)
|
|
15955
|
+
: "";
|
|
15956
|
+
if (!namespaceSuffix) return null;
|
|
15957
|
+
|
|
15958
|
+
const decoded = namespaceIdentityFromToken(namespaceSuffix);
|
|
15959
|
+
if (decoded !== null) return decoded || this.config.defaultNamespace;
|
|
15960
|
+
if (namespaceSuffix.startsWith("ns--")) {
|
|
15961
|
+
const legacyNamespace = namespaceSuffix.slice("ns--".length).trim();
|
|
15962
|
+
return legacyNamespace || null;
|
|
15963
|
+
}
|
|
15964
|
+
return null;
|
|
15965
|
+
}
|
|
15966
|
+
|
|
15967
|
+
private async readQmdResultMemory(
|
|
15968
|
+
resultPath: string,
|
|
15969
|
+
fallbackStorage: StorageManager,
|
|
15970
|
+
recallNamespaces: readonly string[] = [],
|
|
15971
|
+
): Promise<MemoryFile | null> {
|
|
15972
|
+
const parts = qmdCollectionPathParts(resultPath);
|
|
15973
|
+
const storageDirFor = (storage: StorageManager): string | null =>
|
|
15974
|
+
typeof (storage as { dir?: unknown }).dir === "string" &&
|
|
15975
|
+
(storage as { dir?: string }).dir
|
|
15976
|
+
? (storage as { dir: string }).dir
|
|
15977
|
+
: null;
|
|
15978
|
+
const fallbackStorageDir = storageDirFor(fallbackStorage);
|
|
15979
|
+
const coldCollection = this.config.qmdColdCollection ?? "openclaw-engram-cold";
|
|
15980
|
+
if (parts && parts.collection === coldCollection) {
|
|
15981
|
+
const storages: StorageManager[] = [];
|
|
15982
|
+
const seenStorageDirs = new Set<string>();
|
|
15983
|
+
const addStorage = (storage: StorageManager): void => {
|
|
15984
|
+
const storageDir = storageDirFor(storage);
|
|
15985
|
+
const storageKey = storageDir
|
|
15986
|
+
? path.resolve(storageDir)
|
|
15987
|
+
: `storage-without-dir-${storages.length}`;
|
|
15988
|
+
if (seenStorageDirs.has(storageKey)) return;
|
|
15989
|
+
seenStorageDirs.add(storageKey);
|
|
15990
|
+
storages.push(storage);
|
|
15991
|
+
};
|
|
15992
|
+
|
|
15993
|
+
const fallbackNamespace =
|
|
15994
|
+
fallbackStorageDir !== null
|
|
15995
|
+
? this.namespaceFromStorageDir(fallbackStorageDir)
|
|
15996
|
+
: this.config.defaultNamespace;
|
|
15997
|
+
if (
|
|
15998
|
+
recallNamespaces.length === 0 ||
|
|
15999
|
+
!this.config.namespacesEnabled ||
|
|
16000
|
+
recallNamespaces.includes(fallbackNamespace)
|
|
16001
|
+
) {
|
|
16002
|
+
addStorage(fallbackStorage);
|
|
16003
|
+
}
|
|
16004
|
+
|
|
16005
|
+
if (recallNamespaces.length > 0) {
|
|
16006
|
+
for (const namespace of recallNamespaces) {
|
|
16007
|
+
try {
|
|
16008
|
+
addStorage(await this.storageRouter.storageFor(namespace));
|
|
16009
|
+
} catch (err) {
|
|
16010
|
+
log.debug("qmd cold result namespace storage lookup skipped", {
|
|
16011
|
+
path: resultPath,
|
|
16012
|
+
namespace,
|
|
16013
|
+
error: (err as Error).message,
|
|
16014
|
+
});
|
|
16015
|
+
}
|
|
16016
|
+
}
|
|
16017
|
+
}
|
|
16018
|
+
|
|
16019
|
+
for (const storage of storages) {
|
|
16020
|
+
const storageDir = storageDirFor(storage);
|
|
16021
|
+
if (!storageDir) {
|
|
16022
|
+
const memory = await storage.readMemoryByPath(resultPath);
|
|
16023
|
+
if (memory) return memory;
|
|
16024
|
+
continue;
|
|
16025
|
+
}
|
|
16026
|
+
try {
|
|
16027
|
+
const coldRoot = path.join(storageDir, "cold");
|
|
16028
|
+
for (const candidate of qmdResultPathCandidates(
|
|
16029
|
+
coldRoot,
|
|
16030
|
+
parts.relativePath,
|
|
16031
|
+
)) {
|
|
16032
|
+
const memory = await storage.readMemoryByPath(candidate);
|
|
16033
|
+
if (memory) return memory;
|
|
16034
|
+
}
|
|
16035
|
+
} catch (err) {
|
|
16036
|
+
if (err instanceof SecureStoreLockedError) throw err;
|
|
16037
|
+
log.debug("qmd cold result path lookup failed open", {
|
|
16038
|
+
path: resultPath,
|
|
16039
|
+
collection: coldCollection,
|
|
16040
|
+
error: (err as Error).message,
|
|
16041
|
+
});
|
|
16042
|
+
}
|
|
16043
|
+
}
|
|
16044
|
+
return null;
|
|
16045
|
+
}
|
|
16046
|
+
const collectionNamespace = parts
|
|
16047
|
+
? this.qmdCollectionNamespaceFromPrefix(parts.collection)
|
|
16048
|
+
: null;
|
|
16049
|
+
|
|
16050
|
+
if (parts && collectionNamespace) {
|
|
16051
|
+
try {
|
|
16052
|
+
const collectionStorage =
|
|
16053
|
+
await this.storageRouter.storageFor(collectionNamespace);
|
|
16054
|
+
for (const candidate of qmdResultPathCandidates(
|
|
16055
|
+
collectionStorage.dir,
|
|
16056
|
+
parts.relativePath,
|
|
16057
|
+
)) {
|
|
16058
|
+
const memory = await collectionStorage.readMemoryByPath(candidate);
|
|
16059
|
+
if (memory) return memory;
|
|
16060
|
+
}
|
|
16061
|
+
return null;
|
|
16062
|
+
} catch (err) {
|
|
16063
|
+
if (err instanceof SecureStoreLockedError) throw err;
|
|
16064
|
+
log.debug("qmd result namespace path lookup failed open", {
|
|
16065
|
+
path: resultPath,
|
|
16066
|
+
namespace: collectionNamespace,
|
|
16067
|
+
error: (err as Error).message,
|
|
16068
|
+
});
|
|
16069
|
+
return null;
|
|
16070
|
+
}
|
|
16071
|
+
}
|
|
16072
|
+
|
|
16073
|
+
if (path.isAbsolute(resultPath)) {
|
|
16074
|
+
if (!fallbackStorageDir) {
|
|
16075
|
+
return await fallbackStorage.readMemoryByPath(resultPath);
|
|
16076
|
+
}
|
|
16077
|
+
const ownerStorage = await this.storageForAbsoluteQmdResultPath(
|
|
16078
|
+
resultPath,
|
|
16079
|
+
fallbackStorage,
|
|
16080
|
+
recallNamespaces,
|
|
16081
|
+
);
|
|
16082
|
+
if (!ownerStorage) return null;
|
|
16083
|
+
for (const candidate of qmdResultPathCandidates(
|
|
16084
|
+
ownerStorage.dir,
|
|
16085
|
+
resultPath,
|
|
16086
|
+
)) {
|
|
16087
|
+
const memory = await ownerStorage.storage.readMemoryByPath(candidate);
|
|
16088
|
+
if (memory) return memory;
|
|
16089
|
+
}
|
|
16090
|
+
return null;
|
|
16091
|
+
}
|
|
16092
|
+
|
|
16093
|
+
if (!fallbackStorageDir) {
|
|
16094
|
+
return await fallbackStorage.readMemoryByPath(resultPath);
|
|
16095
|
+
}
|
|
16096
|
+
for (const candidate of qmdResultPathCandidates(
|
|
16097
|
+
fallbackStorageDir,
|
|
16098
|
+
resultPath,
|
|
16099
|
+
)) {
|
|
16100
|
+
const memory = await fallbackStorage.readMemoryByPath(candidate);
|
|
16101
|
+
if (memory) return memory;
|
|
16102
|
+
}
|
|
16103
|
+
return null;
|
|
16104
|
+
}
|
|
16105
|
+
|
|
16106
|
+
private async resolveColdQmdResultForRecall(
|
|
16107
|
+
result: QmdSearchResult,
|
|
16108
|
+
fallbackStorage: StorageManager,
|
|
16109
|
+
recallNamespaces: readonly string[] = [],
|
|
16110
|
+
): Promise<{ namespace: string; result: QmdSearchResult } | null> {
|
|
16111
|
+
const memory = await this.readQmdResultMemory(
|
|
16112
|
+
result.path,
|
|
16113
|
+
fallbackStorage,
|
|
16114
|
+
recallNamespaces,
|
|
16115
|
+
);
|
|
16116
|
+
if (!memory) return null;
|
|
16117
|
+
|
|
16118
|
+
let ownerNamespace: string | null = null;
|
|
16119
|
+
if (path.isAbsolute(memory.path)) {
|
|
16120
|
+
const ownerStorage = await this.storageForAbsoluteQmdResultPath(
|
|
16121
|
+
memory.path,
|
|
16122
|
+
fallbackStorage,
|
|
16123
|
+
recallNamespaces,
|
|
16124
|
+
);
|
|
16125
|
+
ownerNamespace = ownerStorage?.namespace ?? null;
|
|
16126
|
+
if (!ownerNamespace && this.config.namespacesEnabled) return null;
|
|
16127
|
+
}
|
|
16128
|
+
ownerNamespace ??= this.namespaceFromPath(memory.path);
|
|
16129
|
+
if (
|
|
16130
|
+
recallNamespaces.length > 0 &&
|
|
16131
|
+
!recallNamespaces.includes(ownerNamespace)
|
|
16132
|
+
) {
|
|
16133
|
+
return null;
|
|
16134
|
+
}
|
|
16135
|
+
|
|
16136
|
+
return {
|
|
16137
|
+
namespace: ownerNamespace,
|
|
16138
|
+
result: {
|
|
16139
|
+
...result,
|
|
16140
|
+
docid: result.docid || memory.frontmatter.id,
|
|
16141
|
+
path: memory.path,
|
|
16142
|
+
snippet: result.snippet || memory.content.slice(0, 400),
|
|
16143
|
+
},
|
|
16144
|
+
};
|
|
16145
|
+
}
|
|
16146
|
+
|
|
16147
|
+
private async storageForAbsoluteQmdResultPath(
|
|
16148
|
+
resultPath: string,
|
|
16149
|
+
fallbackStorage: StorageManager,
|
|
16150
|
+
recallNamespaces: readonly string[] = [],
|
|
16151
|
+
): Promise<{ storage: StorageManager; dir: string; namespace: string } | null> {
|
|
16152
|
+
const resolvedPath = path.resolve(resultPath);
|
|
16153
|
+
const memoryRoot = path.resolve(this.config.memoryDir);
|
|
16154
|
+
const namespacesRoot = path.join(memoryRoot, "namespaces");
|
|
16155
|
+
const fallbackStorageDir =
|
|
16156
|
+
typeof (fallbackStorage as { dir?: unknown }).dir === "string" &&
|
|
16157
|
+
(fallbackStorage as { dir?: string }).dir
|
|
16158
|
+
? (fallbackStorage as { dir: string }).dir
|
|
16159
|
+
: null;
|
|
16160
|
+
const matches: Array<{ storage: StorageManager; dir: string; namespace: string }> = [];
|
|
16161
|
+
const seenDirs = new Set<string>();
|
|
16162
|
+
|
|
16163
|
+
const maybeAddStorage = (storage: StorageManager, namespace: string) => {
|
|
16164
|
+
const storageDir =
|
|
16165
|
+
typeof (storage as { dir?: unknown }).dir === "string" &&
|
|
16166
|
+
(storage as { dir?: string }).dir
|
|
16167
|
+
? (storage as { dir: string }).dir
|
|
16168
|
+
: null;
|
|
16169
|
+
if (!storageDir) return;
|
|
16170
|
+
const candidateRoot = path.resolve(storageDir);
|
|
16171
|
+
if (seenDirs.has(candidateRoot)) return;
|
|
16172
|
+
if (!isPathInsideStorageRoot(candidateRoot, resolvedPath)) return;
|
|
16173
|
+
if (
|
|
16174
|
+
candidateRoot === memoryRoot &&
|
|
16175
|
+
isPathInsideStorageRoot(namespacesRoot, resolvedPath)
|
|
16176
|
+
) {
|
|
16177
|
+
return;
|
|
16178
|
+
}
|
|
16179
|
+
seenDirs.add(candidateRoot);
|
|
16180
|
+
matches.push({ storage, dir: candidateRoot, namespace });
|
|
16181
|
+
};
|
|
16182
|
+
|
|
16183
|
+
const fallbackNamespace =
|
|
16184
|
+
fallbackStorageDir !== null
|
|
16185
|
+
? this.namespaceFromStorageDir(fallbackStorageDir)
|
|
16186
|
+
: this.config.defaultNamespace;
|
|
16187
|
+
maybeAddStorage(fallbackStorage, fallbackNamespace);
|
|
16188
|
+
|
|
16189
|
+
const candidateNamespaces = new Set<string>();
|
|
16190
|
+
candidateNamespaces.add(this.config.defaultNamespace);
|
|
16191
|
+
candidateNamespaces.add(this.config.sharedNamespace);
|
|
16192
|
+
for (const ns of recallNamespaces) {
|
|
16193
|
+
candidateNamespaces.add(ns);
|
|
16194
|
+
}
|
|
16195
|
+
if (isPathInsideStorageRoot(namespacesRoot, resolvedPath)) {
|
|
16196
|
+
const relativeToNamespaces = path.relative(namespacesRoot, resolvedPath);
|
|
16197
|
+
const [namespaceSegment] = relativeToNamespaces.split(/[\\/]/);
|
|
16198
|
+
if (namespaceSegment) {
|
|
16199
|
+
candidateNamespaces.add(
|
|
16200
|
+
namespaceIdentityFromToken(namespaceSegment) ?? namespaceSegment,
|
|
16201
|
+
);
|
|
16202
|
+
}
|
|
16203
|
+
}
|
|
16204
|
+
for (const policy of this.config.namespacePolicies ?? []) {
|
|
16205
|
+
candidateNamespaces.add(policy.name);
|
|
16206
|
+
}
|
|
16207
|
+
|
|
16208
|
+
for (const ns of candidateNamespaces) {
|
|
16209
|
+
if (!ns) continue;
|
|
16210
|
+
try {
|
|
16211
|
+
maybeAddStorage(await this.storageRouter.storageFor(ns), ns);
|
|
16212
|
+
} catch {
|
|
16213
|
+
continue;
|
|
16214
|
+
}
|
|
16215
|
+
}
|
|
16216
|
+
|
|
16217
|
+
matches.sort((a, b) => b.dir.length - a.dir.length);
|
|
16218
|
+
return matches[0] ?? null;
|
|
16219
|
+
}
|
|
16220
|
+
|
|
15485
16221
|
private async applyMemoryWorthRerank(
|
|
15486
16222
|
results: QmdSearchResult[],
|
|
15487
16223
|
namespaces: string[],
|
|
@@ -15557,7 +16293,7 @@ export class Orchestrator {
|
|
|
15557
16293
|
if (reader) {
|
|
15558
16294
|
for (const r of missing) {
|
|
15559
16295
|
try {
|
|
15560
|
-
const memory = await
|
|
16296
|
+
const memory = await this.readQmdResultMemory(r.path, reader, namespaces);
|
|
15561
16297
|
if (!memory) continue;
|
|
15562
16298
|
const fm = memory.frontmatter;
|
|
15563
16299
|
if (fm.mw_success === undefined && fm.mw_fail === undefined) continue;
|
|
@@ -15973,6 +16709,7 @@ export class Orchestrator {
|
|
|
15973
16709
|
* Unset by default so existing call sites are unaffected.
|
|
15974
16710
|
*/
|
|
15975
16711
|
xrayPoolSizeSink?: { size: number };
|
|
16712
|
+
deadlineAtMs?: number | null;
|
|
15976
16713
|
/** Issue #681 — when true, bypass graphTraversalConfidenceFloor. */
|
|
15977
16714
|
includeLowConfidence?: boolean;
|
|
15978
16715
|
}): Promise<QmdSearchResult[]> {
|
|
@@ -15980,6 +16717,50 @@ export class Orchestrator {
|
|
|
15980
16717
|
if (options.xrayPoolSizeSink) options.xrayPoolSizeSink.size = 0;
|
|
15981
16718
|
return [];
|
|
15982
16719
|
}
|
|
16720
|
+
const deadlineRemainingMs = (): number | null =>
|
|
16721
|
+
typeof options.deadlineAtMs === "number"
|
|
16722
|
+
? Math.max(0, options.deadlineAtMs - Date.now())
|
|
16723
|
+
: null;
|
|
16724
|
+
const runColdStepWithinDeadline = async <T>(
|
|
16725
|
+
label: string,
|
|
16726
|
+
fallback: T,
|
|
16727
|
+
task: () => Promise<T>,
|
|
16728
|
+
): Promise<T> => {
|
|
16729
|
+
throwIfRecallAborted(options.abortSignal);
|
|
16730
|
+
const remainingMs = deadlineRemainingMs();
|
|
16731
|
+
if (remainingMs === 0) {
|
|
16732
|
+
log.debug(`cold-tier recall ${label} skipped: shared assembly deadline expired`);
|
|
16733
|
+
return fallback;
|
|
16734
|
+
}
|
|
16735
|
+
if (remainingMs === null) return task();
|
|
16736
|
+
|
|
16737
|
+
let timeoutHandle: NodeJS.Timeout | undefined;
|
|
16738
|
+
let timedOut = false;
|
|
16739
|
+
const taskPromise = task().catch((err) => {
|
|
16740
|
+
if (timedOut) {
|
|
16741
|
+
log.debug(`cold-tier recall ${label} failed after deadline: ${err}`);
|
|
16742
|
+
return fallback;
|
|
16743
|
+
}
|
|
16744
|
+
throw err;
|
|
16745
|
+
});
|
|
16746
|
+
|
|
16747
|
+
try {
|
|
16748
|
+
return await Promise.race<T>([
|
|
16749
|
+
taskPromise,
|
|
16750
|
+
new Promise<T>((resolve) => {
|
|
16751
|
+
timeoutHandle = setTimeout(() => {
|
|
16752
|
+
timedOut = true;
|
|
16753
|
+
log.debug(
|
|
16754
|
+
`cold-tier recall ${label} skipped: shared assembly deadline expired`,
|
|
16755
|
+
);
|
|
16756
|
+
resolve(fallback);
|
|
16757
|
+
}, remainingMs);
|
|
16758
|
+
}),
|
|
16759
|
+
]);
|
|
16760
|
+
} finally {
|
|
16761
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
16762
|
+
}
|
|
16763
|
+
};
|
|
15983
16764
|
|
|
15984
16765
|
const coldQmdEnabled = this.config.qmdColdTierEnabled === true;
|
|
15985
16766
|
const coldCollection =
|
|
@@ -15999,19 +16780,24 @@ export class Orchestrator {
|
|
|
15999
16780
|
false,
|
|
16000
16781
|
0,
|
|
16001
16782
|
);
|
|
16002
|
-
longTerm = await
|
|
16003
|
-
|
|
16004
|
-
|
|
16005
|
-
|
|
16006
|
-
|
|
16007
|
-
|
|
16008
|
-
|
|
16009
|
-
|
|
16010
|
-
|
|
16011
|
-
|
|
16012
|
-
|
|
16013
|
-
|
|
16014
|
-
|
|
16783
|
+
longTerm = await runColdStepWithinDeadline(
|
|
16784
|
+
"qmd lookup",
|
|
16785
|
+
[],
|
|
16786
|
+
() =>
|
|
16787
|
+
this.fetchQmdMemoryResultsWithArtifactTopUp(
|
|
16788
|
+
options.prompt,
|
|
16789
|
+
coldFetchLimit,
|
|
16790
|
+
coldHybridLimit,
|
|
16791
|
+
{
|
|
16792
|
+
namespacesEnabled: this.config.namespacesEnabled,
|
|
16793
|
+
recallNamespaces: options.recallNamespaces,
|
|
16794
|
+
resolveNamespace: (p) => this.namespaceFromPath(p),
|
|
16795
|
+
collection: coldCollection,
|
|
16796
|
+
queryAwarePrefilter: options.queryAwarePrefilter,
|
|
16797
|
+
searchOptions: this.buildConfiguredQmdSearchOptions(options.prompt),
|
|
16798
|
+
abortSignal: options.abortSignal,
|
|
16799
|
+
},
|
|
16800
|
+
),
|
|
16015
16801
|
);
|
|
16016
16802
|
if (longTerm.length > 0) {
|
|
16017
16803
|
log.debug(
|
|
@@ -16021,12 +16807,17 @@ export class Orchestrator {
|
|
|
16021
16807
|
}
|
|
16022
16808
|
}
|
|
16023
16809
|
if (longTerm.length === 0) {
|
|
16024
|
-
longTerm = await
|
|
16025
|
-
|
|
16026
|
-
|
|
16027
|
-
|
|
16028
|
-
|
|
16029
|
-
|
|
16810
|
+
longTerm = await runColdStepWithinDeadline(
|
|
16811
|
+
"archive scan",
|
|
16812
|
+
[],
|
|
16813
|
+
() =>
|
|
16814
|
+
this.searchLongTermArchiveFallback(
|
|
16815
|
+
options.prompt,
|
|
16816
|
+
options.recallNamespaces,
|
|
16817
|
+
options.recallResultLimit,
|
|
16818
|
+
options.queryAwarePrefilter,
|
|
16819
|
+
options.abortSignal,
|
|
16820
|
+
),
|
|
16030
16821
|
);
|
|
16031
16822
|
if (longTerm.length > 0) {
|
|
16032
16823
|
log.debug("cold-tier recall source=archive-scan");
|
|
@@ -16036,9 +16827,57 @@ export class Orchestrator {
|
|
|
16036
16827
|
|
|
16037
16828
|
let results = longTerm;
|
|
16038
16829
|
if (this.config.namespacesEnabled) {
|
|
16039
|
-
|
|
16040
|
-
|
|
16041
|
-
)
|
|
16830
|
+
const recallRoots: string[] = [];
|
|
16831
|
+
const seenRecallRoots = new Set<string>();
|
|
16832
|
+
for (const namespace of options.recallNamespaces) {
|
|
16833
|
+
try {
|
|
16834
|
+
const storage = await this.storageRouter.storageFor(namespace);
|
|
16835
|
+
const storageDir =
|
|
16836
|
+
typeof (storage as { dir?: unknown }).dir === "string" &&
|
|
16837
|
+
(storage as { dir?: string }).dir
|
|
16838
|
+
? (storage as { dir: string }).dir
|
|
16839
|
+
: null;
|
|
16840
|
+
if (!storageDir) continue;
|
|
16841
|
+
const recallRoot = path.resolve(storageDir);
|
|
16842
|
+
if (seenRecallRoots.has(recallRoot)) continue;
|
|
16843
|
+
seenRecallRoots.add(recallRoot);
|
|
16844
|
+
recallRoots.push(recallRoot);
|
|
16845
|
+
} catch (err) {
|
|
16846
|
+
log.debug("cold-tier recall namespace root lookup skipped", {
|
|
16847
|
+
namespace,
|
|
16848
|
+
error: (err as Error).message,
|
|
16849
|
+
});
|
|
16850
|
+
}
|
|
16851
|
+
}
|
|
16852
|
+
const scopedResults: QmdSearchResult[] = [];
|
|
16853
|
+
for (const result of results) {
|
|
16854
|
+
if (options.abortSignal?.aborted || deadlineRemainingMs() === 0) break;
|
|
16855
|
+
const parts = qmdCollectionPathParts(result.path);
|
|
16856
|
+
if (parts?.collection === coldCollection) {
|
|
16857
|
+
const resolvedCold = await this.resolveColdQmdResultForRecall(
|
|
16858
|
+
result,
|
|
16859
|
+
this.storage,
|
|
16860
|
+
options.recallNamespaces,
|
|
16861
|
+
);
|
|
16862
|
+
if (resolvedCold) scopedResults.push(resolvedCold.result);
|
|
16863
|
+
continue;
|
|
16864
|
+
}
|
|
16865
|
+
if (path.isAbsolute(result.path)) {
|
|
16866
|
+
const resolvedPath = path.resolve(result.path);
|
|
16867
|
+
if (
|
|
16868
|
+
recallRoots.some((recallRoot) =>
|
|
16869
|
+
isPathInsideStorageRoot(recallRoot, resolvedPath),
|
|
16870
|
+
)
|
|
16871
|
+
) {
|
|
16872
|
+
scopedResults.push(result);
|
|
16873
|
+
}
|
|
16874
|
+
continue;
|
|
16875
|
+
}
|
|
16876
|
+
if (options.recallNamespaces.includes(this.namespaceFromPath(result.path))) {
|
|
16877
|
+
scopedResults.push(result);
|
|
16878
|
+
}
|
|
16879
|
+
}
|
|
16880
|
+
results = scopedResults;
|
|
16042
16881
|
}
|
|
16043
16882
|
// Artifact isolation contract: generic recall paths must exclude artifacts.
|
|
16044
16883
|
results = results.filter((r) => !isArtifactMemoryPath(r.path));
|
|
@@ -16059,18 +16898,75 @@ export class Orchestrator {
|
|
|
16059
16898
|
memoryResults: results,
|
|
16060
16899
|
recallNamespaces: options.recallNamespaces,
|
|
16061
16900
|
recallResultLimit: options.recallResultLimit,
|
|
16901
|
+
deadlineAtMs: options.deadlineAtMs,
|
|
16062
16902
|
...(options.includeLowConfidence === true ? { includeLowConfidence: true } : {}),
|
|
16063
16903
|
});
|
|
16064
16904
|
results = merged;
|
|
16065
16905
|
}
|
|
16066
16906
|
|
|
16067
|
-
|
|
16907
|
+
const boostInput = await this.filterSearchResultsForRecall(
|
|
16068
16908
|
results,
|
|
16069
|
-
options.recallNamespaces,
|
|
16070
|
-
options.prompt,
|
|
16071
16909
|
undefined,
|
|
16072
|
-
{
|
|
16910
|
+
{
|
|
16911
|
+
allowLifecycleFiltered: true,
|
|
16912
|
+
asOfMs: options.asOfMs,
|
|
16913
|
+
deadlineAtMs: options.deadlineAtMs,
|
|
16914
|
+
abortSignal: options.abortSignal,
|
|
16915
|
+
dropUnresolved: true,
|
|
16916
|
+
recallNamespaces: options.recallNamespaces,
|
|
16917
|
+
},
|
|
16073
16918
|
);
|
|
16919
|
+
results = boostInput.results;
|
|
16920
|
+
const boostTimeoutMs =
|
|
16921
|
+
typeof options.deadlineAtMs === "number"
|
|
16922
|
+
? Math.max(0, options.deadlineAtMs - Date.now())
|
|
16923
|
+
: null;
|
|
16924
|
+
if (boostTimeoutMs !== 0) {
|
|
16925
|
+
let timeoutHandle: NodeJS.Timeout | undefined;
|
|
16926
|
+
try {
|
|
16927
|
+
const boosted = await (boostTimeoutMs !== null
|
|
16928
|
+
? Promise.race<QmdSearchResult[] | { status: "timed_out" }>([
|
|
16929
|
+
this.boostSearchResults(
|
|
16930
|
+
boostInput.results,
|
|
16931
|
+
options.recallNamespaces,
|
|
16932
|
+
options.prompt,
|
|
16933
|
+
boostInput.memoryByPath,
|
|
16934
|
+
{ allowLifecycleFiltered: true, asOfMs: options.asOfMs },
|
|
16935
|
+
),
|
|
16936
|
+
new Promise<{ status: "timed_out" }>((resolve) => {
|
|
16937
|
+
timeoutHandle = setTimeout(
|
|
16938
|
+
() => resolve({ status: "timed_out" }),
|
|
16939
|
+
boostTimeoutMs,
|
|
16940
|
+
);
|
|
16941
|
+
}),
|
|
16942
|
+
])
|
|
16943
|
+
: this.boostSearchResults(
|
|
16944
|
+
boostInput.results,
|
|
16945
|
+
options.recallNamespaces,
|
|
16946
|
+
options.prompt,
|
|
16947
|
+
boostInput.memoryByPath,
|
|
16948
|
+
{ allowLifecycleFiltered: true, asOfMs: options.asOfMs },
|
|
16949
|
+
));
|
|
16950
|
+
if (
|
|
16951
|
+
typeof boosted === "object" &&
|
|
16952
|
+
boosted !== null &&
|
|
16953
|
+
"status" in boosted &&
|
|
16954
|
+
boosted.status === "timed_out"
|
|
16955
|
+
) {
|
|
16956
|
+
log.debug("cold-tier recall boost skipped: shared assembly deadline expired");
|
|
16957
|
+
} else if (Array.isArray(boosted)) {
|
|
16958
|
+
results = boosted;
|
|
16959
|
+
} else {
|
|
16960
|
+
results = boostInput.results;
|
|
16961
|
+
}
|
|
16962
|
+
} catch (err) {
|
|
16963
|
+
log.debug(`cold-tier recall boost failed open: ${err}`);
|
|
16964
|
+
} finally {
|
|
16965
|
+
if (timeoutHandle) clearTimeout(timeoutHandle);
|
|
16966
|
+
}
|
|
16967
|
+
} else {
|
|
16968
|
+
log.debug("cold-tier recall boost skipped: shared assembly deadline already expired");
|
|
16969
|
+
}
|
|
16074
16970
|
|
|
16075
16971
|
if (this.config.rerankEnabled && this.config.rerankProvider === "local") {
|
|
16076
16972
|
const ranked = await rerankLocalOrNoop({
|
|
@@ -16218,104 +17114,133 @@ export class Orchestrator {
|
|
|
16218
17114
|
log.debug(`flushed ${entries.length} access tracking entries`);
|
|
16219
17115
|
}
|
|
16220
17116
|
|
|
16221
|
-
|
|
16222
|
-
* Apply recency, access count, and importance boosting to QMD search results.
|
|
16223
|
-
* Returns re-ranked results.
|
|
16224
|
-
*/
|
|
16225
|
-
private async boostSearchResults(
|
|
17117
|
+
private async loadSearchResultMemoryMap(
|
|
16226
17118
|
results: QmdSearchResult[],
|
|
16227
|
-
_recallNamespaces: string[],
|
|
16228
|
-
prompt?: string,
|
|
16229
17119
|
preloadedMemoryMap?: Map<string, MemoryFile>,
|
|
16230
17120
|
options?: {
|
|
16231
|
-
|
|
16232
|
-
|
|
16233
|
-
|
|
16234
|
-
* Historical recall point in ms-since-epoch (issue #680). When
|
|
16235
|
-
* set, drops candidates that were not authoritative at this
|
|
16236
|
-
* instant per `temporal-validity.isValidAsOf`. Caller is
|
|
16237
|
-
* responsible for parsing/validating the user-supplied ISO
|
|
16238
|
-
* string at the input boundary (CLI / HTTP / MCP).
|
|
16239
|
-
*/
|
|
16240
|
-
asOfMs?: number;
|
|
17121
|
+
deadlineAtMs?: number | null;
|
|
17122
|
+
abortSignal?: AbortSignal;
|
|
17123
|
+
recallNamespaces?: readonly string[];
|
|
16241
17124
|
},
|
|
16242
|
-
): Promise<
|
|
16243
|
-
|
|
16244
|
-
|
|
16245
|
-
|
|
16246
|
-
|
|
16247
|
-
|
|
17125
|
+
): Promise<{
|
|
17126
|
+
memoryByPath: Map<string, MemoryFile>;
|
|
17127
|
+
checkedPaths: Set<string>;
|
|
17128
|
+
unreadablePaths: Set<string>;
|
|
17129
|
+
completed: boolean;
|
|
17130
|
+
}> {
|
|
16248
17131
|
const memoryByPath: Map<string, MemoryFile> = preloadedMemoryMap
|
|
16249
17132
|
? new Map(preloadedMemoryMap)
|
|
16250
17133
|
: new Map();
|
|
17134
|
+
const checkedPaths = new Set<string>();
|
|
17135
|
+
const unreadablePaths = new Set<string>();
|
|
16251
17136
|
|
|
16252
|
-
|
|
16253
|
-
|
|
16254
|
-
|
|
16255
|
-
)
|
|
16256
|
-
|
|
16257
|
-
|
|
16258
|
-
|
|
16259
|
-
|
|
16260
|
-
|
|
17137
|
+
const markChecked = (result: QmdSearchResult): void => {
|
|
17138
|
+
if (result.path) checkedPaths.add(result.path);
|
|
17139
|
+
};
|
|
17140
|
+
const markUnreadable = (result: QmdSearchResult, err: unknown): void => {
|
|
17141
|
+
if (!result.path) return;
|
|
17142
|
+
checkedPaths.add(result.path);
|
|
17143
|
+
unreadablePaths.add(result.path);
|
|
17144
|
+
log.warn("recall safety filter dropped unreadable secure-store candidate", {
|
|
17145
|
+
path: result.path,
|
|
17146
|
+
error: (err as Error).message,
|
|
17147
|
+
});
|
|
17148
|
+
};
|
|
17149
|
+
const deadlineExpired = (): boolean =>
|
|
17150
|
+
typeof options?.deadlineAtMs === "number" &&
|
|
17151
|
+
Date.now() >= options.deadlineAtMs;
|
|
17152
|
+
|
|
17153
|
+
if (options?.deadlineAtMs == null) {
|
|
17154
|
+
const batchSize = options?.abortSignal ? 16 : results.length;
|
|
17155
|
+
for (let offset = 0; offset < results.length; offset += batchSize) {
|
|
17156
|
+
if (options?.abortSignal?.aborted) {
|
|
17157
|
+
return {
|
|
17158
|
+
memoryByPath,
|
|
17159
|
+
checkedPaths,
|
|
17160
|
+
unreadablePaths,
|
|
17161
|
+
completed: false,
|
|
17162
|
+
};
|
|
17163
|
+
}
|
|
17164
|
+
await Promise.all(
|
|
17165
|
+
results.slice(offset, offset + batchSize).map(async (r) => {
|
|
17166
|
+
if (!r.path) return;
|
|
17167
|
+
if (memoryByPath.has(r.path)) {
|
|
17168
|
+
markChecked(r);
|
|
17169
|
+
return;
|
|
17170
|
+
}
|
|
17171
|
+
try {
|
|
17172
|
+
const mem = await this.readQmdResultMemory(
|
|
17173
|
+
r.path,
|
|
17174
|
+
this.storage,
|
|
17175
|
+
options?.recallNamespaces,
|
|
17176
|
+
);
|
|
17177
|
+
markChecked(r);
|
|
17178
|
+
if (mem) memoryByPath.set(r.path, mem);
|
|
17179
|
+
} catch (err) {
|
|
17180
|
+
if (err instanceof SecureStoreLockedError) {
|
|
17181
|
+
markUnreadable(r, err);
|
|
17182
|
+
return;
|
|
17183
|
+
}
|
|
17184
|
+
throw err;
|
|
17185
|
+
}
|
|
17186
|
+
}),
|
|
17187
|
+
);
|
|
16261
17188
|
}
|
|
16262
|
-
promptTags = extractTagsFromPrompt(prompt);
|
|
16263
|
-
}
|
|
16264
17189
|
|
|
16265
|
-
|
|
16266
|
-
|
|
16267
|
-
Promise.all(
|
|
16268
|
-
results.map(async (r) => {
|
|
16269
|
-
if (!r.path || memoryByPath.has(r.path)) return;
|
|
16270
|
-
const mem = await this.storage.readMemoryByPath(r.path);
|
|
16271
|
-
if (mem) memoryByPath.set(r.path, mem);
|
|
16272
|
-
}),
|
|
16273
|
-
),
|
|
16274
|
-
temporalFromDate !== null
|
|
16275
|
-
? queryByDateRangeAsync(this.config.memoryDir, temporalFromDate)
|
|
16276
|
-
: Promise.resolve<Set<string> | null>(null),
|
|
16277
|
-
promptTags.length > 0
|
|
16278
|
-
? queryByTagsAsync(this.config.memoryDir, promptTags)
|
|
16279
|
-
: Promise.resolve<Set<string> | null>(null),
|
|
16280
|
-
]);
|
|
16281
|
-
|
|
16282
|
-
const queryIntent =
|
|
16283
|
-
this.config.intentRoutingEnabled && prompt
|
|
16284
|
-
? inferIntentFromText(prompt)
|
|
16285
|
-
: null;
|
|
17190
|
+
return { memoryByPath, checkedPaths, unreadablePaths, completed: true };
|
|
17191
|
+
}
|
|
16286
17192
|
|
|
16287
|
-
|
|
16288
|
-
|
|
16289
|
-
|
|
16290
|
-
|
|
16291
|
-
|
|
16292
|
-
const maxCandidates = this.config.queryAwareIndexingMaxCandidates;
|
|
16293
|
-
const capSet = (s: Set<string> | null): Set<string> | null => {
|
|
16294
|
-
if (!s) return null;
|
|
16295
|
-
// Intersect with result paths first so out-of-scope paths don't exhaust the budget
|
|
16296
|
-
const scoped = new Set(Array.from(s).filter((p) => resultPaths.has(p)));
|
|
16297
|
-
if (maxCandidates === 0 || scoped.size <= maxCandidates)
|
|
16298
|
-
return scoped;
|
|
16299
|
-
return new Set(Array.from(scoped).slice(0, maxCandidates));
|
|
16300
|
-
};
|
|
16301
|
-
if (temporalFromDate !== null) {
|
|
16302
|
-
temporalCandidates = capSet(rawTemporal);
|
|
17193
|
+
for (const result of results) {
|
|
17194
|
+
if (!result.path) continue;
|
|
17195
|
+
if (memoryByPath.has(result.path)) {
|
|
17196
|
+
markChecked(result);
|
|
17197
|
+
continue;
|
|
16303
17198
|
}
|
|
16304
|
-
if (
|
|
16305
|
-
|
|
17199
|
+
if (options?.abortSignal?.aborted || deadlineExpired()) {
|
|
17200
|
+
return { memoryByPath, checkedPaths, unreadablePaths, completed: false };
|
|
17201
|
+
}
|
|
17202
|
+
try {
|
|
17203
|
+
const mem = await this.readQmdResultMemory(
|
|
17204
|
+
result.path,
|
|
17205
|
+
this.storage,
|
|
17206
|
+
options?.recallNamespaces,
|
|
17207
|
+
);
|
|
17208
|
+
markChecked(result);
|
|
17209
|
+
if (mem) memoryByPath.set(result.path, mem);
|
|
17210
|
+
} catch (err) {
|
|
17211
|
+
if (err instanceof SecureStoreLockedError) {
|
|
17212
|
+
markUnreadable(result, err);
|
|
17213
|
+
continue;
|
|
17214
|
+
}
|
|
17215
|
+
throw err;
|
|
16306
17216
|
}
|
|
16307
17217
|
}
|
|
16308
17218
|
|
|
17219
|
+
return { memoryByPath, checkedPaths, unreadablePaths, completed: true };
|
|
17220
|
+
}
|
|
17221
|
+
|
|
17222
|
+
private filterSearchResultsByRecallSafety(
|
|
17223
|
+
results: QmdSearchResult[],
|
|
17224
|
+
memoryByPath: Map<string, MemoryFile>,
|
|
17225
|
+
options?: {
|
|
17226
|
+
allowLifecycleFiltered?: boolean;
|
|
17227
|
+
allowDedicatedSurface?: boolean;
|
|
17228
|
+
asOfMs?: number;
|
|
17229
|
+
blockedPaths?: Set<string>;
|
|
17230
|
+
},
|
|
17231
|
+
): QmdSearchResult[] {
|
|
16309
17232
|
let lifecycleFilteredCount = 0;
|
|
16310
17233
|
let temporalSupersededFilteredCount = 0;
|
|
16311
17234
|
let dedicatedSurfaceFilteredCount = 0;
|
|
16312
17235
|
let forgottenFilteredCount = 0;
|
|
16313
|
-
|
|
16314
|
-
const
|
|
17236
|
+
let blockedPathFilteredCount = 0;
|
|
17237
|
+
const filtered: QmdSearchResult[] = [];
|
|
16315
17238
|
for (const r of results) {
|
|
17239
|
+
if (r.path && options?.blockedPaths?.has(r.path)) {
|
|
17240
|
+
blockedPathFilteredCount += 1;
|
|
17241
|
+
continue;
|
|
17242
|
+
}
|
|
16316
17243
|
const memory = memoryByPath.get(r.path);
|
|
16317
|
-
let score = r.score;
|
|
16318
|
-
|
|
16319
17244
|
if (memory) {
|
|
16320
17245
|
if (memory.frontmatter.status === "forgotten") {
|
|
16321
17246
|
forgottenFilteredCount += 1;
|
|
@@ -16377,7 +17302,178 @@ export class Orchestrator {
|
|
|
16377
17302
|
dedicatedSurfaceFilteredCount += 1;
|
|
16378
17303
|
continue;
|
|
16379
17304
|
}
|
|
17305
|
+
}
|
|
17306
|
+
filtered.push(r);
|
|
17307
|
+
}
|
|
17308
|
+
if (lifecycleFilteredCount > 0) {
|
|
17309
|
+
log.debug(
|
|
17310
|
+
`lifecycle retrieval filter removed ${lifecycleFilteredCount} stale/archived candidates`,
|
|
17311
|
+
);
|
|
17312
|
+
}
|
|
17313
|
+
if (temporalSupersededFilteredCount > 0) {
|
|
17314
|
+
log.debug(
|
|
17315
|
+
`temporal supersession filter removed ${temporalSupersededFilteredCount} superseded candidates`,
|
|
17316
|
+
);
|
|
17317
|
+
}
|
|
17318
|
+
if (dedicatedSurfaceFilteredCount > 0) {
|
|
17319
|
+
log.debug(
|
|
17320
|
+
`dedicated surface filter removed ${dedicatedSurfaceFilteredCount} dream/procedural candidates from generic recall`,
|
|
17321
|
+
);
|
|
17322
|
+
}
|
|
17323
|
+
if (forgottenFilteredCount > 0) {
|
|
17324
|
+
log.debug(
|
|
17325
|
+
`forgotten status filter removed ${forgottenFilteredCount} candidates from recall`,
|
|
17326
|
+
);
|
|
17327
|
+
}
|
|
17328
|
+
if (blockedPathFilteredCount > 0) {
|
|
17329
|
+
log.debug(
|
|
17330
|
+
`unreadable-path filter removed ${blockedPathFilteredCount} candidates from recall`,
|
|
17331
|
+
);
|
|
17332
|
+
}
|
|
17333
|
+
return filtered;
|
|
17334
|
+
}
|
|
16380
17335
|
|
|
17336
|
+
private async filterSearchResultsForRecall(
|
|
17337
|
+
results: QmdSearchResult[],
|
|
17338
|
+
preloadedMemoryMap?: Map<string, MemoryFile>,
|
|
17339
|
+
options?: {
|
|
17340
|
+
allowLifecycleFiltered?: boolean;
|
|
17341
|
+
allowDedicatedSurface?: boolean;
|
|
17342
|
+
asOfMs?: number;
|
|
17343
|
+
deadlineAtMs?: number | null;
|
|
17344
|
+
abortSignal?: AbortSignal;
|
|
17345
|
+
dropUnresolved?: boolean;
|
|
17346
|
+
recallNamespaces?: readonly string[];
|
|
17347
|
+
},
|
|
17348
|
+
): Promise<{ results: QmdSearchResult[]; memoryByPath: Map<string, MemoryFile> }> {
|
|
17349
|
+
if (results.length === 0) {
|
|
17350
|
+
return {
|
|
17351
|
+
results,
|
|
17352
|
+
memoryByPath: preloadedMemoryMap ? new Map(preloadedMemoryMap) : new Map(),
|
|
17353
|
+
};
|
|
17354
|
+
}
|
|
17355
|
+
const loaded = await this.loadSearchResultMemoryMap(
|
|
17356
|
+
results,
|
|
17357
|
+
preloadedMemoryMap,
|
|
17358
|
+
options,
|
|
17359
|
+
);
|
|
17360
|
+
const candidateResults = loaded.completed
|
|
17361
|
+
? results
|
|
17362
|
+
: results.filter((result) => !result.path || loaded.checkedPaths.has(result.path));
|
|
17363
|
+
if (!loaded.completed) {
|
|
17364
|
+
log.debug(
|
|
17365
|
+
`recall safety filter stopped before validating all candidates (${candidateResults.length}/${results.length} eligible)`,
|
|
17366
|
+
);
|
|
17367
|
+
}
|
|
17368
|
+
const blockedPaths = new Set(loaded.unreadablePaths);
|
|
17369
|
+
if (options?.dropUnresolved === true) {
|
|
17370
|
+
for (const resultPath of loaded.checkedPaths) {
|
|
17371
|
+
if (!loaded.memoryByPath.has(resultPath)) {
|
|
17372
|
+
blockedPaths.add(resultPath);
|
|
17373
|
+
}
|
|
17374
|
+
}
|
|
17375
|
+
}
|
|
17376
|
+
return {
|
|
17377
|
+
results: this.filterSearchResultsByRecallSafety(
|
|
17378
|
+
candidateResults,
|
|
17379
|
+
loaded.memoryByPath,
|
|
17380
|
+
{ ...options, blockedPaths },
|
|
17381
|
+
),
|
|
17382
|
+
memoryByPath: loaded.memoryByPath,
|
|
17383
|
+
};
|
|
17384
|
+
}
|
|
17385
|
+
|
|
17386
|
+
/**
|
|
17387
|
+
* Apply recency, access count, and importance boosting to QMD search results.
|
|
17388
|
+
* Returns re-ranked results.
|
|
17389
|
+
*/
|
|
17390
|
+
private async boostSearchResults(
|
|
17391
|
+
results: QmdSearchResult[],
|
|
17392
|
+
_recallNamespaces: string[],
|
|
17393
|
+
prompt?: string,
|
|
17394
|
+
preloadedMemoryMap?: Map<string, MemoryFile>,
|
|
17395
|
+
options?: {
|
|
17396
|
+
allowLifecycleFiltered?: boolean;
|
|
17397
|
+
allowDedicatedSurface?: boolean;
|
|
17398
|
+
/**
|
|
17399
|
+
* Historical recall point in ms-since-epoch (issue #680). When
|
|
17400
|
+
* set, drops candidates that were not authoritative at this
|
|
17401
|
+
* instant per `temporal-validity.isValidAsOf`. Caller is
|
|
17402
|
+
* responsible for parsing/validating the user-supplied ISO
|
|
17403
|
+
* string at the input boundary (CLI / HTTP / MCP).
|
|
17404
|
+
*/
|
|
17405
|
+
asOfMs?: number;
|
|
17406
|
+
},
|
|
17407
|
+
): Promise<QmdSearchResult[]> {
|
|
17408
|
+
if (results.length === 0) return results;
|
|
17409
|
+
|
|
17410
|
+
const safety = await this.filterSearchResultsForRecall(
|
|
17411
|
+
results,
|
|
17412
|
+
preloadedMemoryMap,
|
|
17413
|
+
{ ...options, recallNamespaces: _recallNamespaces },
|
|
17414
|
+
);
|
|
17415
|
+
const safeResults = safety.results;
|
|
17416
|
+
const memoryByPath = safety.memoryByPath;
|
|
17417
|
+
if (safeResults.length === 0) return safeResults;
|
|
17418
|
+
|
|
17419
|
+
const now = Date.now();
|
|
17420
|
+
|
|
17421
|
+
// Determine temporal/tag query params before index I/O (pure computation).
|
|
17422
|
+
const resultPaths = new Set(
|
|
17423
|
+
safeResults.map((r) => r.path).filter(Boolean) as string[],
|
|
17424
|
+
);
|
|
17425
|
+
let temporalFromDate: string | null = null;
|
|
17426
|
+
let promptTags: string[] = [];
|
|
17427
|
+
if (this.config.queryAwareIndexingEnabled && prompt) {
|
|
17428
|
+
if (isTemporalQuery(prompt)) {
|
|
17429
|
+
temporalFromDate = recencyWindowFromPrompt(prompt, now);
|
|
17430
|
+
}
|
|
17431
|
+
promptTags = extractTagsFromPrompt(prompt);
|
|
17432
|
+
}
|
|
17433
|
+
|
|
17434
|
+
const [rawTemporal, rawTags] = await Promise.all([
|
|
17435
|
+
temporalFromDate !== null
|
|
17436
|
+
? queryByDateRangeAsync(this.config.memoryDir, temporalFromDate)
|
|
17437
|
+
: Promise.resolve<Set<string> | null>(null),
|
|
17438
|
+
promptTags.length > 0
|
|
17439
|
+
? queryByTagsAsync(this.config.memoryDir, promptTags)
|
|
17440
|
+
: Promise.resolve<Set<string> | null>(null),
|
|
17441
|
+
]);
|
|
17442
|
+
|
|
17443
|
+
const queryIntent =
|
|
17444
|
+
this.config.intentRoutingEnabled && prompt
|
|
17445
|
+
? inferIntentFromText(prompt)
|
|
17446
|
+
: null;
|
|
17447
|
+
|
|
17448
|
+
// v8.1: Temporal + Tag prefilter candidate set
|
|
17449
|
+
// Scope to result paths first so cross-namespace paths don't consume the cap.
|
|
17450
|
+
let temporalCandidates: Set<string> | null = null;
|
|
17451
|
+
let tagCandidates: Set<string> | null = null;
|
|
17452
|
+
if (this.config.queryAwareIndexingEnabled && prompt) {
|
|
17453
|
+
const maxCandidates = this.config.queryAwareIndexingMaxCandidates;
|
|
17454
|
+
const capSet = (s: Set<string> | null): Set<string> | null => {
|
|
17455
|
+
if (!s) return null;
|
|
17456
|
+
// Intersect with result paths first so out-of-scope paths don't exhaust the budget
|
|
17457
|
+
const scoped = new Set(Array.from(s).filter((p) => resultPaths.has(p)));
|
|
17458
|
+
if (maxCandidates === 0 || scoped.size <= maxCandidates)
|
|
17459
|
+
return scoped;
|
|
17460
|
+
return new Set(Array.from(scoped).slice(0, maxCandidates));
|
|
17461
|
+
};
|
|
17462
|
+
if (temporalFromDate !== null) {
|
|
17463
|
+
temporalCandidates = capSet(rawTemporal);
|
|
17464
|
+
}
|
|
17465
|
+
if (promptTags.length > 0) {
|
|
17466
|
+
tagCandidates = capSet(rawTags);
|
|
17467
|
+
}
|
|
17468
|
+
}
|
|
17469
|
+
|
|
17470
|
+
const boosted: QmdSearchResult[] = [];
|
|
17471
|
+
const recencyWeight = this.effectiveRecencyWeight();
|
|
17472
|
+
for (const r of safeResults) {
|
|
17473
|
+
const memory = memoryByPath.get(r.path);
|
|
17474
|
+
let score = r.score;
|
|
17475
|
+
|
|
17476
|
+
if (memory) {
|
|
16381
17477
|
// Recency boost: exponential decay over 7 days
|
|
16382
17478
|
if (recencyWeight > 0) {
|
|
16383
17479
|
const createdAt = new Date(memory.frontmatter.created).getTime();
|
|
@@ -16522,26 +17618,6 @@ export class Orchestrator {
|
|
|
16522
17618
|
|
|
16523
17619
|
boosted.push({ ...r, score });
|
|
16524
17620
|
}
|
|
16525
|
-
if (lifecycleFilteredCount > 0) {
|
|
16526
|
-
log.debug(
|
|
16527
|
-
`lifecycle retrieval filter removed ${lifecycleFilteredCount} stale/archived candidates`,
|
|
16528
|
-
);
|
|
16529
|
-
}
|
|
16530
|
-
if (temporalSupersededFilteredCount > 0) {
|
|
16531
|
-
log.debug(
|
|
16532
|
-
`temporal supersession filter removed ${temporalSupersededFilteredCount} superseded candidates`,
|
|
16533
|
-
);
|
|
16534
|
-
}
|
|
16535
|
-
if (dedicatedSurfaceFilteredCount > 0) {
|
|
16536
|
-
log.debug(
|
|
16537
|
-
`dedicated surface filter removed ${dedicatedSurfaceFilteredCount} dream/procedural candidates from generic recall`,
|
|
16538
|
-
);
|
|
16539
|
-
}
|
|
16540
|
-
if (forgottenFilteredCount > 0) {
|
|
16541
|
-
log.debug(
|
|
16542
|
-
`forgotten status filter removed ${forgottenFilteredCount} candidates from recall`,
|
|
16543
|
-
);
|
|
16544
|
-
}
|
|
16545
17621
|
|
|
16546
17622
|
// Re-sort by boosted score
|
|
16547
17623
|
return boosted.sort((a, b) => b.score - a.score);
|
|
@@ -16769,8 +17845,52 @@ export class Orchestrator {
|
|
|
16769
17845
|
}));
|
|
16770
17846
|
}
|
|
16771
17847
|
|
|
17848
|
+
private async graphSeedPathRelativeToStorage(
|
|
17849
|
+
storage: StorageManager,
|
|
17850
|
+
result: QmdSearchResult,
|
|
17851
|
+
recallNamespaces: readonly string[] = [],
|
|
17852
|
+
): Promise<string | null> {
|
|
17853
|
+
const parts = qmdCollectionPathParts(result.path);
|
|
17854
|
+
if (parts) {
|
|
17855
|
+
const memory = await this.readQmdResultMemory(
|
|
17856
|
+
result.path,
|
|
17857
|
+
storage,
|
|
17858
|
+
recallNamespaces,
|
|
17859
|
+
);
|
|
17860
|
+
return memory
|
|
17861
|
+
? graphPathRelativeToStorage(storage.dir, memory.path)
|
|
17862
|
+
: null;
|
|
17863
|
+
}
|
|
17864
|
+
return graphPathRelativeToStorage(storage.dir, result.path);
|
|
17865
|
+
}
|
|
17866
|
+
|
|
17867
|
+
private async graphSeedPathsWithinDeadline(
|
|
17868
|
+
storage: StorageManager,
|
|
17869
|
+
results: QmdSearchResult[],
|
|
17870
|
+
deadlineAtMs: number,
|
|
17871
|
+
recallNamespaces: readonly string[] = [],
|
|
17872
|
+
): Promise<string[]> {
|
|
17873
|
+
const resolved: string[] = [];
|
|
17874
|
+
for (const result of results) {
|
|
17875
|
+
if (Date.now() >= deadlineAtMs) break;
|
|
17876
|
+
const seedPath = await this.graphSeedPathRelativeToStorage(
|
|
17877
|
+
storage,
|
|
17878
|
+
result,
|
|
17879
|
+
recallNamespaces,
|
|
17880
|
+
);
|
|
17881
|
+
if (Date.now() >= deadlineAtMs) break;
|
|
17882
|
+
if (seedPath) resolved.push(seedPath);
|
|
17883
|
+
}
|
|
17884
|
+
return resolved;
|
|
17885
|
+
}
|
|
17886
|
+
|
|
16772
17887
|
private namespaceFromPath(p: string): string {
|
|
16773
17888
|
if (!this.config.namespacesEnabled) return this.config.defaultNamespace;
|
|
17889
|
+
const parts = qmdCollectionPathParts(p);
|
|
17890
|
+
const collectionNamespace = parts
|
|
17891
|
+
? this.qmdCollectionNamespaceFromPrefix(parts.collection)
|
|
17892
|
+
: null;
|
|
17893
|
+
if (collectionNamespace) return collectionNamespace;
|
|
16774
17894
|
const m = p.match(/[\\/]+namespaces[\\/]+([^\\/]+)(?:[\\/]|$)/);
|
|
16775
17895
|
if (!m?.[1]) return this.config.defaultNamespace;
|
|
16776
17896
|
return namespaceIdentityFromToken(m[1]) ?? m[1];
|