@remnic/core 9.3.748 → 9.3.750
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-boundary.d.ts +8 -8
- package/dist/access-boundary.js +15 -15
- package/dist/access-cli.js +39 -39
- package/dist/access-http.d.ts +8 -8
- package/dist/access-http.js +20 -20
- package/dist/access-mcp.d.ts +8 -8
- package/dist/access-mcp.js +19 -19
- package/dist/access-operations-batch.js +16 -16
- package/dist/access-operations.d.ts +8 -8
- package/dist/access-operations.js +18 -18
- package/dist/access-schema.d.ts +2 -2
- package/dist/access-schema.js +5 -5
- package/dist/{access-service-CH1aqqb1.d.ts → access-service-DmAEOJdJ.d.ts} +11 -3
- package/dist/access-service.d.ts +5 -5
- package/dist/access-service.js +14 -14
- package/dist/access-surface-catalog.d.ts +8 -8
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/active-recall.js +8 -1
- package/dist/active-recall.js.map +1 -1
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +7 -7
- package/dist/briefing.d.ts +1 -1
- package/dist/briefing.js +7 -7
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer.d.ts +1 -1
- package/dist/calibration.d.ts +1 -1
- package/dist/capabilities.d.ts +1 -1
- package/dist/{catalog-D_ZDxu8Y.d.ts → catalog-BpOcwHdE.d.ts} +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-consolidation.js +8 -8
- package/dist/causal-trajectory-graph.d.ts +1 -1
- package/dist/{chunk-C5WN74HH.js → chunk-2HAG7JOI.js} +8 -2
- package/dist/chunk-2HAG7JOI.js.map +1 -0
- package/dist/{chunk-GKRLPHPU.js → chunk-2KK5WNQB.js} +2 -2
- package/dist/{chunk-JRDPT6DA.js → chunk-4NWIGAIC.js} +2 -2
- package/dist/{chunk-OBJUKYDO.js → chunk-4XL3CVII.js} +2 -2
- package/dist/{chunk-WWVXXSLF.js → chunk-5PZCDUJ6.js} +4 -4
- package/dist/{chunk-URQLO7BX.js → chunk-6FNZ3NMS.js} +2 -2
- package/dist/{chunk-DRUDC2UX.js → chunk-6MMONPHM.js} +2 -2
- package/dist/{chunk-RB6A7OUN.js → chunk-6SXNQFHB.js} +2 -2
- package/dist/{chunk-ZGZNLRKF.js → chunk-7TWA7DKP.js} +5 -5
- package/dist/{chunk-KMNYXLPL.js → chunk-B35L45HZ.js} +2 -2
- package/dist/{chunk-SIV3JZPP.js → chunk-CGZWGVQF.js} +5 -5
- package/dist/{chunk-ZWGXZ7ID.js → chunk-CKLCFBA3.js} +2377 -2053
- package/dist/chunk-CKLCFBA3.js.map +1 -0
- package/dist/{chunk-CKFLLAIS.js → chunk-DANV2HTH.js} +3 -3
- package/dist/{chunk-NVYF5OB6.js → chunk-DBRVU5PR.js} +3 -3
- package/dist/{chunk-LMFS64GY.js → chunk-E4PFDD3N.js} +4 -4
- package/dist/{chunk-DIHCFJVV.js → chunk-HYI2GP3F.js} +4 -4
- package/dist/{chunk-UIKZZ242.js → chunk-IKHKHADN.js} +2 -2
- package/dist/{chunk-PUR2CKX2.js → chunk-JS2HKKOK.js} +1 -1
- package/dist/{chunk-PUR2CKX2.js.map → chunk-JS2HKKOK.js.map} +1 -1
- package/dist/{chunk-R2OAN6IP.js → chunk-K2JM4DZZ.js} +2 -2
- package/dist/{chunk-3IJGF6K2.js → chunk-K2R3DEEH.js} +2 -2
- package/dist/{chunk-HIHCB2S6.js → chunk-QRRBXD24.js} +2 -2
- package/dist/{chunk-ZPZ4VVLT.js → chunk-SD34EK4Z.js} +2 -2
- package/dist/{chunk-WK3NK6SP.js → chunk-SNEXVX3U.js} +6 -6
- package/dist/{chunk-FVZ2ISIX.js → chunk-TPLE6SNY.js} +7 -7
- package/dist/{chunk-24YDB5QP.js → chunk-UETBEHBA.js} +44 -24
- package/dist/chunk-UETBEHBA.js.map +1 -0
- package/dist/{chunk-5IN26V6W.js → chunk-UU32EAIY.js} +5 -5
- package/dist/{chunk-ZGAJB74V.js → chunk-W3BFT5OJ.js} +2 -2
- package/dist/{chunk-L4N7SVNN.js → chunk-WA7FECSG.js} +11 -11
- package/dist/{chunk-Q3X7GUAQ.js → chunk-WOX2UW4K.js} +2 -2
- package/dist/{chunk-XTEWVF3C.js → chunk-WWSXVOGY.js} +2 -2
- package/dist/{chunk-PWWWLD7D.js → chunk-X64YLC24.js} +153 -27
- package/dist/chunk-X64YLC24.js.map +1 -0
- package/dist/{chunk-P4FW3ZU5.js → chunk-YGGXUNS4.js} +20 -20
- package/dist/{chunk-JHQUIH3W.js → chunk-ZBVQ4ZM4.js} +19 -19
- package/dist/{cli-CE5olss2.d.ts → cli-lmf5RGyQ.d.ts} +3 -3
- package/dist/cli.d.ts +9 -9
- package/dist/cli.js +33 -33
- package/dist/compounding/engine.d.ts +1 -1
- package/dist/compounding/engine.js +7 -7
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.js +8 -1
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.js +7 -7
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/connectors/index.js +7 -7
- package/dist/consolidation-provenance-check.d.ts +1 -1
- package/dist/consolidation-undo.d.ts +1 -1
- package/dist/contradiction/index.d.ts +1 -1
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer.d.ts +1 -1
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-retrieval.d.ts +1 -1
- package/dist/entity-retrieval.js +7 -7
- package/dist/entity-schema.d.ts +1 -1
- package/dist/explicit-capture.d.ts +7 -7
- package/dist/extraction-faithfulness.d.ts +1 -1
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction.d.ts +1 -1
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/graph-dashboard-diff.d.ts +1 -1
- package/dist/graph-dashboard-key.d.ts +1 -1
- package/dist/graph-dashboard-parser.d.ts +1 -1
- package/dist/graph-edge-reinforcement.d.ts +1 -1
- package/dist/graph-snapshot.d.ts +1 -1
- package/dist/graph.d.ts +1 -1
- package/dist/identity-continuity.d.ts +1 -1
- package/dist/importance.d.ts +1 -1
- package/dist/index.d.ts +10 -10
- package/dist/index.js +76 -72
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +1 -1
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/local-model-endpoint.d.ts +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/maintenance/memory-governance.js +7 -7
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +7 -7
- package/dist/maintenance/rebuild-memory-projection.js +8 -8
- package/dist/mcp-memory-inspector-app.d.ts +5 -5
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +1 -1
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +2 -2
- package/dist/namespaces/migrate.js +8 -8
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/search.js +1 -1
- package/dist/namespaces/storage.d.ts +2 -2
- package/dist/namespaces/storage.js +7 -7
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/offline-sync.d.ts +108 -1
- package/dist/offline-sync.js +7 -1
- package/dist/operator-toolkit.d.ts +1 -1
- package/dist/operator-toolkit.js +15 -12
- package/dist/orchestration/compression-guideline-coordinator.d.ts +1 -1
- package/dist/orchestration/maintenance.d.ts +2 -2
- package/dist/orchestration/maintenance.js +9 -9
- package/dist/{orchestrator-CIOqdNzF.d.ts → orchestrator-D7bb8ZGp.d.ts} +200 -257
- package/dist/orchestrator.d.ts +5 -5
- package/dist/orchestrator.js +51 -41
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/provenance.d.ts +1 -1
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +1 -1
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-explain-renderer.js +3 -3
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-state.d.ts +1 -1
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-cli.js +4 -4
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray-renderer.js +3 -3
- package/dist/recall-xray.d.ts +1 -1
- package/dist/recall-xray.js +2 -2
- package/dist/resolve-auth-token.d.ts +1 -1
- package/dist/resume-bundles.js +9 -2
- package/dist/retrieval-agents.d.ts +1 -1
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/factory.js +1 -1
- package/dist/search/index.d.ts +1 -1
- package/dist/search/index.js +1 -1
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/lancedb-backend.js +1 -1
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.js +1 -1
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/orama-backend.js +1 -1
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/{semantic-consolidation-D3aSiu9t.d.ts → semantic-consolidation-CTfbVwCA.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-consolidation.js +8 -8
- package/dist/semantic-rule-promotion.js +7 -7
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/semantic-rule-verifier.js +7 -7
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +1 -1
- package/dist/signal.d.ts +1 -1
- package/dist/storage.d.ts +1 -1
- package/dist/storage.js +6 -6
- package/dist/summarizer.d.ts +1 -1
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/temporal-supersession.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +1 -1
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/trust-score-stage.d.ts +1 -1
- package/dist/trust-score.d.ts +1 -1
- package/dist/{types-CZm6rx9-.d.ts → types-D4NYDtXI.d.ts} +7 -0
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -1
- package/dist/utility-runtime.d.ts +1 -1
- package/dist/verified-recall.js +7 -7
- package/package.json +2 -2
- package/src/access-service.ts +22 -0
- package/src/config.ts +5 -0
- package/src/index.ts +3 -1
- package/src/offline-sync.test.ts +354 -52
- package/src/offline-sync.ts +332 -24
- package/src/orchestration/extraction-run.ts +4 -34
- package/src/orchestration/namespace-read-fanout.ts +484 -0
- package/src/orchestration/orchestrator-helpers.ts +1130 -0
- package/src/orchestration/persistence-index.ts +497 -0
- package/src/orchestration/workspace-ops.ts +789 -0
- package/src/orchestrator.ts +271 -2448
- package/src/types.ts +9 -0
- package/dist/chunk-24YDB5QP.js.map +0 -1
- package/dist/chunk-C5WN74HH.js.map +0 -1
- package/dist/chunk-PWWWLD7D.js.map +0 -1
- package/dist/chunk-ZWGXZ7ID.js.map +0 -1
- /package/dist/{chunk-GKRLPHPU.js.map → chunk-2KK5WNQB.js.map} +0 -0
- /package/dist/{chunk-JRDPT6DA.js.map → chunk-4NWIGAIC.js.map} +0 -0
- /package/dist/{chunk-OBJUKYDO.js.map → chunk-4XL3CVII.js.map} +0 -0
- /package/dist/{chunk-WWVXXSLF.js.map → chunk-5PZCDUJ6.js.map} +0 -0
- /package/dist/{chunk-URQLO7BX.js.map → chunk-6FNZ3NMS.js.map} +0 -0
- /package/dist/{chunk-DRUDC2UX.js.map → chunk-6MMONPHM.js.map} +0 -0
- /package/dist/{chunk-RB6A7OUN.js.map → chunk-6SXNQFHB.js.map} +0 -0
- /package/dist/{chunk-ZGZNLRKF.js.map → chunk-7TWA7DKP.js.map} +0 -0
- /package/dist/{chunk-KMNYXLPL.js.map → chunk-B35L45HZ.js.map} +0 -0
- /package/dist/{chunk-SIV3JZPP.js.map → chunk-CGZWGVQF.js.map} +0 -0
- /package/dist/{chunk-CKFLLAIS.js.map → chunk-DANV2HTH.js.map} +0 -0
- /package/dist/{chunk-NVYF5OB6.js.map → chunk-DBRVU5PR.js.map} +0 -0
- /package/dist/{chunk-LMFS64GY.js.map → chunk-E4PFDD3N.js.map} +0 -0
- /package/dist/{chunk-DIHCFJVV.js.map → chunk-HYI2GP3F.js.map} +0 -0
- /package/dist/{chunk-UIKZZ242.js.map → chunk-IKHKHADN.js.map} +0 -0
- /package/dist/{chunk-R2OAN6IP.js.map → chunk-K2JM4DZZ.js.map} +0 -0
- /package/dist/{chunk-3IJGF6K2.js.map → chunk-K2R3DEEH.js.map} +0 -0
- /package/dist/{chunk-HIHCB2S6.js.map → chunk-QRRBXD24.js.map} +0 -0
- /package/dist/{chunk-ZPZ4VVLT.js.map → chunk-SD34EK4Z.js.map} +0 -0
- /package/dist/{chunk-WK3NK6SP.js.map → chunk-SNEXVX3U.js.map} +0 -0
- /package/dist/{chunk-FVZ2ISIX.js.map → chunk-TPLE6SNY.js.map} +0 -0
- /package/dist/{chunk-5IN26V6W.js.map → chunk-UU32EAIY.js.map} +0 -0
- /package/dist/{chunk-ZGAJB74V.js.map → chunk-W3BFT5OJ.js.map} +0 -0
- /package/dist/{chunk-L4N7SVNN.js.map → chunk-WA7FECSG.js.map} +0 -0
- /package/dist/{chunk-Q3X7GUAQ.js.map → chunk-WOX2UW4K.js.map} +0 -0
- /package/dist/{chunk-XTEWVF3C.js.map → chunk-WWSXVOGY.js.map} +0 -0
- /package/dist/{chunk-P4FW3ZU5.js.map → chunk-YGGXUNS4.js.map} +0 -0
- /package/dist/{chunk-JHQUIH3W.js.map → chunk-ZBVQ4ZM4.js.map} +0 -0
package/src/orchestrator.ts
CHANGED
|
@@ -114,6 +114,75 @@ import { RecallSearchPipelineCoordinator } from "./orchestration/recall-search-p
|
|
|
114
114
|
import { TurnIngestionCoordinator } from "./orchestration/turn-ingestion.js";
|
|
115
115
|
import { RecallIntrospectionCoordinator } from "./orchestration/recall-introspection.js";
|
|
116
116
|
import { OrchestratorInitCoordinator } from "./orchestration/orchestrator-init.js";
|
|
117
|
+
import { PersistenceIndexCoordinator } from "./orchestration/persistence-index.js";
|
|
118
|
+
import { WorkspaceOpsCoordinator } from "./orchestration/workspace-ops.js";
|
|
119
|
+
import { NamespaceReadFanoutCoordinator } from "./orchestration/namespace-read-fanout.js";
|
|
120
|
+
import {
|
|
121
|
+
abortRecallError,
|
|
122
|
+
buildCompressionGuidelinesMarkdown,
|
|
123
|
+
buildQmdIntentHint,
|
|
124
|
+
isArtifactMemoryPath,
|
|
125
|
+
mergeArtifactRecallCandidates,
|
|
126
|
+
tokenizeRecallQuery,
|
|
127
|
+
type BulkImportBatchIngestResult,
|
|
128
|
+
type DaySummaryGatherOptions,
|
|
129
|
+
type GraphRecallSnapshot,
|
|
130
|
+
type IntentDebugSnapshot,
|
|
131
|
+
type QmdRecallSnapshot,
|
|
132
|
+
type QueryAwarePrefilter,
|
|
133
|
+
type RecallInvocationOptions,
|
|
134
|
+
} from "./orchestration/orchestrator-helpers.js";
|
|
135
|
+
// Issue #1526 seam 25: module-level helpers moved to
|
|
136
|
+
// orchestration/orchestrator-helpers.ts; re-exported here so existing
|
|
137
|
+
// importers (coordinators, tests, root shims) keep working.
|
|
138
|
+
export {
|
|
139
|
+
BulkImportBatchPartialFailureError,
|
|
140
|
+
COMPACTION_SIGNAL_MAX_AGE_MS,
|
|
141
|
+
appendMemoryToGraphContext,
|
|
142
|
+
applyQueryAwareCandidateFilter,
|
|
143
|
+
buildCompressionGuidelinesMarkdown,
|
|
144
|
+
buildMemoryPathById,
|
|
145
|
+
computeArtifactCandidateFetchLimit,
|
|
146
|
+
computeArtifactRecallLimit,
|
|
147
|
+
computeQmdHybridFetchLimit,
|
|
148
|
+
defaultWorkspaceDir,
|
|
149
|
+
filterHourlySummaryMarkdownForLocalDay,
|
|
150
|
+
filterRecallCandidates,
|
|
151
|
+
formatDateInTimeZone,
|
|
152
|
+
isArtifactMemoryPath,
|
|
153
|
+
lifecycleRecallScoreAdjustment,
|
|
154
|
+
mapRecallSourceToXrayServedBy,
|
|
155
|
+
mergeArtifactRecallCandidates,
|
|
156
|
+
normalizeIanaTimeZone,
|
|
157
|
+
parseFiniteDate,
|
|
158
|
+
parseGraphRecallRankedResults,
|
|
159
|
+
parseMemoryIntentSnapshot,
|
|
160
|
+
parseQmdRecallResults,
|
|
161
|
+
qmdStartupCollectionCheckWithTimeout,
|
|
162
|
+
raceRecallAbort,
|
|
163
|
+
resolveEffectiveRecallMode,
|
|
164
|
+
resolvePersistedMemoryRelativePath,
|
|
165
|
+
resolveRecallModeDecision,
|
|
166
|
+
resolveRecallModeDecisionAsync,
|
|
167
|
+
resolveRecentThreadMemoryPaths,
|
|
168
|
+
sanitizeSessionKeyForFilename,
|
|
169
|
+
shouldFilterLifecycleRecallCandidate,
|
|
170
|
+
splitTurnsBySourceValidAt,
|
|
171
|
+
summarizeGraphShadowComparison,
|
|
172
|
+
targetSourceValidAtSortMs,
|
|
173
|
+
throwIfRecallAborted,
|
|
174
|
+
tokenizeRecallQuery,
|
|
175
|
+
utcDateKeysForLocalDay,
|
|
176
|
+
type BulkImportBatchIngestResult,
|
|
177
|
+
type DaySummaryGatherOptions,
|
|
178
|
+
type GraphRecallSnapshot,
|
|
179
|
+
type IntentDebugSnapshot,
|
|
180
|
+
type QmdRecallSnapshot,
|
|
181
|
+
type QueryAwarePrefilter,
|
|
182
|
+
type RecallInvocationOptions,
|
|
183
|
+
type RecallModeDecision,
|
|
184
|
+
} from "./orchestration/orchestrator-helpers.js";
|
|
185
|
+
|
|
117
186
|
export { hasIdentityRecoveryIntent, resolveEffectiveIdentityInjectionMode } from "./orchestration/recall-result-formatter.js";
|
|
118
187
|
import {
|
|
119
188
|
GraphRecallCoordinator,
|
|
@@ -468,1105 +537,14 @@ import type {
|
|
|
468
537
|
} from "./types.js";
|
|
469
538
|
import { disposeDefaultArchiveScoring, getDefaultArchiveScoring, memoryFileToScoreItem } from "./recall/archive-scoring.js";
|
|
470
539
|
|
|
471
|
-
export interface BulkImportBatchIngestResult {
|
|
472
|
-
attemptedTurnCount: number;
|
|
473
|
-
extractionCount: number;
|
|
474
|
-
persistedCount: number;
|
|
475
|
-
durableOutputCount: number;
|
|
476
|
-
skippedCount: number;
|
|
477
|
-
failedCount: number;
|
|
478
|
-
postPersistMetadataFailureCount: number;
|
|
479
|
-
processedTurnCount: number;
|
|
480
|
-
}
|
|
481
|
-
|
|
482
|
-
export class BulkImportBatchPartialFailureError extends Error {
|
|
483
|
-
readonly partialResult: BulkImportBatchIngestResult;
|
|
484
|
-
|
|
485
|
-
readonly originalError: unknown;
|
|
486
|
-
|
|
487
|
-
constructor(
|
|
488
|
-
message: string,
|
|
489
|
-
partialResult: BulkImportBatchIngestResult,
|
|
490
|
-
originalError: unknown,
|
|
491
|
-
) {
|
|
492
|
-
super(message);
|
|
493
|
-
this.name = "BulkImportBatchPartialFailureError";
|
|
494
|
-
this.partialResult = partialResult;
|
|
495
|
-
this.originalError = originalError;
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
540
|
|
|
499
541
|
// Issue #1526 seam 15: ExtractionRunResult moved to orchestration/extraction-run.ts.
|
|
500
542
|
export type { ExtractionRunResult } from "./orchestration/extraction-run.js";
|
|
501
543
|
|
|
502
|
-
export interface GraphRecallSnapshot {
|
|
503
|
-
recordedAt: string;
|
|
504
|
-
mode: RecallPlanMode | string;
|
|
505
|
-
queryHash: string;
|
|
506
|
-
queryLength: number;
|
|
507
|
-
namespaces: string[];
|
|
508
|
-
seedCount: number;
|
|
509
|
-
expandedCount: number;
|
|
510
|
-
seeds: string[];
|
|
511
|
-
expanded: GraphRecallExpandedEntry[];
|
|
512
|
-
status?: "completed" | "skipped" | "aborted";
|
|
513
|
-
reason?: string;
|
|
514
|
-
shadowMode?: boolean;
|
|
515
|
-
queryIntent?: MemoryIntent;
|
|
516
|
-
seedResults?: GraphRecallRankedResult[];
|
|
517
|
-
finalResults?: GraphRecallRankedResult[];
|
|
518
|
-
shadowComparison?: GraphRecallShadowComparison;
|
|
519
|
-
}
|
|
520
|
-
|
|
521
|
-
export interface IntentDebugSnapshot {
|
|
522
|
-
recordedAt: string;
|
|
523
|
-
promptHash: string;
|
|
524
|
-
promptLength: number;
|
|
525
|
-
retrievalQueryHash: string;
|
|
526
|
-
retrievalQueryLength: number;
|
|
527
|
-
plannerEnabled: boolean;
|
|
528
|
-
plannedMode: RecallPlanMode;
|
|
529
|
-
effectiveMode: RecallPlanMode;
|
|
530
|
-
recallResultLimit: number;
|
|
531
|
-
queryIntent: MemoryIntent;
|
|
532
|
-
graphExpandedIntentDetected: boolean;
|
|
533
|
-
graphDecision: {
|
|
534
|
-
status: "not_requested" | "skipped" | "completed" | "aborted";
|
|
535
|
-
reason?: string;
|
|
536
|
-
shadowMode: boolean;
|
|
537
|
-
qmdAvailable: boolean;
|
|
538
|
-
graphRecallEnabled: boolean;
|
|
539
|
-
multiGraphMemoryEnabled: boolean;
|
|
540
|
-
};
|
|
541
|
-
}
|
|
542
|
-
|
|
543
|
-
export interface QmdRecallSnapshot {
|
|
544
|
-
recordedAt: string;
|
|
545
|
-
queryHash: string;
|
|
546
|
-
queryLength: number;
|
|
547
|
-
collection?: string;
|
|
548
|
-
namespaces: string[];
|
|
549
|
-
fetchLimit: number;
|
|
550
|
-
primaryResultCount: number;
|
|
551
|
-
hybridResultCount: number;
|
|
552
|
-
queryAwareSeedCount: number;
|
|
553
|
-
resultCount: number;
|
|
554
|
-
intentHint?: string;
|
|
555
|
-
explainEnabled: boolean;
|
|
556
|
-
hybridTopUpUsed: boolean;
|
|
557
|
-
hybridTopUpSkippedReason?: string;
|
|
558
|
-
results: QmdSearchResult[];
|
|
559
|
-
}
|
|
560
|
-
|
|
561
|
-
export interface RecallModeDecision {
|
|
562
|
-
plannedMode: RecallPlanMode;
|
|
563
|
-
effectiveMode: RecallPlanMode;
|
|
564
|
-
graphExpandedIntentDetected: boolean;
|
|
565
|
-
graphReason?: string;
|
|
566
|
-
/**
|
|
567
|
-
* Where `plannedMode` came from (issue #1367 / Option C). `"heuristic"` for
|
|
568
|
-
* the regex planner; `"llm"` when the LLM planner classified it; and
|
|
569
|
-
* `"heuristic-fallback"` when the LLM was enabled but errored/timed out and we
|
|
570
|
-
* fell back. Absent on the synchronous heuristic-only path.
|
|
571
|
-
*/
|
|
572
|
-
plannerSource?: "heuristic" | "llm" | "heuristic-fallback";
|
|
573
|
-
/** Short rationale from the planner (for telemetry / x-ray). */
|
|
574
|
-
plannerReason?: string;
|
|
575
|
-
/** Wall-clock spent in the LLM planner call, when one was made. */
|
|
576
|
-
plannerLatencyMs?: number;
|
|
577
|
-
/** True when the LLM planner was enabled but fell back to the heuristic. */
|
|
578
|
-
plannerFallbackUsed?: boolean;
|
|
579
|
-
/** Model that served the LLM classification, when one was used. */
|
|
580
|
-
plannerModelUsed?: string;
|
|
581
|
-
/**
|
|
582
|
-
* The regex-heuristic baseline mode, captured whenever the LLM planner ran
|
|
583
|
-
* (any source). Lets operators compare planned-vs-heuristic during rollout —
|
|
584
|
-
* distinct from `plannedMode`, which on the LLM path is the LLM's choice.
|
|
585
|
-
*/
|
|
586
|
-
plannerHeuristicMode?: RecallPlanMode;
|
|
587
|
-
/**
|
|
588
|
-
* In shadow mode, the mode the LLM *would* have chosen (recorded for
|
|
589
|
-
* comparison) while `effectiveMode` stays on the heuristic decision.
|
|
590
|
-
*/
|
|
591
|
-
shadowLlmMode?: RecallPlanMode;
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
/**
|
|
595
|
-
* Map the orchestrator's internal `recallSource` strings to the
|
|
596
|
-
* X-ray `servedBy` vocabulary (issue #570 PR 1). The X-ray tier
|
|
597
|
-
* ladder intentionally flattens QMD / embedding / cold-fallback to
|
|
598
|
-
* the `hybrid` tier because they all materialize through the same
|
|
599
|
-
* hybrid BM25+vector pipeline from the caller's perspective. The
|
|
600
|
-
* `recent_scan` path gets its own dedicated tier because it bypasses
|
|
601
|
-
* the hybrid pipeline entirely. `none` is treated as `hybrid` on the
|
|
602
|
-
* theory that a query that returned nothing still routed through the
|
|
603
|
-
* hybrid pipeline — but callers should normally gate capture on
|
|
604
|
-
* `recalledMemoryIds.length > 0`.
|
|
605
|
-
*/
|
|
606
|
-
export function mapRecallSourceToXrayServedBy(
|
|
607
|
-
source:
|
|
608
|
-
| "none"
|
|
609
|
-
| "hot_qmd"
|
|
610
|
-
| "hot_embedding"
|
|
611
|
-
| "cold_fallback"
|
|
612
|
-
| "recent_scan",
|
|
613
|
-
): RecallXrayServedBy {
|
|
614
|
-
// Exhaustive switch: every current union member is explicitly
|
|
615
|
-
// listed so TypeScript surfaces a compile error if a new source is
|
|
616
|
-
// added without a deliberate mapping. The `never`-typed fallthrough
|
|
617
|
-
// keeps the function total at runtime — if the caller passes an
|
|
618
|
-
// unexpected value that slipped past the type system (e.g. a JSON
|
|
619
|
-
// deserialization), we still fall back to `hybrid`.
|
|
620
|
-
switch (source) {
|
|
621
|
-
case "recent_scan":
|
|
622
|
-
return "recent-scan";
|
|
623
|
-
case "hot_qmd":
|
|
624
|
-
case "hot_embedding":
|
|
625
|
-
case "cold_fallback":
|
|
626
|
-
case "none":
|
|
627
|
-
return "hybrid";
|
|
628
|
-
}
|
|
629
|
-
const _exhaustive: never = source;
|
|
630
|
-
void _exhaustive;
|
|
631
|
-
return "hybrid";
|
|
632
|
-
}
|
|
633
|
-
|
|
634
|
-
export interface RecallInvocationOptions {
|
|
635
|
-
namespace?: string;
|
|
636
|
-
topK?: number;
|
|
637
|
-
mode?: RecallPlanMode;
|
|
638
|
-
abortSignal?: AbortSignal;
|
|
639
|
-
/**
|
|
640
|
-
* Capture a `RecallXraySnapshot` for this recall (issue #570). When
|
|
641
|
-
* `true`, the orchestrator builds a snapshot from the data it has
|
|
642
|
-
* already gathered and stashes it in memory, accessible via
|
|
643
|
-
* `getLastXraySnapshot()`. When `false` or absent, nothing is
|
|
644
|
-
* captured and recall behavior is unchanged (schema-only slice).
|
|
645
|
-
*/
|
|
646
|
-
xrayCapture?: boolean;
|
|
647
|
-
/**
|
|
648
|
-
* Per-invocation override for `recallBudgetChars` (issue #570 PR 3/4).
|
|
649
|
-
* Flows through `getRecallBudgetChars()` for this recall only — no
|
|
650
|
-
* shared config mutation, so concurrent recalls on the same
|
|
651
|
-
* orchestrator are not affected (CLAUDE.md rule 47: no shared
|
|
652
|
-
* mutable state across async boundaries). Must be a non-negative
|
|
653
|
-
* finite integer; non-conforming values are ignored and the
|
|
654
|
-
* configured budget is used.
|
|
655
|
-
*/
|
|
656
|
-
budgetCharsOverride?: number;
|
|
657
|
-
/**
|
|
658
|
-
* Per-invocation principal override (issue #570 PR 4). When set,
|
|
659
|
-
* the orchestrator uses this principal for ACL / namespace checks
|
|
660
|
-
* instead of `resolvePrincipal(sessionKey, config)`. This is the
|
|
661
|
-
* escape hatch for access surfaces (HTTP / MCP) that have already
|
|
662
|
-
* authenticated the caller upstream — threading an unmapped
|
|
663
|
-
* principal through the session-key-based resolver would otherwise
|
|
664
|
-
* collapse it to `"default"` and produce false denials in
|
|
665
|
-
* namespace-enabled deployments (CLAUDE.md rule 42).
|
|
666
|
-
*/
|
|
667
|
-
principalOverride?: string;
|
|
668
|
-
/**
|
|
669
|
-
* Historical recall point (issue #680). When set, the orchestrator
|
|
670
|
-
* filters out memories whose `valid_at` is after this timestamp OR
|
|
671
|
-
* whose `invalid_at` is at-or-before this timestamp, so callers see
|
|
672
|
-
* the corpus as it existed at `asOf`. ISO 8601 string; comparisons
|
|
673
|
-
* use `Date.parse()` so timezone-aware values round-trip correctly
|
|
674
|
-
* (CLAUDE.md gotcha — never compare ISO strings lexicographically).
|
|
675
|
-
* Invalid values must be rejected at input boundaries (CLAUDE.md
|
|
676
|
-
* rule 51); the orchestrator does NOT silently fall back here.
|
|
677
|
-
*/
|
|
678
|
-
asOf?: string;
|
|
679
|
-
/**
|
|
680
|
-
* Issue #681 — when `true`, bypasses `graphTraversalConfidenceFloor`
|
|
681
|
-
* and includes edges below the floor in graph traversal. Useful for
|
|
682
|
-
* diagnostic recall queries that need to surface results that would
|
|
683
|
-
* normally be pruned by confidence decay. Default `false`.
|
|
684
|
-
*/
|
|
685
|
-
includeLowConfidence?: boolean;
|
|
686
|
-
/**
|
|
687
|
-
* User-aware context scopes active for this recall. Used by X-ray
|
|
688
|
-
* provenance safety checks so boundary-scoped memories are evaluated
|
|
689
|
-
* against the caller's real context.
|
|
690
|
-
*/
|
|
691
|
-
currentContextScopes?: readonly unknown[];
|
|
692
|
-
}
|
|
693
|
-
|
|
694
|
-
export type QueryAwarePrefilter = {
|
|
695
|
-
candidatePaths: Set<string> | null;
|
|
696
|
-
temporalFromDate: string | null;
|
|
697
|
-
matchedTags: string[];
|
|
698
|
-
expandedTags: string[];
|
|
699
|
-
combination: "none" | "temporal" | "tag" | "intersection" | "union";
|
|
700
|
-
filteredToFullSearch: boolean;
|
|
701
|
-
};
|
|
702
|
-
|
|
703
|
-
// Recall-specific abort helpers. Thin wrappers over the shared
|
|
704
|
-
// `abort-error.ts` module so every abort in the codebase shares the
|
|
705
|
-
// same `name === "AbortError"` classification contract (`isAbortError`
|
|
706
|
-
// works uniformly). We keep the "recall aborted" default message for
|
|
707
|
-
// back-compat with call-site logs; callers that pass an explicit
|
|
708
|
-
// message (e.g. "extraction aborted (before_extract)") are unaffected.
|
|
709
|
-
const abortRecallError = sharedAbortError;
|
|
710
|
-
|
|
711
|
-
export function throwIfRecallAborted(
|
|
712
|
-
signal?: AbortSignal,
|
|
713
|
-
message = "recall aborted",
|
|
714
|
-
): void {
|
|
715
|
-
sharedThrowIfAborted(signal, message);
|
|
716
|
-
}
|
|
717
|
-
|
|
718
|
-
export async function raceRecallAbort<T>(
|
|
719
|
-
promise: Promise<T>,
|
|
720
|
-
signal?: AbortSignal,
|
|
721
|
-
message = "recall aborted",
|
|
722
|
-
): Promise<T> {
|
|
723
|
-
throwIfRecallAborted(signal, message);
|
|
724
|
-
if (!signal) return promise;
|
|
725
|
-
|
|
726
|
-
let onAbort: (() => void) | null = null;
|
|
727
|
-
const abortPromise = new Promise<T>((_resolve, reject) => {
|
|
728
|
-
onAbort = () => reject(abortRecallError(message));
|
|
729
|
-
signal.addEventListener("abort", onAbort, { once: true });
|
|
730
|
-
});
|
|
731
|
-
|
|
732
|
-
try {
|
|
733
|
-
return await Promise.race([promise, abortPromise]);
|
|
734
|
-
} finally {
|
|
735
|
-
if (onAbort) {
|
|
736
|
-
signal.removeEventListener("abort", onAbort);
|
|
737
|
-
}
|
|
738
|
-
}
|
|
739
|
-
}
|
|
740
|
-
|
|
741
|
-
/** Maximum age (ms) before a compaction-reset signal file is considered stale and removed. */
|
|
742
|
-
export const COMPACTION_SIGNAL_MAX_AGE_MS = 60 * 60 * 1000; // 1 hour
|
|
743
|
-
const DEFAULT_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS = 10_000;
|
|
744
|
-
|
|
745
|
-
type DaySummaryGatherOptions = {
|
|
746
|
-
timeZone?: string;
|
|
747
|
-
now?: Date;
|
|
748
|
-
};
|
|
749
|
-
|
|
750
|
-
function normalizeIanaTimeZone(value: unknown): string | undefined {
|
|
751
|
-
if (typeof value !== "string") return undefined;
|
|
752
|
-
const trimmed = value.trim();
|
|
753
|
-
if (!trimmed) return undefined;
|
|
754
|
-
try {
|
|
755
|
-
Intl.DateTimeFormat(undefined, { timeZone: trimmed });
|
|
756
|
-
return trimmed;
|
|
757
|
-
} catch {
|
|
758
|
-
return undefined;
|
|
759
|
-
}
|
|
760
|
-
}
|
|
761
|
-
|
|
762
|
-
function formatDateInTimeZone(date: Date, timeZone: string): string {
|
|
763
|
-
const parts = new Intl.DateTimeFormat("en-US", {
|
|
764
|
-
timeZone,
|
|
765
|
-
year: "numeric",
|
|
766
|
-
month: "2-digit",
|
|
767
|
-
day: "2-digit",
|
|
768
|
-
}).formatToParts(date);
|
|
769
|
-
const get = (type: string): string => parts.find((part) => part.type === type)?.value ?? "";
|
|
770
|
-
return `${get("year")}-${get("month")}-${get("day")}`;
|
|
771
|
-
}
|
|
772
|
-
|
|
773
|
-
function utcDateKey(date: Date): string {
|
|
774
|
-
return date.toISOString().slice(0, 10);
|
|
775
|
-
}
|
|
776
|
-
|
|
777
|
-
function utcDateKeysAround(date: Date): string[] {
|
|
778
|
-
const dayMs = 86_400_000;
|
|
779
|
-
const keys = [
|
|
780
|
-
utcDateKey(new Date(date.getTime() - dayMs)),
|
|
781
|
-
utcDateKey(date),
|
|
782
|
-
utcDateKey(new Date(date.getTime() + dayMs)),
|
|
783
|
-
];
|
|
784
|
-
return keys.filter((value, index, array) => array.indexOf(value) === index);
|
|
785
|
-
}
|
|
786
|
-
|
|
787
|
-
function utcDateKeysForLocalDay(date: Date, timeZone: string): string[] {
|
|
788
|
-
const targetLocalDate = formatDateInTimeZone(date, timeZone);
|
|
789
|
-
const keys = new Set<string>();
|
|
790
|
-
const hourMs = 3_600_000;
|
|
791
|
-
const scanStart = date.getTime() - 48 * hourMs;
|
|
792
|
-
const scanEnd = date.getTime() + 48 * hourMs;
|
|
793
|
-
for (let ms = scanStart; ms <= scanEnd; ms += hourMs) {
|
|
794
|
-
const candidate = new Date(ms);
|
|
795
|
-
if (formatDateInTimeZone(candidate, timeZone) === targetLocalDate) {
|
|
796
|
-
keys.add(utcDateKey(candidate));
|
|
797
|
-
}
|
|
798
|
-
}
|
|
799
|
-
return keys.size > 0 ? [...keys].sort() : utcDateKeysAround(date);
|
|
800
|
-
}
|
|
801
|
-
|
|
802
|
-
function parseFiniteDate(value: unknown): Date | null {
|
|
803
|
-
if (typeof value !== "string" || value.trim().length === 0) return null;
|
|
804
|
-
const parsed = new Date(value);
|
|
805
|
-
return Number.isFinite(parsed.getTime()) ? parsed : null;
|
|
806
|
-
}
|
|
807
|
-
|
|
808
|
-
function filterHourlySummaryMarkdownForLocalDay(
|
|
809
|
-
raw: string,
|
|
810
|
-
utcDate: string,
|
|
811
|
-
timeZone: string,
|
|
812
|
-
targetLocalDate: string,
|
|
813
|
-
): string | null {
|
|
814
|
-
const hourHeaderPattern = /^## ([01]\d|2[0-3]):00[ \t]*$/gm;
|
|
815
|
-
const matches = Array.from(raw.matchAll(hourHeaderPattern));
|
|
816
|
-
if (matches.length === 0) return null;
|
|
817
|
-
|
|
818
|
-
const firstSectionStart = matches[0]?.index ?? 0;
|
|
819
|
-
const preamble = raw.slice(0, firstSectionStart).trim();
|
|
820
|
-
const sections: string[] = [];
|
|
821
|
-
for (let index = 0; index < matches.length; index += 1) {
|
|
822
|
-
const match = matches[index];
|
|
823
|
-
const hour = match[1];
|
|
824
|
-
if (!hour) continue;
|
|
825
|
-
const sectionTimestamp = parseFiniteDate(`${utcDate}T${hour}:00:00.000Z`);
|
|
826
|
-
if (
|
|
827
|
-
!sectionTimestamp ||
|
|
828
|
-
formatDateInTimeZone(sectionTimestamp, timeZone) !== targetLocalDate
|
|
829
|
-
) {
|
|
830
|
-
continue;
|
|
831
|
-
}
|
|
832
|
-
const sectionStart = match.index ?? 0;
|
|
833
|
-
const sectionEnd = matches[index + 1]?.index ?? raw.length;
|
|
834
|
-
const section = raw.slice(sectionStart, sectionEnd).trim();
|
|
835
|
-
if (section.length > 0) sections.push(section);
|
|
836
|
-
}
|
|
837
|
-
|
|
838
|
-
if (sections.length === 0) return null;
|
|
839
|
-
return [preamble, ...sections]
|
|
840
|
-
.filter((section) => section.length > 0)
|
|
841
|
-
.join("\n\n");
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
type SearchCollectionState = "present" | "missing" | "unknown" | "skipped";
|
|
845
|
-
|
|
846
|
-
function qmdStartupCollectionCheckTimeoutMs(): number {
|
|
847
|
-
const raw =
|
|
848
|
-
process.env.REMNIC_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS ??
|
|
849
|
-
process.env.ENGRAM_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS;
|
|
850
|
-
if (raw === undefined) return DEFAULT_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS;
|
|
851
|
-
const parsed = Number(raw);
|
|
852
|
-
return Number.isFinite(parsed) && parsed >= 1_000
|
|
853
|
-
? Math.floor(parsed)
|
|
854
|
-
: DEFAULT_QMD_STARTUP_COLLECTION_CHECK_TIMEOUT_MS;
|
|
855
|
-
}
|
|
856
544
|
|
|
857
|
-
export async function qmdStartupCollectionCheckWithTimeout(
|
|
858
|
-
promise: Promise<SearchCollectionState>,
|
|
859
|
-
controller: AbortController,
|
|
860
|
-
label: string,
|
|
861
|
-
): Promise<SearchCollectionState> {
|
|
862
|
-
const timeoutMs = qmdStartupCollectionCheckTimeoutMs();
|
|
863
|
-
let timer: NodeJS.Timeout | undefined;
|
|
864
|
-
let settled = false;
|
|
865
|
-
|
|
866
|
-
const timeoutPromise = new Promise<SearchCollectionState>((resolve) => {
|
|
867
|
-
timer = setTimeout(() => {
|
|
868
|
-
if (settled) return;
|
|
869
|
-
controller.abort();
|
|
870
|
-
log.warn(
|
|
871
|
-
`QMD startup collection check for ${label} timed out after ${timeoutMs}ms; keeping search enabled fail-open`,
|
|
872
|
-
);
|
|
873
|
-
resolve("unknown");
|
|
874
|
-
}, timeoutMs);
|
|
875
|
-
timer.unref?.();
|
|
876
|
-
});
|
|
877
|
-
|
|
878
|
-
const checkedPromise = promise
|
|
879
|
-
.catch((err): SearchCollectionState => {
|
|
880
|
-
log.warn(
|
|
881
|
-
`QMD startup collection check for ${label} failed; keeping search enabled fail-open: ${err}`,
|
|
882
|
-
);
|
|
883
|
-
return "unknown";
|
|
884
|
-
})
|
|
885
|
-
.finally(() => {
|
|
886
|
-
settled = true;
|
|
887
|
-
if (timer) clearTimeout(timer);
|
|
888
|
-
});
|
|
889
|
-
|
|
890
|
-
return await Promise.race([checkedPromise, timeoutPromise]);
|
|
891
|
-
}
|
|
892
|
-
|
|
893
|
-
/** Default workspace directory when no per-agent or config workspace is available. */
|
|
894
|
-
export function defaultWorkspaceDir(): string {
|
|
895
|
-
return path.join(os.homedir(), ".openclaw", "workspace");
|
|
896
|
-
}
|
|
897
|
-
|
|
898
|
-
/**
|
|
899
|
-
* Produce a collision-resistant, filesystem-safe identifier from a session key.
|
|
900
|
-
*
|
|
901
|
-
* Session keys follow colon-delimited forms (e.g., `agent:gpucodebot:main`).
|
|
902
|
-
* A naive replace (`:` → `_`) is lossy: different keys like `agent:alpha` and
|
|
903
|
-
* `agent/alpha` would collide. Instead we append a short SHA-256 hash of the
|
|
904
|
-
* original key to the human-readable sanitized prefix, guaranteeing uniqueness
|
|
905
|
-
* while keeping filenames debuggable.
|
|
906
|
-
*
|
|
907
|
-
* Format: `<sanitized>-<12-char-hex-hash>`
|
|
908
|
-
* Example: `agent:gpucodebot:main` → `agent_gpucodebot_main-a1b2c3d4e5f6`
|
|
909
|
-
*/
|
|
910
|
-
export function sanitizeSessionKeyForFilename(sessionKey: string): string {
|
|
911
|
-
const readable = sessionKey.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
912
|
-
const hash = createHash("sha256")
|
|
913
|
-
.update(sessionKey)
|
|
914
|
-
.digest("hex")
|
|
915
|
-
.slice(0, 12);
|
|
916
|
-
return `${readable}-${hash}`;
|
|
917
|
-
}
|
|
918
|
-
|
|
919
|
-
function sourceValidAtMs(turn: BufferTurn): number | null {
|
|
920
|
-
if (typeof turn.sourceValidAt !== "string") return null;
|
|
921
|
-
return parseFlexibleIsoTimestamp(turn.sourceValidAt.trim());
|
|
922
|
-
}
|
|
923
|
-
|
|
924
|
-
const SOURCE_VALID_AT_CONTEXT_TURNS = 2;
|
|
925
|
-
|
|
926
|
-
function sourceValidAtSliceKey(turn: BufferTurn, index: number): string {
|
|
927
|
-
const validAtMs = sourceValidAtMs(turn);
|
|
928
|
-
return validAtMs === null ? `unknown:${index}` : String(validAtMs);
|
|
929
|
-
}
|
|
930
|
-
|
|
931
|
-
function asExtractionContextTurn(turn: BufferTurn): BufferTurn {
|
|
932
|
-
return { ...turn, extractionContextOnly: true };
|
|
933
|
-
}
|
|
934
|
-
|
|
935
|
-
function asExtractionTargetTurn(turn: BufferTurn): BufferTurn {
|
|
936
|
-
const { extractionContextOnly: _contextOnly, ...targetTurn } = turn;
|
|
937
|
-
return targetTurn;
|
|
938
|
-
}
|
|
939
|
-
|
|
940
|
-
function sourceValidAtContextTurns(
|
|
941
|
-
turns: readonly BufferTurn[],
|
|
942
|
-
targetStart: number,
|
|
943
|
-
targetEnd: number,
|
|
944
|
-
targetValidAtMs: number | null,
|
|
945
|
-
): BufferTurn[] {
|
|
946
|
-
if (targetValidAtMs === null) return [];
|
|
947
|
-
return turns
|
|
948
|
-
.flatMap((turn, index) => {
|
|
949
|
-
if (index >= targetStart && index < targetEnd) return [];
|
|
950
|
-
const contextValidAtMs = sourceValidAtMs(turn);
|
|
951
|
-
if (contextValidAtMs === null || contextValidAtMs > targetValidAtMs) {
|
|
952
|
-
return [];
|
|
953
|
-
}
|
|
954
|
-
return [{ turn, index, validAtMs: contextValidAtMs }];
|
|
955
|
-
})
|
|
956
|
-
.sort((a, b) => {
|
|
957
|
-
if (a.validAtMs < b.validAtMs) return -1;
|
|
958
|
-
if (a.validAtMs > b.validAtMs) return 1;
|
|
959
|
-
if (a.index === b.index) return 0;
|
|
960
|
-
return a.index < b.index ? -1 : 1;
|
|
961
|
-
})
|
|
962
|
-
.slice(-SOURCE_VALID_AT_CONTEXT_TURNS)
|
|
963
|
-
.map(({ turn }) => asExtractionContextTurn(turn));
|
|
964
|
-
}
|
|
965
|
-
|
|
966
|
-
export function targetSourceValidAtSortMs(turns: readonly BufferTurn[]): number {
|
|
967
|
-
let latestMs: number | null = null;
|
|
968
|
-
for (const turn of turns) {
|
|
969
|
-
if (turn.extractionContextOnly === true) continue;
|
|
970
|
-
const validAtMs = sourceValidAtMs(turn);
|
|
971
|
-
if (validAtMs === null) continue;
|
|
972
|
-
if (latestMs === null || validAtMs > latestMs) {
|
|
973
|
-
latestMs = validAtMs;
|
|
974
|
-
}
|
|
975
|
-
}
|
|
976
|
-
return latestMs ?? Number.POSITIVE_INFINITY;
|
|
977
|
-
}
|
|
978
|
-
|
|
979
|
-
function sortSourceValidAtSlicesChronologically(
|
|
980
|
-
slices: BufferTurn[][],
|
|
981
|
-
): BufferTurn[][] {
|
|
982
|
-
return slices
|
|
983
|
-
.map((turns, order) => ({
|
|
984
|
-
turns,
|
|
985
|
-
order,
|
|
986
|
-
targetValidAtMs: targetSourceValidAtSortMs(turns),
|
|
987
|
-
}))
|
|
988
|
-
.sort((a, b) => {
|
|
989
|
-
if (a.targetValidAtMs < b.targetValidAtMs) return -1;
|
|
990
|
-
if (a.targetValidAtMs > b.targetValidAtMs) return 1;
|
|
991
|
-
if (a.order === b.order) return 0;
|
|
992
|
-
return a.order < b.order ? -1 : 1;
|
|
993
|
-
})
|
|
994
|
-
.map((slice) => slice.turns);
|
|
995
|
-
}
|
|
996
|
-
|
|
997
|
-
export function splitTurnsBySourceValidAt(
|
|
998
|
-
turns: readonly BufferTurn[],
|
|
999
|
-
options: { includeContext?: boolean } = {},
|
|
1000
|
-
): BufferTurn[][] {
|
|
1001
|
-
if (turns.length === 0) return [];
|
|
1002
|
-
if (!turns.some((turn) => sourceValidAtMs(turn) !== null)) {
|
|
1003
|
-
return [[...turns]];
|
|
1004
|
-
}
|
|
1005
|
-
|
|
1006
|
-
const slices: BufferTurn[][] = [];
|
|
1007
|
-
let start = 0;
|
|
1008
|
-
while (start < turns.length) {
|
|
1009
|
-
const targetValidAtMs = sourceValidAtMs(turns[start]);
|
|
1010
|
-
const activeKey = sourceValidAtSliceKey(turns[start], start);
|
|
1011
|
-
let end = start + 1;
|
|
1012
|
-
while (
|
|
1013
|
-
end < turns.length &&
|
|
1014
|
-
sourceValidAtSliceKey(turns[end], end) === activeKey
|
|
1015
|
-
) {
|
|
1016
|
-
end += 1;
|
|
1017
|
-
}
|
|
1018
|
-
|
|
1019
|
-
const contextTurns =
|
|
1020
|
-
options.includeContext === false
|
|
1021
|
-
? []
|
|
1022
|
-
: sourceValidAtContextTurns(turns, start, end, targetValidAtMs);
|
|
1023
|
-
slices.push([
|
|
1024
|
-
...contextTurns,
|
|
1025
|
-
...turns.slice(start, end).map(asExtractionTargetTurn),
|
|
1026
|
-
]);
|
|
1027
|
-
start = end;
|
|
1028
|
-
}
|
|
1029
|
-
return sortSourceValidAtSlicesChronologically(slices);
|
|
1030
|
-
}
|
|
1031
|
-
|
|
1032
|
-
export function isArtifactMemoryPath(filePath: string): boolean {
|
|
1033
|
-
return /(?:^|[\\/])artifacts(?:[\\/]|$)/i.test(filePath);
|
|
1034
|
-
}
|
|
1035
545
|
// Issue #1526 seam 15: deriveTopicsFromExtraction moved to orchestration/extraction-run.ts.
|
|
1036
546
|
export { deriveTopicsFromExtraction } from "./orchestration/extraction-run.js";
|
|
1037
|
-
export function buildCompressionGuidelinesMarkdown(
|
|
1038
|
-
events: MemoryActionEvent[],
|
|
1039
|
-
generatedAtIso: string = new Date().toISOString(),
|
|
1040
|
-
): string {
|
|
1041
|
-
return buildCompressionGuidelinesMarkdownV2(events, generatedAtIso);
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
export function filterRecallCandidates(
|
|
1045
|
-
candidates: QmdSearchResult[],
|
|
1046
|
-
options: {
|
|
1047
|
-
namespacesEnabled: boolean;
|
|
1048
|
-
recallNamespaces: string[];
|
|
1049
|
-
resolveNamespace: (path: string) => string;
|
|
1050
|
-
limit: number;
|
|
1051
|
-
},
|
|
1052
|
-
): QmdSearchResult[] {
|
|
1053
|
-
const scopedByNamespace = options.namespacesEnabled
|
|
1054
|
-
? candidates.filter((r) =>
|
|
1055
|
-
options.recallNamespaces.includes(options.resolveNamespace(r.path)),
|
|
1056
|
-
)
|
|
1057
|
-
: candidates;
|
|
1058
|
-
return scopedByNamespace
|
|
1059
|
-
.filter((r) => !isArtifactMemoryPath(r.path))
|
|
1060
|
-
.slice(0, Math.max(0, options.limit));
|
|
1061
|
-
}
|
|
1062
|
-
|
|
1063
|
-
export function applyQueryAwareCandidateFilter(
|
|
1064
|
-
candidates: QmdSearchResult[],
|
|
1065
|
-
candidatePaths: Set<string> | null,
|
|
1066
|
-
): QmdSearchResult[] {
|
|
1067
|
-
if (!candidatePaths) return candidates;
|
|
1068
|
-
if (candidatePaths.size === 0) return [];
|
|
1069
|
-
const filtered = candidates.filter((candidate) =>
|
|
1070
|
-
candidatePaths.has(candidate.path),
|
|
1071
|
-
);
|
|
1072
|
-
return filtered.length > 0 ? filtered : candidates;
|
|
1073
|
-
}
|
|
1074
|
-
|
|
1075
|
-
export function tokenizeRecallQuery(prompt: string): string[] {
|
|
1076
|
-
return prompt
|
|
1077
|
-
.toLowerCase()
|
|
1078
|
-
.split(/[^a-z0-9]+/i)
|
|
1079
|
-
.map((t) => t.trim())
|
|
1080
|
-
.filter((t) => t.length >= 3);
|
|
1081
|
-
}
|
|
1082
|
-
|
|
1083
|
-
function hasLifecycleMetadata(frontmatter: MemoryFrontmatter): boolean {
|
|
1084
|
-
return (
|
|
1085
|
-
frontmatter.lifecycleState !== undefined ||
|
|
1086
|
-
frontmatter.verificationState !== undefined ||
|
|
1087
|
-
frontmatter.policyClass !== undefined ||
|
|
1088
|
-
frontmatter.lastValidatedAt !== undefined ||
|
|
1089
|
-
frontmatter.decayScore !== undefined ||
|
|
1090
|
-
frontmatter.heatScore !== undefined
|
|
1091
|
-
);
|
|
1092
|
-
}
|
|
1093
|
-
|
|
1094
|
-
export function shouldFilterLifecycleRecallCandidate(
|
|
1095
|
-
frontmatter: MemoryFrontmatter,
|
|
1096
|
-
options: {
|
|
1097
|
-
lifecyclePolicyEnabled: boolean;
|
|
1098
|
-
lifecycleFilterStaleEnabled: boolean;
|
|
1099
|
-
},
|
|
1100
|
-
): boolean {
|
|
1101
|
-
if (!options.lifecyclePolicyEnabled || !options.lifecycleFilterStaleEnabled)
|
|
1102
|
-
return false;
|
|
1103
|
-
if (!hasLifecycleMetadata(frontmatter)) return false;
|
|
1104
|
-
const lifecycleState = resolveLifecycleState(frontmatter);
|
|
1105
|
-
return lifecycleState === "stale" || lifecycleState === "archived";
|
|
1106
|
-
}
|
|
1107
|
-
|
|
1108
|
-
export function lifecycleRecallScoreAdjustment(
|
|
1109
|
-
frontmatter: MemoryFrontmatter,
|
|
1110
|
-
options: {
|
|
1111
|
-
lifecyclePolicyEnabled: boolean;
|
|
1112
|
-
},
|
|
1113
|
-
): number {
|
|
1114
|
-
if (!options.lifecyclePolicyEnabled) return 0;
|
|
1115
|
-
if (!hasLifecycleMetadata(frontmatter)) return 0;
|
|
1116
|
-
|
|
1117
|
-
let delta = 0;
|
|
1118
|
-
const lifecycleState = resolveLifecycleState(frontmatter);
|
|
1119
|
-
switch (lifecycleState) {
|
|
1120
|
-
case "active":
|
|
1121
|
-
delta += 0.05;
|
|
1122
|
-
break;
|
|
1123
|
-
case "validated":
|
|
1124
|
-
delta += 0.03;
|
|
1125
|
-
break;
|
|
1126
|
-
case "candidate":
|
|
1127
|
-
delta -= 0.01;
|
|
1128
|
-
break;
|
|
1129
|
-
case "stale":
|
|
1130
|
-
delta -= 0.06;
|
|
1131
|
-
break;
|
|
1132
|
-
case "archived":
|
|
1133
|
-
delta -= 0.08;
|
|
1134
|
-
break;
|
|
1135
|
-
}
|
|
1136
|
-
if (frontmatter.verificationState === "disputed") {
|
|
1137
|
-
delta -= 0.12;
|
|
1138
|
-
}
|
|
1139
|
-
return delta;
|
|
1140
|
-
}
|
|
1141
|
-
|
|
1142
|
-
export function computeArtifactRecallLimit(
|
|
1143
|
-
recallMode: RecallPlanMode,
|
|
1144
|
-
recallResultLimit: number,
|
|
1145
|
-
verbatimArtifactsMaxRecall: number,
|
|
1146
|
-
): number {
|
|
1147
|
-
if (recallMode === "no_recall") return 0;
|
|
1148
|
-
if (Math.max(0, recallResultLimit) === 0) return 0;
|
|
1149
|
-
const base = Math.max(0, verbatimArtifactsMaxRecall);
|
|
1150
|
-
if (recallMode === "minimal") {
|
|
1151
|
-
return Math.min(base, Math.max(0, recallResultLimit));
|
|
1152
|
-
}
|
|
1153
|
-
return base;
|
|
1154
|
-
}
|
|
1155
|
-
|
|
1156
|
-
export function resolveEffectiveRecallMode(options: {
|
|
1157
|
-
plannerEnabled: boolean;
|
|
1158
|
-
graphRecallEnabled: boolean;
|
|
1159
|
-
multiGraphMemoryEnabled: boolean;
|
|
1160
|
-
graphExpandedIntentEnabled?: boolean;
|
|
1161
|
-
prompt: string;
|
|
1162
|
-
}): RecallPlanMode {
|
|
1163
|
-
return resolveRecallModeDecision(options).effectiveMode;
|
|
1164
|
-
}
|
|
1165
|
-
|
|
1166
|
-
interface RecallModeGraphOptions {
|
|
1167
|
-
plannerEnabled: boolean;
|
|
1168
|
-
graphRecallEnabled: boolean;
|
|
1169
|
-
multiGraphMemoryEnabled: boolean;
|
|
1170
|
-
graphExpandedIntentEnabled?: boolean;
|
|
1171
|
-
prompt: string;
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
/**
|
|
1175
|
-
* Apply the graph-mode overlay + gating to a planner-produced mode.
|
|
1176
|
-
*
|
|
1177
|
-
* Shared by the heuristic ({@link resolveRecallModeDecision}) and LLM
|
|
1178
|
-
* ({@link resolveRecallModeDecisionAsync}) paths so graph promotion and the
|
|
1179
|
-
* "graph disabled → fall back to full" gating behave identically regardless of
|
|
1180
|
-
* which planner produced `plannedModeRaw` (gotcha #39).
|
|
1181
|
-
*/
|
|
1182
|
-
function finalizeRecallModeDecision(
|
|
1183
|
-
plannedModeRaw: RecallPlanMode,
|
|
1184
|
-
options: RecallModeGraphOptions,
|
|
1185
|
-
): RecallModeDecision {
|
|
1186
|
-
let plannedMode: RecallPlanMode = plannedModeRaw;
|
|
1187
|
-
const graphExpandedIntentDetected =
|
|
1188
|
-
options.plannerEnabled &&
|
|
1189
|
-
options.graphExpandedIntentEnabled === true &&
|
|
1190
|
-
hasBroadGraphIntent(options.prompt);
|
|
1191
|
-
if (plannedMode !== "graph_mode" && graphExpandedIntentDetected) {
|
|
1192
|
-
plannedMode = "graph_mode";
|
|
1193
|
-
}
|
|
1194
|
-
if (
|
|
1195
|
-
plannedMode === "graph_mode" &&
|
|
1196
|
-
(!options.graphRecallEnabled || !options.multiGraphMemoryEnabled)
|
|
1197
|
-
) {
|
|
1198
|
-
return {
|
|
1199
|
-
plannedMode,
|
|
1200
|
-
effectiveMode: "full",
|
|
1201
|
-
graphExpandedIntentDetected,
|
|
1202
|
-
graphReason: !options.graphRecallEnabled
|
|
1203
|
-
? "graph recall disabled by config"
|
|
1204
|
-
: "multi-graph memory disabled by config",
|
|
1205
|
-
};
|
|
1206
|
-
}
|
|
1207
|
-
return {
|
|
1208
|
-
plannedMode,
|
|
1209
|
-
effectiveMode: plannedMode,
|
|
1210
|
-
graphExpandedIntentDetected,
|
|
1211
|
-
};
|
|
1212
|
-
}
|
|
1213
|
-
|
|
1214
|
-
export function resolveRecallModeDecision(options: RecallModeGraphOptions): RecallModeDecision {
|
|
1215
|
-
const plannedMode: RecallPlanMode = options.plannerEnabled
|
|
1216
|
-
? planRecallMode(options.prompt)
|
|
1217
|
-
: "full";
|
|
1218
|
-
return finalizeRecallModeDecision(plannedMode, options);
|
|
1219
|
-
}
|
|
1220
|
-
|
|
1221
|
-
/**
|
|
1222
|
-
* Async recall-mode decision with optional LLM-based planning (issue #1367 /
|
|
1223
|
-
* Option C). Falls back to the heuristic decision when the LLM planner is
|
|
1224
|
-
* disabled, in shadow mode, or unavailable/failed — so this is always safe to
|
|
1225
|
-
* await on the recall hot path. Provider-agnostic: the LLM call routes through
|
|
1226
|
-
* the gateway/fallback chain.
|
|
1227
|
-
*
|
|
1228
|
-
* `recallPlannerEnabled === false` keeps the legacy "always full" behavior and
|
|
1229
|
-
* skips the LLM entirely (the planner as a whole is off).
|
|
1230
|
-
*/
|
|
1231
|
-
export async function resolveRecallModeDecisionAsync(
|
|
1232
|
-
options: RecallModeGraphOptions & {
|
|
1233
|
-
config: PluginConfig;
|
|
1234
|
-
/**
|
|
1235
|
-
* Recall-operation capability gates (issue #1523). REQUIRED: the recall
|
|
1236
|
-
* orchestrator always passes a resolved set — the LLM planner gate reads
|
|
1237
|
-
* `caps.recallPlannerLlm`, never re-derives from config.
|
|
1238
|
-
*/
|
|
1239
|
-
caps: CapabilitySet;
|
|
1240
|
-
hints?: string[];
|
|
1241
|
-
llm?: FallbackLlmClient;
|
|
1242
|
-
signal?: AbortSignal;
|
|
1243
|
-
},
|
|
1244
|
-
): Promise<RecallModeDecision> {
|
|
1245
|
-
const heuristicDecision = resolveRecallModeDecision(options);
|
|
1246
|
-
|
|
1247
|
-
// Planner globally off, or LLM planning not opted into → heuristic only.
|
|
1248
|
-
// Read the resolved capability (issue #1523) — never re-derive from config.
|
|
1249
|
-
const plannerLlmEnabled = options.caps.recallPlannerLlm;
|
|
1250
|
-
if (!options.plannerEnabled || !plannerLlmEnabled) {
|
|
1251
|
-
return heuristicDecision;
|
|
1252
|
-
}
|
|
1253
|
-
|
|
1254
|
-
const { planRecallModeLLM } = await import("./recall-planner-llm.js");
|
|
1255
|
-
const planned = await planRecallModeLLM(
|
|
1256
|
-
options.prompt,
|
|
1257
|
-
options.hints,
|
|
1258
|
-
options.config,
|
|
1259
|
-
options.caps,
|
|
1260
|
-
options.llm,
|
|
1261
|
-
options.signal,
|
|
1262
|
-
);
|
|
1263
|
-
|
|
1264
|
-
// Shadow mode: record what the LLM would have chosen but keep the heuristic
|
|
1265
|
-
// effective decision (safe rollout / comparison — gotcha #30).
|
|
1266
|
-
if (options.config.recallPlannerShadowMode) {
|
|
1267
|
-
return {
|
|
1268
|
-
...heuristicDecision,
|
|
1269
|
-
plannerSource: planned.source,
|
|
1270
|
-
plannerReason: `shadow:${planned.reason}`,
|
|
1271
|
-
plannerLatencyMs: planned.latencyMs,
|
|
1272
|
-
plannerFallbackUsed: planned.fallbackUsed,
|
|
1273
|
-
plannerModelUsed: planned.modelUsed,
|
|
1274
|
-
plannerHeuristicMode: planned.heuristicMode,
|
|
1275
|
-
shadowLlmMode: planned.mode,
|
|
1276
|
-
};
|
|
1277
|
-
}
|
|
1278
|
-
|
|
1279
|
-
const llmDecision = finalizeRecallModeDecision(planned.mode, options);
|
|
1280
|
-
return {
|
|
1281
|
-
...llmDecision,
|
|
1282
|
-
plannerSource: planned.source,
|
|
1283
|
-
plannerReason: planned.reason,
|
|
1284
|
-
plannerLatencyMs: planned.latencyMs,
|
|
1285
|
-
plannerFallbackUsed: planned.fallbackUsed,
|
|
1286
|
-
plannerModelUsed: planned.modelUsed,
|
|
1287
|
-
plannerHeuristicMode: planned.heuristicMode,
|
|
1288
|
-
};
|
|
1289
|
-
}
|
|
1290
|
-
|
|
1291
|
-
export function computeArtifactCandidateFetchLimit(
|
|
1292
|
-
targetCount: number,
|
|
1293
|
-
): number {
|
|
1294
|
-
const cappedTarget = Math.max(0, targetCount);
|
|
1295
|
-
if (cappedTarget === 0) return 0;
|
|
1296
|
-
const headroom = Math.max(8, cappedTarget * 4);
|
|
1297
|
-
return Math.min(200, cappedTarget + headroom);
|
|
1298
|
-
}
|
|
1299
|
-
|
|
1300
|
-
export function computeQmdHybridFetchLimit(
|
|
1301
|
-
recallFetchLimit: number,
|
|
1302
|
-
artifactsEnabled: boolean,
|
|
1303
|
-
maxArtifactRecall: number,
|
|
1304
|
-
): number {
|
|
1305
|
-
const cappedRecallLimit = Math.max(0, recallFetchLimit);
|
|
1306
|
-
if (cappedRecallLimit === 0) return 0;
|
|
1307
|
-
if (!artifactsEnabled) return cappedRecallLimit;
|
|
1308
|
-
// Overscan when artifacts are enabled, then filter artifact paths before
|
|
1309
|
-
// re-applying the recall cap to avoid artifact-dominated top-N starvation.
|
|
1310
|
-
const artifactHeadroom = Math.max(20, Math.max(0, maxArtifactRecall) * 8);
|
|
1311
|
-
return Math.min(400, cappedRecallLimit + artifactHeadroom);
|
|
1312
|
-
}
|
|
1313
|
-
export function summarizeGraphShadowComparison(
|
|
1314
|
-
baseline: QmdSearchResult[],
|
|
1315
|
-
merged: QmdSearchResult[],
|
|
1316
|
-
topN: number,
|
|
1317
|
-
): {
|
|
1318
|
-
baselineCount: number;
|
|
1319
|
-
graphCount: number;
|
|
1320
|
-
overlapCount: number;
|
|
1321
|
-
overlapRatio: number;
|
|
1322
|
-
averageOverlapDelta: number;
|
|
1323
|
-
} {
|
|
1324
|
-
const limit = Math.max(0, Math.floor(topN));
|
|
1325
|
-
const baselineTop = limit > 0 ? baseline.slice(0, limit) : [];
|
|
1326
|
-
const graphTop = limit > 0 ? merged.slice(0, limit) : [];
|
|
1327
|
-
const baselineByPath = new Map(
|
|
1328
|
-
baselineTop.map((item) => [item.path, item.score]),
|
|
1329
|
-
);
|
|
1330
|
-
const graphByPath = new Map(graphTop.map((item) => [item.path, item.score]));
|
|
1331
|
-
|
|
1332
|
-
let overlapCount = 0;
|
|
1333
|
-
let overlapDeltaSum = 0;
|
|
1334
|
-
for (const [p, baselineScore] of baselineByPath.entries()) {
|
|
1335
|
-
const graphScore = graphByPath.get(p);
|
|
1336
|
-
if (typeof graphScore !== "number") continue;
|
|
1337
|
-
overlapCount += 1;
|
|
1338
|
-
overlapDeltaSum += graphScore - baselineScore;
|
|
1339
|
-
}
|
|
1340
|
-
|
|
1341
|
-
const baselineCount = baselineTop.length;
|
|
1342
|
-
return {
|
|
1343
|
-
baselineCount,
|
|
1344
|
-
graphCount: graphTop.length,
|
|
1345
|
-
overlapCount,
|
|
1346
|
-
overlapRatio: baselineCount > 0 ? overlapCount / baselineCount : 0,
|
|
1347
|
-
averageOverlapDelta: overlapCount > 0 ? overlapDeltaSum / overlapCount : 0,
|
|
1348
|
-
};
|
|
1349
|
-
}
|
|
1350
|
-
|
|
1351
|
-
export function parseGraphRecallRankedResults(
|
|
1352
|
-
value: unknown,
|
|
1353
|
-
): GraphRecallRankedResult[] {
|
|
1354
|
-
if (!Array.isArray(value)) return [];
|
|
1355
|
-
const parsed: GraphRecallRankedResult[] = [];
|
|
1356
|
-
for (const entry of value) {
|
|
1357
|
-
if (!entry || typeof entry !== "object") continue;
|
|
1358
|
-
const candidate = entry as Partial<GraphRecallRankedResult>;
|
|
1359
|
-
if (
|
|
1360
|
-
typeof candidate.path !== "string" ||
|
|
1361
|
-
typeof candidate.score !== "number"
|
|
1362
|
-
)
|
|
1363
|
-
continue;
|
|
1364
|
-
parsed.push({
|
|
1365
|
-
path: candidate.path,
|
|
1366
|
-
score: candidate.score,
|
|
1367
|
-
docid: typeof candidate.docid === "string" ? candidate.docid : undefined,
|
|
1368
|
-
sourceLabels: Array.isArray(candidate.sourceLabels)
|
|
1369
|
-
? candidate.sourceLabels.filter(
|
|
1370
|
-
(item): item is string => typeof item === "string",
|
|
1371
|
-
)
|
|
1372
|
-
: [],
|
|
1373
|
-
});
|
|
1374
|
-
}
|
|
1375
|
-
return parsed.slice(0, 64);
|
|
1376
|
-
}
|
|
1377
|
-
|
|
1378
|
-
export function parseMemoryIntentSnapshot(value: unknown): MemoryIntent {
|
|
1379
|
-
const candidate =
|
|
1380
|
-
value && typeof value === "object" ? (value as Partial<MemoryIntent>) : {};
|
|
1381
|
-
return {
|
|
1382
|
-
goal: typeof candidate.goal === "string" ? candidate.goal : "unknown",
|
|
1383
|
-
actionType:
|
|
1384
|
-
typeof candidate.actionType === "string"
|
|
1385
|
-
? candidate.actionType
|
|
1386
|
-
: "unknown",
|
|
1387
|
-
entityTypes: Array.isArray(candidate.entityTypes)
|
|
1388
|
-
? candidate.entityTypes.filter(
|
|
1389
|
-
(item): item is string => typeof item === "string",
|
|
1390
|
-
)
|
|
1391
|
-
: [],
|
|
1392
|
-
taskInitiation: candidate.taskInitiation === true,
|
|
1393
|
-
};
|
|
1394
|
-
}
|
|
1395
|
-
|
|
1396
|
-
function buildQmdIntentHint(intent: MemoryIntent): string | undefined {
|
|
1397
|
-
const parts: string[] = [];
|
|
1398
|
-
if (intent.goal !== "unknown") {
|
|
1399
|
-
parts.push(`goal:${intent.goal.replace(/_/g, " ")}`);
|
|
1400
|
-
}
|
|
1401
|
-
if (intent.actionType !== "unknown") {
|
|
1402
|
-
parts.push(`action:${intent.actionType.replace(/_/g, " ")}`);
|
|
1403
|
-
}
|
|
1404
|
-
if (intent.entityTypes.length > 0) {
|
|
1405
|
-
parts.push(`entities:${intent.entityTypes.join(",")}`);
|
|
1406
|
-
}
|
|
1407
|
-
if (intent.taskInitiation === true) {
|
|
1408
|
-
parts.push("task_initiation");
|
|
1409
|
-
}
|
|
1410
|
-
return parts.length > 0 ? parts.join(" ") : undefined;
|
|
1411
|
-
}
|
|
1412
547
|
|
|
1413
|
-
export function parseQmdRecallResults(value: unknown): QmdSearchResult[] {
|
|
1414
|
-
if (!Array.isArray(value)) return [];
|
|
1415
|
-
const parsed: QmdSearchResult[] = [];
|
|
1416
|
-
for (const entry of value) {
|
|
1417
|
-
if (!entry || typeof entry !== "object") continue;
|
|
1418
|
-
const candidate = entry as Partial<QmdSearchResult>;
|
|
1419
|
-
if (
|
|
1420
|
-
typeof candidate.path !== "string" ||
|
|
1421
|
-
typeof candidate.score !== "number"
|
|
1422
|
-
)
|
|
1423
|
-
continue;
|
|
1424
|
-
parsed.push({
|
|
1425
|
-
docid: typeof candidate.docid === "string" ? candidate.docid : "",
|
|
1426
|
-
path: candidate.path,
|
|
1427
|
-
snippet: typeof candidate.snippet === "string" ? candidate.snippet : "",
|
|
1428
|
-
score: candidate.score,
|
|
1429
|
-
explain: parseQmdExplain(candidate.explain),
|
|
1430
|
-
transport:
|
|
1431
|
-
candidate.transport === "daemon" ||
|
|
1432
|
-
candidate.transport === "subprocess" ||
|
|
1433
|
-
candidate.transport === "hybrid" ||
|
|
1434
|
-
candidate.transport === "scoped_prefilter"
|
|
1435
|
-
? candidate.transport
|
|
1436
|
-
: undefined,
|
|
1437
|
-
});
|
|
1438
|
-
}
|
|
1439
|
-
return parsed.slice(0, 32);
|
|
1440
|
-
}
|
|
1441
|
-
|
|
1442
|
-
export function mergeArtifactRecallCandidates(
|
|
1443
|
-
candidatesByNamespace: MemoryFile[][],
|
|
1444
|
-
limit: number,
|
|
1445
|
-
): MemoryFile[] {
|
|
1446
|
-
const cappedLimit = Math.max(0, limit);
|
|
1447
|
-
if (cappedLimit === 0) return [];
|
|
1448
|
-
|
|
1449
|
-
const out: MemoryFile[] = [];
|
|
1450
|
-
const seen = new Set<string>();
|
|
1451
|
-
let offset = 0;
|
|
1452
|
-
while (out.length < cappedLimit) {
|
|
1453
|
-
let hasAnyCandidateAtOffset = false;
|
|
1454
|
-
for (const list of candidatesByNamespace) {
|
|
1455
|
-
if (offset >= list.length) continue;
|
|
1456
|
-
hasAnyCandidateAtOffset = true;
|
|
1457
|
-
const item = list[offset];
|
|
1458
|
-
const dedupeKey = `${item.frontmatter.id}:${item.frontmatter.sourceMemoryId ?? ""}:${item.content}`;
|
|
1459
|
-
if (seen.has(dedupeKey)) continue;
|
|
1460
|
-
seen.add(dedupeKey);
|
|
1461
|
-
out.push(item);
|
|
1462
|
-
if (out.length >= cappedLimit) break;
|
|
1463
|
-
}
|
|
1464
|
-
if (!hasAnyCandidateAtOffset) break;
|
|
1465
|
-
offset += 1;
|
|
1466
|
-
}
|
|
1467
|
-
return out;
|
|
1468
|
-
}
|
|
1469
|
-
|
|
1470
|
-
export function resolveRecentThreadMemoryPaths(options: {
|
|
1471
|
-
threadEpisodeIds: string[];
|
|
1472
|
-
currentMemoryId: string;
|
|
1473
|
-
allMemsForGraph: MemoryFile[] | null | undefined;
|
|
1474
|
-
pathById?: Map<string, string>;
|
|
1475
|
-
storageDir: string;
|
|
1476
|
-
maxRecent: number;
|
|
1477
|
-
}): string[] {
|
|
1478
|
-
const maxRecent = Math.max(0, options.maxRecent);
|
|
1479
|
-
if (options.threadEpisodeIds.length === 0 || maxRecent === 0) return [];
|
|
1480
|
-
const pathById =
|
|
1481
|
-
options.pathById ??
|
|
1482
|
-
buildMemoryPathById(options.allMemsForGraph, options.storageDir);
|
|
1483
|
-
if (pathById.size === 0) return [];
|
|
1484
|
-
|
|
1485
|
-
// #1635 (defensive): skip pending_review ids from legacy episode sets.
|
|
1486
|
-
const pendingReviewIds = new Set<string>(
|
|
1487
|
-
(options.allMemsForGraph ?? [])
|
|
1488
|
-
.filter((m) => m.frontmatter.status === "pending_review" && m.frontmatter.id)
|
|
1489
|
-
.map((m) => m.frontmatter.id as string),
|
|
1490
|
-
);
|
|
1491
|
-
|
|
1492
|
-
return options.threadEpisodeIds
|
|
1493
|
-
.filter((id) => id !== options.currentMemoryId)
|
|
1494
|
-
.filter((id) => !pendingReviewIds.has(id))
|
|
1495
|
-
.slice(-maxRecent)
|
|
1496
|
-
.map((id) => pathById.get(id))
|
|
1497
|
-
.filter((p): p is string => typeof p === "string" && p.length > 0);
|
|
1498
|
-
}
|
|
1499
|
-
|
|
1500
|
-
export function buildMemoryPathById(
|
|
1501
|
-
allMemsForGraph: MemoryFile[] | null | undefined,
|
|
1502
|
-
storageDir: string,
|
|
1503
|
-
): Map<string, string> {
|
|
1504
|
-
const pathById = new Map<string, string>();
|
|
1505
|
-
for (const mem of allMemsForGraph ?? []) {
|
|
1506
|
-
const id = mem.frontmatter.id;
|
|
1507
|
-
if (!id) continue;
|
|
1508
|
-
pathById.set(id, path.relative(storageDir, mem.path));
|
|
1509
|
-
}
|
|
1510
|
-
return pathById;
|
|
1511
|
-
}
|
|
1512
|
-
|
|
1513
|
-
export function appendMemoryToGraphContext(options: {
|
|
1514
|
-
allMemsForGraph: MemoryFile[] | null | undefined;
|
|
1515
|
-
storageDir: string;
|
|
1516
|
-
memoryRelPath: string;
|
|
1517
|
-
memoryId: string;
|
|
1518
|
-
category: MemoryFile["frontmatter"]["category"];
|
|
1519
|
-
content: string;
|
|
1520
|
-
entityRef: string | undefined;
|
|
1521
|
-
}): void {
|
|
1522
|
-
if (!Array.isArray(options.allMemsForGraph)) return;
|
|
1523
|
-
|
|
1524
|
-
const nowIso = new Date().toISOString();
|
|
1525
|
-
options.allMemsForGraph.push({
|
|
1526
|
-
path: path.join(options.storageDir, options.memoryRelPath),
|
|
1527
|
-
content: options.content,
|
|
1528
|
-
frontmatter: {
|
|
1529
|
-
id: options.memoryId,
|
|
1530
|
-
category: options.category,
|
|
1531
|
-
created: nowIso,
|
|
1532
|
-
updated: nowIso,
|
|
1533
|
-
source: "extraction",
|
|
1534
|
-
confidence: 0.8,
|
|
1535
|
-
confidenceTier: "implied",
|
|
1536
|
-
tags: [],
|
|
1537
|
-
entityRef: options.entityRef,
|
|
1538
|
-
status: "active",
|
|
1539
|
-
},
|
|
1540
|
-
});
|
|
1541
|
-
}
|
|
1542
|
-
|
|
1543
|
-
export function resolvePersistedMemoryRelativePath(options: {
|
|
1544
|
-
memoryId: string;
|
|
1545
|
-
pathById: Map<string, string>;
|
|
1546
|
-
category: string;
|
|
1547
|
-
}): string {
|
|
1548
|
-
const persisted = options.pathById.get(options.memoryId);
|
|
1549
|
-
if (persisted) return persisted;
|
|
1550
|
-
if (options.category === "correction") {
|
|
1551
|
-
return path.join("corrections", `${options.memoryId}.md`);
|
|
1552
|
-
}
|
|
1553
|
-
// Pick the subtree that matches the StorageManager.writeMemory routing
|
|
1554
|
-
// so fallback paths (used before memoryPathById has seen the fresh
|
|
1555
|
-
// write) agree with where the file actually lives. Routing goes through
|
|
1556
|
-
// the shared categoryDirName() chokepoint (utils/category-dir.ts) so
|
|
1557
|
-
// every category — decisions/, preferences/, reasoning-traces/, ... —
|
|
1558
|
-
// resolves to the same dir the writer used; otherwise graph edges point
|
|
1559
|
-
// at the wrong subtree and graph expansion silently drops those nodes
|
|
1560
|
-
// when readMemoryByPath cannot resolve them (issue #564 PR 3 / #1546).
|
|
1561
|
-
const subtree = categoryDirName(options.category);
|
|
1562
|
-
const idParts = options.memoryId.split("-");
|
|
1563
|
-
const maybeTimestamp = Number(idParts[1]);
|
|
1564
|
-
if (Number.isFinite(maybeTimestamp) && maybeTimestamp > 0) {
|
|
1565
|
-
const day = new Date(maybeTimestamp).toISOString().slice(0, 10);
|
|
1566
|
-
return path.join(subtree, day, `${options.memoryId}.md`);
|
|
1567
|
-
}
|
|
1568
|
-
return path.join(subtree, `${options.memoryId}.md`);
|
|
1569
|
-
}
|
|
1570
548
|
|
|
1571
549
|
export class Orchestrator {
|
|
1572
550
|
|
|
@@ -1705,7 +683,8 @@ export class Orchestrator {
|
|
|
1705
683
|
statuses: Map<string, "active" | "superseded" | "archived" | "missing">;
|
|
1706
684
|
}
|
|
1707
685
|
>();
|
|
1708
|
-
|
|
686
|
+
/** Read by NamespaceReadFanoutCoordinator (seam 26), hence not `private`. */
|
|
687
|
+
static readonly ARTIFACT_STATUS_CACHE_TTL_MS = 60_000;
|
|
1709
688
|
|
|
1710
689
|
// Access tracking buffer (Phase 1A)
|
|
1711
690
|
// Maps memoryId -> {count, lastAccessed} for batched updates
|
|
@@ -2272,98 +1251,17 @@ export class Orchestrator {
|
|
|
2272
1251
|
resolvedStorageDir: string,
|
|
2273
1252
|
configured: Set<string>,
|
|
2274
1253
|
): { namespace: string; identityToken: string; storageDir: string } {
|
|
2275
|
-
|
|
1254
|
+
return this.namespaceReadFanoutCoordinator.preferNamespaceStorageDirHintOwner(
|
|
2276
1255
|
current,
|
|
2277
|
-
resolvedStorageDir,
|
|
2278
|
-
configured,
|
|
2279
|
-
);
|
|
2280
|
-
const candidateRank = this.namespaceStorageDirHintOwnershipRank(
|
|
2281
1256
|
candidate,
|
|
2282
1257
|
resolvedStorageDir,
|
|
2283
1258
|
configured,
|
|
2284
1259
|
);
|
|
2285
|
-
if (candidateRank < currentRank) return candidate;
|
|
2286
|
-
if (candidateRank > currentRank) return current;
|
|
2287
|
-
|
|
2288
|
-
const byName = candidate.namespace.localeCompare(current.namespace);
|
|
2289
|
-
if (byName < 0) return candidate;
|
|
2290
|
-
if (byName > 0) return current;
|
|
2291
|
-
return candidate.identityToken.localeCompare(current.identityToken) < 0
|
|
2292
|
-
? candidate
|
|
2293
|
-
: current;
|
|
2294
1260
|
}
|
|
2295
1261
|
|
|
2296
1262
|
private loadNamespaceStorageDirHintsFromCatalog(): void {
|
|
2297
|
-
|
|
2298
|
-
this.namespaceStorageDirHintsLoaded = true;
|
|
2299
|
-
const catalogPath = path.join(this.config.memoryDir, "state", "namespaces.jsonl");
|
|
2300
|
-
if (!existsSync(catalogPath)) return;
|
|
2301
|
-
|
|
2302
|
-
let body: string;
|
|
2303
|
-
try {
|
|
2304
|
-
body = readFileSync(catalogPath, "utf8");
|
|
2305
|
-
} catch {
|
|
2306
|
-
return;
|
|
2307
|
-
}
|
|
2308
|
-
|
|
2309
|
-
const compactedByNamespace = new Map<
|
|
2310
|
-
string,
|
|
2311
|
-
{ namespace: string; identityToken: string; storageDir: string }
|
|
2312
|
-
>();
|
|
2313
|
-
for (const line of body.split(/\r?\n/)) {
|
|
2314
|
-
const trimmed = line.trim();
|
|
2315
|
-
if (!trimmed) continue;
|
|
2316
|
-
try {
|
|
2317
|
-
const parsed = JSON.parse(trimmed) as unknown;
|
|
2318
|
-
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) continue;
|
|
2319
|
-
const record = parsed as Record<string, unknown>;
|
|
2320
|
-
if (
|
|
2321
|
-
typeof record.namespace !== "string" ||
|
|
2322
|
-
typeof record.storageDir !== "string" ||
|
|
2323
|
-
typeof record.identityToken !== "string"
|
|
2324
|
-
) {
|
|
2325
|
-
continue;
|
|
2326
|
-
}
|
|
2327
|
-
const namespace = normalizeNamespaceIdentity(record.namespace);
|
|
2328
|
-
if (!namespace || record.identityToken !== namespaceIdentityToken(namespace)) continue;
|
|
2329
|
-
compactedByNamespace.set(namespace, {
|
|
2330
|
-
namespace,
|
|
2331
|
-
identityToken: record.identityToken,
|
|
2332
|
-
storageDir: record.storageDir,
|
|
2333
|
-
});
|
|
2334
|
-
} catch {
|
|
2335
|
-
// Catalog hints are best-effort. The catalog reader still owns full recovery.
|
|
2336
|
-
}
|
|
2337
|
-
}
|
|
2338
|
-
|
|
2339
|
-
const configured = new Set(
|
|
2340
|
-
this.configuredNamespaceList().map((namespace) => normalizeNamespaceIdentity(namespace)),
|
|
1263
|
+
return this.namespaceReadFanoutCoordinator.loadNamespaceStorageDirHintsFromCatalog(
|
|
2341
1264
|
);
|
|
2342
|
-
const preferredByStorageDir = new Map<
|
|
2343
|
-
string,
|
|
2344
|
-
{ namespace: string; identityToken: string; storageDir: string }
|
|
2345
|
-
>();
|
|
2346
|
-
for (const record of compactedByNamespace.values()) {
|
|
2347
|
-
if (!this.storageDirMatchesNamespaceHint(record.namespace, record.storageDir)) {
|
|
2348
|
-
continue;
|
|
2349
|
-
}
|
|
2350
|
-
const resolvedStorageDir = path.resolve(record.storageDir);
|
|
2351
|
-
const current = preferredByStorageDir.get(resolvedStorageDir);
|
|
2352
|
-
preferredByStorageDir.set(
|
|
2353
|
-
resolvedStorageDir,
|
|
2354
|
-
current
|
|
2355
|
-
? this.preferNamespaceStorageDirHintOwner(
|
|
2356
|
-
current,
|
|
2357
|
-
record,
|
|
2358
|
-
resolvedStorageDir,
|
|
2359
|
-
configured,
|
|
2360
|
-
)
|
|
2361
|
-
: record,
|
|
2362
|
-
);
|
|
2363
|
-
}
|
|
2364
|
-
for (const record of preferredByStorageDir.values()) {
|
|
2365
|
-
this.rememberNamespaceStorageDirHint(record.namespace, record.storageDir);
|
|
2366
|
-
}
|
|
2367
1265
|
}
|
|
2368
1266
|
|
|
2369
1267
|
private async maintenanceNamespaces(
|
|
@@ -2439,68 +1337,9 @@ export class Orchestrator {
|
|
|
2439
1337
|
searchOptions?: SearchQueryOptions;
|
|
2440
1338
|
execution?: SearchExecutionOptions;
|
|
2441
1339
|
}): Promise<QmdSearchResult[]> {
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
options.namespaces.length === 0
|
|
2446
|
-
) {
|
|
2447
|
-
return [];
|
|
2448
|
-
}
|
|
2449
|
-
const namespaces = resolveNamespaceCapabilities(this.config).namespaces
|
|
2450
|
-
? Array.from(
|
|
2451
|
-
new Set(
|
|
2452
|
-
(options.namespaces?.length
|
|
2453
|
-
? options.namespaces
|
|
2454
|
-
: this.configuredNamespaceList()
|
|
2455
|
-
)
|
|
2456
|
-
.map((value) => value.trim())
|
|
2457
|
-
.filter(Boolean),
|
|
2458
|
-
),
|
|
2459
|
-
)
|
|
2460
|
-
: [this.config.defaultNamespace];
|
|
2461
|
-
|
|
2462
|
-
if (!resolveNamespaceCapabilities(this.config).namespaces) {
|
|
2463
|
-
switch (options.mode) {
|
|
2464
|
-
case "hybrid":
|
|
2465
|
-
return await this.qmd.hybridSearch(
|
|
2466
|
-
options.query,
|
|
2467
|
-
undefined,
|
|
2468
|
-
options.maxResults,
|
|
2469
|
-
options.execution,
|
|
2470
|
-
);
|
|
2471
|
-
case "bm25":
|
|
2472
|
-
return await this.qmd.bm25Search(
|
|
2473
|
-
options.query,
|
|
2474
|
-
undefined,
|
|
2475
|
-
options.maxResults,
|
|
2476
|
-
options.execution,
|
|
2477
|
-
);
|
|
2478
|
-
case "vector":
|
|
2479
|
-
return await this.qmd.vectorSearch(
|
|
2480
|
-
options.query,
|
|
2481
|
-
undefined,
|
|
2482
|
-
options.maxResults,
|
|
2483
|
-
options.execution,
|
|
2484
|
-
);
|
|
2485
|
-
default:
|
|
2486
|
-
return await this.qmd.search(
|
|
2487
|
-
options.query,
|
|
2488
|
-
undefined,
|
|
2489
|
-
options.maxResults,
|
|
2490
|
-
options.searchOptions,
|
|
2491
|
-
options.execution,
|
|
2492
|
-
);
|
|
2493
|
-
}
|
|
2494
|
-
}
|
|
2495
|
-
|
|
2496
|
-
return await this.namespaceSearchRouter.searchAcrossNamespaces({
|
|
2497
|
-
query: options.query,
|
|
2498
|
-
namespaces,
|
|
2499
|
-
maxResults: options.maxResults,
|
|
2500
|
-
mode: options.mode,
|
|
2501
|
-
searchOptions: options.searchOptions,
|
|
2502
|
-
execution: options.execution,
|
|
2503
|
-
});
|
|
1340
|
+
return this.namespaceReadFanoutCoordinator.searchAcrossNamespaces(
|
|
1341
|
+
options,
|
|
1342
|
+
);
|
|
2504
1343
|
}
|
|
2505
1344
|
|
|
2506
1345
|
async searchHealthForNamespace(
|
|
@@ -2549,17 +1388,20 @@ export class Orchestrator {
|
|
|
2549
1388
|
targetStorage: StorageManager,
|
|
2550
1389
|
content: string,
|
|
2551
1390
|
): Promise<boolean> {
|
|
2552
|
-
|
|
2553
|
-
|
|
1391
|
+
return this.persistenceIndexCoordinator.hasContentHashDedup(
|
|
1392
|
+
targetStorage,
|
|
1393
|
+
content,
|
|
1394
|
+
);
|
|
2554
1395
|
}
|
|
2555
1396
|
|
|
2556
1397
|
private async addContentHashDedup(
|
|
2557
1398
|
targetStorage: StorageManager,
|
|
2558
1399
|
content: string,
|
|
2559
1400
|
): Promise<void> {
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
1401
|
+
return this.persistenceIndexCoordinator.addContentHashDedup(
|
|
1402
|
+
targetStorage,
|
|
1403
|
+
content,
|
|
1404
|
+
);
|
|
2563
1405
|
}
|
|
2564
1406
|
|
|
2565
1407
|
private async removeContentHashForMemory(
|
|
@@ -2567,50 +1409,13 @@ export class Orchestrator {
|
|
|
2567
1409
|
memory: MemoryFile,
|
|
2568
1410
|
context: string,
|
|
2569
1411
|
): Promise<void> {
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
index.removeByHash(memory.frontmatter.contentHash);
|
|
2575
|
-
return;
|
|
2576
|
-
}
|
|
2577
|
-
|
|
2578
|
-
log.warn(
|
|
2579
|
-
`[${context}] removing hash for legacy memory ${memory.frontmatter.id ?? "(unknown)"} via content fallback - no contentHash in frontmatter`,
|
|
1412
|
+
return this.persistenceIndexCoordinator.removeContentHashForMemory(
|
|
1413
|
+
targetStorage,
|
|
1414
|
+
memory,
|
|
1415
|
+
context,
|
|
2580
1416
|
);
|
|
2581
|
-
index.remove(memory.content);
|
|
2582
1417
|
}
|
|
2583
1418
|
|
|
2584
|
-
/**
|
|
2585
|
-
* Issue #1671 — backfill bi-temporal bounds onto an existing promoted/deduped
|
|
2586
|
-
* copy that was written BEFORE the source fact carried a resolved
|
|
2587
|
-
* `invalid_at`/`observedAt`/`eventTimeSource`.
|
|
2588
|
-
*
|
|
2589
|
-
* On re-extraction/backfill, a fact may now carry a resolved end bound (e.g.
|
|
2590
|
-
* "until June 2025") that the existing copy lacks because it was promoted
|
|
2591
|
-
* before bi-temporal wiring existed. Without this backfill, recall keeps
|
|
2592
|
-
* surfacing an expired fact even though the source copy now expires correctly.
|
|
2593
|
-
*
|
|
2594
|
-
* Finds the active fact in `targetStorage` matching `dedupContent`, then
|
|
2595
|
-
* patches the temporal frontmatter the existing copy is missing. Best-effort
|
|
2596
|
-
* / fail-open — any I/O error is logged and swallowed so the dedup
|
|
2597
|
-
* short-circuit is never blocked by a backfill failure.
|
|
2598
|
-
*
|
|
2599
|
-
* Matching: the stored `frontmatter.contentHash` is compared against
|
|
2600
|
-
* `ContentHashIndex.computeHash(dedupContent)` first (the exact hash the
|
|
2601
|
-
* content-hash index uses), then falls back to stripping citations and
|
|
2602
|
-
* comparing normalized bodies. This handles inline-attribution deployments
|
|
2603
|
-
* where the persisted body carries a citation marker the dedup key does not.
|
|
2604
|
-
*
|
|
2605
|
-
* I/O gate: only triggers when `invalidAt` is present (the end bound that
|
|
2606
|
-
* actually changes recall behavior by expiring the fact). `observedAt` and
|
|
2607
|
-
* `eventTimeSource` alone don'\''t expire a fact, so backfilling them without
|
|
2608
|
-
* an end bound would cause a full readAllMemories scan on every dedup hit
|
|
2609
|
-
* under biTemporal for no recall benefit.
|
|
2610
|
-
*
|
|
2611
|
-
* Only patches fields the existing copy LACKS — never overwrites a bound the
|
|
2612
|
-
* copy already carries.
|
|
2613
|
-
*/
|
|
2614
1419
|
private async backfillTemporalBoundsOnDedupHit(
|
|
2615
1420
|
targetStorage: StorageManager,
|
|
2616
1421
|
dedupContent: string,
|
|
@@ -2626,130 +1431,17 @@ export class Orchestrator {
|
|
|
2626
1431
|
},
|
|
2627
1432
|
entityRef?: string,
|
|
2628
1433
|
): Promise<void> {
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
// run a full readAllMemories on every bi-temporal dedup hit for no benefit
|
|
2636
|
-
// (review cursor PRRT_OvHk / codex PRRT_OvHxVH). observedAt and
|
|
2637
|
-
// eventTimeSource alone never change recall.
|
|
2638
|
-
const hasExtractedStart =
|
|
2639
|
-
bounds.validFrom !== undefined && bounds.eventTimeSource === "extracted";
|
|
2640
|
-
if (!bounds.invalidAt && !hasExtractedStart) return;
|
|
2641
|
-
try {
|
|
2642
|
-
const incomingHash = ContentHashIndex.computeHash(dedupContent);
|
|
2643
|
-
const normalizedIncoming = ContentHashIndex.normalizeContent(dedupContent);
|
|
2644
|
-
// Normalize the entity for same-entity scoping when provided — two
|
|
2645
|
-
// entities can share identical fact text, and patching a different
|
|
2646
|
-
// entity'\''s fact would corrupt its temporal bounds (cursor review).
|
|
2647
|
-
const incomingEntityNorm = entityRef
|
|
2648
|
-
? normalizeSupersessionKey(entityRef)
|
|
2649
|
-
: undefined;
|
|
2650
|
-
const all = await targetStorage.readAllMemories();
|
|
2651
|
-
const existing = all.find((m) => {
|
|
2652
|
-
if (m.frontmatter.category !== "fact") return false;
|
|
2653
|
-
if ((m.frontmatter.status ?? "active") !== "active") return false;
|
|
2654
|
-
// Same-entity guard: reject only when the stored fact carries a
|
|
2655
|
-
// DIFFERENT entity (two entities can share identical fact text, so
|
|
2656
|
-
// patching the other entity's copy would corrupt its bounds — codex
|
|
2657
|
-
// P2 PRRT_OvB4A). Legacy facts written before entity linkage (no
|
|
2658
|
-
// entityRef) have no entity to conflict with, so they stay eligible
|
|
2659
|
-
// for backfill (cursor PRRT_OvKnV: the guard must NOT silently
|
|
2660
|
-
// no-op for older promoted copies that predate entity linkage).
|
|
2661
|
-
if (
|
|
2662
|
-
incomingEntityNorm &&
|
|
2663
|
-
m.frontmatter.entityRef &&
|
|
2664
|
-
normalizeSupersessionKey(m.frontmatter.entityRef) !== incomingEntityNorm
|
|
2665
|
-
) {
|
|
2666
|
-
return false;
|
|
2667
|
-
}
|
|
2668
|
-
// Prefer the stored contentHash (what the hash index actually keys
|
|
2669
|
-
// on) — it is computed from contentHashSource (the raw/enriched
|
|
2670
|
-
// body before citation), matching the dedupContent the caller passes.
|
|
2671
|
-
if (m.frontmatter.contentHash) {
|
|
2672
|
-
return m.frontmatter.contentHash === incomingHash;
|
|
2673
|
-
}
|
|
2674
|
-
// Legacy facts without a stored hash: strip citations then compare
|
|
2675
|
-
// normalized bodies so inline-attribution markers don'\''t prevent
|
|
2676
|
-
// a match.
|
|
2677
|
-
return (
|
|
2678
|
-
ContentHashIndex.normalizeContent(
|
|
2679
|
-
stripCitationForTemplate(m.content ?? "", this.config.inlineSourceAttributionFormat),
|
|
2680
|
-
) === normalizedIncoming
|
|
2681
|
-
);
|
|
2682
|
-
});
|
|
2683
|
-
if (!existing) return;
|
|
2684
|
-
// Build a patch containing ONLY the fields the existing copy lacks.
|
|
2685
|
-
const patch: Partial<MemoryFrontmatter> = {};
|
|
2686
|
-
const fm = existing.frontmatter;
|
|
2687
|
-
if (bounds.invalidAt && (!fm.invalid_at || fm.invalid_at.length === 0)) {
|
|
2688
|
-
patch.invalid_at = bounds.invalidAt;
|
|
2689
|
-
}
|
|
2690
|
-
if (bounds.observedAt && (!fm.observedAt || fm.observedAt.length === 0)) {
|
|
2691
|
-
patch.observedAt = bounds.observedAt;
|
|
2692
|
-
}
|
|
2693
|
-
if (
|
|
2694
|
-
bounds.eventTimeSource &&
|
|
2695
|
-
(!fm.eventTimeSource || fm.eventTimeSource.length === 0)
|
|
2696
|
-
) {
|
|
2697
|
-
patch.eventTimeSource = bounds.eventTimeSource;
|
|
2698
|
-
}
|
|
2699
|
-
// #1707 thread 2 — per-fact-anchored start bound. A re-extracted
|
|
2700
|
-
// duplicate whose event time resolves a real start bound must carry
|
|
2701
|
-
// that anchor onto the existing copy so as-of recall uses the corrected
|
|
2702
|
-
// valid_at instead of a stale batch-anchored value. Only an EXTRACTED
|
|
2703
|
-
// bound corrects; an "assumed" bound is just the ingestion anchor.
|
|
2704
|
-
//
|
|
2705
|
-
// No-clobber via equality, not provenance inference: exact-content dedup
|
|
2706
|
-
// re-extracts the SAME event-time expression, which #1670 per-fact
|
|
2707
|
-
// anchoring resolves deterministically to the same validFrom — so for
|
|
2708
|
-
// stable content the incoming validFrom EQUALS the copy's valid_at and
|
|
2709
|
-
// we skip the redundant write (the only no-clobber that holds without a
|
|
2710
|
-
// fragile provenance heuristic — review codex PRRT_Ov7LKC). When they
|
|
2711
|
-
// differ (a prior batch/assumed anchor, end-only assumed start, or a
|
|
2712
|
-
// non-deterministic re-resolution), the extracted validFrom is the
|
|
2713
|
-
// authoritative correction and overwrites. The eventTimeSource upgrade
|
|
2714
|
-
// below records that the start is now extracted-anchored.
|
|
2715
|
-
if (
|
|
2716
|
-
bounds.validFrom &&
|
|
2717
|
-
bounds.eventTimeSource === "extracted" &&
|
|
2718
|
-
fm.valid_at !== bounds.validFrom
|
|
2719
|
-
) {
|
|
2720
|
-
patch.valid_at = bounds.validFrom;
|
|
2721
|
-
// Mark the copy extracted-anchored in the SAME patch so its provenance
|
|
2722
|
-
// reflects the correction (review cursor PRRT_OvHM / codex PRRT_OvHxVD):
|
|
2723
|
-
// without this, a copy upgraded from "assumed" would keep "assumed"
|
|
2724
|
-
// provenance while carrying an extracted start. (The earlier
|
|
2725
|
-
// eventTimeSource block only fills an EMPTY source.)
|
|
2726
|
-
if (fm.eventTimeSource !== "extracted") {
|
|
2727
|
-
patch.eventTimeSource = "extracted";
|
|
2728
|
-
}
|
|
2729
|
-
}
|
|
2730
|
-
if (Object.keys(patch).length === 0) return;
|
|
2731
|
-
const ok = await targetStorage.writeMemoryFrontmatter(existing, patch);
|
|
2732
|
-
if (ok) {
|
|
2733
|
-
log.debug(
|
|
2734
|
-
`bitemporal-backfill: patched ${Object.keys(patch).join(",")} onto existing fact ${fm.id ?? "(unknown)"} in ${targetStorage.dir}`,
|
|
2735
|
-
);
|
|
2736
|
-
}
|
|
2737
|
-
} catch (err) {
|
|
2738
|
-
log.warn(
|
|
2739
|
-
`bitemporal-backfill: failed open for ${targetStorage.dir}: ${err}`,
|
|
2740
|
-
);
|
|
2741
|
-
}
|
|
1434
|
+
return this.persistenceIndexCoordinator.backfillTemporalBoundsOnDedupHit(
|
|
1435
|
+
targetStorage,
|
|
1436
|
+
dedupContent,
|
|
1437
|
+
bounds,
|
|
1438
|
+
entityRef,
|
|
1439
|
+
);
|
|
2742
1440
|
}
|
|
2743
1441
|
|
|
2744
1442
|
private async saveContentHashIndexes(): Promise<void> {
|
|
2745
|
-
|
|
2746
|
-
|
|
2747
|
-
for (const index of this.contentHashIndexesByStorageDir.values()) {
|
|
2748
|
-
indexes.add(index);
|
|
2749
|
-
}
|
|
2750
|
-
for (const index of indexes) {
|
|
2751
|
-
await index.save();
|
|
2752
|
-
}
|
|
1443
|
+
return this.persistenceIndexCoordinator.saveContentHashIndexes(
|
|
1444
|
+
);
|
|
2753
1445
|
}
|
|
2754
1446
|
|
|
2755
1447
|
constructor(config: PluginConfig) {
|
|
@@ -3194,92 +1886,10 @@ export class Orchestrator {
|
|
|
3194
1886
|
storage: StorageManager,
|
|
3195
1887
|
sourceIds: string[],
|
|
3196
1888
|
): Promise<Map<string, "active" | "superseded" | "archived" | "missing">> {
|
|
3197
|
-
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
snapshot !== undefined &&
|
|
3202
|
-
Date.now() - snapshot.loadedAtMs <=
|
|
3203
|
-
Orchestrator.ARTIFACT_STATUS_CACHE_TTL_MS &&
|
|
3204
|
-
snapshot.statusVersion === currentStatusVersion;
|
|
3205
|
-
|
|
3206
|
-
const rebuildSnapshot = async () => {
|
|
3207
|
-
const MAX_STABLE_READ_ATTEMPTS = 3;
|
|
3208
|
-
let latestStatuses = new Map<
|
|
3209
|
-
string,
|
|
3210
|
-
"active" | "superseded" | "archived" | "missing"
|
|
3211
|
-
>();
|
|
3212
|
-
let latestVersionAfter = storage.getMemoryStatusVersion();
|
|
3213
|
-
|
|
3214
|
-
for (let attempt = 0; attempt < MAX_STABLE_READ_ATTEMPTS; attempt += 1) {
|
|
3215
|
-
const versionBefore = storage.getMemoryStatusVersion();
|
|
3216
|
-
const allMemories = await storage.readAllMemories();
|
|
3217
|
-
const versionAfter = storage.getMemoryStatusVersion();
|
|
3218
|
-
latestVersionAfter = versionAfter;
|
|
3219
|
-
latestStatuses = new Map(
|
|
3220
|
-
allMemories.map((m) => [
|
|
3221
|
-
m.frontmatter.id,
|
|
3222
|
-
(m.frontmatter.status ?? "active") as
|
|
3223
|
-
| "active"
|
|
3224
|
-
| "superseded"
|
|
3225
|
-
| "archived"
|
|
3226
|
-
| "missing",
|
|
3227
|
-
]),
|
|
3228
|
-
);
|
|
3229
|
-
|
|
3230
|
-
if (versionAfter === versionBefore) {
|
|
3231
|
-
const rebuilt = {
|
|
3232
|
-
loadedAtMs: Date.now(),
|
|
3233
|
-
statusVersion: versionAfter,
|
|
3234
|
-
statuses: latestStatuses,
|
|
3235
|
-
};
|
|
3236
|
-
this.artifactSourceStatusCache.set(storage, rebuilt);
|
|
3237
|
-
return rebuilt;
|
|
3238
|
-
}
|
|
3239
|
-
}
|
|
3240
|
-
|
|
3241
|
-
// Sustained write churn: return latest read without caching a potentially torn snapshot.
|
|
3242
|
-
return {
|
|
3243
|
-
loadedAtMs: Date.now(),
|
|
3244
|
-
statusVersion: latestVersionAfter,
|
|
3245
|
-
statuses: latestStatuses,
|
|
3246
|
-
};
|
|
3247
|
-
};
|
|
3248
|
-
|
|
3249
|
-
if (!isFresh) {
|
|
3250
|
-
snapshot = await rebuildSnapshot();
|
|
3251
|
-
} else {
|
|
3252
|
-
// Warm cache may miss brand-new sourceMemoryId values created after snapshot build.
|
|
3253
|
-
// Refresh once on-demand when unseen IDs are requested.
|
|
3254
|
-
const hasUnknownSourceIds = sourceIds.some(
|
|
3255
|
-
(id) => !snapshot?.statuses.has(id),
|
|
3256
|
-
);
|
|
3257
|
-
if (hasUnknownSourceIds) {
|
|
3258
|
-
snapshot = await rebuildSnapshot();
|
|
3259
|
-
}
|
|
3260
|
-
}
|
|
3261
|
-
|
|
3262
|
-
// Persist negative lookups in the cached snapshot so stale source IDs do not
|
|
3263
|
-
// trigger repeated full snapshot rebuilds on every matching recall.
|
|
3264
|
-
for (const id of sourceIds) {
|
|
3265
|
-
if (!snapshot?.statuses.has(id)) {
|
|
3266
|
-
snapshot?.statuses.set(id, "missing");
|
|
3267
|
-
}
|
|
3268
|
-
}
|
|
3269
|
-
|
|
3270
|
-
const statuses = new Map<
|
|
3271
|
-
string,
|
|
3272
|
-
"active" | "superseded" | "archived" | "missing"
|
|
3273
|
-
>();
|
|
3274
|
-
for (const id of sourceIds) {
|
|
3275
|
-
const status = snapshot?.statuses.get(id);
|
|
3276
|
-
if (status) {
|
|
3277
|
-
statuses.set(id, status);
|
|
3278
|
-
} else {
|
|
3279
|
-
statuses.set(id, "missing");
|
|
3280
|
-
}
|
|
3281
|
-
}
|
|
3282
|
-
return statuses;
|
|
1889
|
+
return this.namespaceReadFanoutCoordinator.resolveArtifactSourceStatuses(
|
|
1890
|
+
storage,
|
|
1891
|
+
sourceIds,
|
|
1892
|
+
);
|
|
3283
1893
|
}
|
|
3284
1894
|
|
|
3285
1895
|
/**
|
|
@@ -3370,29 +1980,8 @@ export class Orchestrator {
|
|
|
3370
1980
|
return this.orchestratorInitCoordinator.startupSearchSync(
|
|
3371
1981
|
signal,
|
|
3372
1982
|
);
|
|
3373
|
-
}
|
|
3374
|
-
|
|
3375
|
-
/**
|
|
3376
|
-
* Run the pattern-reinforcement maintenance job (issue #687 PR 2/4).
|
|
3377
|
-
*
|
|
3378
|
-
* Cadence-gated on `patternReinforcementCadenceMs` so every caller
|
|
3379
|
-
* (orchestrator cron path, MCP tool, CLI) shares a single floor —
|
|
3380
|
-
* none can call this on a hot loop and burn the corpus. When the
|
|
3381
|
-
* feature is disabled or the cadence has not elapsed, returns a
|
|
3382
|
-
* synthetic "skipped" result rather than throwing.
|
|
3383
|
-
*
|
|
3384
|
-
* Cadence tracking is per-namespace so a tenant-scoped MCP run in
|
|
3385
|
-
* one namespace does not silence a cron run in another (PR #730
|
|
3386
|
-
* review feedback, Codex P2). Pass `force: true` for ad-hoc
|
|
3387
|
-
* operator runs that must bypass the cadence floor — mirrors the
|
|
3388
|
-
* pattern used by other maintenance MCP tools.
|
|
3389
|
-
*
|
|
3390
|
-
* `force` deliberately does NOT bypass the master
|
|
3391
|
-
* `patternReinforcementEnabled` flag (PR #730 review feedback,
|
|
3392
|
-
* Cursor Medium). Operators who have explicitly disabled the
|
|
3393
|
-
* feature must not have their corpus mutated by an MCP tool call —
|
|
3394
|
-
* the only way to run the job is to enable the feature in config.
|
|
3395
|
-
*/
|
|
1983
|
+
}
|
|
1984
|
+
|
|
3396
1985
|
async runPatternReinforcement(options: {
|
|
3397
1986
|
force?: boolean;
|
|
3398
1987
|
namespace?: string;
|
|
@@ -3402,69 +1991,16 @@ export class Orchestrator {
|
|
|
3402
1991
|
namespace: string;
|
|
3403
1992
|
result?: PatternReinforcementResult;
|
|
3404
1993
|
}> {
|
|
3405
|
-
|
|
3406
|
-
|
|
3407
|
-
// force=true. `force` only relaxes the cadence floor below.
|
|
3408
|
-
if (!resolveConsolidationCapabilities(this.config).patternReinforcement) {
|
|
3409
|
-
return { ran: false, skippedReason: "disabled", namespace: cadenceKey };
|
|
3410
|
-
}
|
|
3411
|
-
const cadence = this.config.patternReinforcementCadenceMs;
|
|
3412
|
-
const lastAt = this.lastPatternReinforcementAtByNs.get(cadenceKey);
|
|
3413
|
-
if (
|
|
3414
|
-
!options.force &&
|
|
3415
|
-
cadence > 0 &&
|
|
3416
|
-
lastAt !== undefined &&
|
|
3417
|
-
Date.now() - lastAt < cadence
|
|
3418
|
-
) {
|
|
3419
|
-
return { ran: false, skippedReason: "cadence", namespace: cadenceKey };
|
|
3420
|
-
}
|
|
3421
|
-
const storage = options.namespace
|
|
3422
|
-
? await this.getStorage(options.namespace)
|
|
3423
|
-
: this.storage;
|
|
3424
|
-
const result = await runPatternReinforcement(storage, {
|
|
3425
|
-
categories: this.config.patternReinforcementCategories,
|
|
3426
|
-
minCount: this.config.patternReinforcementMinCount,
|
|
3427
|
-
});
|
|
3428
|
-
this.lastPatternReinforcementAtByNs.set(cadenceKey, Date.now());
|
|
3429
|
-
log.debug(
|
|
3430
|
-
`pattern reinforcement [ns=${cadenceKey || "(default)"}]: clusters=${result.clustersFound} canonicalsUpdated=${result.canonicalsUpdated} duplicatesSuperseded=${result.duplicatesSuperseded}`,
|
|
1994
|
+
return this.workspaceOpsCoordinator.runPatternReinforcement(
|
|
1995
|
+
options,
|
|
3431
1996
|
);
|
|
3432
|
-
return { ran: true, result, namespace: cadenceKey };
|
|
3433
1997
|
}
|
|
3434
1998
|
|
|
3435
|
-
/**
|
|
3436
|
-
* Fan out pattern reinforcement across all maintained namespaces (issue #1500).
|
|
3437
|
-
* Delegates per-namespace execution to {@link runPatternReinforcement} while
|
|
3438
|
-
* the planner handles discovery, budgeting, locking, and status recording.
|
|
3439
|
-
* When namespaces are disabled, runs once against default storage.
|
|
3440
|
-
*/
|
|
3441
1999
|
async runPatternReinforcementFanout(options: {
|
|
3442
2000
|
force?: boolean;
|
|
3443
2001
|
} = {}): Promise<NamespaceMaintenanceSummary> {
|
|
3444
|
-
return this.
|
|
3445
|
-
|
|
3446
|
-
async (ctx) => {
|
|
3447
|
-
const result = await this.runPatternReinforcement({
|
|
3448
|
-
namespace: ctx.candidate.namespace,
|
|
3449
|
-
force: options.force,
|
|
3450
|
-
});
|
|
3451
|
-
// runPatternReinforcement has its own per-namespace cadence gate
|
|
3452
|
-
// (lastPatternReinforcementAtByNs). When it throttles (ran:false),
|
|
3453
|
-
// signal skip so the planner records state:"skipped" and does NOT
|
|
3454
|
-
// touch lastMaintenanceAt — otherwise a throttled namespace would
|
|
3455
|
-
// look maintained while pattern reinforcement never ran (#1500
|
|
3456
|
-
// review: cadence-skip accuracy).
|
|
3457
|
-
if (!result.ran) {
|
|
3458
|
-
return {
|
|
3459
|
-
skipped: true,
|
|
3460
|
-
skipReason: result.skippedReason ?? "throttled",
|
|
3461
|
-
};
|
|
3462
|
-
}
|
|
3463
|
-
return result.result
|
|
3464
|
-
? { itemCount: result.result.clustersFound }
|
|
3465
|
-
: { itemCount: 0 };
|
|
3466
|
-
},
|
|
3467
|
-
{ enabled: resolveConsolidationCapabilities(this.config).patternReinforcement },
|
|
2002
|
+
return this.workspaceOpsCoordinator.runPatternReinforcementFanout(
|
|
2003
|
+
options,
|
|
3468
2004
|
);
|
|
3469
2005
|
}
|
|
3470
2006
|
|
|
@@ -3578,75 +2114,8 @@ export class Orchestrator {
|
|
|
3578
2114
|
}
|
|
3579
2115
|
|
|
3580
2116
|
async maybeRunFileHygiene(): Promise<void> {
|
|
3581
|
-
|
|
3582
|
-
|
|
3583
|
-
|
|
3584
|
-
const now = Date.now();
|
|
3585
|
-
if (now - this.lastFileHygieneRunAtMs < hygiene.runMinIntervalMs) return;
|
|
3586
|
-
this.lastFileHygieneRunAtMs = now;
|
|
3587
|
-
|
|
3588
|
-
// Rotation first (keeps bootstrap files small).
|
|
3589
|
-
if (hygiene.rotateEnabled) {
|
|
3590
|
-
for (const rel of hygiene.rotatePaths) {
|
|
3591
|
-
const abs = path.isAbsolute(rel)
|
|
3592
|
-
? rel
|
|
3593
|
-
: path.join(this.config.workspaceDir, rel);
|
|
3594
|
-
try {
|
|
3595
|
-
const raw = await readFile(abs, "utf-8");
|
|
3596
|
-
if (raw.length > hygiene.rotateMaxBytes) {
|
|
3597
|
-
const archiveDir = path.join(
|
|
3598
|
-
this.config.workspaceDir,
|
|
3599
|
-
hygiene.archiveDir,
|
|
3600
|
-
);
|
|
3601
|
-
const base = path.basename(abs);
|
|
3602
|
-
const prefix =
|
|
3603
|
-
base
|
|
3604
|
-
.toUpperCase()
|
|
3605
|
-
.replace(/\.MD$/i, "")
|
|
3606
|
-
.replace(/[^A-Z0-9]+/g, "-") || "FILE";
|
|
3607
|
-
const { newContent } = await rotateMarkdownFileToArchive({
|
|
3608
|
-
filePath: abs,
|
|
3609
|
-
archiveDir,
|
|
3610
|
-
archivePrefix: prefix,
|
|
3611
|
-
keepTailChars: hygiene.rotateKeepTailChars,
|
|
3612
|
-
});
|
|
3613
|
-
await writeFile(abs, newContent, "utf-8");
|
|
3614
|
-
}
|
|
3615
|
-
} catch {
|
|
3616
|
-
// ignore missing/unreadable targets
|
|
3617
|
-
}
|
|
3618
|
-
}
|
|
3619
|
-
}
|
|
3620
|
-
|
|
3621
|
-
// Lint (warn before truncation risk).
|
|
3622
|
-
if (hygiene.lintEnabled) {
|
|
3623
|
-
const warnings = await lintWorkspaceFiles({
|
|
3624
|
-
workspaceDir: this.config.workspaceDir,
|
|
3625
|
-
paths: hygiene.lintPaths,
|
|
3626
|
-
budgetBytes: hygiene.lintBudgetBytes,
|
|
3627
|
-
warnRatio: hygiene.lintWarnRatio,
|
|
3628
|
-
});
|
|
3629
|
-
for (const w of warnings) {
|
|
3630
|
-
log.warn(w.message);
|
|
3631
|
-
}
|
|
3632
|
-
|
|
3633
|
-
if (hygiene.warningsLogEnabled && warnings.length > 0) {
|
|
3634
|
-
const fp = path.join(this.config.memoryDir, hygiene.warningsLogPath);
|
|
3635
|
-
await mkdir(path.dirname(fp), { recursive: true });
|
|
3636
|
-
const stamp = new Date().toISOString();
|
|
3637
|
-
const block =
|
|
3638
|
-
`\n\n## ${stamp}\n\n` +
|
|
3639
|
-
warnings.map((w) => `- ${w.message}`).join("\n") +
|
|
3640
|
-
"\n";
|
|
3641
|
-
let existing = "";
|
|
3642
|
-
try {
|
|
3643
|
-
existing = await readFile(fp, "utf-8");
|
|
3644
|
-
} catch {
|
|
3645
|
-
existing = "# Engram File Hygiene Warnings\n";
|
|
3646
|
-
}
|
|
3647
|
-
await writeFile(fp, existing + block, "utf-8");
|
|
3648
|
-
}
|
|
3649
|
-
}
|
|
2117
|
+
return this.workspaceOpsCoordinator.maybeRunFileHygiene(
|
|
2118
|
+
);
|
|
3650
2119
|
}
|
|
3651
2120
|
|
|
3652
2121
|
async runBootstrap(options: BootstrapOptions): Promise<BootstrapResult> {
|
|
@@ -3797,282 +2266,22 @@ export class Orchestrator {
|
|
|
3797
2266
|
return this.generateDaySummary(gathered);
|
|
3798
2267
|
}
|
|
3799
2268
|
|
|
3800
|
-
/**
|
|
3801
|
-
* Read today's facts and hourly summaries from storage, returning them
|
|
3802
|
-
* as a formatted string suitable for generateDaySummary().
|
|
3803
|
-
*/
|
|
3804
2269
|
async gatherTodayFacts(
|
|
3805
2270
|
namespace?: string,
|
|
3806
2271
|
options: DaySummaryGatherOptions = {},
|
|
3807
2272
|
): Promise<string> {
|
|
3808
|
-
|
|
3809
|
-
namespace
|
|
3810
|
-
|
|
3811
|
-
: this.config.defaultNamespace;
|
|
3812
|
-
const storage = await this.storageRouter.storageFor(ns);
|
|
3813
|
-
const configuredTimeZone = normalizeIanaTimeZone(options.timeZone)
|
|
3814
|
-
?? normalizeIanaTimeZone(this.config.daySummaryTimezone);
|
|
3815
|
-
const timeZone =
|
|
3816
|
-
configuredTimeZone ?? Intl.DateTimeFormat().resolvedOptions().timeZone;
|
|
3817
|
-
const now = options.now instanceof Date && Number.isFinite(options.now.getTime())
|
|
3818
|
-
? options.now
|
|
3819
|
-
: new Date();
|
|
3820
|
-
const targetLocalDate = formatDateInTimeZone(now, timeZone);
|
|
3821
|
-
// Facts are stored under UTC date directories, while the summary target is
|
|
3822
|
-
// a local calendar day. Scan the UTC-date envelope that overlaps the local
|
|
3823
|
-
// day, then filter parseable fact timestamps to that configured local day.
|
|
3824
|
-
const datesToScan = utcDateKeysForLocalDay(now, timeZone);
|
|
3825
|
-
const MAX_CHARS = 100_000;
|
|
3826
|
-
|
|
3827
|
-
// --- Read memory files from each category dir × date directory ---
|
|
3828
|
-
// Iterate every recall category dir (RECALL_FALLBACK_DIRS — single source
|
|
3829
|
-
// of truth) so the day summary includes decisions/, moments/, ... not just
|
|
3830
|
-
// facts/ (#1546). corrections/ is flat, so corrections/<date>/ never exists
|
|
3831
|
-
// and is skipped by the ENOENT guard — preserving the prior exclusion. The
|
|
3832
|
-
// per-file created→local-day filter below is unchanged.
|
|
3833
|
-
//
|
|
3834
|
-
// Symlink/containment hardening (mirrors scanDir / the CLI walker): the
|
|
3835
|
-
// gathered contents feed the day-summary LLM input, so a symlinked category
|
|
3836
|
-
// dir (decisions/ → outside memoryDir) must not be followed and leak files.
|
|
3837
|
-
// Resolve the store root once; skip symlinked / out-of-root dirs and
|
|
3838
|
-
// entries; skip the scan gracefully if the root can't be resolved.
|
|
3839
|
-
const facts: MemoryFile[] = [];
|
|
3840
|
-
let memoryRootReal: string | null = null;
|
|
3841
|
-
try {
|
|
3842
|
-
memoryRootReal = await realpath(storage.dir);
|
|
3843
|
-
} catch {
|
|
3844
|
-
memoryRootReal = null;
|
|
3845
|
-
}
|
|
3846
|
-
for (const categoryDir of RECALL_FALLBACK_DIRS) {
|
|
3847
|
-
if (memoryRootReal === null) break;
|
|
3848
|
-
for (const date of datesToScan) {
|
|
3849
|
-
const dateDir = path.join(storage.dir, categoryDir, date);
|
|
3850
|
-
try {
|
|
3851
|
-
const dirStat = await lstat(dateDir);
|
|
3852
|
-
if (dirStat.isSymbolicLink() || !dirStat.isDirectory()) continue;
|
|
3853
|
-
assertPathInsideRoot(memoryRootReal, await realpath(dateDir), dateDir);
|
|
3854
|
-
const entries = await readdir(dateDir, { withFileTypes: true });
|
|
3855
|
-
for (const entry of entries) {
|
|
3856
|
-
if (entry.isSymbolicLink()) continue;
|
|
3857
|
-
if (!entry.name.endsWith(".md")) continue;
|
|
3858
|
-
const fullPath = path.join(dateDir, entry.name);
|
|
3859
|
-
try {
|
|
3860
|
-
assertPathInsideRoot(memoryRootReal, await realpath(fullPath), fullPath);
|
|
3861
|
-
const raw = await readFile(fullPath, "utf-8");
|
|
3862
|
-
const fmMatch = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
|
3863
|
-
if (!fmMatch) continue;
|
|
3864
|
-
const fmBlock = fmMatch[1];
|
|
3865
|
-
const content = fmMatch[2].trim();
|
|
3866
|
-
const fm: Record<string, string> = {};
|
|
3867
|
-
for (const line of fmBlock.split("\n")) {
|
|
3868
|
-
const colonIdx = line.indexOf(":");
|
|
3869
|
-
if (colonIdx === -1) continue;
|
|
3870
|
-
fm[line.slice(0, colonIdx).trim()] = line
|
|
3871
|
-
.slice(colonIdx + 1)
|
|
3872
|
-
.trim();
|
|
3873
|
-
}
|
|
3874
|
-
const created = fm.created || "unknown";
|
|
3875
|
-
const createdAt = parseFiniteDate(created);
|
|
3876
|
-
if (
|
|
3877
|
-
createdAt &&
|
|
3878
|
-
formatDateInTimeZone(createdAt, timeZone) !== targetLocalDate
|
|
3879
|
-
) {
|
|
3880
|
-
continue;
|
|
3881
|
-
}
|
|
3882
|
-
facts.push({
|
|
3883
|
-
path: fullPath,
|
|
3884
|
-
frontmatter: {
|
|
3885
|
-
id: fm.id || path.basename(entry.name, ".md"),
|
|
3886
|
-
category: (fm.category as any) || "fact",
|
|
3887
|
-
created,
|
|
3888
|
-
updated: fm.updated || created,
|
|
3889
|
-
source: fm.source || "unknown",
|
|
3890
|
-
confidence: parseFloat(fm.confidence || "0.8"),
|
|
3891
|
-
confidenceTier: (fm.confidenceTier as any) || "implied",
|
|
3892
|
-
tags: [],
|
|
3893
|
-
},
|
|
3894
|
-
content,
|
|
3895
|
-
});
|
|
3896
|
-
} catch {
|
|
3897
|
-
// Skip unreadable files
|
|
3898
|
-
}
|
|
3899
|
-
}
|
|
3900
|
-
} catch {
|
|
3901
|
-
// Absent dir (ENOENT), symlinked/out-of-root dir, or containment
|
|
3902
|
-
// violation — skip this category/date without aborting the summary.
|
|
3903
|
-
}
|
|
3904
|
-
}
|
|
3905
|
-
}
|
|
3906
|
-
|
|
3907
|
-
// Sort facts by created timestamp (most recent last) so truncation keeps newest
|
|
3908
|
-
facts.sort((a, b) => {
|
|
3909
|
-
if (a.frontmatter.created === b.frontmatter.created) return 0;
|
|
3910
|
-
return a.frontmatter.created < b.frontmatter.created ? -1 : 1;
|
|
3911
|
-
});
|
|
3912
|
-
|
|
3913
|
-
// --- Read hourly summaries for the scanned dates ---
|
|
3914
|
-
const hourlySummaries: string[] = [];
|
|
3915
|
-
const hourlyBaseDir = path.join(storage.dir, "summaries", "hourly");
|
|
3916
|
-
try {
|
|
3917
|
-
const sessionKeys = await readdir(hourlyBaseDir, { withFileTypes: true });
|
|
3918
|
-
for (const sk of sessionKeys) {
|
|
3919
|
-
if (!sk.isDirectory()) continue;
|
|
3920
|
-
for (const date of datesToScan) {
|
|
3921
|
-
const summaryFile = path.join(hourlyBaseDir, sk.name, `${date}.md`);
|
|
3922
|
-
try {
|
|
3923
|
-
const raw = await readFile(summaryFile, "utf-8");
|
|
3924
|
-
const filtered = filterHourlySummaryMarkdownForLocalDay(
|
|
3925
|
-
raw,
|
|
3926
|
-
date,
|
|
3927
|
-
timeZone,
|
|
3928
|
-
targetLocalDate,
|
|
3929
|
-
);
|
|
3930
|
-
if (filtered) {
|
|
3931
|
-
hourlySummaries.push(filtered);
|
|
3932
|
-
}
|
|
3933
|
-
} catch {
|
|
3934
|
-
// No summary file for this session/date
|
|
3935
|
-
}
|
|
3936
|
-
}
|
|
3937
|
-
}
|
|
3938
|
-
} catch {
|
|
3939
|
-
// No hourly summaries directory
|
|
3940
|
-
}
|
|
3941
|
-
|
|
3942
|
-
// --- Format and truncate ---
|
|
3943
|
-
let formatted = formatDaySummaryMemories(facts);
|
|
3944
|
-
if (hourlySummaries.length > 0) {
|
|
3945
|
-
formatted +=
|
|
3946
|
-
"\n\n---\n## Hourly Summaries\n\n" +
|
|
3947
|
-
hourlySummaries.join("\n\n---\n\n");
|
|
3948
|
-
}
|
|
3949
|
-
|
|
3950
|
-
// Truncate intelligently if over budget: drop oldest facts first
|
|
3951
|
-
if (formatted.length > MAX_CHARS) {
|
|
3952
|
-
// Re-build with fewer facts, keeping most recent
|
|
3953
|
-
while (facts.length > 1 && formatted.length > MAX_CHARS) {
|
|
3954
|
-
facts.shift(); // drop oldest
|
|
3955
|
-
formatted = formatDaySummaryMemories(facts);
|
|
3956
|
-
if (hourlySummaries.length > 0) {
|
|
3957
|
-
formatted +=
|
|
3958
|
-
"\n\n---\n## Hourly Summaries\n\n" +
|
|
3959
|
-
hourlySummaries.join("\n\n---\n\n");
|
|
3960
|
-
}
|
|
3961
|
-
}
|
|
3962
|
-
// If still over, hard truncate
|
|
3963
|
-
if (formatted.length > MAX_CHARS) {
|
|
3964
|
-
formatted = formatted.slice(0, MAX_CHARS);
|
|
3965
|
-
}
|
|
3966
|
-
}
|
|
3967
|
-
|
|
3968
|
-
log.info(
|
|
3969
|
-
`gatherTodayFacts: collected ${facts.length} facts, ${hourlySummaries.length} hourly summaries for ${targetLocalDate} (${timeZone}, ${formatted.length} chars)`,
|
|
2273
|
+
return this.workspaceOpsCoordinator.gatherTodayFacts(
|
|
2274
|
+
namespace,
|
|
2275
|
+
options,
|
|
3970
2276
|
);
|
|
3971
|
-
|
|
3972
|
-
return formatted;
|
|
3973
2277
|
}
|
|
3974
2278
|
|
|
3975
2279
|
previewMemoryActionEvent(
|
|
3976
2280
|
event: Omit<MemoryActionEvent, "timestamp"> & { timestamp?: string },
|
|
3977
2281
|
): MemoryActionEvent {
|
|
3978
|
-
|
|
3979
|
-
|
|
3980
|
-
? event.namespace
|
|
3981
|
-
: this.config.defaultNamespace;
|
|
3982
|
-
const eligibility = parseMemoryActionEligibilityContext(
|
|
3983
|
-
event.policyEligibility,
|
|
3984
|
-
);
|
|
3985
|
-
const policy = evaluateMemoryActionPolicy({
|
|
3986
|
-
action: event.action,
|
|
3987
|
-
eligibility,
|
|
3988
|
-
options: {
|
|
3989
|
-
actionsEnabled: resolveCompressionCapabilities(this.config).contextCompressionActions,
|
|
3990
|
-
maxCompressionTokensPerHour: this.config.maxCompressionTokensPerHour,
|
|
3991
|
-
},
|
|
3992
|
-
});
|
|
3993
|
-
const dryRun = event.dryRun === true;
|
|
3994
|
-
|
|
3995
|
-
const normalizedOutcome = dryRun
|
|
3996
|
-
? event.outcome === "failed"
|
|
3997
|
-
? "failed"
|
|
3998
|
-
: "skipped"
|
|
3999
|
-
: policy.decision === "allow"
|
|
4000
|
-
? event.outcome
|
|
4001
|
-
: event.outcome === "failed"
|
|
4002
|
-
? "failed"
|
|
4003
|
-
: "skipped";
|
|
4004
|
-
const sourceSessionKey =
|
|
4005
|
-
typeof event.sourceSessionKey === "string" &&
|
|
4006
|
-
event.sourceSessionKey.length > 0
|
|
4007
|
-
? event.sourceSessionKey
|
|
4008
|
-
: typeof event.sessionKey === "string" && event.sessionKey.length > 0
|
|
4009
|
-
? event.sessionKey
|
|
4010
|
-
: undefined;
|
|
4011
|
-
const outputMemoryIds = Array.isArray(event.outputMemoryIds)
|
|
4012
|
-
? Array.from(
|
|
4013
|
-
new Set(
|
|
4014
|
-
event.outputMemoryIds.filter(
|
|
4015
|
-
(value): value is string =>
|
|
4016
|
-
typeof value === "string" && value.length > 0,
|
|
4017
|
-
),
|
|
4018
|
-
),
|
|
4019
|
-
)
|
|
4020
|
-
: [];
|
|
4021
|
-
|
|
4022
|
-
const reasonParts = [
|
|
4023
|
-
event.reason,
|
|
4024
|
-
`policy:${policy.decision}`,
|
|
4025
|
-
policy.rationale,
|
|
4026
|
-
].filter(
|
|
4027
|
-
(part): part is string => typeof part === "string" && part.length > 0,
|
|
2282
|
+
return this.workspaceOpsCoordinator.previewMemoryActionEvent(
|
|
2283
|
+
event,
|
|
4028
2284
|
);
|
|
4029
|
-
|
|
4030
|
-
return {
|
|
4031
|
-
...event,
|
|
4032
|
-
schemaVersion: event.schemaVersion ?? 1,
|
|
4033
|
-
actionId:
|
|
4034
|
-
typeof event.actionId === "string" && event.actionId.length > 0
|
|
4035
|
-
? event.actionId
|
|
4036
|
-
: `memact-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
|
4037
|
-
outcome: normalizedOutcome,
|
|
4038
|
-
status:
|
|
4039
|
-
event.status ??
|
|
4040
|
-
(dryRun && policy.decision === "allow" && event.outcome !== "failed"
|
|
4041
|
-
? "validated"
|
|
4042
|
-
: normalizedOutcome === "applied"
|
|
4043
|
-
? "applied"
|
|
4044
|
-
: "rejected"),
|
|
4045
|
-
actor:
|
|
4046
|
-
typeof event.actor === "string" && event.actor.length > 0
|
|
4047
|
-
? event.actor
|
|
4048
|
-
: "engram",
|
|
4049
|
-
subsystem:
|
|
4050
|
-
typeof event.subsystem === "string" && event.subsystem.length > 0
|
|
4051
|
-
? event.subsystem
|
|
4052
|
-
: "memory_action",
|
|
4053
|
-
reason: reasonParts.join(" | "),
|
|
4054
|
-
namespace,
|
|
4055
|
-
sessionKey: sourceSessionKey ?? event.sessionKey,
|
|
4056
|
-
sourceSessionKey,
|
|
4057
|
-
inputSummary:
|
|
4058
|
-
typeof event.inputSummary === "string" && event.inputSummary.length > 0
|
|
4059
|
-
? event.inputSummary
|
|
4060
|
-
: undefined,
|
|
4061
|
-
outputMemoryIds,
|
|
4062
|
-
dryRun,
|
|
4063
|
-
policyVersion:
|
|
4064
|
-
typeof event.policyVersion === "string" &&
|
|
4065
|
-
event.policyVersion.length > 0
|
|
4066
|
-
? event.policyVersion
|
|
4067
|
-
: "memory-action-policy.v1",
|
|
4068
|
-
timestamp:
|
|
4069
|
-
typeof event.timestamp === "string" && event.timestamp.length > 0
|
|
4070
|
-
? event.timestamp
|
|
4071
|
-
: new Date().toISOString(),
|
|
4072
|
-
policyDecision: policy.decision,
|
|
4073
|
-
policyRationale: policy.rationale,
|
|
4074
|
-
policyEligibility: eligibility,
|
|
4075
|
-
};
|
|
4076
2285
|
}
|
|
4077
2286
|
|
|
4078
2287
|
async appendMemoryActionEvent(
|
|
@@ -4233,63 +2442,9 @@ export class Orchestrator {
|
|
|
4233
2442
|
return this.conversationIndexCoordinator.rebuild(sessionKey, hours, opts);
|
|
4234
2443
|
}
|
|
4235
2444
|
|
|
4236
|
-
/**
|
|
4237
|
-
* Validate local LLM model availability and context window compatibility.
|
|
4238
|
-
* Warns the user if there's a mismatch.
|
|
4239
|
-
*/
|
|
4240
2445
|
private async validateLocalLlmModel(): Promise<void> {
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
const modelInfo = await this.localLlm.getLoadedModelInfo();
|
|
4244
|
-
if (!modelInfo) {
|
|
4245
|
-
log.warn(
|
|
4246
|
-
"Local LLM validation: Could not query model info from server",
|
|
4247
|
-
);
|
|
4248
|
-
log.warn(
|
|
4249
|
-
"Local LLM validation: Could not query model info. " +
|
|
4250
|
-
"Ensure LM Studio/Ollama is running with the model loaded.",
|
|
4251
|
-
);
|
|
4252
|
-
return;
|
|
4253
|
-
}
|
|
4254
|
-
|
|
4255
|
-
// Check for context window mismatch
|
|
4256
|
-
const configuredMaxContext = this.config.localLlmMaxContext;
|
|
4257
|
-
|
|
4258
|
-
if (modelInfo.contextWindow) {
|
|
4259
|
-
log.debug(
|
|
4260
|
-
`Local LLM: ${modelInfo.id} loaded with ${modelInfo.contextWindow.toLocaleString()} token context window`,
|
|
4261
|
-
);
|
|
4262
|
-
|
|
4263
|
-
if (
|
|
4264
|
-
configuredMaxContext &&
|
|
4265
|
-
configuredMaxContext > modelInfo.contextWindow
|
|
4266
|
-
) {
|
|
4267
|
-
log.warn(
|
|
4268
|
-
`Local LLM context mismatch: engram configured for ${configuredMaxContext.toLocaleString()} tokens, ` +
|
|
4269
|
-
`but ${modelInfo.id} only supports ${modelInfo.contextWindow.toLocaleString()}. ` +
|
|
4270
|
-
`Reducing to ${modelInfo.contextWindow.toLocaleString()} to avoid errors.`,
|
|
4271
|
-
);
|
|
4272
|
-
// Update the config in-memory to match actual capability
|
|
4273
|
-
// (This is a temporary fix - user should update their config)
|
|
4274
|
-
(this.config as { localLlmMaxContext?: number }).localLlmMaxContext =
|
|
4275
|
-
modelInfo.contextWindow;
|
|
4276
|
-
}
|
|
4277
|
-
} else {
|
|
4278
|
-
log.debug(
|
|
4279
|
-
`Local LLM: ${modelInfo.id} loaded (context window not reported by server)`,
|
|
4280
|
-
);
|
|
4281
|
-
|
|
4282
|
-
if (!configuredMaxContext) {
|
|
4283
|
-
log.warn(
|
|
4284
|
-
"Local LLM: Server did not report context window. " +
|
|
4285
|
-
"If you get 'context length exceeded' errors, set localLlmMaxContext in your config. " +
|
|
4286
|
-
"Common defaults: LM Studio (32K), Ollama (2K-128K depending on model).",
|
|
4287
|
-
);
|
|
4288
|
-
}
|
|
4289
|
-
}
|
|
4290
|
-
} catch (err) {
|
|
4291
|
-
log.warn(`Local LLM validation failed: ${err}`);
|
|
4292
|
-
}
|
|
2446
|
+
return this.workspaceOpsCoordinator.validateLocalLlmModel(
|
|
2447
|
+
);
|
|
4293
2448
|
}
|
|
4294
2449
|
|
|
4295
2450
|
async recall(
|
|
@@ -4556,15 +2711,11 @@ export class Orchestrator {
|
|
|
4556
2711
|
recallNamespaces: string[],
|
|
4557
2712
|
targetCount: number,
|
|
4558
2713
|
): Promise<MemoryFile[]> {
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
this.fetchActiveArtifactsForNamespace(namespace, prompt, targetCount),
|
|
4564
|
-
),
|
|
2714
|
+
return this.namespaceReadFanoutCoordinator.recallArtifactsAcrossNamespaces(
|
|
2715
|
+
prompt,
|
|
2716
|
+
recallNamespaces,
|
|
2717
|
+
targetCount,
|
|
4565
2718
|
);
|
|
4566
|
-
|
|
4567
|
-
return mergeArtifactRecallCandidates(filteredByNamespace, targetCount);
|
|
4568
2719
|
}
|
|
4569
2720
|
|
|
4570
2721
|
private scopeQueryAwarePaths(
|
|
@@ -4604,51 +2755,12 @@ export class Orchestrator {
|
|
|
4604
2755
|
allowArchived?: boolean;
|
|
4605
2756
|
},
|
|
4606
2757
|
): Promise<QmdSearchResult[]> {
|
|
4607
|
-
|
|
4608
|
-
|
|
4609
|
-
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
|
|
4613
|
-
Array.from(candidatePaths).map(async (memoryPath) => {
|
|
4614
|
-
const namespace = resolveNamespaceCapabilities(this.config).namespaces
|
|
4615
|
-
? this.namespaceFromPath(memoryPath)
|
|
4616
|
-
: this.config.defaultNamespace;
|
|
4617
|
-
const storage = await this.storageRouter.storageFor(namespace);
|
|
4618
|
-
return await storage.readMemoryByPath(memoryPath);
|
|
4619
|
-
}),
|
|
4620
|
-
)
|
|
4621
|
-
).filter((memory): memory is MemoryFile => memory !== null);
|
|
4622
|
-
|
|
4623
|
-
const results: QmdSearchResult[] = [];
|
|
4624
|
-
for (const memory of memories) {
|
|
4625
|
-
const status = memory.frontmatter.status ?? "active";
|
|
4626
|
-
if (!options?.allowArchived && status !== "active") continue;
|
|
4627
|
-
|
|
4628
|
-
const haystack = [
|
|
4629
|
-
memory.content,
|
|
4630
|
-
memory.frontmatter.category,
|
|
4631
|
-
...(memory.frontmatter.tags ?? []),
|
|
4632
|
-
]
|
|
4633
|
-
.join(" ")
|
|
4634
|
-
.toLowerCase();
|
|
4635
|
-
let hits = 0;
|
|
4636
|
-
for (const token of tokens) {
|
|
4637
|
-
if (haystack.includes(token)) hits += 1;
|
|
4638
|
-
}
|
|
4639
|
-
const score = tokens.length > 0 ? hits / tokens.length : 0.01;
|
|
4640
|
-
if (tokens.length > 0 && hits === 0) continue;
|
|
4641
|
-
|
|
4642
|
-
results.push({
|
|
4643
|
-
docid: memory.frontmatter.id,
|
|
4644
|
-
path: memory.path,
|
|
4645
|
-
score,
|
|
4646
|
-
snippet: memory.content.slice(0, 400).replace(/\n/g, " "),
|
|
4647
|
-
transport: "scoped_prefilter",
|
|
4648
|
-
});
|
|
4649
|
-
}
|
|
4650
|
-
|
|
4651
|
-
return results.sort((a, b) => b.score - a.score).slice(0, cappedLimit);
|
|
2758
|
+
return this.namespaceReadFanoutCoordinator.searchScopedMemoryCandidates(
|
|
2759
|
+
candidatePaths,
|
|
2760
|
+
query,
|
|
2761
|
+
limit,
|
|
2762
|
+
options,
|
|
2763
|
+
);
|
|
4652
2764
|
}
|
|
4653
2765
|
|
|
4654
2766
|
private async fetchQmdMemoryResultsWithArtifactTopUp(
|
|
@@ -4742,26 +2854,9 @@ export class Orchestrator {
|
|
|
4742
2854
|
private async resolveStateDirForNamespace(
|
|
4743
2855
|
namespace: string,
|
|
4744
2856
|
): Promise<string> {
|
|
4745
|
-
|
|
4746
|
-
|
|
4747
|
-
}
|
|
4748
|
-
if (namespace !== this.config.defaultNamespace) {
|
|
4749
|
-
return path.join(this.config.memoryDir, "namespaces", namespace, "state");
|
|
4750
|
-
}
|
|
4751
|
-
const candidate = path.join(
|
|
4752
|
-
this.config.memoryDir,
|
|
4753
|
-
"namespaces",
|
|
4754
|
-
this.config.defaultNamespace,
|
|
2857
|
+
return this.namespaceReadFanoutCoordinator.resolveStateDirForNamespace(
|
|
2858
|
+
namespace,
|
|
4755
2859
|
);
|
|
4756
|
-
try {
|
|
4757
|
-
const candidateStat = await stat(candidate);
|
|
4758
|
-
if (candidateStat.isDirectory()) {
|
|
4759
|
-
return path.join(candidate, "state");
|
|
4760
|
-
}
|
|
4761
|
-
} catch {
|
|
4762
|
-
// Fall back to the legacy root when the migrated default namespace directory is absent.
|
|
4763
|
-
}
|
|
4764
|
-
return path.join(this.config.memoryDir, "state");
|
|
4765
2860
|
}
|
|
4766
2861
|
|
|
4767
2862
|
private buildGraphRecallRankedResults(
|
|
@@ -4862,7 +2957,6 @@ export class Orchestrator {
|
|
|
4862
2957
|
}
|
|
4863
2958
|
|
|
4864
2959
|
|
|
4865
|
-
|
|
4866
2960
|
/**
|
|
4867
2961
|
* Clock source for the shared post-retrieval assembly/enrichment budget. The
|
|
4868
2962
|
* deadline is set from this value and every expiry check reads it back, so a
|
|
@@ -5146,6 +3240,105 @@ export class Orchestrator {
|
|
|
5146
3240
|
return this._orchestratorInitCoordinator;
|
|
5147
3241
|
}
|
|
5148
3242
|
|
|
3243
|
+
/**
|
|
3244
|
+
* Persistence-index coordinator (issue #1526 seam 23). Owns post-persist
|
|
3245
|
+
* bookkeeping (content-hash dedup, temporal indexes, graph edges,
|
|
3246
|
+
* semantic dedup lookup). Lazy + accessor-wired (late-binding rule).
|
|
3247
|
+
*/
|
|
3248
|
+
private _persistenceIndexCoordinator: PersistenceIndexCoordinator | undefined;
|
|
3249
|
+
|
|
3250
|
+
private get persistenceIndexCoordinator(): PersistenceIndexCoordinator {
|
|
3251
|
+
if (!this._persistenceIndexCoordinator) {
|
|
3252
|
+
// eslint-disable-next-line @typescript-eslint/no-this-alias
|
|
3253
|
+
const self = this;
|
|
3254
|
+
this._persistenceIndexCoordinator = new PersistenceIndexCoordinator({
|
|
3255
|
+
get config() { return self.config; },
|
|
3256
|
+
get contentHashIndex() { return self.contentHashIndex; },
|
|
3257
|
+
contentHashIndexForStorage: (targetStorage) => self.contentHashIndexForStorage(targetStorage),
|
|
3258
|
+
get contentHashIndexesByStorageDir() { return self.contentHashIndexesByStorageDir; },
|
|
3259
|
+
get embeddingFallback() { return self.embeddingFallback; },
|
|
3260
|
+
graphIndexFor: (storage) => self.graphIndexFor(storage),
|
|
3261
|
+
readAllMemoriesForNamespaces: (namespaces) => self.readAllMemoriesForNamespaces(namespaces),
|
|
3262
|
+
semanticDedupScopeFor: (targetStorage) => self.semanticDedupScopeFor(targetStorage),
|
|
3263
|
+
});
|
|
3264
|
+
}
|
|
3265
|
+
return this._persistenceIndexCoordinator;
|
|
3266
|
+
}
|
|
3267
|
+
|
|
3268
|
+
/**
|
|
3269
|
+
* Workspace-ops coordinator (issue #1526 seam 24). Owns periodic
|
|
3270
|
+
* workspace/operations surfaces. Lazy + accessor-wired (late-binding
|
|
3271
|
+
* rule, seams 18–23).
|
|
3272
|
+
*/
|
|
3273
|
+
private _workspaceOpsCoordinator: WorkspaceOpsCoordinator | undefined;
|
|
3274
|
+
|
|
3275
|
+
private get workspaceOpsCoordinator(): WorkspaceOpsCoordinator {
|
|
3276
|
+
if (!this._workspaceOpsCoordinator) {
|
|
3277
|
+
// eslint-disable-next-line @typescript-eslint/no-this-alias
|
|
3278
|
+
const self = this;
|
|
3279
|
+
this._workspaceOpsCoordinator = new WorkspaceOpsCoordinator({
|
|
3280
|
+
get accessTrackingBuffer() { return self.accessTrackingBuffer; },
|
|
3281
|
+
bulkImportWriteNamespace: () => self.bulkImportWriteNamespace(),
|
|
3282
|
+
get config() { return self.config; },
|
|
3283
|
+
get extraction() { return self.extraction; },
|
|
3284
|
+
getStorage: (namespace) => self.getStorage(namespace),
|
|
3285
|
+
getStorageForNamespace: (namespace) => self.getStorageForNamespace(namespace),
|
|
3286
|
+
get judgeDeferCounts() { return self.judgeDeferCounts; },
|
|
3287
|
+
get judgeVerdictCache() { return self.judgeVerdictCache; },
|
|
3288
|
+
get lastFileHygieneRunAtMs() { return self.lastFileHygieneRunAtMs; },
|
|
3289
|
+
set lastFileHygieneRunAtMs(value) { self.lastFileHygieneRunAtMs = value; },
|
|
3290
|
+
get lastPatternReinforcementAtByNs() { return self.lastPatternReinforcementAtByNs; },
|
|
3291
|
+
get localLlm() { return self.localLlm; },
|
|
3292
|
+
maintenanceNamespaces: (jobName, budgetMode) => self.maintenanceNamespaces(jobName, budgetMode),
|
|
3293
|
+
namespaceFromPath: (p) => self.namespaceFromPath(p),
|
|
3294
|
+
get qmd() { return self.qmd; },
|
|
3295
|
+
readAllMemoriesForNamespaces: (namespaces) => self.readAllMemoriesForNamespaces(namespaces),
|
|
3296
|
+
runNamespaceMaintenanceFanoutForJob: (jobName, runner, options) => self.runNamespaceMaintenanceFanoutForJob(jobName, runner, options),
|
|
3297
|
+
runPatternReinforcement: (options) => self.runPatternReinforcement(options),
|
|
3298
|
+
get storage() { return self.storage; },
|
|
3299
|
+
get storageRouter() { return self.storageRouter; },
|
|
3300
|
+
get wearablesServiceInstance() { return self.wearablesServiceInstance; },
|
|
3301
|
+
set wearablesServiceInstance(value) { self.wearablesServiceInstance = value; },
|
|
3302
|
+
});
|
|
3303
|
+
}
|
|
3304
|
+
return this._workspaceOpsCoordinator;
|
|
3305
|
+
}
|
|
3306
|
+
|
|
3307
|
+
/**
|
|
3308
|
+
* Namespace read-fanout coordinator (issue #1526 seam 26). Owns
|
|
3309
|
+
* namespace-scoped read fanout (hints, cross-namespace search/reads,
|
|
3310
|
+
* artifact statuses). Lazy + accessor-wired (late-binding rule).
|
|
3311
|
+
*/
|
|
3312
|
+
private _namespaceReadFanoutCoordinator: NamespaceReadFanoutCoordinator | undefined;
|
|
3313
|
+
|
|
3314
|
+
private get namespaceReadFanoutCoordinator(): NamespaceReadFanoutCoordinator {
|
|
3315
|
+
if (!this._namespaceReadFanoutCoordinator) {
|
|
3316
|
+
// eslint-disable-next-line @typescript-eslint/no-this-alias
|
|
3317
|
+
const self = this;
|
|
3318
|
+
this._namespaceReadFanoutCoordinator = new NamespaceReadFanoutCoordinator({
|
|
3319
|
+
get artifactSourceStatusCache() { return self.artifactSourceStatusCache; },
|
|
3320
|
+
get config() { return self.config; },
|
|
3321
|
+
configuredNamespaceList: () => self.configuredNamespaceList(),
|
|
3322
|
+
fetchActiveArtifactsForNamespace: (namespace, prompt, targetCount) => self.fetchActiveArtifactsForNamespace(namespace, prompt, targetCount),
|
|
3323
|
+
loadNamespaceStorageDirHintsFromCatalog: () => self.loadNamespaceStorageDirHintsFromCatalog(),
|
|
3324
|
+
get namespaceCatalog() { return self.namespaceCatalog; },
|
|
3325
|
+
namespaceFromPath: (p) => self.namespaceFromPath(p),
|
|
3326
|
+
get namespaceSearchRouter() { return self.namespaceSearchRouter; },
|
|
3327
|
+
namespaceStorageDirHintOwnershipRank: (record, resolvedStorageDir, configured) => self.namespaceStorageDirHintOwnershipRank(record, resolvedStorageDir, configured),
|
|
3328
|
+
get namespaceStorageDirHints() { return self.namespaceStorageDirHints; },
|
|
3329
|
+
get namespaceStorageDirHintsLoaded() { return self.namespaceStorageDirHintsLoaded; },
|
|
3330
|
+
set namespaceStorageDirHintsLoaded(value) { self.namespaceStorageDirHintsLoaded = value; },
|
|
3331
|
+
preferNamespaceStorageDirHintOwner: (current, candidate, resolvedStorageDir, configured) => self.preferNamespaceStorageDirHintOwner(current, candidate, resolvedStorageDir, configured),
|
|
3332
|
+
get qmd() { return self.qmd; },
|
|
3333
|
+
qmdCollectionNamespaceFromPrefix: (collectionPrefix) => self.qmdCollectionNamespaceFromPrefix(collectionPrefix),
|
|
3334
|
+
rememberNamespaceStorageDirHint: (namespace, storageDir) => self.rememberNamespaceStorageDirHint(namespace, storageDir),
|
|
3335
|
+
storageDirMatchesNamespaceHint: (namespace, storageDir) => self.storageDirMatchesNamespaceHint(namespace, storageDir),
|
|
3336
|
+
get storageRouter() { return self.storageRouter; },
|
|
3337
|
+
});
|
|
3338
|
+
}
|
|
3339
|
+
return this._namespaceReadFanoutCoordinator;
|
|
3340
|
+
}
|
|
3341
|
+
|
|
5149
3342
|
private async recallInternal(
|
|
5150
3343
|
prompt: string,
|
|
5151
3344
|
sessionKey?: string,
|
|
@@ -5280,60 +3473,9 @@ export class Orchestrator {
|
|
|
5280
3473
|
return this.config.defaultNamespace;
|
|
5281
3474
|
}
|
|
5282
3475
|
|
|
5283
|
-
/**
|
|
5284
|
-
* Lazily-constructed wearables service (Limitless / Bee / Omi
|
|
5285
|
-
* transcript ingestion). All wearables surfaces — CLI, MCP tools,
|
|
5286
|
-
* HTTP routes — share this one instance so sync state, search, and
|
|
5287
|
-
* memory writes stay consistent. Writes are pinned to the same
|
|
5288
|
-
* deterministic namespace bulk-import uses.
|
|
5289
|
-
*/
|
|
5290
3476
|
getWearablesService(): WearablesService {
|
|
5291
|
-
|
|
5292
|
-
|
|
5293
|
-
config: this.config.wearables,
|
|
5294
|
-
getStorage: async () =>
|
|
5295
|
-
await this.getStorageForNamespace(this.bulkImportWriteNamespace()),
|
|
5296
|
-
extract: (turns) => this.extraction.extract(turns),
|
|
5297
|
-
// Smart memoryMode runs candidates through the SAME extraction
|
|
5298
|
-
// judge (cache + defer counters included) the live extraction
|
|
5299
|
-
// pipeline uses, so wearable facts get identical LLM-as-judge
|
|
5300
|
-
// durability gating.
|
|
5301
|
-
judgeFacts: (candidates) =>
|
|
5302
|
-
judgeFactDurability(
|
|
5303
|
-
candidates,
|
|
5304
|
-
this.config,
|
|
5305
|
-
this.localLlm,
|
|
5306
|
-
new FallbackLlmClient(
|
|
5307
|
-
this.config.gatewayConfig,
|
|
5308
|
-
fallbackLlmRuntimeContextFromConfig(this.config),
|
|
5309
|
-
),
|
|
5310
|
-
this.judgeVerdictCache,
|
|
5311
|
-
this.judgeDeferCounts,
|
|
5312
|
-
),
|
|
5313
|
-
searchBackend: {
|
|
5314
|
-
search: async (query, maxResults) => {
|
|
5315
|
-
if (!this.qmd.isAvailable()) return null;
|
|
5316
|
-
try {
|
|
5317
|
-
const results = await this.qmd.search(query, undefined, maxResults);
|
|
5318
|
-
return results.map((result) => ({
|
|
5319
|
-
path: result.path,
|
|
5320
|
-
score: result.score,
|
|
5321
|
-
preview: result.snippet,
|
|
5322
|
-
}));
|
|
5323
|
-
} catch {
|
|
5324
|
-
// Backend hiccup → tell the service "unavailable" so it
|
|
5325
|
-
// runs its bounded scan fallback instead of returning a
|
|
5326
|
-
// silent empty result (CLAUDE.md rule 34).
|
|
5327
|
-
return null;
|
|
5328
|
-
}
|
|
5329
|
-
},
|
|
5330
|
-
},
|
|
5331
|
-
reindexSearch: async () => {
|
|
5332
|
-
await this.qmd.update();
|
|
5333
|
-
},
|
|
5334
|
-
});
|
|
5335
|
-
}
|
|
5336
|
-
return this.wearablesServiceInstance;
|
|
3477
|
+
return this.workspaceOpsCoordinator.getWearablesService(
|
|
3478
|
+
);
|
|
5337
3479
|
}
|
|
5338
3480
|
|
|
5339
3481
|
async ingestBulkImportBatch(
|
|
@@ -5593,22 +3735,12 @@ export class Orchestrator {
|
|
|
5593
3735
|
storage: StorageManager,
|
|
5594
3736
|
memoryId: string,
|
|
5595
3737
|
): Promise<void> {
|
|
5596
|
-
|
|
5597
|
-
|
|
5598
|
-
const memory = await storage.getMemoryById(memoryId);
|
|
5599
|
-
if (!memory) return;
|
|
5600
|
-
await this.embeddingFallback.indexFile(
|
|
3738
|
+
return this.persistenceIndexCoordinator.indexPersistedMemory(
|
|
3739
|
+
storage,
|
|
5601
3740
|
memoryId,
|
|
5602
|
-
memory.content,
|
|
5603
|
-
memory.path,
|
|
5604
3741
|
);
|
|
5605
3742
|
}
|
|
5606
3743
|
|
|
5607
|
-
/**
|
|
5608
|
-
* Build a graph edge for a persisted memory (v8.2).
|
|
5609
|
-
* Shared helper used by both the chunked and non-chunked write paths to avoid duplication.
|
|
5610
|
-
* Fail-open: caller wraps in try/catch.
|
|
5611
|
-
*/
|
|
5612
3744
|
private async buildGraphEdge(
|
|
5613
3745
|
storage: StorageManager,
|
|
5614
3746
|
memoryRelPath: string,
|
|
@@ -5622,65 +3754,19 @@ export class Orchestrator {
|
|
|
5622
3754
|
fallbackCausalPredecessor: string | undefined,
|
|
5623
3755
|
graphCaps: GraphConstructionCapabilitySet = resolveGraphConstructionCapabilities(this.config),
|
|
5624
3756
|
): Promise<void> {
|
|
5625
|
-
|
|
5626
|
-
|
|
5627
|
-
|
|
5628
|
-
try {
|
|
5629
|
-
const allMems = allMemsForGraph ?? [];
|
|
5630
|
-
for (const m of allMems) {
|
|
5631
|
-
if (m.frontmatter.entityRef === entityRef) {
|
|
5632
|
-
const rel = path.relative(storage.dir, m.path);
|
|
5633
|
-
if (rel !== memoryRelPath) entitySiblings.push(rel);
|
|
5634
|
-
}
|
|
5635
|
-
}
|
|
5636
|
-
} catch {
|
|
5637
|
-
/* fail-open */
|
|
5638
|
-
}
|
|
5639
|
-
}
|
|
5640
|
-
// Recent thread memories for time graph
|
|
5641
|
-
const recentInThread: string[] = [];
|
|
5642
|
-
if (threadIdForEdge && threadEpisodeIdsForGraph?.length) {
|
|
5643
|
-
try {
|
|
5644
|
-
recentInThread.push(
|
|
5645
|
-
...resolveRecentThreadMemoryPaths({
|
|
5646
|
-
threadEpisodeIds: threadEpisodeIdsForGraph,
|
|
5647
|
-
currentMemoryId: memoryId,
|
|
5648
|
-
allMemsForGraph,
|
|
5649
|
-
pathById: memoryPathById,
|
|
5650
|
-
storageDir: storage.dir,
|
|
5651
|
-
maxRecent: 3,
|
|
5652
|
-
}),
|
|
5653
|
-
);
|
|
5654
|
-
} catch {
|
|
5655
|
-
/* fail-open */
|
|
5656
|
-
}
|
|
5657
|
-
}
|
|
5658
|
-
if (
|
|
5659
|
-
recentInThread.length === 0 &&
|
|
5660
|
-
graphCaps.graphWriteSessionAdjacency &&
|
|
5661
|
-
fallbackCausalPredecessor &&
|
|
5662
|
-
fallbackCausalPredecessor !== memoryRelPath
|
|
5663
|
-
) {
|
|
5664
|
-
recentInThread.push(fallbackCausalPredecessor);
|
|
5665
|
-
}
|
|
5666
|
-
const causalPredecessor =
|
|
5667
|
-
recentInThread[recentInThread.length - 1] ?? fallbackCausalPredecessor;
|
|
5668
|
-
await this.graphIndexFor(storage).onMemoryWritten({
|
|
5669
|
-
memoryPath: memoryRelPath,
|
|
3757
|
+
return this.persistenceIndexCoordinator.buildGraphEdge(
|
|
3758
|
+
storage,
|
|
3759
|
+
memoryRelPath,
|
|
5670
3760
|
entityRef,
|
|
5671
|
-
|
|
5672
|
-
|
|
5673
|
-
|
|
5674
|
-
|
|
5675
|
-
|
|
5676
|
-
|
|
5677
|
-
|
|
5678
|
-
|
|
5679
|
-
|
|
5680
|
-
causalGraph: graphCaps.causalGraph,
|
|
5681
|
-
multiGraphMemory: graphCaps.multiGraphMemory,
|
|
5682
|
-
},
|
|
5683
|
-
});
|
|
3761
|
+
memoryId,
|
|
3762
|
+
factContent,
|
|
3763
|
+
allMemsForGraph,
|
|
3764
|
+
memoryPathById,
|
|
3765
|
+
threadIdForEdge,
|
|
3766
|
+
threadEpisodeIdsForGraph,
|
|
3767
|
+
fallbackCausalPredecessor,
|
|
3768
|
+
graphCaps,
|
|
3769
|
+
);
|
|
5684
3770
|
}
|
|
5685
3771
|
|
|
5686
3772
|
private graphIndexFor(storage: StorageManager): GraphIndex {
|
|
@@ -5692,89 +3778,14 @@ export class Orchestrator {
|
|
|
5692
3778
|
return created;
|
|
5693
3779
|
}
|
|
5694
3780
|
|
|
5695
|
-
/**
|
|
5696
|
-
* Batch-update temporal and tag indexes after extraction (v8.1).
|
|
5697
|
-
* Reads each persisted memory's path + frontmatter and adds them to
|
|
5698
|
-
* state/index_time.json and state/index_tags.json.
|
|
5699
|
-
* Fail-open: any error is logged but does not abort extraction.
|
|
5700
|
-
*/
|
|
5701
3781
|
private async updateTemporalTagIndexes(
|
|
5702
3782
|
storage: StorageManager,
|
|
5703
3783
|
persistedIds: string[],
|
|
5704
3784
|
): Promise<void> {
|
|
5705
|
-
|
|
5706
|
-
|
|
5707
|
-
|
|
5708
|
-
|
|
5709
|
-
// Enabling only parallelRetrievalEnabled without queryAwareIndexingEnabled would silently
|
|
5710
|
-
// produce an empty temporal index, leaving the temporal agent with no data to work from.
|
|
5711
|
-
if (
|
|
5712
|
-
!resolveIndexingCapabilities(this.config).queryAwareIndexing &&
|
|
5713
|
-
!caps.parallelRetrieval
|
|
5714
|
-
)
|
|
5715
|
-
return;
|
|
5716
|
-
// Check for missing indexes BEFORE the early-return so first-time enablement
|
|
5717
|
-
// can bootstrap the full corpus even when this extraction turn persisted nothing.
|
|
5718
|
-
const needsFullRebuild = !indexesExist(this.config.memoryDir);
|
|
5719
|
-
if (!needsFullRebuild && persistedIds.length === 0) return;
|
|
5720
|
-
try {
|
|
5721
|
-
// Read the corpus once to avoid N separate full-corpus scans.
|
|
5722
|
-
// On full rebuild with namespaces enabled, span all configured namespaces so
|
|
5723
|
-
// memories written to other namespaces before the index existed are also captured.
|
|
5724
|
-
const allMemories =
|
|
5725
|
-
needsFullRebuild && resolveNamespaceCapabilities(this.config).namespaces
|
|
5726
|
-
? await this.readAllMemoriesForNamespaces(
|
|
5727
|
-
Array.from(
|
|
5728
|
-
new Set<string>([
|
|
5729
|
-
this.config.defaultNamespace,
|
|
5730
|
-
this.config.sharedNamespace,
|
|
5731
|
-
...this.config.namespacePolicies.map((p) => p.name),
|
|
5732
|
-
]),
|
|
5733
|
-
),
|
|
5734
|
-
)
|
|
5735
|
-
: await storage.readAllMemories();
|
|
5736
|
-
|
|
5737
|
-
// Bootstrap: index only active (non-archived, non-superseded) memories.
|
|
5738
|
-
// Incremental: index only the newly persisted IDs.
|
|
5739
|
-
const pool = needsFullRebuild
|
|
5740
|
-
? allMemories.filter((m) => isActiveMemoryStatus(m.frontmatter.status))
|
|
5741
|
-
: (() => {
|
|
5742
|
-
const idSet = new Set(persistedIds);
|
|
5743
|
-
return allMemories.filter((m) => idSet.has(m.frontmatter.id));
|
|
5744
|
-
})();
|
|
5745
|
-
|
|
5746
|
-
const entries: Array<{
|
|
5747
|
-
path: string;
|
|
5748
|
-
createdAt: string;
|
|
5749
|
-
tags: string[];
|
|
5750
|
-
}> = [];
|
|
5751
|
-
for (const mem of pool) {
|
|
5752
|
-
if (mem.path && mem.frontmatter?.created) {
|
|
5753
|
-
entries.push({
|
|
5754
|
-
path: mem.path,
|
|
5755
|
-
createdAt: mem.frontmatter.created,
|
|
5756
|
-
tags: mem.frontmatter.tags ?? [],
|
|
5757
|
-
});
|
|
5758
|
-
}
|
|
5759
|
-
}
|
|
5760
|
-
if (needsFullRebuild) {
|
|
5761
|
-
// Always write empty indexes on full rebuild — even when the active pool
|
|
5762
|
-
// is empty (e.g. store contains only archived/superseded entries).
|
|
5763
|
-
// This marks bootstrap completion so indexesExist() returns true and
|
|
5764
|
-
// subsequent extractions skip the full-corpus scan.
|
|
5765
|
-
clearIndexes(this.config.memoryDir);
|
|
5766
|
-
if (entries.length > 0) {
|
|
5767
|
-
indexMemoriesBatch(this.config.memoryDir, entries);
|
|
5768
|
-
}
|
|
5769
|
-
log.info(
|
|
5770
|
-
`temporal-index: bootstrapped from ${entries.length} active memories`,
|
|
5771
|
-
);
|
|
5772
|
-
} else if (entries.length > 0) {
|
|
5773
|
-
indexMemoriesBatch(this.config.memoryDir, entries);
|
|
5774
|
-
}
|
|
5775
|
-
} catch (err) {
|
|
5776
|
-
log.debug(`temporal-index update failed (non-fatal): ${err}`);
|
|
5777
|
-
}
|
|
3785
|
+
return this.persistenceIndexCoordinator.updateTemporalTagIndexes(
|
|
3786
|
+
storage,
|
|
3787
|
+
persistedIds,
|
|
3788
|
+
);
|
|
5778
3789
|
}
|
|
5779
3790
|
|
|
5780
3791
|
/** IDs of facts persisted in the last extraction */
|
|
@@ -5823,93 +3834,16 @@ export class Orchestrator {
|
|
|
5823
3834
|
): Promise<number> {
|
|
5824
3835
|
return this.lifecyclePolicyCoordinator.runLifecyclePolicyPass(allMemories, storage);
|
|
5825
3836
|
}
|
|
5826
|
-
/**
|
|
5827
|
-
|
|
3837
|
+
/**
|
|
3838
|
+
* Threshold (bytes) at which IDENTITY.md reflections get auto-consolidated.
|
|
3839
|
+
* Read by WorkspaceOpsCoordinator (seam 24) via `Orchestrator.…`, hence
|
|
3840
|
+
* not `private`.
|
|
3841
|
+
*/
|
|
3842
|
+
static readonly IDENTITY_CONSOLIDATE_THRESHOLD = 8_000;
|
|
5828
3843
|
|
|
5829
3844
|
private async autoConsolidateIdentity(): Promise<void> {
|
|
5830
|
-
|
|
5831
|
-
|
|
5832
|
-
// auto-consolidation, otherwise its identity file grows unbounded and is never
|
|
5833
|
-
// consolidated. Falls back to the configured set on any catalog read failure.
|
|
5834
|
-
const namespaces = resolveNamespaceCapabilities(this.config).namespaces
|
|
5835
|
-
? await this.maintenanceNamespaces()
|
|
5836
|
-
: [this.config.defaultNamespace];
|
|
5837
|
-
|
|
5838
|
-
for (const namespace of namespaces) {
|
|
5839
|
-
const storage = await this.storageRouter.storageFor(namespace);
|
|
5840
|
-
const identityNamespace =
|
|
5841
|
-
resolveNamespaceCapabilities(this.config).namespaces &&
|
|
5842
|
-
namespace !== this.config.defaultNamespace
|
|
5843
|
-
? namespace
|
|
5844
|
-
: undefined;
|
|
5845
|
-
const reflectionsContent =
|
|
5846
|
-
(await storage.readIdentityReflections()) ?? "";
|
|
5847
|
-
|
|
5848
|
-
const existingIdentity = await storage.readIdentity(
|
|
5849
|
-
this.config.workspaceDir,
|
|
5850
|
-
identityNamespace,
|
|
5851
|
-
);
|
|
5852
|
-
const headerEnd =
|
|
5853
|
-
existingIdentity.indexOf("## Learned Patterns") !== -1
|
|
5854
|
-
? existingIdentity.indexOf("## Learned Patterns")
|
|
5855
|
-
: existingIdentity.indexOf("## Reflection");
|
|
5856
|
-
const staticHeader =
|
|
5857
|
-
(headerEnd !== -1
|
|
5858
|
-
? existingIdentity.slice(0, headerEnd)
|
|
5859
|
-
: existingIdentity
|
|
5860
|
-
).trimEnd() || "# IDENTITY";
|
|
5861
|
-
const identityContent = `${staticHeader}\n\n${reflectionsContent.trim()}\n`;
|
|
5862
|
-
if (identityContent.length < Orchestrator.IDENTITY_CONSOLIDATE_THRESHOLD)
|
|
5863
|
-
continue;
|
|
5864
|
-
|
|
5865
|
-
log.info(
|
|
5866
|
-
`IDENTITY(${namespace}) is ${identityContent.length} chars — auto-consolidating reflections`,
|
|
5867
|
-
);
|
|
5868
|
-
const result = await this.extraction.consolidateIdentity(
|
|
5869
|
-
identityContent,
|
|
5870
|
-
"## Reflection",
|
|
5871
|
-
);
|
|
5872
|
-
|
|
5873
|
-
if (!result || result.learnedPatterns.length === 0) {
|
|
5874
|
-
log.warn(
|
|
5875
|
-
`identity consolidation produced no patterns for namespace=${namespace}`,
|
|
5876
|
-
);
|
|
5877
|
-
continue;
|
|
5878
|
-
}
|
|
5879
|
-
|
|
5880
|
-
const patternsSection = [
|
|
5881
|
-
"## Learned Patterns (consolidated from reflections, " +
|
|
5882
|
-
new Date().toISOString().slice(0, 10) +
|
|
5883
|
-
")",
|
|
5884
|
-
"",
|
|
5885
|
-
...result.learnedPatterns.map((p) => `- ${p}`),
|
|
5886
|
-
"",
|
|
5887
|
-
].join("\n");
|
|
5888
|
-
|
|
5889
|
-
const newContent = staticHeader + "\n\n" + patternsSection + "\n";
|
|
5890
|
-
|
|
5891
|
-
await storage.writeIdentity(
|
|
5892
|
-
this.config.workspaceDir,
|
|
5893
|
-
newContent,
|
|
5894
|
-
identityNamespace,
|
|
5895
|
-
);
|
|
5896
|
-
await storage.writeIdentityReflections("");
|
|
5897
|
-
// NRcCL (codex P2): record a per-namespace catalog write for THIS namespace
|
|
5898
|
-
// after the identity files are updated. This fan-out can mutate a dynamic
|
|
5899
|
-
// namespace via `writeIdentity`/`writeIdentityReflections`, but the
|
|
5900
|
-
// consolidation pass's only consolidated touch covers `this.storage` (the
|
|
5901
|
-
// default) and only fires when `memoryItemMutated` was set by OTHER work — so
|
|
5902
|
-
// a namespace whose sole mutation in the pass is identity consolidation would
|
|
5903
|
-
// otherwise keep a stale `lastWriteAt`, making `listNamespaces({ writtenSince })`
|
|
5904
|
-
// and catalog-recency consumers miss the write. Best-effort and
|
|
5905
|
-
// failure-tolerant (the storage chokepoint (#1522) swallows errors, never crashing the
|
|
5906
|
-
// consolidation; gotcha #13, rule #40). No double-count with the consolidated
|
|
5907
|
-
// touch above: that one is gated on `memoryItemMutated` (which identity
|
|
5908
|
-
// consolidation does not set), and `markWrite` is idempotent regardless.
|
|
5909
|
-
log.info(
|
|
5910
|
-
`IDENTITY(${namespace}) consolidated: ${identityContent.length} → ${newContent.length} chars, ${result.learnedPatterns.length} patterns`,
|
|
5911
|
-
);
|
|
5912
|
-
}
|
|
3845
|
+
return this.workspaceOpsCoordinator.autoConsolidateIdentity(
|
|
3846
|
+
);
|
|
5913
3847
|
}
|
|
5914
3848
|
|
|
5915
3849
|
// Issue #1526: recall result formatting moved to RecallResultFormatter. Thin
|
|
@@ -6198,60 +4132,16 @@ export class Orchestrator {
|
|
|
6198
4132
|
);
|
|
6199
4133
|
}
|
|
6200
4134
|
|
|
6201
|
-
/**
|
|
6202
|
-
* Issue #373 — nearest-neighbor lookup for the write-time semantic dedup
|
|
6203
|
-
* guard. Returns the top-K embedding hits against the currently indexed
|
|
6204
|
-
* memories, or an empty array when the embedding backend is unavailable.
|
|
6205
|
-
* Intentionally does NOT throw; `decideSemanticDedup` treats both "empty"
|
|
6206
|
-
* and "error" outcomes as fail-open (keep the candidate).
|
|
6207
|
-
*
|
|
6208
|
-
* PR #399 P1 fix: when namespaces are enabled the lookup must be scoped
|
|
6209
|
-
* to the SAME namespace as the fact being written. Otherwise a
|
|
6210
|
-
* high-similarity memory from another namespace can suppress a write in
|
|
6211
|
-
* the target namespace — cross-tenant data loss. Callers pass the target
|
|
6212
|
-
* storage so we can translate its root directory into the correct index
|
|
6213
|
-
* path prefix (and, for the legacy default-namespace layout at
|
|
6214
|
-
* `memoryDir` root, an exclusion list for `namespaces/*`).
|
|
6215
|
-
*/
|
|
6216
4135
|
async semanticDedupLookup(
|
|
6217
4136
|
content: string,
|
|
6218
4137
|
limit: number,
|
|
6219
4138
|
targetStorage: StorageManager,
|
|
6220
4139
|
): Promise<SemanticDedupHit[]> {
|
|
6221
|
-
|
|
6222
|
-
|
|
6223
|
-
|
|
6224
|
-
|
|
6225
|
-
|
|
6226
|
-
//
|
|
6227
|
-
// Contract after this fix:
|
|
6228
|
-
// • embeddingFallbackEnabled=false → throw (feature not configured;
|
|
6229
|
-
// caller treats this as backend_unavailable and fails open).
|
|
6230
|
-
// • isAvailable() returns false → throw (provider is reachable but
|
|
6231
|
-
// reports itself unavailable; distinct from empty index).
|
|
6232
|
-
// • search() throws → re-throw (network/provider error).
|
|
6233
|
-
// • search() returns [] → return [] (empty index, not a
|
|
6234
|
-
// backend failure; decideSemanticDedup reports no_candidates).
|
|
6235
|
-
if (!resolveMemoryLifecycleCapabilities(this.config).embeddingFallback) {
|
|
6236
|
-
throw new Error("semantic dedup: embedding backend not configured");
|
|
6237
|
-
}
|
|
6238
|
-
if (!(await this.embeddingFallback.isAvailable())) {
|
|
6239
|
-
log.debug("semantic dedup: embedding backend unavailable, skipping");
|
|
6240
|
-
throw new Error("semantic dedup: embedding backend unavailable");
|
|
6241
|
-
}
|
|
6242
|
-
// search() may throw — let it propagate so decideSemanticDedup catches it
|
|
6243
|
-
// and returns reason="backend_unavailable". Pass throwOnTimeout:true so
|
|
6244
|
-
// EmbeddingTimeoutError is re-thrown here (Round 10 fix, Ui1J+Ui1L: the
|
|
6245
|
-
// recall-path caller searchEmbeddingFallback does NOT pass this flag,
|
|
6246
|
-
// keeping its fail-open [] contract on timeout).
|
|
6247
|
-
const scope = this.semanticDedupScopeFor(targetStorage);
|
|
6248
|
-
const hits = await this.embeddingFallback.search(content, limit, { ...scope, throwOnTimeout: true });
|
|
6249
|
-
if (!Array.isArray(hits) || hits.length === 0) return [];
|
|
6250
|
-
return hits.map((hit) => ({
|
|
6251
|
-
id: hit.id,
|
|
6252
|
-
score: hit.score,
|
|
6253
|
-
path: hit.path,
|
|
6254
|
-
}));
|
|
4140
|
+
return this.persistenceIndexCoordinator.semanticDedupLookup(
|
|
4141
|
+
content,
|
|
4142
|
+
limit,
|
|
4143
|
+
targetStorage,
|
|
4144
|
+
);
|
|
6255
4145
|
}
|
|
6256
4146
|
|
|
6257
4147
|
/**
|
|
@@ -6404,52 +4294,9 @@ export class Orchestrator {
|
|
|
6404
4294
|
}
|
|
6405
4295
|
}
|
|
6406
4296
|
|
|
6407
|
-
/**
|
|
6408
|
-
* Flush access tracking buffer to disk.
|
|
6409
|
-
* Called during consolidation or when buffer is full.
|
|
6410
|
-
*/
|
|
6411
4297
|
async flushAccessTracking(): Promise<void> {
|
|
6412
|
-
|
|
6413
|
-
|
|
6414
|
-
// Build entries from buffer, merging with existing counts
|
|
6415
|
-
const entries: AccessTrackingEntry[] = [];
|
|
6416
|
-
const namespaces = resolveNamespaceCapabilities(this.config).namespaces
|
|
6417
|
-
? Array.from(
|
|
6418
|
-
new Set<string>([
|
|
6419
|
-
this.config.defaultNamespace,
|
|
6420
|
-
this.config.sharedNamespace,
|
|
6421
|
-
...this.config.namespacePolicies.map((p) => p.name),
|
|
6422
|
-
]),
|
|
6423
|
-
)
|
|
6424
|
-
: [this.config.defaultNamespace];
|
|
6425
|
-
const memories = await this.readAllMemoriesForNamespaces(namespaces);
|
|
6426
|
-
const memoryMap = new Map(memories.map((m) => [m.frontmatter.id, m]));
|
|
6427
|
-
|
|
6428
|
-
for (const [memoryId, update] of this.accessTrackingBuffer) {
|
|
6429
|
-
const memory = memoryMap.get(memoryId);
|
|
6430
|
-
const existingCount = memory?.frontmatter.accessCount ?? 0;
|
|
6431
|
-
entries.push({
|
|
6432
|
-
memoryId,
|
|
6433
|
-
newCount: existingCount + update.count,
|
|
6434
|
-
lastAccessed: update.lastAccessed,
|
|
6435
|
-
});
|
|
6436
|
-
}
|
|
6437
|
-
|
|
6438
|
-
const byNamespace = new Map<string, AccessTrackingEntry[]>();
|
|
6439
|
-
for (const e of entries) {
|
|
6440
|
-
const m = memoryMap.get(e.memoryId);
|
|
6441
|
-
if (!m) continue;
|
|
6442
|
-
const ns = this.namespaceFromPath(m.path);
|
|
6443
|
-
const list = byNamespace.get(ns) ?? [];
|
|
6444
|
-
list.push(e);
|
|
6445
|
-
byNamespace.set(ns, list);
|
|
6446
|
-
}
|
|
6447
|
-
for (const [ns, list] of byNamespace) {
|
|
6448
|
-
const sm = await this.storageRouter.storageFor(ns);
|
|
6449
|
-
await sm.flushAccessTracking(list);
|
|
6450
|
-
}
|
|
6451
|
-
this.accessTrackingBuffer.clear();
|
|
6452
|
-
log.debug(`flushed ${entries.length} access tracking entries`);
|
|
4298
|
+
return this.workspaceOpsCoordinator.flushAccessTracking(
|
|
4299
|
+
);
|
|
6453
4300
|
}
|
|
6454
4301
|
|
|
6455
4302
|
private async loadSearchResultMemoryMap(
|
|
@@ -6698,29 +4545,15 @@ export class Orchestrator {
|
|
|
6698
4545
|
}
|
|
6699
4546
|
|
|
6700
4547
|
private namespaceFromPath(p: string): string {
|
|
6701
|
-
|
|
6702
|
-
|
|
6703
|
-
|
|
6704
|
-
? this.qmdCollectionNamespaceFromPrefix(parts.collection)
|
|
6705
|
-
: null;
|
|
6706
|
-
if (collectionNamespace) return collectionNamespace;
|
|
6707
|
-
const m = p.match(/[\\/]+namespaces[\\/]+([^\\/]+)(?:[\\/]|$)/);
|
|
6708
|
-
if (!m?.[1]) return this.config.defaultNamespace;
|
|
6709
|
-
return namespaceIdentityFromToken(m[1]) ?? m[1];
|
|
4548
|
+
return this.namespaceReadFanoutCoordinator.namespaceFromPath(
|
|
4549
|
+
p,
|
|
4550
|
+
);
|
|
6710
4551
|
}
|
|
6711
4552
|
|
|
6712
4553
|
private storageDirNamespace(storageDir: string): string {
|
|
6713
|
-
|
|
6714
|
-
|
|
6715
|
-
|
|
6716
|
-
// loaded lazily via the callback (only after early returns, matching the
|
|
6717
|
-
// original behavior — codex P2).
|
|
6718
|
-
return resolveNamespaceFromStorageDir(storageDir, {
|
|
6719
|
-
config: this.config,
|
|
6720
|
-
configuredNamespaces: this.configuredNamespaceList(),
|
|
6721
|
-
hints: this.namespaceStorageDirHints,
|
|
6722
|
-
loadHints: () => this.loadNamespaceStorageDirHintsFromCatalog(),
|
|
6723
|
-
});
|
|
4554
|
+
return this.namespaceReadFanoutCoordinator.storageDirNamespace(
|
|
4555
|
+
storageDir,
|
|
4556
|
+
);
|
|
6724
4557
|
}
|
|
6725
4558
|
|
|
6726
4559
|
// #1522: catalog touch methods removed — touches now happen at the storage chokepoint.
|
|
@@ -6746,26 +4579,16 @@ export class Orchestrator {
|
|
|
6746
4579
|
private async readAllMemoriesForNamespaces(
|
|
6747
4580
|
namespaces: string[],
|
|
6748
4581
|
): Promise<MemoryFile[]> {
|
|
6749
|
-
|
|
6750
|
-
|
|
6751
|
-
uniq.map(async (ns) => {
|
|
6752
|
-
const sm = await this.storageRouter.storageFor(ns);
|
|
6753
|
-
return sm.readAllMemories();
|
|
6754
|
-
}),
|
|
4582
|
+
return this.namespaceReadFanoutCoordinator.readAllMemoriesForNamespaces(
|
|
4583
|
+
namespaces,
|
|
6755
4584
|
);
|
|
6756
|
-
return lists.flat();
|
|
6757
4585
|
}
|
|
6758
4586
|
|
|
6759
4587
|
private async readArchivedMemoriesForNamespaces(
|
|
6760
4588
|
namespaces: string[],
|
|
6761
4589
|
): Promise<MemoryFile[]> {
|
|
6762
|
-
|
|
6763
|
-
|
|
6764
|
-
uniq.map(async (ns) => {
|
|
6765
|
-
const sm = await this.storageRouter.storageFor(ns);
|
|
6766
|
-
return sm.readArchivedMemories();
|
|
6767
|
-
}),
|
|
4590
|
+
return this.namespaceReadFanoutCoordinator.readArchivedMemoriesForNamespaces(
|
|
4591
|
+
namespaces,
|
|
6768
4592
|
);
|
|
6769
|
-
return lists.flat();
|
|
6770
4593
|
}
|
|
6771
4594
|
}
|