@remnic/core 9.3.744 → 9.3.746

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. package/dist/access-boundary.d.ts +17 -15
  2. package/dist/access-boundary.js +14 -15
  3. package/dist/access-cli.js +92 -33
  4. package/dist/access-cli.js.map +1 -1
  5. package/dist/access-http.d.ts +17 -15
  6. package/dist/access-http.js +20 -21
  7. package/dist/access-mcp.d.ts +17 -15
  8. package/dist/access-mcp.js +18 -19
  9. package/dist/access-operations-batch.js +16 -17
  10. package/dist/access-operations.d.ts +17 -15
  11. package/dist/access-operations.js +17 -18
  12. package/dist/access-schema.js +3 -3
  13. package/dist/{access-service-OT3Gk79W.d.ts → access-service-Cx16nmCv.d.ts} +3 -30
  14. package/dist/access-service.d.ts +7 -5
  15. package/dist/access-service.js +13 -14
  16. package/dist/access-surface-catalog.d.ts +17 -15
  17. package/dist/action-confidence.d.ts +1 -1
  18. package/dist/active-memory-bridge.d.ts +1 -1
  19. package/dist/active-recall.d.ts +1 -1
  20. package/dist/{auto-sync-XHSKKQAH.js → auto-sync-IEBKPGAM.js} +2 -2
  21. package/dist/behavior-learner.d.ts +1 -1
  22. package/dist/behavior-signals.d.ts +1 -1
  23. package/dist/bootstrap.d.ts +6 -4
  24. package/dist/briefing.d.ts +1 -1
  25. package/dist/briefing.js +3 -3
  26. package/dist/buffer-surprise-report.d.ts +1 -1
  27. package/dist/buffer.d.ts +1 -1
  28. package/dist/calibration.d.ts +1 -1
  29. package/dist/capabilities.d.ts +1 -1
  30. package/dist/{catalog-CX0CjOaf.d.ts → catalog-D_ZDxu8Y.d.ts} +1 -1
  31. package/dist/causal-behavior.d.ts +1 -1
  32. package/dist/causal-consolidation.d.ts +1 -1
  33. package/dist/causal-consolidation.js +4 -4
  34. package/dist/causal-trajectory-graph.d.ts +1 -1
  35. package/dist/{chunk-WPLRJ4XM.js → chunk-24YDB5QP.js} +2098 -230
  36. package/dist/chunk-24YDB5QP.js.map +1 -0
  37. package/dist/{chunk-6XG3NQRC.js → chunk-3IJGF6K2.js} +2 -2
  38. package/dist/{chunk-U5WVRMCA.js → chunk-5IN26V6W.js} +2 -2
  39. package/dist/{chunk-XEADGIAA.js → chunk-DIHCFJVV.js} +7 -7
  40. package/dist/{chunk-C7LDPZJJ.js → chunk-EHYB325U.js} +227 -227
  41. package/dist/chunk-EHYB325U.js.map +1 -0
  42. package/dist/{chunk-E7RGJQWP.js → chunk-HIHCB2S6.js} +2 -2
  43. package/dist/{chunk-JFCS6PQU.js → chunk-JHQUIH3W.js} +5 -5
  44. package/dist/{chunk-MXDG6SRH.js → chunk-JRDPT6DA.js} +2 -2
  45. package/dist/{chunk-BYLC4DKI.js → chunk-KMNYXLPL.js} +2 -2
  46. package/dist/{chunk-TQPSOTC7.js → chunk-L4N7SVNN.js} +8 -10
  47. package/dist/{chunk-TQPSOTC7.js.map → chunk-L4N7SVNN.js.map} +1 -1
  48. package/dist/{chunk-DYK27X7V.js → chunk-LMFS64GY.js} +3 -3
  49. package/dist/{chunk-FWXLKH2H.js → chunk-NVYF5OB6.js} +3 -3
  50. package/dist/{chunk-G5FP7VSQ.js → chunk-P4FW3ZU5.js} +58 -60
  51. package/dist/{chunk-G5FP7VSQ.js.map → chunk-P4FW3ZU5.js.map} +1 -1
  52. package/dist/{chunk-SODQQCAZ.js → chunk-PJE6D3NH.js} +10362 -4909
  53. package/dist/chunk-PJE6D3NH.js.map +1 -0
  54. package/dist/{chunk-56F6EAQ6.js → chunk-Q3X7GUAQ.js} +2 -2
  55. package/dist/{chunk-7N7S2RKF.js → chunk-R2OAN6IP.js} +2 -2
  56. package/dist/{chunk-CHELXDTO.js → chunk-RB6A7OUN.js} +2 -2
  57. package/dist/{chunk-3EA5LIFJ.js → chunk-SIV3JZPP.js} +2 -2
  58. package/dist/{chunk-IP5SANO2.js → chunk-UIKZZ242.js} +2 -2
  59. package/dist/{chunk-UGFPGWJB.js → chunk-URQLO7BX.js} +2 -2
  60. package/dist/{chunk-5UNSGRCR.js → chunk-WK3NK6SP.js} +162 -16
  61. package/dist/chunk-WK3NK6SP.js.map +1 -0
  62. package/dist/{chunk-VE54E6RP.js → chunk-WWVXXSLF.js} +4 -6
  63. package/dist/{chunk-VE54E6RP.js.map → chunk-WWVXXSLF.js.map} +1 -1
  64. package/dist/{chunk-WO22UEM3.js → chunk-XTEWVF3C.js} +2 -2
  65. package/dist/{chunk-FCQ7ZYCQ.js → chunk-ZGAJB74V.js} +2 -2
  66. package/dist/{chunk-NBP3YHHS.js → chunk-ZGZNLRKF.js} +2 -2
  67. package/dist/{chunk-C7C5WIIF.js → chunk-ZPZ4VVLT.js} +2 -2
  68. package/dist/{cli-BQiTZ7Zs.d.ts → cli-D1y3vooB.d.ts} +3 -3
  69. package/dist/cli.d.ts +9 -7
  70. package/dist/cli.js +44 -46
  71. package/dist/compounding/engine.d.ts +1 -1
  72. package/dist/compounding/engine.js +3 -3
  73. package/dist/compounding/preference-consolidator.d.ts +1 -1
  74. package/dist/compression-optimizer.d.ts +1 -1
  75. package/dist/config.d.ts +1 -1
  76. package/dist/connectors/codex-materialize-runner.d.ts +1 -1
  77. package/dist/connectors/codex-materialize-runner.js +3 -3
  78. package/dist/connectors/codex-materialize.d.ts +1 -1
  79. package/dist/connectors/index.d.ts +1 -1
  80. package/dist/connectors/index.js +3 -3
  81. package/dist/consolidation-provenance-check.d.ts +1 -1
  82. package/dist/consolidation-undo.d.ts +1 -1
  83. package/dist/contradiction/index.d.ts +1 -1
  84. package/dist/contradiction/index.js +4 -4
  85. package/dist/conversation-index/backend.d.ts +1 -1
  86. package/dist/conversation-index/chunker.d.ts +1 -1
  87. package/dist/conversation-index/faiss-adapter.d.ts +1 -1
  88. package/dist/conversation-index/indexer.d.ts +1 -1
  89. package/dist/conversation-index/search.d.ts +1 -1
  90. package/dist/day-summary.d.ts +1 -1
  91. package/dist/delinearize.d.ts +1 -1
  92. package/dist/direct-answer-wiring.d.ts +1 -1
  93. package/dist/direct-answer.d.ts +1 -1
  94. package/dist/embedding-fallback.d.ts +1 -1
  95. package/dist/enrichment/index.d.ts +1 -1
  96. package/dist/entity-retrieval.d.ts +1 -1
  97. package/dist/entity-retrieval.js +3 -3
  98. package/dist/entity-schema.d.ts +1 -1
  99. package/dist/explicit-capture.d.ts +6 -4
  100. package/dist/extraction-faithfulness.d.ts +1 -1
  101. package/dist/extraction-judge-telemetry.d.ts +1 -1
  102. package/dist/extraction-judge-training.d.ts +1 -1
  103. package/dist/extraction-judge.d.ts +1 -1
  104. package/dist/extraction.d.ts +1 -1
  105. package/dist/fallback-llm.d.ts +1 -1
  106. package/dist/graph-dashboard-diff.d.ts +1 -1
  107. package/dist/graph-dashboard-key.d.ts +1 -1
  108. package/dist/graph-dashboard-parser.d.ts +1 -1
  109. package/dist/graph-edge-reinforcement.d.ts +1 -1
  110. package/dist/graph-snapshot.d.ts +1 -1
  111. package/dist/graph.d.ts +1 -1
  112. package/dist/identity-continuity.d.ts +1 -1
  113. package/dist/importance.d.ts +1 -1
  114. package/dist/index.d.ts +12 -10
  115. package/dist/index.js +271 -5362
  116. package/dist/index.js.map +1 -1
  117. package/dist/intent.d.ts +1 -1
  118. package/dist/lcm/engine.d.ts +1 -1
  119. package/dist/lcm/index.d.ts +1 -1
  120. package/dist/lcm/tools.d.ts +1 -1
  121. package/dist/lifecycle.d.ts +1 -1
  122. package/dist/live-connectors-runner.d.ts +1 -1
  123. package/dist/local-llm.d.ts +1 -1
  124. package/dist/local-model-endpoint.d.ts +1 -1
  125. package/dist/maintenance/memory-governance.d.ts +1 -1
  126. package/dist/maintenance/memory-governance.js +3 -3
  127. package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
  128. package/dist/maintenance/rebuild-memory-projection.js +4 -4
  129. package/dist/mcp-memory-inspector-app.d.ts +16 -14
  130. package/dist/memory-action-policy.d.ts +1 -1
  131. package/dist/memory-cache.d.ts +1 -1
  132. package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
  133. package/dist/memory-projection-store.d.ts +1 -1
  134. package/dist/memory-provenance.d.ts +1 -1
  135. package/dist/memory-worth-outcomes.d.ts +1 -1
  136. package/dist/models-json.d.ts +1 -1
  137. package/dist/namespaces/migrate.d.ts +2 -2
  138. package/dist/namespaces/migrate.js +4 -4
  139. package/dist/namespaces/principal.d.ts +1 -1
  140. package/dist/namespaces/search.d.ts +1 -1
  141. package/dist/namespaces/storage.d.ts +2 -2
  142. package/dist/namespaces/storage.js +3 -3
  143. package/dist/native-knowledge.d.ts +1 -1
  144. package/dist/operator-toolkit.d.ts +1 -1
  145. package/dist/operator-toolkit.js +10 -11
  146. package/dist/orchestration/compression-guideline-coordinator.d.ts +1 -1
  147. package/dist/orchestration/maintenance.d.ts +2 -2
  148. package/dist/orchestration/maintenance.js +5 -5
  149. package/dist/{orchestrator-BPQGHpJz.d.ts → orchestrator-CnCE3Hvq.d.ts} +180 -12
  150. package/dist/orchestrator.d.ts +6 -4
  151. package/dist/orchestrator.js +119 -16
  152. package/dist/patterns-cli.d.ts +1 -1
  153. package/dist/policy-runtime.d.ts +1 -1
  154. package/dist/provenance.d.ts +1 -1
  155. package/dist/qmd-recall-cache.d.ts +1 -1
  156. package/dist/qmd.d.ts +1 -1
  157. package/dist/recall-disclosure-escalation.d.ts +1 -1
  158. package/dist/recall-explain-renderer.d.ts +1 -1
  159. package/dist/recall-planner-llm.d.ts +1 -1
  160. package/dist/recall-state.d.ts +1 -1
  161. package/dist/recall-tag-filter.d.ts +1 -1
  162. package/dist/recall-xray-cli.d.ts +1 -1
  163. package/dist/recall-xray-renderer.d.ts +1 -1
  164. package/dist/recall-xray.d.ts +1 -1
  165. package/dist/resolve-auth-token.d.ts +1 -1
  166. package/dist/retrieval-agents.d.ts +1 -1
  167. package/dist/retrieval-tiers.d.ts +1 -1
  168. package/dist/routing/engine.d.ts +1 -1
  169. package/dist/routing/store.d.ts +1 -1
  170. package/dist/search/embed-helper.d.ts +1 -1
  171. package/dist/search/factory.d.ts +1 -1
  172. package/dist/search/index.d.ts +1 -1
  173. package/dist/search/lancedb-backend.d.ts +1 -1
  174. package/dist/search/meilisearch-backend.d.ts +1 -1
  175. package/dist/search/noop-backend.d.ts +1 -1
  176. package/dist/search/orama-backend.d.ts +1 -1
  177. package/dist/search/port.d.ts +1 -1
  178. package/dist/search/remote-backend.d.ts +1 -1
  179. package/dist/{semantic-consolidation-ByzGwC9c.d.ts → semantic-consolidation-D3aSiu9t.d.ts} +1 -1
  180. package/dist/semantic-consolidation.d.ts +2 -2
  181. package/dist/semantic-consolidation.js +4 -4
  182. package/dist/semantic-rule-promotion.js +3 -3
  183. package/dist/semantic-rule-verifier.d.ts +1 -1
  184. package/dist/semantic-rule-verifier.js +3 -3
  185. package/dist/session-observer-bands.d.ts +1 -1
  186. package/dist/session-observer-state.d.ts +1 -1
  187. package/dist/shared-context/manager.d.ts +1 -1
  188. package/dist/signal.d.ts +1 -1
  189. package/dist/storage.d.ts +1 -1
  190. package/dist/storage.js +2 -2
  191. package/dist/summarizer.d.ts +1 -1
  192. package/dist/summary-snapshot.d.ts +1 -1
  193. package/dist/temporal-supersession.d.ts +1 -1
  194. package/dist/temporal-validity.d.ts +1 -1
  195. package/dist/threading.d.ts +1 -1
  196. package/dist/tier-migration.d.ts +1 -1
  197. package/dist/tier-routing.d.ts +1 -1
  198. package/dist/topics.d.ts +1 -1
  199. package/dist/transcript.d.ts +1 -1
  200. package/dist/transfer/backup.js +2 -2
  201. package/dist/transfer/capsule-export.js +2 -2
  202. package/dist/transfer/capsule-import.js +1 -1
  203. package/dist/trust-score-stage.d.ts +1 -1
  204. package/dist/trust-score.d.ts +1 -1
  205. package/dist/{types-5Jik3r4G.d.ts → types-CZm6rx9-.d.ts} +1 -1
  206. package/dist/types.d.ts +1 -1
  207. package/dist/utility-runtime.d.ts +1 -1
  208. package/dist/verified-recall.js +3 -3
  209. package/package.json +2 -2
  210. package/src/index.ts +2 -0
  211. package/src/orchestration/consolidation-run.ts +633 -0
  212. package/src/orchestration/extraction-persist.ts +2785 -0
  213. package/src/orchestration/extraction-run.ts +874 -0
  214. package/src/orchestrator.ts +206 -3815
  215. package/dist/chunk-5UNSGRCR.js.map +0 -1
  216. package/dist/chunk-A2KKLCOS.js +0 -1889
  217. package/dist/chunk-A2KKLCOS.js.map +0 -1
  218. package/dist/chunk-AVBT5WEW.js +0 -153
  219. package/dist/chunk-AVBT5WEW.js.map +0 -1
  220. package/dist/chunk-C7LDPZJJ.js.map +0 -1
  221. package/dist/chunk-SODQQCAZ.js.map +0 -1
  222. package/dist/chunk-WPLRJ4XM.js.map +0 -1
  223. /package/dist/{auto-sync-XHSKKQAH.js.map → auto-sync-IEBKPGAM.js.map} +0 -0
  224. /package/dist/{chunk-6XG3NQRC.js.map → chunk-3IJGF6K2.js.map} +0 -0
  225. /package/dist/{chunk-U5WVRMCA.js.map → chunk-5IN26V6W.js.map} +0 -0
  226. /package/dist/{chunk-XEADGIAA.js.map → chunk-DIHCFJVV.js.map} +0 -0
  227. /package/dist/{chunk-E7RGJQWP.js.map → chunk-HIHCB2S6.js.map} +0 -0
  228. /package/dist/{chunk-JFCS6PQU.js.map → chunk-JHQUIH3W.js.map} +0 -0
  229. /package/dist/{chunk-MXDG6SRH.js.map → chunk-JRDPT6DA.js.map} +0 -0
  230. /package/dist/{chunk-BYLC4DKI.js.map → chunk-KMNYXLPL.js.map} +0 -0
  231. /package/dist/{chunk-DYK27X7V.js.map → chunk-LMFS64GY.js.map} +0 -0
  232. /package/dist/{chunk-FWXLKH2H.js.map → chunk-NVYF5OB6.js.map} +0 -0
  233. /package/dist/{chunk-56F6EAQ6.js.map → chunk-Q3X7GUAQ.js.map} +0 -0
  234. /package/dist/{chunk-7N7S2RKF.js.map → chunk-R2OAN6IP.js.map} +0 -0
  235. /package/dist/{chunk-CHELXDTO.js.map → chunk-RB6A7OUN.js.map} +0 -0
  236. /package/dist/{chunk-3EA5LIFJ.js.map → chunk-SIV3JZPP.js.map} +0 -0
  237. /package/dist/{chunk-IP5SANO2.js.map → chunk-UIKZZ242.js.map} +0 -0
  238. /package/dist/{chunk-UGFPGWJB.js.map → chunk-URQLO7BX.js.map} +0 -0
  239. /package/dist/{chunk-WO22UEM3.js.map → chunk-XTEWVF3C.js.map} +0 -0
  240. /package/dist/{chunk-FCQ7ZYCQ.js.map → chunk-ZGAJB74V.js.map} +0 -0
  241. /package/dist/{chunk-NBP3YHHS.js.map → chunk-ZGZNLRKF.js.map} +0 -0
  242. /package/dist/{chunk-C7C5WIIF.js.map → chunk-ZPZ4VVLT.js.map} +0 -0
@@ -81,7 +81,6 @@ import { NoopSearchBackend } from "./search/noop-backend.js";
81
81
  import {
82
82
  StorageManager,
83
83
  ContentHashIndex,
84
- normalizeAttributePairs,
85
84
  } from "./storage.js";
86
85
  import { sanitizeMemoryContent } from "./sanitize.js";
87
86
  import { ThreadingManager } from "./threading.js";
@@ -107,6 +106,9 @@ import { RecallRerankCoordinator } from "./orchestration/recall-rerank-coordinat
107
106
  import { RecallSectionCoordinator } from "./orchestration/recall-section-coordinator.js";
108
107
  import { QmdResultResolver, qmdCollectionPathParts, qmdResultPathCandidates } from "./orchestration/qmd-result-resolver.js";
109
108
  import { ContradictionLinkingCoordinator } from "./orchestration/contradiction-linking-coordinator.js";
109
+ import { ExtractionRunCoordinator, type ExtractionRunResult } from "./orchestration/extraction-run.js";
110
+ import { ConsolidationRunCoordinator } from "./orchestration/consolidation-run.js";
111
+ import { ExtractionPersistCoordinator } from "./orchestration/extraction-persist.js";
110
112
  export { hasIdentityRecoveryIntent, resolveEffectiveIdentityInjectionMode } from "./orchestration/recall-result-formatter.js";
111
113
  import {
112
114
  GraphRecallCoordinator,
@@ -489,13 +491,8 @@ export class BulkImportBatchPartialFailureError extends Error {
489
491
  }
490
492
  }
491
493
 
492
- interface ExtractionRunResult {
493
- status: "completed" | "skipped";
494
- reason?: string;
495
- persistedCount: number;
496
- durableOutputCount: number;
497
- postPersistMetadataFailed?: boolean;
498
- }
494
+ // Issue #1526 seam 15: ExtractionRunResult moved to orchestration/extraction-run.ts.
495
+ export type { ExtractionRunResult } from "./orchestration/extraction-run.js";
499
496
 
500
497
  export interface GraphRecallSnapshot {
501
498
  recordedAt: string;
@@ -914,20 +911,6 @@ export function sanitizeSessionKeyForFilename(sessionKey: string): string {
914
911
  return `${readable}-${hash}`;
915
912
  }
916
913
 
917
- function latestSourceValidAtFromTurns(turns: readonly BufferTurn[]): string | undefined {
918
- let latestMs: number | null = null;
919
- for (const turn of turns) {
920
- if (turn.extractionContextOnly === true) continue;
921
- if (typeof turn.sourceValidAt !== "string") continue;
922
- const parsed = parseFlexibleIsoTimestamp(turn.sourceValidAt.trim());
923
- if (parsed === null) continue;
924
- if (latestMs === null || parsed > latestMs) {
925
- latestMs = parsed;
926
- }
927
- }
928
- return latestMs === null ? undefined : new Date(latestMs).toISOString();
929
- }
930
-
931
914
  function sourceValidAtMs(turn: BufferTurn): number | null {
932
915
  if (typeof turn.sourceValidAt !== "string") return null;
933
916
  return parseFlexibleIsoTimestamp(turn.sourceValidAt.trim());
@@ -1044,24 +1027,8 @@ function splitTurnsBySourceValidAt(
1044
1027
  export function isArtifactMemoryPath(filePath: string): boolean {
1045
1028
  return /(?:^|[\\/])artifacts(?:[\\/]|$)/i.test(filePath);
1046
1029
  }
1047
-
1048
- export function deriveTopicsFromExtraction(result: ExtractionResult): string[] {
1049
- const topics = new Set<string>();
1050
- for (const fact of result.facts ?? []) {
1051
- for (const tag of fact.tags ?? []) {
1052
- if (tag && tag.length >= 2) topics.add(tag.toLowerCase());
1053
- }
1054
- if (fact.entityRef) topics.add(fact.entityRef.toLowerCase());
1055
- if (fact.category) topics.add(fact.category);
1056
- }
1057
- for (const entity of (result as any).entities ?? []) {
1058
- if (typeof entity.name === "string" && entity.name.length >= 2) {
1059
- topics.add(entity.name.toLowerCase());
1060
- }
1061
- }
1062
- return [...topics].slice(0, 16);
1063
- }
1064
-
1030
+ // Issue #1526 seam 15: deriveTopicsFromExtraction moved to orchestration/extraction-run.ts.
1031
+ export { deriveTopicsFromExtraction } from "./orchestration/extraction-run.js";
1065
1032
  export function buildCompressionGuidelinesMarkdown(
1066
1033
  events: MemoryActionEvent[],
1067
1034
  generatedAtIso: string = new Date().toISOString(),
@@ -1769,6 +1736,128 @@ export class Orchestrator {
1769
1736
  * hands it to `enqueue`.
1770
1737
  */
1771
1738
  readonly extractionQueueCoordinator: ExtractionQueueCoordinator;
1739
+ /**
1740
+ * Extraction-run pipeline coordinator (issue #1526 seam 15). Owns
1741
+ * `runExtraction` + dedupe fingerprint helpers + processed-fingerprint
1742
+ * recording. The orchestrator delegates and injects its own methods.
1743
+ */
1744
+ private _extractionRunCoordinator: ExtractionRunCoordinator | undefined;
1745
+
1746
+ /**
1747
+ * Lazy getter: creates the ExtractionRunCoordinator on first access using
1748
+ * the orchestrator's live field references. Supports Object.create(prototype)
1749
+ * tests that set fields post-construction without invoking the constructor.
1750
+ */
1751
+ private get extractionRunCoordinator(): ExtractionRunCoordinator {
1752
+ if (!this._extractionRunCoordinator) {
1753
+ this._extractionRunCoordinator = new ExtractionRunCoordinator({
1754
+ config: this.config,
1755
+ getBuffer: () => this.buffer,
1756
+ getExtraction: () => this.extraction,
1757
+ getStorageRouter: () => this.storageRouter,
1758
+ getThreading: () => this.threading,
1759
+ persistExtraction: (result, storage, threadId, sourceContext, baseNamespace, scopeProfileWritePlan, sourceText, graphCaps, lifecycleCaps) =>
1760
+ this.persistExtraction(result, storage, threadId, sourceContext, baseNamespace, scopeProfileWritePlan, sourceText, graphCaps, lifecycleCaps),
1761
+ maybeCapturePassiveCorrections: (turns, opts) => this.maybeCapturePassiveCorrections(turns, opts),
1762
+ resolveSelfNamespace: (sessionKey) => this.resolveSelfNamespace(sessionKey),
1763
+ getCodingContextForSession: (sessionKey) => this.getCodingContextForSession(sessionKey),
1764
+ applyCodingNamespaceOverlay: (sessionKey, namespace) => this.applyCodingNamespaceOverlay(sessionKey, namespace),
1765
+ boxBuilderFor: (storage) => this.boxBuilderFor(storage),
1766
+ appendPersistedThreadEpisodes: (threadId, ids) => this.appendPersistedThreadEpisodes(threadId, ids),
1767
+ maybeScheduleConsolidation: (nonZero) => this.maybeScheduleConsolidation(nonZero),
1768
+ requestQmdMaintenance: () => this.requestQmdMaintenance(),
1769
+ runTierMigrationCycle: (storage, trigger, options) => this.runTierMigrationCycle(storage, trigger, options),
1770
+ getLastPersistExtractionDeferredCount: () => this.lastPersistExtractionDeferredCount,
1771
+ recordProcessedExtractionFingerprint: (storage, fingerprint, preloadedMeta) =>
1772
+ this.recordProcessedExtractionFingerprint(storage, fingerprint, preloadedMeta),
1773
+ });
1774
+ }
1775
+ return this._extractionRunCoordinator;
1776
+ }
1777
+ /**
1778
+ * Consolidation-run coordinator (issue #1526 seam 17). Owns
1779
+ * the full consolidation maintenance pass (LLM merge/invalidate/update,
1780
+ * entity merge, commitment/TTL cleanup, lifecycle policy, compression
1781
+ * guideline learning, tier migration, fact archival, semantic consolidation,
1782
+ * identity consolidation, profile consolidation, summarization, topic
1783
+ * extraction, TMT rebuild). The orchestrator delegates and injects
1784
+ * its own methods + coordinators.
1785
+ */
1786
+ private _consolidationRunCoordinator: ConsolidationRunCoordinator | undefined;
1787
+
1788
+ private get consolidationRunCoordinator(): ConsolidationRunCoordinator {
1789
+ if (!this._consolidationRunCoordinator) {
1790
+ this._consolidationRunCoordinator = new ConsolidationRunCoordinator({
1791
+ config: this.config,
1792
+ getStorage: () => this.storage,
1793
+ getStorageRouter: () => this.storageRouter,
1794
+ getExtraction: () => this.extraction,
1795
+ embeddingFallback: this.embeddingFallback,
1796
+ tmtBuilder: this.tmtBuilder,
1797
+ consolidationObservers: this.consolidationObservers,
1798
+ getAccessTrackingBuffer: () => this.accessTrackingBuffer,
1799
+ lifecyclePolicyCoordinator: this.lifecyclePolicyCoordinator,
1800
+ compressionGuidelineCoordinator: this.compressionGuidelineCoordinator,
1801
+ semanticConsolidationCoordinator: this.semanticConsolidationCoordinator,
1802
+ entitySynthesisCoordinator: this.entitySynthesisCoordinator,
1803
+ recallSectionCoordinator: this.recallSectionCoordinator,
1804
+ tierMigrationCoordinator: this.tierMigrationCoordinator,
1805
+ flushAccessTracking: () => this.flushAccessTracking(),
1806
+ indexPersistedMemory: (storage, memoryId) => this.indexPersistedMemory(storage, memoryId),
1807
+ autoConsolidateIdentity: () => this.autoConsolidateIdentity(),
1808
+ fastChatCompletion: (messages, options) => this.fastChatCompletion(messages, options),
1809
+ });
1810
+ }
1811
+ return this._consolidationRunCoordinator;
1812
+ }
1813
+
1814
+ /**
1815
+ * Extraction-persist coordinator (issue #1526 seam 16). Owns the
1816
+ * `persistExtraction` pipeline. Lazy: created on first access so
1817
+ * Object.create(prototype) tests that set fields post-construction work.
1818
+ */
1819
+ private _extractionPersistCoordinator: ExtractionPersistCoordinator | undefined;
1820
+
1821
+ private get extractionPersistCoordinator(): ExtractionPersistCoordinator {
1822
+ if (!this._extractionPersistCoordinator) {
1823
+ this._extractionPersistCoordinator = new ExtractionPersistCoordinator({
1824
+ config: this.config,
1825
+ getStorageRouter: () => this.storageRouter,
1826
+ getThreading: () => this.threading,
1827
+ getLocalLlm: () => this.localLlm,
1828
+ getQmd: () => this.qmd,
1829
+ getJudgeVerdictCache: () => this.judgeVerdictCache,
1830
+ getJudgeDeferCounts: () => this.judgeDeferCounts,
1831
+ getFaithfulnessCounters: () => this.faithfulnessCounters,
1832
+ getEmbeddingFallback: () => this.embeddingFallback,
1833
+ setLastPersistExtractionDeferredCount: (v) => { this.lastPersistExtractionDeferredCount = v; },
1834
+ setLastPersistExtractionPendingReviewIds: (ids) => { this.lastPersistExtractionPendingReviewIds = ids; },
1835
+ addContentHashDedup: (targetStorage, content) => this.addContentHashDedup(targetStorage, content),
1836
+ hasContentHashDedup: (targetStorage, content) => this.hasContentHashDedup(targetStorage, content),
1837
+ backfillTemporalBoundsOnDedupHit: (targetStorage, dedupContent, bounds, entityRef) =>
1838
+ this.backfillTemporalBoundsOnDedupHit(targetStorage, dedupContent, bounds, entityRef),
1839
+ saveContentHashIndexes: () => this.saveContentHashIndexes(),
1840
+ artifactTypeForCategory: (category) => this.artifactTypeForCategory(category),
1841
+ loadRoutingRules: () => this.loadRoutingRules(),
1842
+ routeEngineOptions: () => this.routeEngineOptions(),
1843
+ semanticDedupLookup: (content, limit, targetStorage) =>
1844
+ this.semanticDedupLookup(content, limit, targetStorage),
1845
+ checkForContradiction: (content, category, namespaceScope) =>
1846
+ this.checkForContradiction(content, category, namespaceScope),
1847
+ applyDeferredContradictionResolve: (contradiction, storage, newMemoryId, postWriteGuard) =>
1848
+ this.applyDeferredContradictionResolve(contradiction, storage, newMemoryId, postWriteGuard),
1849
+ suggestLinksForMemory: (content, category, namespaceScope) =>
1850
+ this.suggestLinksForMemory(content, category, namespaceScope),
1851
+ storageDirNamespace: (storageDir) => this.storageDirNamespace(storageDir),
1852
+ indexPersistedMemory: (storage, memoryId) => this.indexPersistedMemory(storage, memoryId),
1853
+ buildGraphEdge: (storage, memoryRelPath, entityRef, memoryId, factContent, allMemsForGraph, memoryPathById, threadIdForEdge, threadEpisodeIdsForGraph, fallbackCausalPredecessor, graphCaps) =>
1854
+ this.buildGraphEdge(storage, memoryRelPath, entityRef, memoryId, factContent, allMemsForGraph, memoryPathById, threadIdForEdge, threadEpisodeIdsForGraph, fallbackCausalPredecessor, graphCaps),
1855
+ updateTemporalTagIndexes: (storage, persistedIds) =>
1856
+ this.updateTemporalTagIndexes(storage, persistedIds),
1857
+ });
1858
+ }
1859
+ return this._extractionPersistCoordinator;
1860
+ }
1772
1861
  /**
1773
1862
  * Issue #1526: recall result formatting + identity continuity section moved
1774
1863
  * to RecallResultFormatter.
@@ -2751,6 +2840,7 @@ export class Orchestrator {
2751
2840
  });
2752
2841
  // Issue #1526: background extraction queue lives on its own coordinator.
2753
2842
  this.extractionQueueCoordinator = new ExtractionQueueCoordinator();
2843
+
2754
2844
  this.recallResultFormatter = new RecallResultFormatter(this.config);
2755
2845
  const conversationIndexRuntime = createConversationIndexRuntime(config, {
2756
2846
  getQmd: () => this.conversationQmd,
@@ -11983,67 +12073,18 @@ export class Orchestrator {
11983
12073
  }
11984
12074
 
11985
12075
  private normalizeExtractionFingerprintTurns(turns: BufferTurn[]): string[] {
11986
- if (!Array.isArray(turns) || turns.length === 0) return [];
11987
- return turns
11988
- .filter((turn) => turn.role === "user" || turn.role === "assistant")
11989
- .map((turn) => {
11990
- if (
11991
- typeof turn.turnFingerprint === "string" &&
11992
- turn.turnFingerprint.length > 0
11993
- ) {
11994
- return `fp:${turn.turnFingerprint}`;
11995
- }
11996
- return `${turn.role}:${(turn.content ?? "").replace(/\s+/g, " ").trim().slice(0, this.config.extractionMaxTurnChars)}`;
11997
- })
11998
- .filter((value) => value.length > 0);
12076
+ return this.extractionRunCoordinator.normalizeExtractionFingerprintTurns(turns);
11999
12077
  }
12000
12078
 
12001
- private buildExtractionFingerprint(
12002
- turns: BufferTurn[],
12003
- bufferKey: string,
12004
- ): string | null {
12005
- const normalized = this.normalizeExtractionFingerprintTurns(turns).join("\n");
12006
- if (!normalized) return null;
12007
- return createHash("sha256")
12008
- .update(`${bufferKey}\n${normalized}`)
12009
- .digest("hex");
12079
+ private buildExtractionFingerprint(turns: BufferTurn[], bufferKey: string): string | null {
12080
+ return this.extractionRunCoordinator.buildExtractionFingerprint(turns, bufferKey);
12010
12081
  }
12011
12082
 
12012
12083
  private shouldQueueExtraction(
12013
12084
  turns: BufferTurn[],
12014
12085
  options: { commit?: boolean; bufferKey?: string } = {},
12015
12086
  ): boolean {
12016
- const lifecycleCaps = resolveMemoryLifecycleCapabilities(this.config);
12017
- if (!lifecycleCaps.extractionDedupe) return true;
12018
- if (!Array.isArray(turns) || turns.length === 0) return false;
12019
-
12020
- const bufferKey = options.bufferKey ?? turns[0]?.sessionKey ?? "default";
12021
- const fingerprint = this.buildExtractionFingerprint(turns, bufferKey);
12022
- if (!fingerprint) return false;
12023
- const now = Date.now();
12024
- const seenAt = this.recentExtractionFingerprints.get(fingerprint);
12025
- if (seenAt && now - seenAt < this.config.extractionDedupeWindowMs) {
12026
- log.debug("extraction dedupe: skipped duplicate buffered turn set");
12027
- return false;
12028
- }
12029
-
12030
- if (options.commit !== false) {
12031
- this.recentExtractionFingerprints.set(fingerprint, now);
12032
- }
12033
- // Keep this cache bounded to avoid unbounded growth.
12034
- if (
12035
- options.commit !== false &&
12036
- this.recentExtractionFingerprints.size > 200
12037
- ) {
12038
- const entries = Array.from(
12039
- this.recentExtractionFingerprints.entries(),
12040
- ).sort((a, b) => a[1] - b[1]);
12041
- for (const [key] of entries.slice(0, entries.length - 200)) {
12042
- this.recentExtractionFingerprints.delete(key);
12043
- }
12044
- }
12045
-
12046
- return true;
12087
+ return this.extractionRunCoordinator.shouldQueueExtraction(turns, options);
12047
12088
  }
12048
12089
 
12049
12090
  /**
@@ -12179,3204 +12220,91 @@ export class Orchestrator {
12179
12220
  return this._passiveCorrectionService;
12180
12221
  }
12181
12222
 
12182
- private async runExtraction(
12183
- turns: BufferTurn[],
12184
- options: {
12185
- clearBufferAfterExtraction?: boolean;
12186
- skipCharThreshold?: boolean;
12187
- skipUserTurnThreshold?: boolean;
12188
- deadlineMs?: number;
12189
- bufferKey?: string;
12190
- abortSignal?: AbortSignal;
12191
- failOnExtractionFailure?: boolean;
12192
- /**
12193
- * Explicit namespace override for the write path (#460). When set,
12194
- * extraction writes go to this namespace instead of the one derived
12195
- * from `defaultNamespaceForPrincipal(resolvePrincipal(sessionKey))`.
12196
- * The resolved `principal` is still threaded into memory metadata
12197
- * for provenance; only the storage target is overridden.
12198
- */
12199
- writeNamespaceOverride?: string;
12200
- /**
12201
- * Pin the provenance principal instead of deriving it from
12202
- * `resolvePrincipal(sessionKey)` (#1495 thread 1). When set, this is the
12203
- * identity an access surface already authenticated; used so observed-turn
12204
- * provenance is correct even though `turn.sessionKey` is the ORIGINAL
12205
- * (un-prefixed) key and storage is pinned via `writeNamespaceOverride`.
12206
- */
12207
- principalOverride?: string;
12208
- } = {},
12223
+ async runExtraction(
12224
+ ...args: Parameters<ExtractionRunCoordinator["runExtraction"]>
12209
12225
  ): Promise<ExtractionRunResult> {
12210
- log.debug(`running extraction on ${turns.length} turns`);
12211
- const clearBufferAfterExtraction =
12212
- options.clearBufferAfterExtraction ?? true;
12213
- const skipCharThreshold = options.skipCharThreshold ?? false;
12214
- const skipUserTurnThreshold = options.skipUserTurnThreshold ?? false;
12215
- const deadlineMs =
12216
- typeof options.deadlineMs === "number" &&
12217
- Number.isFinite(options.deadlineMs)
12218
- ? options.deadlineMs
12219
- : undefined;
12220
- const bufferKey = options.bufferKey ?? turns[0]?.sessionKey ?? "default";
12221
- const throwIfDeadlineExceeded = (stage: string): void => {
12222
- if (typeof deadlineMs === "number" && Date.now() > deadlineMs) {
12223
- throw new Error(`replay extraction deadline exceeded (${stage})`);
12224
- }
12225
- };
12226
- const throwIfAborted = (stage: string): void => {
12227
- throwIfRecallAborted(options.abortSignal, `extraction aborted (${stage})`);
12228
- };
12229
- const clearBuffer = async (options?: { ignoreAbort?: boolean }) => {
12230
- if (options?.ignoreAbort !== true) {
12231
- throwIfAborted("before_clear_buffer");
12232
- }
12233
- if (clearBufferAfterExtraction) {
12234
- await this.buffer.clearAfterExtraction(bufferKey, turns);
12235
- }
12236
- };
12237
-
12238
- // Skip extraction for cron job sessions - these are system operations, not user conversations
12239
- const sessionKey = turns[0]?.sessionKey ?? "";
12240
- if (sessionKey.includes(":cron:")) {
12241
- log.debug(`skipping extraction for cron session: ${sessionKey}`);
12242
- await clearBuffer();
12243
- return {
12244
- status: "skipped",
12245
- reason: "cron_session",
12246
- persistedCount: 0,
12247
- durableOutputCount: 0,
12248
- };
12249
- }
12226
+ return this.extractionRunCoordinator.runExtraction(...args);
12227
+ }
12250
12228
 
12251
- const normalizedTurns = turns
12252
- .filter(
12253
- (t) =>
12254
- (t.role === "user" || t.role === "assistant") &&
12255
- typeof t.content === "string",
12256
- )
12257
- .map((t) => ({
12258
- ...t,
12259
- content: t.content.trim().slice(0, this.config.extractionMaxTurnChars),
12260
- }))
12261
- .filter((t) => t.content.length > 0);
12262
- const targetTurns = normalizedTurns.filter(
12263
- (turn) => turn.extractionContextOnly !== true,
12229
+ private async recordProcessedExtractionFingerprint(
12230
+ storage: StorageManager,
12231
+ fingerprint: string,
12232
+ preloadedMeta?: Awaited<ReturnType<StorageManager["loadMeta"]>>,
12233
+ ): Promise<void> {
12234
+ return this.extractionRunCoordinator.recordProcessedExtractionFingerprint(
12235
+ storage, fingerprint, preloadedMeta,
12264
12236
  );
12265
- if (targetTurns.length === 0) {
12266
- log.debug("skipping extraction: no non-context turns after normalization");
12267
- // Context-only turns may still contain corrections (review: "context-only
12268
- // turns skip capture"). Scan normalizedTurns before clearing the buffer.
12269
- if (normalizedTurns.length > 0) {
12270
- await this.maybeCapturePassiveCorrections(normalizedTurns as BufferTurn[], { sessionKey, principal: resolvePrincipal(sessionKey, this.config), namespace: this.resolveSelfNamespace(sessionKey), bufferKey, isLiveSession: clearBufferAfterExtraction });
12271
- }
12272
- await clearBuffer();
12273
- return { status: "skipped", reason: "empty_normalized_turns", persistedCount: 0, durableOutputCount: 0 };
12274
- }
12275
- const sourceValidAt = latestSourceValidAtFromTurns(targetTurns);
12276
- throwIfDeadlineExceeded("before_extract");
12277
- throwIfAborted("before_extract");
12237
+ }
12278
12238
 
12279
- const userTurns = targetTurns.filter((t) => t.role === "user");
12280
- const totalChars = targetTurns.reduce(
12281
- (sum, t) => sum + t.content.length,
12282
- 0,
12283
- );
12284
- const belowCharThreshold = totalChars < this.config.extractionMinChars;
12285
- const belowUserTurnThreshold =
12286
- !skipUserTurnThreshold &&
12287
- userTurns.length < this.config.extractionMinUserTurns;
12288
- if ((!skipCharThreshold && belowCharThreshold) || belowUserTurnThreshold) {
12289
- log.debug(
12290
- `skipping extraction: below threshold (totalChars=${totalChars}, userTurns=${userTurns.length})`,
12291
- );
12292
- await clearBuffer();
12293
- // Passive correction capture runs even when extraction is skipped for
12294
- // being below the char/user-turn threshold (review: "skipped extraction
12295
- // skips capture"). A short correction like "stop using Vim" is under
12296
- // extractionMinChars but is exactly the high-value case this feature
12297
- // exists for. The write namespace resolves through the SAME scope-profile
12298
- // plan as the full extraction path so corrections target the active
12299
- // profile's write layer, not just resolveSelfNamespace (review:
12300
- // "below-threshold wrong namespace"). The ACL in passiveCorrectionService
12301
- // authorizes the actual plan/apply.
12302
- {
12303
- const capturePrincipal =
12304
- typeof options.principalOverride === "string" && options.principalOverride.length > 0
12305
- ? options.principalOverride
12306
- : resolvePrincipal(sessionKey, this.config);
12307
- const captureWO = typeof options.writeNamespaceOverride === "string" && options.writeNamespaceOverride.length > 0
12308
- ? options.writeNamespaceOverride : undefined;
12309
- const captureCodingCtx = sessionKey ? this.getCodingContextForSession(sessionKey) : null;
12310
- const captureCodingOv = resolveCodingNamespaceOverlay(captureCodingCtx, this.config.codingMode, this.config.defaultNamespace);
12311
- const captureScopePlan = resolveScopeProfilePlan({ config: this.config, principal: capturePrincipal, codingContext: captureCodingCtx, codingOverlay: captureCodingOv });
12312
- const captureNamespace = captureWO ?? captureScopePlan?.writeNamespace ?? this.resolveSelfNamespace(sessionKey);
12313
- await this.maybeCapturePassiveCorrections(normalizedTurns as BufferTurn[], {
12314
- sessionKey, principal: capturePrincipal, namespace: captureNamespace, bufferKey, isLiveSession: clearBufferAfterExtraction,
12315
- });
12316
- }
12317
- return {
12318
- status: "skipped",
12319
- reason: "below_threshold",
12320
- persistedCount: 0,
12321
- durableOutputCount: 0,
12322
- };
12323
- }
12239
+ private async runTierMigrationCycle(
12240
+ storage: StorageManager,
12241
+ trigger: "extraction" | "maintenance" | "manual",
12242
+ options?: {
12243
+ dryRun?: boolean;
12244
+ limitOverride?: number;
12245
+ force?: boolean;
12246
+ },
12247
+ ): Promise<TierMigrationCycleSummary> {
12248
+ return this.tierMigrationCoordinator.runCycle(storage, trigger, options);
12249
+ }
12324
12250
 
12325
- // Provenance principal honours the access-surface override (#1495 thread 1,
12326
- // mirroring the recall path's `principalOverride`, issue #570 PR 4). Access
12327
- // surfaces that authenticated the caller at the transport layer pass their
12328
- // resolved principal so provenance uses the SAME identity the surface
12329
- // authorized, instead of `resolvePrincipal(sessionKey)` — which on a
12330
- // namespace-prefixed key would collapse to `default`. The ORIGINAL,
12331
- // un-prefixed session key still drives threading.
12332
- const principal =
12333
- typeof options.principalOverride === "string" &&
12334
- options.principalOverride.length > 0
12335
- ? options.principalOverride
12336
- : resolvePrincipal(sessionKey, this.config);
12337
- // Write path — explicit callers still win. Otherwise, an active hosted
12338
- // scope profile owns the extraction write target so hook-captured turns land
12339
- // in the same layer that profile recall searches. Without a profile, preserve
12340
- // the existing coding-agent overlay behavior (issue #569).
12341
- const explicitWriteNamespace =
12342
- typeof options.writeNamespaceOverride === "string" &&
12343
- options.writeNamespaceOverride.length > 0
12344
- ? options.writeNamespaceOverride
12345
- : undefined;
12346
- const codingContextForWrite = sessionKey
12347
- ? this.getCodingContextForSession(sessionKey)
12348
- : null;
12349
- const codingOverlayForWrite = resolveCodingNamespaceOverlay(
12350
- codingContextForWrite,
12351
- this.config.codingMode,
12352
- this.config.defaultNamespace,
12353
- );
12354
- const scopeProfileGatePlan = resolveScopeProfilePlan({
12355
- config: this.config,
12356
- principal,
12357
- codingContext: codingContextForWrite,
12358
- codingOverlay: codingOverlayForWrite,
12251
+ async getTierMigrationStatus(): Promise<TierMigrationStatusSnapshot> {
12252
+ return this.tierMigrationStatus.get();
12253
+ }
12254
+
12255
+ async runTierMigrationNow(options?: {
12256
+ dryRun?: boolean;
12257
+ limit?: number;
12258
+ }): Promise<TierMigrationCycleSummary> {
12259
+ return this.runTierMigrationCycle(this.storage, "manual", {
12260
+ dryRun: options?.dryRun === true,
12261
+ limitOverride: options?.limit,
12262
+ force: false,
12359
12263
  });
12360
- const scopeProfileWritePlan = explicitWriteNamespace ? null : scopeProfileGatePlan;
12361
- if (scopeProfileWritePlan) {
12362
- const selectedLayer = scopeProfileWritePlan.layers.find(
12363
- (layer) => layer.id === scopeProfileWritePlan.writeLayer,
12364
- );
12365
- const writeNamespaceReadable = scopeProfileWritePlan.readNamespaces.includes(
12366
- scopeProfileWritePlan.writeNamespace,
12367
- );
12368
- if (!selectedLayer?.writable || !writeNamespaceReadable) {
12369
- log.warn(
12370
- `runExtraction: skipping scope profile ${scopeProfileWritePlan.profileId} because write layer ${scopeProfileWritePlan.writeLayer} is not writable inside the profile read stack`,
12371
- );
12372
- await clearBuffer();
12373
- return {
12374
- status: "skipped",
12375
- reason: "scope_profile_no_writable_layer",
12376
- persistedCount: 0,
12377
- durableOutputCount: 0,
12378
- };
12379
- }
12380
- }
12381
- const selfNamespace =
12382
- explicitWriteNamespace ??
12383
- scopeProfileWritePlan?.writeNamespace ??
12384
- this.applyCodingNamespaceOverlay(
12385
- sessionKey,
12386
- defaultNamespaceForPrincipal(principal, this.config),
12387
- );
12388
- const storage = await this.storageRouter.storageFor(selfNamespace);
12389
- const shouldPersistProcessedFingerprint = targetTurns.some(
12390
- (turn) => turn.persistProcessedFingerprint === true,
12391
- );
12392
- const extractionFingerprint = this.buildExtractionFingerprint(
12393
- targetTurns,
12394
- bufferKey,
12395
- );
12396
- let meta =
12397
- extractionFingerprint && shouldPersistProcessedFingerprint
12398
- ? await storage.loadMeta()
12399
- : null;
12400
- if (
12401
- extractionFingerprint &&
12402
- shouldPersistProcessedFingerprint &&
12403
- (meta?.processedExtractionFingerprints ?? []).some(
12404
- (entry) => entry.fingerprint === extractionFingerprint,
12405
- )
12406
- ) {
12407
- log.debug(
12408
- `runExtraction: skipping already-processed extraction fingerprint for ${bufferKey}`,
12409
- );
12410
- await clearBuffer();
12411
- return {
12412
- status: "skipped",
12413
- reason: "processed_fingerprint",
12414
- persistedCount: 0,
12415
- durableOutputCount: 0,
12416
- };
12417
- }
12264
+ }
12418
12265
 
12419
- // Pass existing entity names so the LLM can reuse them instead of inventing variants
12420
- const existingEntities = await storage.listEntityNames();
12421
- const result = await raceRecallAbort(
12422
- this.extraction.extract(
12423
- normalizedTurns,
12424
- existingEntities,
12425
- ),
12426
- options.abortSignal,
12427
- "extraction aborted (during_extract)",
12266
+ private maybeScheduleConsolidation(nonZeroExtraction: boolean): void {
12267
+ this.maintenanceScheduler.maybeScheduleConsolidation(
12268
+ nonZeroExtraction,
12269
+ () => this.runConsolidation(),
12428
12270
  );
12429
- throwIfDeadlineExceeded("before_persist");
12430
- throwIfAborted("before_persist");
12431
-
12432
- // Defensive: validate extraction result before processing. Explicit
12433
- // fail-closed callers, such as flush-plan import, must not observe
12434
- // malformed extractor output as a successful skip.
12435
- if (!result) {
12436
- log.warn("runExtraction: extraction returned null/undefined");
12437
- if (options.failOnExtractionFailure) {
12438
- throw new Error("extraction failed: invalid_extraction_result");
12439
- }
12440
- await clearBuffer();
12441
- return {
12442
- status: "skipped",
12443
- reason: "invalid_extraction_result",
12444
- persistedCount: 0,
12445
- durableOutputCount: 0,
12446
- };
12447
- }
12448
- const invalidExtractionResultFields = [
12449
- ["facts", result.facts],
12450
- ["entities", result.entities],
12451
- ["questions", result.questions],
12452
- ["profileUpdates", result.profileUpdates],
12453
- ]
12454
- .filter(([, value]) => !Array.isArray(value))
12455
- .map(([field]) => field);
12456
- if (invalidExtractionResultFields.length > 0) {
12457
- log.warn(
12458
- "runExtraction: extraction returned invalid collection fields",
12459
- {
12460
- invalidFields: invalidExtractionResultFields,
12461
- resultKeys:
12462
- typeof result === "object" && result !== null
12463
- ? Object.keys(result)
12464
- : [],
12465
- },
12466
- );
12467
- if (options.failOnExtractionFailure) {
12468
- throw new Error("extraction failed: invalid_extraction_result");
12469
- }
12470
- await clearBuffer();
12471
- return {
12472
- status: "skipped",
12473
- reason: "invalid_extraction_result",
12474
- persistedCount: 0,
12475
- durableOutputCount: 0,
12476
- };
12477
- }
12478
- const extractionFailure =
12479
- typeof result.extractionFailure === "string" &&
12480
- result.extractionFailure.trim().length > 0
12481
- ? result.extractionFailure
12482
- : undefined;
12483
- if (options.failOnExtractionFailure && extractionFailure) {
12484
- throw new Error(`extraction failed: ${extractionFailure}`);
12485
- }
12486
- if (
12487
- result.facts.length === 0 &&
12488
- result.entities.length === 0 &&
12489
- result.questions.length === 0 &&
12490
- result.profileUpdates.length === 0
12491
- ) {
12492
- log.debug(
12493
- "runExtraction: extraction produced no durable outputs; skipping persistence",
12494
- );
12495
- if (extractionFailure) {
12496
- log.warn(
12497
- "runExtraction: extraction reported failure with no durable outputs; not marking fingerprint processed",
12498
- { extractionFailure },
12499
- );
12500
- }
12501
- if (
12502
- extractionFingerprint &&
12503
- shouldPersistProcessedFingerprint &&
12504
- !extractionFailure
12505
- ) {
12506
- meta ??= await storage.loadMeta();
12507
- await this.recordProcessedExtractionFingerprint(
12508
- storage,
12509
- extractionFingerprint,
12510
- meta,
12511
- );
12512
- meta.extractionCount += 1;
12513
- meta.lastExtractionAt = new Date().toISOString();
12514
- await storage.saveMeta(meta);
12515
- }
12516
- // Correction-only turns that meet char/user-turn thresholds but yield
12517
- // zero facts still need passive capture (review: "empty extraction skips
12518
- // capture"). selfNamespace/principal already resolved above.
12519
- await this.maybeCapturePassiveCorrections(normalizedTurns as BufferTurn[], { sessionKey, principal, namespace: selfNamespace, bufferKey, isLiveSession: clearBufferAfterExtraction });
12520
- await clearBuffer();
12521
- return { status: "skipped", reason: "empty_extraction_result", persistedCount: 0, durableOutputCount: 0 };
12522
- }
12271
+ }
12523
12272
 
12524
- let threadIdForExtraction: string | null = null;
12525
- if (resolvePresentationCapabilities(this.config).threading && turns.length > 0) {
12526
- const lastTurn = turns[turns.length - 1];
12527
- try {
12528
- threadIdForExtraction = await this.threading.processTurn(lastTurn, []);
12529
- } catch (err) {
12530
- // Fail-open: threading errors must not block memory persistence.
12531
- log.warn(
12532
- "[threading] processTurn failed before persistence (non-fatal)",
12533
- err,
12534
- );
12535
- }
12536
- }
12273
+ requestQmdMaintenance(): void {
12274
+ this.maintenanceScheduler.requestQmdMaintenanceForTool("__internal__");
12275
+ }
12276
+
12277
+ /**
12278
+ * Public entrypoint for tool-driven QMD maintenance requests.
12279
+ * Delegates to the maintenance scheduler (issue #1526 PR1).
12280
+ */
12281
+ requestQmdMaintenanceForTool(reason: string): void {
12282
+ this.maintenanceScheduler.requestQmdMaintenanceForTool(reason);
12283
+ }
12537
12284
 
12538
- const persistedIds = await this.persistExtraction(
12285
+ private async persistExtraction(
12286
+ result: ExtractionResult,
12287
+ storage: StorageManager,
12288
+ threadIdForExtraction?: string | null,
12289
+ sourceContext?: { sessionKey?: string; principal?: string; validAt?: string },
12290
+ baseNamespace?: string,
12291
+ scopeProfileWritePlan?: ResolvedScopeProfilePlan | null,
12292
+ /** Verbatim source turn text the facts were extracted from (faithfulness gate #1576). */
12293
+ sourceText?: string,
12294
+ graphCaps: GraphConstructionCapabilitySet = resolveGraphConstructionCapabilities(this.config),
12295
+ lifecycleCaps: MemoryLifecycleCapabilitySet = resolveMemoryLifecycleCapabilities(this.config),
12296
+ ): Promise<string[]> {
12297
+ return this.extractionPersistCoordinator.persistExtraction(
12539
12298
  result,
12540
12299
  storage,
12541
12300
  threadIdForExtraction,
12542
- { sessionKey, principal, validAt: sourceValidAt },
12543
- // Pass the KNOWN base namespace (NHIdx) so the catalog write touch records the
12544
- // real namespace rather than a guess decoded from the storage dir.
12545
- selfNamespace,
12546
- scopeProfileGatePlan,
12547
- // Verbatim source turns for the faithfulness gate (#1576) so it can
12548
- // locate a quote per fact when #1575 spans are absent.
12549
- normalizedTurns.map((t) => t.content).join("\n\n"),
12301
+ sourceContext,
12302
+ baseNamespace,
12303
+ scopeProfileWritePlan,
12304
+ sourceText,
12305
+ graphCaps,
12306
+ lifecycleCaps,
12550
12307
  );
12551
- let postPersistMetadataFailed = false;
12552
- meta ??= await storage.loadMeta();
12553
- if (extractionFingerprint && shouldPersistProcessedFingerprint) {
12554
- try {
12555
- await this.recordProcessedExtractionFingerprint(
12556
- storage,
12557
- extractionFingerprint,
12558
- meta,
12559
- );
12560
- } catch (error) {
12561
- log.warn(
12562
- "runExtraction: failed to persist processed extraction fingerprint; continuing with buffer clear",
12563
- error,
12564
- );
12565
- postPersistMetadataFailed = true;
12566
- }
12567
- }
12568
- // Persist extraction counters and processed fingerprints before running
12569
- // follow-on helpers so replay dedupe survives any later non-essential
12570
- // failure. If this aggregate meta write fails, still clear the buffer:
12571
- // the durable memories are already written and replaying the same turns
12572
- // would duplicate them.
12573
- meta.extractionCount += 1;
12574
- meta.lastExtractionAt = new Date().toISOString();
12575
- meta.totalMemories += Array.isArray(result?.facts)
12576
- ? result.facts.length
12577
- : 0;
12578
- meta.totalEntities += Array.isArray(result?.entities)
12579
- ? result.entities.length
12580
- : 0;
12581
- try {
12582
- await storage.saveMeta(meta);
12583
- } catch (error) {
12584
- log.warn(
12585
- "runExtraction: failed to save extraction metadata after durable persistence; continuing with buffer clear",
12586
- error,
12587
- );
12588
- postPersistMetadataFailed = true;
12589
- }
12590
-
12591
- const durableOutputCount =
12592
- result.facts.length +
12593
- result.entities.length +
12594
- result.questions.length +
12595
- result.profileUpdates.length;
12596
-
12597
- // Buffer retention for defer verdicts (issue #562, PR 2). When the judge
12598
- // deferred at least one candidate, retain the tail of the current turn
12599
- // window so the next extraction pass has the surrounding context that
12600
- // may disambiguate the deferred fact. Non-defer runs clear the slot.
12601
- //
12602
- // Gated on:
12603
- // - `clearBufferAfterExtraction` — replay / bulk-import paths call
12604
- // `runExtraction` with this false and do not operate on live buffer
12605
- // state. Writing retention there would create synthetic buffer
12606
- // entries and cross-contaminate future live extractions.
12607
- // - NOT `extractionJudgeShadow` — in shadow mode the judge is only
12608
- // advisory; facts are still persisted regardless of verdict, so
12609
- // retaining the turn window on top of a persisted write would both
12610
- // waste buffer space and cause the same facts to re-enter the
12611
- // pipeline on the next pass.
12612
- try {
12613
- if (
12614
- clearBufferAfterExtraction &&
12615
- !this.config.extractionJudgeShadow
12616
- ) {
12617
- const deferredCount = this.lastPersistExtractionDeferredCount;
12618
- if (deferredCount > 0 && normalizedTurns.length > 0) {
12619
- await this.buffer.retainDeferredTurns(
12620
- bufferKey,
12621
- normalizedTurns as BufferTurn[],
12622
- 10,
12623
- );
12624
- } else {
12625
- await this.buffer.retainDeferredTurns(bufferKey, [], 0);
12626
- }
12627
- }
12628
- } catch (err) {
12629
- // Fail-open: retention is a nice-to-have. If it fails the judge will
12630
- // still cap deferrals and convert to reject on the next pass.
12631
- log.debug(
12632
- `extraction-judge: defer retention failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`,
12633
- );
12634
- }
12635
-
12636
- await clearBuffer({ ignoreAbort: true });
12637
-
12638
- // Passive correction capture (issue #1581) — detect corrections expressed
12639
- // passively in the extracted turns and route to the Correction Contract.
12640
- // Runs AFTER persistence + buffer clear so a capture failure never blocks
12641
- // the extraction return. `clearBufferAfterExtraction` gates live-session
12642
- // auto-apply (replay/import → queue-only).
12643
- await this.maybeCapturePassiveCorrections(normalizedTurns as BufferTurn[], {
12644
- sessionKey,
12645
- principal,
12646
- namespace: selfNamespace,
12647
- bufferKey,
12648
- isLiveSession: clearBufferAfterExtraction,
12649
- });
12650
-
12651
- // Build memory box from this extraction (v8.0 Phase 2A)
12652
- // Topics are derived from the current extraction's facts and entities only —
12653
- // not from readAllMemories() — so box topics accurately reflect the current
12654
- // session window and the call is free of expensive full-corpus I/O.
12655
- if (resolvePresentationCapabilities(this.config).memoryBoxes && persistedIds.length > 0) {
12656
- const extractionTopics = deriveTopicsFromExtraction(result);
12657
- // Derive episodic metadata from buffer turns (REMem-inspired)
12658
- const firstUserTurn = turns.find((t) => t.role === "user");
12659
- const boxGoal =
12660
- firstUserTurn?.content?.slice(0, 100)?.trim() || undefined;
12661
- await this.boxBuilderFor(storage)
12662
- .onExtraction({
12663
- topics: extractionTopics,
12664
- memoryIds: persistedIds,
12665
- timestamp: new Date().toISOString(),
12666
- goal: boxGoal,
12667
- })
12668
- .catch((err) =>
12669
- log.warn("[boxes] onExtraction failed (non-fatal)", err),
12670
- );
12671
- }
12672
-
12673
- // Batch-append persisted IDs so non-fact memories (entities/questions) are
12674
- // always attached to the thread. The helper excludes pending_review ids (#1635).
12675
- if (
12676
- resolvePresentationCapabilities(this.config).threading &&
12677
- threadIdForExtraction &&
12678
- persistedIds.length > 0
12679
- ) {
12680
- await this.appendPersistedThreadEpisodes(
12681
- threadIdForExtraction,
12682
- persistedIds,
12683
- );
12684
- }
12685
-
12686
- // Thread title update for the already-established thread context.
12687
- if (resolvePresentationCapabilities(this.config).threading && threadIdForExtraction) {
12688
- const conversationContent = turns.map((t) => t.content).join(" ");
12689
- try {
12690
- await this.threading.updateThreadTitle(
12691
- threadIdForExtraction,
12692
- conversationContent,
12693
- );
12694
- } catch (err) {
12695
- log.warn(
12696
- "[threading] updateThreadTitle failed after persistence (non-fatal)",
12697
- err,
12698
- );
12699
- }
12700
- }
12701
-
12702
- // Check if consolidation is needed (debounced + non-zero gated).
12703
- const nonZeroExtraction = durableOutputCount > 0;
12704
- try {
12705
- // The increment of nonZeroExtractionsSinceConsolidation moved into
12706
- // the scheduler with the rest of the cadence state (issue #1526 PR1).
12707
- this.maybeScheduleConsolidation(nonZeroExtraction);
12708
- } catch (err) {
12709
- log.warn(
12710
- "runExtraction: consolidation scheduling failed after persistence (non-fatal)",
12711
- err,
12712
- );
12713
- }
12714
-
12715
- try {
12716
- this.requestQmdMaintenance();
12717
- } catch (err) {
12718
- log.warn(
12719
- "runExtraction: QMD maintenance scheduling failed after persistence (non-fatal)",
12720
- err,
12721
- );
12722
- }
12723
-
12724
- try {
12725
- await this.runTierMigrationCycle(storage, "extraction");
12726
- } catch (err) {
12727
- log.warn(
12728
- "runExtraction: tier migration failed after persistence (non-fatal)",
12729
- err,
12730
- );
12731
- }
12732
-
12733
- return {
12734
- status: "completed",
12735
- persistedCount: persistedIds.length,
12736
- durableOutputCount,
12737
- postPersistMetadataFailed,
12738
- };
12739
- }
12740
-
12741
- private async recordProcessedExtractionFingerprint(
12742
- storage: StorageManager,
12743
- fingerprint: string,
12744
- preloadedMeta?: Awaited<ReturnType<StorageManager["loadMeta"]>>,
12745
- ): Promise<void> {
12746
- const meta = preloadedMeta ?? (await storage.loadMeta());
12747
- const observedAt = new Date().toISOString();
12748
- const seen = new Map(
12749
- (meta.processedExtractionFingerprints ?? []).map((entry) => [
12750
- entry.fingerprint,
12751
- entry.observedAt,
12752
- ]),
12753
- );
12754
- seen.set(fingerprint, observedAt);
12755
- meta.processedExtractionFingerprints = Array.from(seen.entries())
12756
- .map(([value, at]) => ({ fingerprint: value, observedAt: at }))
12757
- .sort((left, right) => left.observedAt.localeCompare(right.observedAt))
12758
- .slice(-500);
12759
- if (!preloadedMeta) {
12760
- await storage.saveMeta(meta);
12761
- }
12762
- }
12763
-
12764
- private async runTierMigrationCycle(
12765
- storage: StorageManager,
12766
- trigger: "extraction" | "maintenance" | "manual",
12767
- options?: {
12768
- dryRun?: boolean;
12769
- limitOverride?: number;
12770
- force?: boolean;
12771
- },
12772
- ): Promise<TierMigrationCycleSummary> {
12773
- return this.tierMigrationCoordinator.runCycle(storage, trigger, options);
12774
- }
12775
-
12776
- async getTierMigrationStatus(): Promise<TierMigrationStatusSnapshot> {
12777
- return this.tierMigrationStatus.get();
12778
- }
12779
-
12780
- async runTierMigrationNow(options?: {
12781
- dryRun?: boolean;
12782
- limit?: number;
12783
- }): Promise<TierMigrationCycleSummary> {
12784
- return this.runTierMigrationCycle(this.storage, "manual", {
12785
- dryRun: options?.dryRun === true,
12786
- limitOverride: options?.limit,
12787
- force: false,
12788
- });
12789
- }
12790
-
12791
- private maybeScheduleConsolidation(nonZeroExtraction: boolean): void {
12792
- this.maintenanceScheduler.maybeScheduleConsolidation(
12793
- nonZeroExtraction,
12794
- () => this.runConsolidation(),
12795
- );
12796
- }
12797
-
12798
- requestQmdMaintenance(): void {
12799
- this.maintenanceScheduler.requestQmdMaintenanceForTool("__internal__");
12800
- }
12801
-
12802
- /**
12803
- * Public entrypoint for tool-driven QMD maintenance requests.
12804
- * Delegates to the maintenance scheduler (issue #1526 PR1).
12805
- */
12806
- requestQmdMaintenanceForTool(reason: string): void {
12807
- this.maintenanceScheduler.requestQmdMaintenanceForTool(reason);
12808
- }
12809
-
12810
- private async persistExtraction(
12811
- result: ExtractionResult,
12812
- storage: StorageManager,
12813
- threadIdForExtraction?: string | null,
12814
- sourceContext?: { sessionKey?: string; principal?: string; validAt?: string },
12815
- baseNamespace?: string,
12816
- scopeProfileWritePlan?: ResolvedScopeProfilePlan | null,
12817
- /** Verbatim source turn text the facts were extracted from (faithfulness gate #1576). */
12818
- sourceText?: string,
12819
- graphCaps: GraphConstructionCapabilitySet = resolveGraphConstructionCapabilities(this.config),
12820
- lifecycleCaps: MemoryLifecycleCapabilitySet = resolveMemoryLifecycleCapabilities(this.config),
12821
- ): Promise<string[]> {
12822
- // Inline source attribution (issue #369). When enabled, every extracted
12823
- // fact is rewritten to carry a compact provenance tag inside its body so
12824
- // the citation survives hostile memory text, copy/paste, and LLM quoting.
12825
- // The helper is a no-op when the feature flag is off, so legacy pipelines
12826
- // see zero behavioral change.
12827
- const citationEnabled = resolvePipelineProcessingCapabilities(this.config).inlineSourceAttribution === true;
12828
- const citationTemplate = this.config.inlineSourceAttributionFormat;
12829
- // The stable fields (agent, session) are computed once; `ts` is intentionally
12830
- // omitted here and added fresh per invocation so each fact in a large batch
12831
- // gets its own insertion timestamp rather than sharing a single batch-start time.
12832
- const citationContextBase: Omit<CitationContext, "ts"> = citationEnabled
12833
- ? {
12834
- agent: sourceContext?.principal,
12835
- session: sourceContext?.sessionKey,
12836
- }
12837
- : {};
12838
- const applyInlineCitation = (content: string): string => {
12839
- if (!citationEnabled) return content;
12840
- if (typeof content !== "string" || content.length === 0) return content;
12841
- // Build a fresh CitationContext per call so `ts` reflects the actual
12842
- // insertion time of each individual fact rather than the batch-start time.
12843
- const citationContext: CitationContext = {
12844
- ...citationContextBase,
12845
- ts: new Date().toISOString(),
12846
- };
12847
- // `attachCitation` already calls `hasCitationForTemplate` internally and
12848
- // is a no-op when the content already carries a citation (default or
12849
- // custom template). The outer check was redundant and has been removed
12850
- // to avoid a maintenance hazard where the two guard paths could diverge.
12851
- return attachCitation(content, citationContext, citationTemplate);
12852
- };
12853
- const persistedIds: string[] = [];
12854
- const supersessionOrderingAt = (validAt?: string): string =>
12855
- validAt && validAt.length > 0 ? validAt : new Date().toISOString();
12856
- // #1635: pending_review persisted ids, excluded from the thread episode set below.
12857
- const pendingReviewPersistedIds: string[] = [];
12858
- const persistedIdsByStorage = new Map<
12859
- string,
12860
- { storage: StorageManager; ids: string[] }
12861
- >();
12862
- const trackPersistedId = (
12863
- targetStorage: StorageManager,
12864
- id: string,
12865
- options: {
12866
- includeReturnedIds?: boolean;
12867
- /** #1635: keep this id out of the persisted thread episode set. */
12868
- pendingReview?: boolean;
12869
- } = {},
12870
- ): void => {
12871
- if (options.includeReturnedIds !== false) {
12872
- persistedIds.push(id);
12873
- }
12874
- if (options.pendingReview) {
12875
- pendingReviewPersistedIds.push(id);
12876
- }
12877
- const key = targetStorage.dir;
12878
- const existing = persistedIdsByStorage.get(key);
12879
- if (existing) {
12880
- existing.ids.push(id);
12881
- return;
12882
- }
12883
- persistedIdsByStorage.set(key, { storage: targetStorage, ids: [id] });
12884
- };
12885
- let dedupedCount = 0;
12886
- // Counter for facts skipped by the importance write-gate (issue #372).
12887
- // Emitted via the `importance_gated` metric below and rolled into the
12888
- // final `persisted:` log line so operators can tune the threshold.
12889
- let importanceGatedCount = 0;
12890
- // UUI2: short-circuit semantic dedup after first backend-unavailable signal
12891
- // within this batch. Once any fact in the batch gets reason="backend_unavailable"
12892
- // (meaning the embedding backend is degraded), subsequent facts skip the
12893
- // lookup entirely and proceed directly to write. This prevents N-fact batches
12894
- // from paying N × timeout when the backend is down. The flag resets per-batch
12895
- // (declared here, inside persistExtraction) so a transient hiccup in one
12896
- // batch does not permanently disable dedup in future batches.
12897
- let batchBackendUnavailable = false;
12898
- // #1669: per-namespace redaction-rule cache for this persist pass. A
12899
- // `never_store` / redaction_rule correction persists patterns under each
12900
- // namespace's state/corrections/redaction-rules/ dir; we consult them
12901
- // before a fact reaches the storage write chokepoint so matching content
12902
- // is withheld entirely rather than landing as pending_review. Cached per
12903
- // dir so a multi-fact batch over one namespace reads the dir once.
12904
- let redactionGatedCount = 0;
12905
- const redactionRulesByDir = new Map<string, CompiledRedactionRule[]>();
12906
- const redactionRulesFor = async (...dirs: string[]): Promise<CompiledRedactionRule[]> => {
12907
- const out: CompiledRedactionRule[] = [];
12908
- for (const d of dirs) {
12909
- let r = redactionRulesByDir.get(d);
12910
- if (!r) { r = await loadRedactionRules(d); redactionRulesByDir.set(d, r); }
12911
- out.push(...r);
12912
- }
12913
- return out;
12914
- };
12915
- const behaviorSignalsByStorage = new Map<
12916
- string,
12917
- { storage: StorageManager; events: BehaviorSignalEvent[] }
12918
- >();
12919
- const trackBehaviorSignals = (
12920
- targetStorage: StorageManager,
12921
- events: BehaviorSignalEvent[],
12922
- ): void => {
12923
- if (events.length === 0) return;
12924
- const key = targetStorage.dir;
12925
- const existing = behaviorSignalsByStorage.get(key);
12926
- if (existing) {
12927
- existing.events.push(...events);
12928
- return;
12929
- }
12930
- behaviorSignalsByStorage.set(key, {
12931
- storage: targetStorage,
12932
- events: [...events],
12933
- });
12934
- };
12935
- const confidenceTierOrder = [
12936
- "explicit",
12937
- "implied",
12938
- "inferred",
12939
- "speculative",
12940
- ] as const;
12941
- const sharedProfileLayer = scopeProfileWritePlan?.layers.find(
12942
- (layer) =>
12943
- layer.id === "serverShared" &&
12944
- layer.namespace === this.config.sharedNamespace,
12945
- );
12946
- const sharedPromotionTarget = scopeProfileWritePlan?.promotionTargets.find(
12947
- (target) =>
12948
- target.target === "serverShared" &&
12949
- target.namespace === this.config.sharedNamespace,
12950
- );
12951
- const profileAllowsSharedWrites =
12952
- !scopeProfileWritePlan ||
12953
- Boolean(
12954
- scopeProfileWritePlan.profile.readOrder.includes("serverShared") &&
12955
- scopeProfileWritePlan.readNamespaces.includes(this.config.sharedNamespace) &&
12956
- sharedProfileLayer?.readable &&
12957
- sharedProfileLayer.writable &&
12958
- sharedPromotionTarget?.authorized,
12959
- );
12960
- // #1713: hoist namespaces-enabled once for the scope-routing mirrors
12961
- // (pre-judge + write-loop) so no new scattered config.*Enabled read is
12962
- // introduced (ratchet scatteredConfigFlagReads; see #1523).
12963
- const namespacesEnabled = resolveNamespaceCapabilities(this.config).namespaces;
12964
- const profileAutoPromotionAllows = (
12965
- category: string,
12966
- confidence: number,
12967
- ): boolean => {
12968
- if (!scopeProfileWritePlan) return false;
12969
- const actualTier = confidenceTier(confidence);
12970
- const actualRank = confidenceTierOrder.indexOf(actualTier);
12971
- if (actualRank === -1) return false;
12972
- const autoPromote = scopeProfileWritePlan.profile.autoPromote;
12973
- if (!autoPromote.enabled) return false;
12974
- if (!autoPromote.categories.includes(category as any)) return false;
12975
- const minimumRank = confidenceTierOrder.indexOf(autoPromote.minConfidenceTier);
12976
- return minimumRank !== -1 && actualRank <= minimumRank;
12977
- };
12978
- const sharedAutoPromotionAllows = (
12979
- category: string,
12980
- confidence: number,
12981
- ): boolean => {
12982
- if (!scopeProfileWritePlan) {
12983
- const actualTier = confidenceTier(confidence);
12984
- const actualRank = confidenceTierOrder.indexOf(actualTier);
12985
- if (actualRank === -1) return false;
12986
- if (!resolveRecallEnhancementCapabilities(this.config).autoPromoteToShared) return false;
12987
- if (!this.config.autoPromoteToSharedCategories.includes(category as any))
12988
- return false;
12989
- const minimumRank = confidenceTierOrder.indexOf(
12990
- this.config.autoPromoteMinConfidenceTier,
12991
- );
12992
- return minimumRank !== -1 && actualRank <= minimumRank;
12993
- }
12994
- return (
12995
- scopeProfileWritePlan.profile.autoPromote.targets.includes("serverShared") &&
12996
- profileAutoPromotionAllows(category, confidence)
12997
- );
12998
- };
12999
- const shouldPromoteToShared = (
13000
- targetStorage: StorageManager,
13001
- category: string,
13002
- confidence: number,
13003
- ): boolean => {
13004
- if (
13005
- !resolveNamespaceCapabilities(this.config).namespaces ||
13006
- !profileAllowsSharedWrites ||
13007
- !sharedAutoPromotionAllows(category, confidence)
13008
- )
13009
- return false;
13010
- if (
13011
- this.storageDirNamespace(targetStorage.dir) ===
13012
- this.config.sharedNamespace
13013
- )
13014
- return false;
13015
- return true;
13016
- };
13017
- const promoteMemoryToProfileTargets = async (options: {
13018
- sourceStorage: StorageManager;
13019
- category: string;
13020
- content: string;
13021
- confidence: number;
13022
- tags: string[];
13023
- entityRef?: string;
13024
- structuredAttributes?: Record<string, string>;
13025
- sourceMemoryId: string;
13026
- importance?: ReturnType<typeof scoreImportance>;
13027
- intentGoal?: string;
13028
- intentActionType?: string;
13029
- intentEntityTypes?: string[];
13030
- memoryKind?: MemoryFrontmatter["memoryKind"];
13031
- validAt?: string;
13032
- // #1578 — bi-temporal bounds + ingestion provenance forwarded to profile-
13033
- // target copies (same defect class as shared promotion; cursor bugbot).
13034
- invalidAt?: string;
13035
- observedAt?: string;
13036
- eventTimeSource?: "extracted" | "assumed";
13037
- source: string;
13038
- sources?: ProvenanceSource[];
13039
- provenance?: "verified" | "unverified" | "none";
13040
- }): Promise<void> => {
13041
- if (
13042
- !scopeProfileWritePlan ||
13043
- !profileAutoPromotionAllows(options.category, options.confidence)
13044
- )
13045
- return;
13046
- const autoTargets = new Set(scopeProfileWritePlan.profile.autoPromote.targets);
13047
- const targets = scopeProfileWritePlan.promotionTargets.filter(
13048
- (target) =>
13049
- target.target !== "serverShared" &&
13050
- autoTargets.has(target.target) &&
13051
- target.authorized &&
13052
- target.namespace,
13053
- );
13054
- if (targets.length === 0) return;
13055
- const rawContent =
13056
- citationEnabled && hasCitationForTemplate(options.content, citationTemplate)
13057
- ? stripCitationForTemplate(options.content, citationTemplate)
13058
- : options.content;
13059
- const citedContent = applyInlineCitation(rawContent);
13060
- const sanitizedBase = sanitizeMemoryContent(rawContent);
13061
- const dedupContent =
13062
- options.category === "fact" &&
13063
- options.structuredAttributes &&
13064
- Object.keys(options.structuredAttributes).length > 0
13065
- ? `${sanitizedBase.text}\n[Attributes: ${normalizeAttributePairs(options.structuredAttributes)}]`
13066
- : sanitizedBase.text;
13067
- for (const target of targets) {
13068
- if (!target.namespace) continue;
13069
- try {
13070
- const targetStorage = await this.storageRouter.storageFor(target.namespace);
13071
- if (targetStorage.dir === options.sourceStorage.dir) continue;
13072
- if (
13073
- options.category === "fact" &&
13074
- (await targetStorage.hasFactContentHash(dedupContent))
13075
- ) {
13076
- // #1671 — backfill bi-temporal bounds the existing promoted copy
13077
- // lacks (re-extraction with a now-resolved invalidAt). Best-effort,
13078
- // fail-open; the helper gates on invalidAt to avoid I/O when no
13079
- // end bound is present.
13080
- if (options.invalidAt || options.validAt) {
13081
- await this.backfillTemporalBoundsOnDedupHit(
13082
- targetStorage,
13083
- dedupContent,
13084
- {
13085
- invalidAt: options.invalidAt,
13086
- // #1707 thread 2 — carry the corrected start bound.
13087
- validFrom: options.validAt,
13088
- ...(options.observedAt ? { observedAt: options.observedAt } : {}),
13089
- ...(options.eventTimeSource ? { eventTimeSource: options.eventTimeSource } : {}),
13090
- },
13091
- options.entityRef,
13092
- );
13093
- }
13094
- continue;
13095
- }
13096
- const targetPromotion = await targetStorage.writeMemory(
13097
- options.category as any,
13098
- citedContent,
13099
- {
13100
- confidence: options.confidence,
13101
- tags: [...options.tags, `${target.target}-promotion`],
13102
- entityRef: options.entityRef,
13103
- structuredAttributes: options.structuredAttributes,
13104
- source: `${options.source}-${target.target}-promotion`,
13105
- importance: options.importance,
13106
- lineage: [options.sourceMemoryId],
13107
- sourceMemoryId: options.sourceMemoryId,
13108
- intentGoal: options.intentGoal,
13109
- intentActionType: options.intentActionType,
13110
- intentEntityTypes: options.intentEntityTypes,
13111
- memoryKind: options.memoryKind,
13112
- validAt: options.validAt,
13113
- // #1578 — forward bi-temporal bounds + ingestion provenance.
13114
- ...(options.invalidAt ? { invalidAt: options.invalidAt } : {}),
13115
- ...(options.observedAt ? { observedAt: options.observedAt } : {}),
13116
- ...(options.eventTimeSource ? { eventTimeSource: options.eventTimeSource } : {}),
13117
- contentHashSource: options.category === "fact" ? dedupContent : rawContent,
13118
- ...(options.sources && options.sources.length > 0 ? { sources: options.sources } : {}),
13119
- ...(options.provenance ? { provenance: options.provenance } : {}),
13120
- },
13121
- );
13122
- const promotedId = targetPromotion.id;
13123
- // #1645: if the TARGET namespace's own tombstone blocked this promotion,
13124
- // the row lands pending_review — do NOT supersede active target memories.
13125
- if (
13126
- !targetPromotion.tombstoneBlocked &&
13127
- lifecycleCaps.temporalSupersession &&
13128
- options.category === "fact" &&
13129
- options.entityRef &&
13130
- options.structuredAttributes &&
13131
- Object.keys(options.structuredAttributes).length > 0
13132
- ) {
13133
- try {
13134
- await applyTemporalSupersession({
13135
- storage: targetStorage,
13136
- newMemoryId: promotedId,
13137
- entityRef: options.entityRef,
13138
- structuredAttributes: options.structuredAttributes,
13139
- createdAt: supersessionOrderingAt(options.validAt),
13140
- enabled: !(options.eventTimeSource === "extracted" && !options.validAt),
13141
- });
13142
- } catch (profileSupersessionErr) {
13143
- log.warn(
13144
- `persistExtraction: ${target.target} promotion temporal supersession failed open for promoted ${promotedId}: ${profileSupersessionErr}`,
13145
- );
13146
- }
13147
- }
13148
- // #1645 TV6: a tombstone-blocked promotion is pending_review (no
13149
- // active copy) — skip catalog/index/behavior like postWriteGuard.
13150
- if (!targetPromotion.tombstoneBlocked) {
13151
- trackPersistedId(targetStorage, promotedId, { includeReturnedIds: false });
13152
- await this.indexPersistedMemory(targetStorage, promotedId);
13153
- trackBehaviorSignals(
13154
- targetStorage,
13155
- buildBehaviorSignalsForMemory({
13156
- memoryId: promotedId,
13157
- category: options.category as any,
13158
- content: options.content,
13159
- namespace: target.namespace,
13160
- confidence: options.confidence,
13161
- source: "extraction",
13162
- }),
13163
- );
13164
- }
13165
- } catch (err) {
13166
- log.warn(
13167
- `persistExtraction: ${target.target} promotion failed open for ${options.sourceMemoryId}: ${err}`,
13168
- );
13169
- }
13170
- }
13171
- };
13172
- const promoteMemoryToShared = async (options: {
13173
- sourceStorage: StorageManager;
13174
- category: string;
13175
- content: string;
13176
- confidence: number;
13177
- tags: string[];
13178
- entityRef?: string;
13179
- structuredAttributes?: Record<string, string>;
13180
- sourceMemoryId: string;
13181
- importance?: ReturnType<typeof scoreImportance>;
13182
- intentGoal?: string;
13183
- intentActionType?: string;
13184
- intentEntityTypes?: string[];
13185
- memoryKind?: MemoryFrontmatter["memoryKind"];
13186
- validAt?: string;
13187
- // #1578 — bi-temporal bounds + ingestion provenance forwarded to the
13188
- // shared-namespace copy so shared recall honours the same invalid_at
13189
- // window as the source fact (cursor bugbot).
13190
- invalidAt?: string;
13191
- observedAt?: string;
13192
- eventTimeSource?: "extracted" | "assumed";
13193
- source: string;
13194
- /** Claim-level provenance spans (issue #1575 PR 2). */
13195
- sources?: ProvenanceSource[];
13196
- provenance?: "verified" | "unverified" | "none";
13197
- }): Promise<void> => {
13198
- await promoteMemoryToProfileTargets(options);
13199
- if (
13200
- !shouldPromoteToShared(
13201
- options.sourceStorage,
13202
- options.category,
13203
- options.confidence,
13204
- )
13205
- )
13206
- return;
13207
- try {
13208
- const sharedStorage = await this.storageRouter.storageFor(
13209
- this.config.sharedNamespace,
13210
- );
13211
- // Dedup gate: canonicalize content before hashing.
13212
- //
13213
- // Issue #369 (PR #401): When inline attribution is enabled,
13214
- // `applyInlineCitation` appends a timestamp-bearing marker (e.g.
13215
- // `[Source: ..., ts=2026-04-11T...]`). Because the timestamp changes
13216
- // on every call, hashing cited content produces a unique hash each
13217
- // time — defeating dedup entirely and allowing the same logical fact to
13218
- // be promoted repeatedly. Also, both promotion call sites pass
13219
- // `fact.content`, which can already carry an inline citation (e.g. a
13220
- // relayed or reprocessed fact). Strip any pre-existing citation so the
13221
- // dedup key matches the hash stored from the original un-cited write.
13222
- //
13223
- // PR #402 round-6 (Fix #2 / chatgpt-codex P1 PRRT_kwDORJXyws56U74n):
13224
- // Compute the enriched content before the hash-dedup check so the
13225
- // lookup uses the same content that writeMemory will actually store.
13226
- // When structuredAttributes are present, writeMemory appends an
13227
- // "[Attributes: ...]" suffix before hashing; hasFactContentHash must
13228
- // receive the same enriched body or the check is against a different
13229
- // hash and dedup fails to fire (letting duplicates through) or fires
13230
- // when it shouldn't (collapsing memories with different enrichments).
13231
- // Fix #1 (P2 PRRT_kwDORJXyws56VHZc): use normalizeAttributePairs so
13232
- // key order and casing are canonical — identical to the enrichment
13233
- // applied by storage.writeMemory — preventing spurious hash misses
13234
- // when attribute maps arrive with different insertion orders or casing.
13235
- //
13236
- // Fix #4 (Low PRRT_kwDORJXyws56VHth): sanitize the base content before
13237
- // building dedupContent. writeMemory runs sanitizeMemoryContent on the
13238
- // enriched body before hashing; if sanitization redacts the content to
13239
- // REDACTED_PLACEHOLDER the stored hash is for the redacted form, not
13240
- // the raw form. Computing dedupContent from sanitized.text here ensures
13241
- // the hash lookup and the normalizedIncoming comparison both use the
13242
- // same content that writeMemory will actually store.
13243
- //
13244
- // Combined fix: strip any pre-existing citation FIRST to obtain
13245
- // rawContent (the canonical body), then sanitize rawContent (not
13246
- // options.content) when building dedupContent, so that citation
13247
- // stripping and sanitization are applied in a consistent order.
13248
- const rawContent =
13249
- citationEnabled &&
13250
- hasCitationForTemplate(options.content, citationTemplate)
13251
- ? stripCitationForTemplate(options.content, citationTemplate)
13252
- : options.content;
13253
- const citedContent = applyInlineCitation(rawContent);
13254
- const sanitizedBase = sanitizeMemoryContent(rawContent);
13255
- const dedupContent =
13256
- options.category === "fact" &&
13257
- options.structuredAttributes &&
13258
- Object.keys(options.structuredAttributes).length > 0
13259
- ? `${sanitizedBase.text}\n[Attributes: ${normalizeAttributePairs(options.structuredAttributes)}]`
13260
- : sanitizedBase.text;
13261
- if (
13262
- options.category === "fact" &&
13263
- (await sharedStorage.hasFactContentHash(dedupContent))
13264
- ) {
13265
- // #1671 — backfill bi-temporal bounds onto the existing shared copy
13266
- // before the supersession short-circuit. Covers all return paths below
13267
- // (supersession-hit, catch-skip, and the no-supersession short-circuit)
13268
- // in one shot. Best-effort / fail-open; the helper gates on invalidAt.
13269
- if (options.invalidAt || options.validAt) {
13270
- await this.backfillTemporalBoundsOnDedupHit(
13271
- sharedStorage,
13272
- dedupContent,
13273
- {
13274
- invalidAt: options.invalidAt,
13275
- // #1707 thread 2 — carry the corrected start bound.
13276
- validFrom: options.validAt,
13277
- ...(options.observedAt ? { observedAt: options.observedAt } : {}),
13278
- ...(options.eventTimeSource ? { eventTimeSource: options.eventTimeSource } : {}),
13279
- },
13280
- options.entityRef,
13281
- );
13282
- }
13283
- // Uj6H fix: shared-namespace temporal supersession must also run when
13284
- // the hash-dedup short-circuit fires. Without this, an existing shared
13285
- // fact whose structuredAttributes are stale (or an older conflicting
13286
- // shared fact that is still active) never gets retired — supersession
13287
- // only ran in the post-writeMemory block which is unreachable here.
13288
- //
13289
- // Strategy: scan the shared namespace for the existing fact whose
13290
- // normalized content matches the incoming content, then run
13291
- // applyTemporalSupersession against it using the same logic that
13292
- // would have run post-writeMemory. This is a best-effort / fail-open
13293
- // step — if the lookup fails we skip silently (same as the normal path).
13294
- if (
13295
- lifecycleCaps.temporalSupersession &&
13296
- options.entityRef &&
13297
- options.structuredAttributes &&
13298
- Object.keys(options.structuredAttributes).length > 0
13299
- ) {
13300
- // PR #402 round-7 (Fix #2 / Codex P1 PRRT_kwDORJXyws56VALC):
13301
- // Track whether matchingFact lookup completed before the try block
13302
- // so the catch block can distinguish an early-lookup failure (where
13303
- // we don't know if a duplicate exists) from a post-lookup supersession
13304
- // failure (where we confirmed a duplicate and must skip the write).
13305
- let hashDedupMatchingFact: MemoryFile | undefined;
13306
- let hashDedupLookupComplete = false;
13307
- try {
13308
- // Fix #2 (P2 PRRT_kwDORJXyws56VHZf): dedupContent is now built
13309
- // from sanitizedBase.text (see fix #4 above), so normalizedIncoming
13310
- // uses the same sanitized+normalized content that writeMemory hashes
13311
- // and that hasFactContentHash just matched. Previously this used the
13312
- // raw options.content, which diverged from the stored hash when
13313
- // sanitization redacted the content, causing the candidate lookup to
13314
- // return undefined and leaving stale facts active.
13315
- const normalizedIncoming = ContentHashIndex.normalizeContent(dedupContent);
13316
- const allShared = await sharedStorage.readAllMemories();
13317
- // PR #402 round-12 (Finding Uybg): restrict hash-dedup matching to
13318
- // the SAME entity. Content-hash equality alone can collide across
13319
- // entities when two entities share identical fact text. Using an
13320
- // unrelated entity's existing fact as `newMemoryId` would anchor
13321
- // supersession to that entity's record and corrupt its
13322
- // `supersededBy` links. Only consider facts whose normalized
13323
- // `entityRef` matches the incoming entity.
13324
- const incomingEntityNorm = normalizeSupersessionKey(options.entityRef);
13325
- hashDedupMatchingFact = allShared.find((m) => {
13326
- if (m.frontmatter.category !== "fact") return false;
13327
- if ((m.frontmatter.status ?? "active") !== "active") return false;
13328
- // Same-entity guard: skip if entity doesn't match.
13329
- if (!m.frontmatter.entityRef) return false;
13330
- if (normalizeSupersessionKey(m.frontmatter.entityRef) !== incomingEntityNorm) {
13331
- log.debug(
13332
- `persistExtraction: hash-dedup skipping cross-entity match (incoming="${incomingEntityNorm}" candidate="${normalizeSupersessionKey(m.frontmatter.entityRef)}")`,
13333
- );
13334
- return false;
13335
- }
13336
- // PR #402 round-7 (Fix #2): compare stored fact's full body
13337
- // (including any appended "[Attributes: ...]" suffix) against the
13338
- // enriched normalizedIncoming so the candidate selected is the one
13339
- // whose hash actually matched in hasFactContentHash.
13340
- return ContentHashIndex.normalizeContent(m.content ?? "") === normalizedIncoming;
13341
- });
13342
- hashDedupLookupComplete = true;
13343
- if (hashDedupMatchingFact) {
13344
- // Finding UvU1 (PR #402 round-11): anchor supersession to the
13345
- // incoming event's time, not the existing fact's persisted
13346
- // `created`. For source-dated replay/import, this is the
13347
- // source valid_at; otherwise it is the current wall-clock. The
13348
- // matching fact may be an old shared copy whose `created`
13349
- // predates the incoming promotion event — using it as
13350
- // `createdAt` would make the new memory appear older than the
13351
- // existing one, preventing supersession from firing.
13352
- // PR #402 round-12 (Finding Uyui): the matching fact is an
13353
- // existing OLD memory — its persisted `frontmatter.created` is
13354
- // stale relative to the incoming promotion event. Pass
13355
- // `useCallerTimestamp: true` so the function uses
13356
- // `createdAt` as the ordering anchor instead of the old fact's
13357
- // timestamp, ensuring supersession fires correctly even when
13358
- // the matching fact predates conflicting candidates.
13359
- const hashDedupSupersession = await applyTemporalSupersession({
13360
- storage: sharedStorage,
13361
- newMemoryId: hashDedupMatchingFact.frontmatter.id,
13362
- entityRef: options.entityRef,
13363
- structuredAttributes: options.structuredAttributes,
13364
- createdAt: supersessionOrderingAt(options.validAt),
13365
- enabled: !(options.eventTimeSource === "extracted" && !options.validAt),
13366
- useCallerTimestamp: true,
13367
- });
13368
- // Catalog touch (issue #1499 — codex P2 NElSf): this dedup branch
13369
- // returns WITHOUT the post-write catalog touch (now at the storage chokepoint #1522),
13370
- // but `applyTemporalSupersession` mutated the shared namespace
13371
- // (it rewrote frontmatter to retire stale shared facts). When any
13372
- // ids were actually superseded, the shared namespace changed, so we
13373
- // must record the write — otherwise the shared record's
13374
- // `lastWriteAt` stays stale and `writtenSince` maintenance / QMD
13375
- // fanout skips the namespace after a supersession-only update.
13376
- // Best-effort and failure-tolerant (the storage chokepoint swallows
13377
- // errors); only touch when work happened to avoid spurious writes.
13378
- if (hashDedupSupersession.supersededIds.length > 0) {
13379
- }
13380
- // Active matching fact exists — normal short-circuit is safe.
13381
- return;
13382
- }
13383
- // No active same-entity shared fact found with this content hash.
13384
- // This can happen when the previously-written shared fact has since
13385
- // been superseded (e.g. Austin → NYC → Austin reversion): the hash
13386
- // index still records the hash but the fact is no longer active.
13387
- // Fall through to the write path below so a new active shared
13388
- // memory is created, then supersession fires post-write as usual.
13389
- log.debug(
13390
- `persistExtraction: hash-dedup found no active same-entity shared fact for ${options.sourceMemoryId}; falling through to write`,
13391
- );
13392
- } catch (hashDedupSupersessionErr) {
13393
- log.warn(
13394
- `persistExtraction: shared-namespace supersession on hash-dedup path failed open for ${options.sourceMemoryId}: ${hashDedupSupersessionErr}`,
13395
- );
13396
- // PR #402 round-7 (Fix #1 / cursor Medium PRRT_kwDORJXyws56U_ig):
13397
- // Only skip the write if we CONFIRMED a matching active shared fact
13398
- // before the error occurred (hashDedupLookupComplete is true AND
13399
- // hashDedupMatchingFact is set). If the error was thrown before
13400
- // matchingFact was resolved — e.g. readAllMemories() threw — we
13401
- // cannot assume a duplicate exists, and unconditionally returning
13402
- // would permanently lose the shared promotion. Fall through to the
13403
- // write path so the fact is not silently dropped.
13404
- if (hashDedupLookupComplete && hashDedupMatchingFact) {
13405
- // A matching active shared fact was confirmed — skip the write to
13406
- // avoid duplicating content that is already present. The existing
13407
- // fact remains active and the supersession failure is logged above.
13408
- return;
13409
- }
13410
- // Lookup did not complete or no candidate was found — we cannot
13411
- // confirm a duplicate. Fall through to the write + post-write
13412
- // supersession path so the shared promotion is not lost.
13413
- log.debug(
13414
- `persistExtraction: hash-dedup catch: lookup incomplete or no candidate found for ${options.sourceMemoryId}; falling through to write`,
13415
- );
13416
- }
13417
- } else {
13418
- // temporalSupersessionEnabled is off or no entity/attributes — keep
13419
- // the original short-circuit behaviour.
13420
- return;
13421
- }
13422
- }
13423
- const sharedPromotion = await sharedStorage.writeMemory(
13424
- options.category as any,
13425
- citedContent,
13426
- {
13427
- confidence: options.confidence,
13428
- tags: [...options.tags, "shared-promotion"],
13429
- entityRef: options.entityRef,
13430
- structuredAttributes: options.structuredAttributes,
13431
- source: `${options.source}-shared-promotion`,
13432
- importance: options.importance,
13433
- lineage: [options.sourceMemoryId],
13434
- sourceMemoryId: options.sourceMemoryId,
13435
- intentGoal: options.intentGoal,
13436
- intentActionType: options.intentActionType,
13437
- intentEntityTypes: options.intentEntityTypes,
13438
- memoryKind: options.memoryKind,
13439
- validAt: options.validAt,
13440
- // #1578 — forward bi-temporal bounds + ingestion provenance.
13441
- ...(options.invalidAt ? { invalidAt: options.invalidAt } : {}),
13442
- ...(options.observedAt ? { observedAt: options.observedAt } : {}),
13443
- ...(options.eventTimeSource ? { eventTimeSource: options.eventTimeSource } : {}),
13444
- contentHashSource: options.category === "fact" ? dedupContent : rawContent,
13445
- // Claim-level provenance spans (issue #1575 PR 2).
13446
- ...(options.sources && options.sources.length > 0 ? { sources: options.sources } : {}),
13447
- ...(options.provenance ? { provenance: options.provenance } : {}),
13448
- },
13449
- );
13450
- const promotedId = sharedPromotion.id;
13451
- // #1645: if the shared namespace's own tombstone blocked this promotion,
13452
- // leave the row pending_review but do NOT supersede active shared memories.
13453
- // PR #402 Finding 3 fix: run temporal supersession against the shared
13454
- // namespace after the promoted write lands so stale shared-namespace
13455
- // copies of the same entity attribute are retired. Without this,
13456
- // source-namespace supersession leaves the shared copy active and
13457
- // shared recall continues returning the stale state. Reuses the same
13458
- // applyTemporalSupersession helper — no logic duplication.
13459
- if (
13460
- !sharedPromotion.tombstoneBlocked &&
13461
- lifecycleCaps.temporalSupersession &&
13462
- options.entityRef &&
13463
- options.structuredAttributes &&
13464
- Object.keys(options.structuredAttributes).length > 0
13465
- ) {
13466
- try {
13467
- await applyTemporalSupersession({
13468
- storage: sharedStorage,
13469
- newMemoryId: promotedId,
13470
- entityRef: options.entityRef,
13471
- structuredAttributes: options.structuredAttributes,
13472
- createdAt: supersessionOrderingAt(options.validAt),
13473
- enabled: !(options.eventTimeSource === "extracted" && !options.validAt),
13474
- });
13475
- } catch (sharedSupersessionErr) {
13476
- log.warn(
13477
- `persistExtraction: shared-namespace temporal supersession failed open for promoted ${promotedId}: ${sharedSupersessionErr}`,
13478
- );
13479
- }
13480
- }
13481
- // Catalog touch (issue #1499, Issue B + ordering sweep): a shared-
13482
- // namespace promotion is the ONLY write the shared namespace receives on
13483
- // this path, so without this the shared record's lastWriteAt stays stale
13484
- // and `writtenSince` filters / maintenance fanout skip it. Record AFTER
13485
- // the promoted write and the shared temporal-supersession attempt so the
13486
- // catalog timestamp never precedes a later durable frontmatter mutation in
13487
- // the same promotion pass. The hot-path source-namespace touch uses a
13488
- // different storage dir, so this does not double-count the source.
13489
- // Best-effort and failure-tolerant — it must never crash the promotion.
13490
- // #1645 TV6: same guard as the profile-target promotion above.
13491
- if (!sharedPromotion.tombstoneBlocked) {
13492
- trackPersistedId(sharedStorage, promotedId, {
13493
- includeReturnedIds: false,
13494
- });
13495
- await this.indexPersistedMemory(sharedStorage, promotedId);
13496
- trackBehaviorSignals(
13497
- sharedStorage,
13498
- buildBehaviorSignalsForMemory({
13499
- memoryId: promotedId,
13500
- category: options.category as any,
13501
- content: options.content,
13502
- namespace: this.config.sharedNamespace,
13503
- confidence: options.confidence,
13504
- source: "extraction",
13505
- }),
13506
- );
13507
- }
13508
- } catch (err) {
13509
- log.warn(
13510
- `persistExtraction: shared promotion failed open for ${options.sourceMemoryId}: ${err}`,
13511
- );
13512
- }
13513
- };
13514
- // #1707 thread 1 — backfill temporal bounds onto promotion copies when the
13515
- // SOURCE-namespace dedup short-circuit fires. That branch patches the
13516
- // source copy then `continue`s before the promotion dedup paths run, so
13517
- // promoted shared/profile copies written before the source fact carried
13518
- // resolved bounds stay stale (cross-namespace recall surfaces an expired
13519
- // fact). This mirrors the promotion-target resolution used by the two
13520
- // promote closures above and calls the same fail-open helper against each
13521
- // target storage. Backfill-only: never writes a new promoted copy.
13522
- const backfillTemporalBoundsOnPromotionCopies = async (args: {
13523
- sourceStorage: StorageManager;
13524
- content: string;
13525
- category: string;
13526
- confidence: number;
13527
- entityRef?: string;
13528
- structuredAttributes?: Record<string, string>;
13529
- bounds: {
13530
- invalidAt?: string;
13531
- validFrom?: string;
13532
- observedAt?: string;
13533
- eventTimeSource?: "extracted" | "assumed";
13534
- };
13535
- }): Promise<void> => {
13536
- // Mirror the helper's I/O gate: only resolve promotion targets when
13537
- // there is an end bound OR an EXTRACTED start bound. An "assumed"
13538
- // validFrom is just the ingestion anchor (no recall effect), so
13539
- // skipping it avoids resolving target storages on every bi-temporal
13540
- // duplicate (review cursor PRRT_OvHk).
13541
- const hasExtractedStart =
13542
- args.bounds.validFrom !== undefined &&
13543
- args.bounds.eventTimeSource === "extracted";
13544
- if (!args.bounds.invalidAt && !hasExtractedStart) return;
13545
- // Build the same dedupContent the promotion functions hash on so the
13546
- // content-hash lookup in the helper matches what was stored.
13547
- const rawContent =
13548
- citationEnabled && hasCitationForTemplate(args.content, citationTemplate)
13549
- ? stripCitationForTemplate(args.content, citationTemplate)
13550
- : args.content;
13551
- const sanitizedBase = sanitizeMemoryContent(rawContent);
13552
- const dedupContent =
13553
- args.category === "fact" &&
13554
- args.structuredAttributes &&
13555
- Object.keys(args.structuredAttributes).length > 0
13556
- ? `${sanitizedBase.text}\n[Attributes: ${normalizeAttributePairs(args.structuredAttributes)}]`
13557
- : sanitizedBase.text;
13558
- // Profile targets. NOTE: we do NOT gate on profileAutoPromotionAllows —
13559
- // a promoted profile copy may exist from an EARLIER extraction with
13560
- // higher confidence or older auto-promote settings, so the current
13561
- // duplicate's confidence must not skip backfilling an already-existing
13562
- // promoted copy (review codex PRRT_Ov7LKF). The helper no-ops when no
13563
- // matching copy is found, so resolving all configured auto-promote
13564
- // targets is safe.
13565
- if (scopeProfileWritePlan) {
13566
- const autoTargets = new Set(
13567
- scopeProfileWritePlan.profile.autoPromote.targets,
13568
- );
13569
- const profileTargets = scopeProfileWritePlan.promotionTargets.filter(
13570
- (target) =>
13571
- target.target !== "serverShared" &&
13572
- autoTargets.has(target.target) &&
13573
- target.authorized &&
13574
- target.namespace,
13575
- );
13576
- for (const target of profileTargets) {
13577
- if (!target.namespace) continue;
13578
- try {
13579
- const targetStorage = await this.storageRouter.storageFor(
13580
- target.namespace,
13581
- );
13582
- if (targetStorage.dir === args.sourceStorage.dir) continue;
13583
- await this.backfillTemporalBoundsOnDedupHit(
13584
- targetStorage,
13585
- dedupContent,
13586
- args.bounds,
13587
- args.entityRef,
13588
- );
13589
- } catch (err) {
13590
- log.warn(
13591
- `bitemporal-backfill: profile-target backfill failed open for ${target.target}: ${err}`,
13592
- );
13593
- }
13594
- }
13595
- }
13596
- // Shared target. A shared copy may exist from an earlier extraction
13597
- // regardless of the current confidence, so do not gate on
13598
- // shouldPromoteToShared (review codex PRRT_Ov7LKF) — BUT still respect
13599
- // shared-write AUTHORIZATION: a scoped profile that does not authorize
13600
- // serverShared writes must not have its shared namespace backfilled
13601
- // (review codex PRRT_Ov7dHR). profileAllowsSharedWrites is true for the
13602
- // legacy no-scope case and encodes the readable/writable/authorized
13603
- // checks under a scope profile.
13604
- if (profileAllowsSharedWrites) {
13605
- try {
13606
- const sharedStorage = await this.storageRouter.storageFor(
13607
- this.config.sharedNamespace,
13608
- );
13609
- if (sharedStorage.dir !== args.sourceStorage.dir) {
13610
- await this.backfillTemporalBoundsOnDedupHit(
13611
- sharedStorage,
13612
- dedupContent,
13613
- args.bounds,
13614
- args.entityRef,
13615
- );
13616
- }
13617
- } catch (err) {
13618
- log.warn(
13619
- `bitemporal-backfill: shared-target backfill failed open: ${err}`,
13620
- );
13621
- }
13622
- }
13623
- };
13624
-
13625
- // Defensive: validate result and facts array
13626
- if (!result || !Array.isArray(result.facts)) {
13627
- log.warn(
13628
- "persistExtraction: result or result.facts is invalid, skipping",
13629
- { resultType: typeof result, factsType: typeof result?.facts },
13630
- );
13631
- return persistedIds;
13632
- }
13633
-
13634
- // Chunking config from plugin settings
13635
- const chunkingConfig: ChunkingConfig = {
13636
- targetTokens: this.config.chunkingTargetTokens,
13637
- minTokens: this.config.chunkingMinTokens,
13638
- overlapSentences: this.config.chunkingOverlapSentences,
13639
- };
13640
-
13641
- const rawEntities = Array.isArray((result as any).entities)
13642
- ? (result as any).entities
13643
- : [];
13644
- const rawQuestions = Array.isArray((result as any).questions)
13645
- ? (result as any).questions
13646
- : [];
13647
- const rawProfileUpdates = Array.isArray((result as any).profileUpdates)
13648
- ? (result as any).profileUpdates
13649
- : [];
13650
-
13651
- const facts = result.facts.slice(0, this.config.extractionMaxFactsPerRun);
13652
- const entities = rawEntities.slice(
13653
- 0,
13654
- this.config.extractionMaxEntitiesPerRun,
13655
- );
13656
- const questions = rawQuestions.slice(
13657
- 0,
13658
- this.config.extractionMaxQuestionsPerRun,
13659
- );
13660
- const profileUpdates = rawProfileUpdates.slice(
13661
- 0,
13662
- this.config.extractionMaxProfileUpdatesPerRun,
13663
- );
13664
-
13665
- if (
13666
- facts.length < result.facts.length ||
13667
- entities.length < result.entities.length ||
13668
- questions.length < result.questions.length ||
13669
- profileUpdates.length < result.profileUpdates.length
13670
- ) {
13671
- log.warn(
13672
- "persistExtraction: capped extraction payload to guardrails " +
13673
- `(facts ${facts.length}/${result.facts.length}, entities ${entities.length}/${result.entities.length}, ` +
13674
- `questions ${questions.length}/${result.questions.length}, profile ${profileUpdates.length}/${result.profileUpdates.length})`,
13675
- );
13676
- }
13677
-
13678
- // v8.2: pre-load all memories once for entity-sibling graph edges (avoids per-fact disk scan)
13679
- type GraphStorageContext = {
13680
- allMemsForGraph: Awaited<
13681
- ReturnType<typeof storage.readAllMemories>
13682
- > | null;
13683
- memoryPathById: Map<string, string>;
13684
- previousPersistedRelPath?: string;
13685
- };
13686
- const graphContextByStorageDir = new Map<string, GraphStorageContext>();
13687
- const ensureGraphContext = async (
13688
- targetStorage: StorageManager,
13689
- ): Promise<GraphStorageContext> => {
13690
- const existing = graphContextByStorageDir.get(targetStorage.dir);
13691
- if (existing) return existing;
13692
- const created: GraphStorageContext = {
13693
- allMemsForGraph: null,
13694
- memoryPathById: new Map<string, string>(),
13695
- };
13696
- if (graphCaps.multiGraphMemory) {
13697
- try {
13698
- created.allMemsForGraph = await targetStorage.readAllMemories();
13699
- for (const [id, relPath] of buildMemoryPathById(
13700
- created.allMemsForGraph,
13701
- targetStorage.dir,
13702
- )) {
13703
- created.memoryPathById.set(id, relPath);
13704
- }
13705
- } catch {
13706
- /* fail-open */
13707
- }
13708
- }
13709
- graphContextByStorageDir.set(targetStorage.dir, created);
13710
- return created;
13711
- };
13712
- let threadEpisodeIdsForGraph: string[] | undefined;
13713
- if (graphCaps.multiGraphMemory && threadIdForExtraction) {
13714
- try {
13715
- const thread = await this.threading.loadThread(threadIdForExtraction);
13716
- threadEpisodeIdsForGraph = thread?.episodeIds
13717
- ? [...thread.episodeIds]
13718
- : [];
13719
- } catch {
13720
- /* fail-open */
13721
- }
13722
- }
13723
- const routeRules = await this.loadRoutingRules();
13724
- const routeOptions = this.routeEngineOptions();
13725
-
13726
- // Pre-routing pass: compute the routed category for every fact BEFORE
13727
- // building judge candidates. Route rules may override f.category (e.g.
13728
- // via taxonomy remapping), and the judge must evaluate against the
13729
- // *final* category that will actually be persisted — not the raw
13730
- // extraction-time category. The per-fact write loop below reuses
13731
- // these pre-computed results so routing is evaluated exactly once per
13732
- // fact (no duplicated logic).
13733
- const preRoutedCategories: Array<string | undefined> = new Array(facts.length);
13734
- // #1713 Item 1: collect routed target namespaces so the pre-judge redaction
13735
- // filter can consult rules from cross-namespace targets, not just the source.
13736
- // #1713 P2 (cursor): track per-fact routed namespace for per-fact pre-judge rules
13737
- const preRoutedNamespaceByFact: Array<string | undefined> = new Array(facts.length);
13738
- if (routeRules.length > 0) {
13739
- for (let fi = 0; fi < facts.length; fi++) {
13740
- const f = facts[fi];
13741
- if (
13742
- !f ||
13743
- typeof f.content !== "string" ||
13744
- !f.content.trim() ||
13745
- typeof f.category !== "string" ||
13746
- !f.category.trim()
13747
- ) {
13748
- continue;
13749
- }
13750
- try {
13751
- const tags = Array.isArray(f.tags) ? f.tags : [];
13752
- const routeText = `${f.category} ${tags.join(" ")} ${f.content}`;
13753
- const selected = selectRouteRule(routeText, routeRules, routeOptions);
13754
- if (selected?.target.category) {
13755
- preRoutedCategories[fi] = selected.target.category;
13756
- }
13757
- if (selected?.target.namespace) {
13758
- preRoutedNamespaceByFact[fi] = selected.target.namespace;
13759
- }
13760
- } catch {
13761
- // Fail-open: routing errors fall through to the extracted category.
13762
- }
13763
- }
13764
- }
13765
-
13766
- // Extraction judge gate (issue #376). When enabled, batch-evaluate all
13767
- // candidate facts for durability before the per-fact write loop.
13768
- // The verdicts map is keyed by candidate index — we maintain a
13769
- // candidateIndexToFactIndex mapping so the write loop can look up
13770
- // verdicts by original fact index.
13771
- //
13772
- // Candidates are built using the *routed* category (preRoutedCategories)
13773
- // so the judge evaluates durability against the same category that will
13774
- // be persisted, not the raw extraction-time category.
13775
- let judgeVerdictsByFactIndex: Map<number, import("./extraction-judge.js").JudgeVerdict> | null = null;
13776
- let judgeGatedCount = 0;
13777
- // Reset the side-channel defer count at the start of every
13778
- // persistExtraction call so stale state from a prior call cannot leak
13779
- // into the caller's buffer-retention decision.
13780
- this.lastPersistExtractionDeferredCount = 0;
13781
- if (lifecycleCaps.extractionJudge) {
13782
- // #1669 P1 + #1713 Item 1: pre-filter redacted facts from judge candidates
13783
- // so never-store content is not persisted as judge training data.
13784
- // Consult source + shared + routed target namespace rules so a
13785
- // never-store pattern registered under a cross-namespace target is
13786
- // caught at the batch pre-filter point, not just at the persist gate.
13787
- let preJudgeRedactionRules: CompiledRedactionRule[] = [];
13788
- try {
13789
- // #1713: base rules are source-only. Per-fact routed target rules
13790
- // (including shared when a fact is routed there) are checked in the
13791
- // fact loop, matching the write-time gate's per-fact scoping exactly.
13792
- preJudgeRedactionRules = await redactionRulesFor(storage.dir);
13793
- } catch { /* fail open */ }
13794
- try {
13795
- const judgeCandidates: JudgeCandidate[] = [];
13796
- const candidateToFactIndex: number[] = [];
13797
- for (let fi = 0; fi < facts.length; fi++) {
13798
- const f = facts[fi];
13799
- if (
13800
- !f ||
13801
- typeof f.content !== "string" ||
13802
- !f.content.trim() ||
13803
- typeof f.category !== "string" ||
13804
- !f.category.trim()
13805
- ) {
13806
- continue;
13807
- }
13808
- // Use the routed category when available so the judge sees the
13809
- // final persisted category, not the raw extraction-time value.
13810
- // Cast to MemoryCategory — routing targets are always valid
13811
- // category slugs defined in the taxonomy; the fallback is the
13812
- // original ExtractedFact.category which is already typed.
13813
- const judgeCategory = (preRoutedCategories[fi] ?? f.category) as import("./types.js").MemoryCategory;
13814
- if (judgeCategory === "procedure") {
13815
- continue;
13816
- }
13817
- const tags = Array.isArray(f.tags) ? f.tags : [];
13818
- const imp = scoreImportance(
13819
- f.content,
13820
- judgeCategory,
13821
- tags,
13822
- );
13823
- // Pre-filter: skip facts below importance threshold to avoid
13824
- // wasting LLM calls on facts that will be filtered anyway in
13825
- // the per-fact write loop (issue #376 review finding).
13826
- if (
13827
- !isAboveImportanceThreshold(
13828
- imp.level,
13829
- this.config.extractionMinImportanceLevel,
13830
- )
13831
- ) {
13832
- continue;
13833
- }
13834
- // #1713 P2 (cursor): per-fact rules — load this fact's routed target
13835
- // namespace rules FIRST, then check. This catches target-only rules
13836
- // even when the base (source+shared) rules are empty (threads acc58c42
13837
- // + PBJEe/PBKAj).
13838
- let factRedactionRules = preJudgeRedactionRules;
13839
- let factNs = preRoutedNamespaceByFact[fi];
13840
- // #1713 (codex PRRT_kwDORJXyws6PBj5X): scope classification can route
13841
- // a scope=global fact to the shared namespace independent of routing
13842
- // rules. If so, the pre-judge filter must consult the shared
13843
- // namespace's rules too, or a never-store pattern under shared is
13844
- // missed at pre-judge and only caught at the write gate — letting the
13845
- // content reach the extraction judge/training path. Mirror the write
13846
- // loop's scope-routing conditions exactly.
13847
- if (
13848
- !factNs &&
13849
- lifecycleCaps.extractionScopeClassification &&
13850
- namespacesEnabled &&
13851
- f.scope === "global" &&
13852
- profileAllowsSharedWrites &&
13853
- this.storageDirNamespace(storage.dir) !== this.config.sharedNamespace
13854
- ) {
13855
- factNs = this.config.sharedNamespace;
13856
- }
13857
- if (factNs) {
13858
- try {
13859
- const factDir = (await this.storageRouter.storageFor(factNs)).dir;
13860
- if (factDir !== storage.dir) {
13861
- const targetRules = await redactionRulesFor(factDir);
13862
- if (targetRules.length) factRedactionRules = [...preJudgeRedactionRules, ...targetRules];
13863
- }
13864
- } catch { /* fail open */ }
13865
- }
13866
- if (factRedactionRules.length > 0) {
13867
- const rc = f.content + (f.structuredAttributes ? " " + JSON.stringify(f.structuredAttributes) : "")
13868
- + (f.procedureSteps ? " " + f.procedureSteps.map((s) => `${s.intent} ${s.expectedOutcome ?? ""} ${s.toolCall ? `${s.toolCall.kind} ${s.toolCall.signature}` : ""}`.trim()).join(" ") : "");
13869
- if (contentMatchesRedactionRules(rc, factRedactionRules)) continue;
13870
- }
13871
- judgeCandidates.push({
13872
- text: f.content,
13873
- category: judgeCategory,
13874
- confidence: typeof f.confidence === "number" ? f.confidence : 0.7,
13875
- tags,
13876
- importanceLevel: imp.level,
13877
- });
13878
- candidateToFactIndex.push(fi);
13879
- }
13880
- // Telemetry + training-pair emit (issue #562 PR 3 + PR 4). The
13881
- // orchestrator wires two fire-and-forget writers behind a single
13882
- // callback so `judgeFactDurability` does not need to know about
13883
- // either ledger. Both handlers are skipped when their flags are
13884
- // off; the combined callback itself is undefined when both are
13885
- // disabled so there is zero overhead in the default configuration.
13886
- const judgeTelemetryOpts = {
13887
- enabled: lifecycleCaps.extractionJudgeTelemetry,
13888
- memoryDir: this.config.memoryDir,
13889
- };
13890
- const judgeTrainingOpts = {
13891
- enabled: this.config.collectJudgeTrainingPairs === true,
13892
- ...(this.config.judgeTrainingDir
13893
- ? { directory: this.config.judgeTrainingDir }
13894
- : {}),
13895
- };
13896
- const judgeTelemetryHandler =
13897
- judgeTelemetryOpts.enabled || judgeTrainingOpts.enabled
13898
- ? (obs: import("./extraction-judge.js").JudgeVerdictObservation) => {
13899
- const ts = new Date().toISOString();
13900
- const verdictKind = getVerdictKind(obs.verdict);
13901
- if (judgeTelemetryOpts.enabled) {
13902
- const event: import("./extraction-judge-telemetry.js").JudgeVerdictEvent = {
13903
- version: 1,
13904
- category: EXTRACTION_JUDGE_VERDICT_CATEGORY,
13905
- ts,
13906
- verdictKind,
13907
- reason: obs.verdict.reason,
13908
- deferrals: obs.priorDeferrals,
13909
- elapsedMs: obs.elapsedMs,
13910
- candidateCategory: obs.candidate.category,
13911
- confidence: obs.candidate.confidence,
13912
- contentHash: obs.contentHash,
13913
- fromCache: obs.source === "cache",
13914
- ...(obs.source === "llm-cap-rejected"
13915
- ? { deferCapTriggered: true }
13916
- : {}),
13917
- };
13918
- void recordJudgeVerdict(event, judgeTelemetryOpts);
13919
- }
13920
- if (judgeTrainingOpts.enabled) {
13921
- const pair: import("./extraction-judge-training.js").JudgeTrainingPair = {
13922
- version: 1,
13923
- ts,
13924
- candidateText: obs.candidate.text,
13925
- candidateCategory: obs.candidate.category,
13926
- ...(typeof obs.candidate.confidence === "number"
13927
- ? { candidateConfidence: obs.candidate.confidence }
13928
- : {}),
13929
- verdictKind,
13930
- reason: obs.verdict.reason,
13931
- priorDeferrals: obs.priorDeferrals,
13932
- };
13933
- void recordJudgeTrainingPair(pair, judgeTrainingOpts);
13934
- }
13935
- }
13936
- : undefined;
13937
- const judgeResult = await judgeFactDurability(
13938
- judgeCandidates,
13939
- this.config,
13940
- this.localLlm,
13941
- new FallbackLlmClient(
13942
- this.config.gatewayConfig,
13943
- fallbackLlmRuntimeContextFromConfig(this.config),
13944
- ),
13945
- this.judgeVerdictCache,
13946
- this.judgeDeferCounts,
13947
- judgeTelemetryHandler,
13948
- );
13949
- // Remap candidate-indexed verdicts to original fact indexes
13950
- judgeVerdictsByFactIndex = new Map();
13951
- for (const [candidateIdx, verdict] of judgeResult.verdicts) {
13952
- const factIdx = candidateToFactIndex[candidateIdx];
13953
- if (factIdx !== undefined) {
13954
- judgeVerdictsByFactIndex.set(factIdx, verdict);
13955
- }
13956
- }
13957
- log.info(
13958
- `extraction-judge: ${judgeResult.verdicts.size}/${judgeCandidates.length} facts evaluated, ` +
13959
- `${judgeResult.cached} cached, ${judgeResult.judged} judged, ` +
13960
- `${judgeResult.deferred} deferred` +
13961
- (judgeResult.deferredCappedToReject > 0
13962
- ? ` (${judgeResult.deferredCappedToReject} cap-rejected)`
13963
- : "") +
13964
- `, ${judgeResult.elapsed}ms`,
13965
- );
13966
- // Expose defer count to the caller (issue #562 PR 2) so it can decide
13967
- // whether to retain buffer turns for the next extraction pass.
13968
- this.lastPersistExtractionDeferredCount = judgeResult.deferred;
13969
- } catch (err) {
13970
- // Fail-open: if the entire judge pipeline errors, proceed without filtering
13971
- log.warn(
13972
- `extraction-judge: pipeline error, proceeding without filtering (fail-open): ${err instanceof Error ? err.message : String(err)}`,
13973
- );
13974
- }
13975
- }
13976
-
13977
- // Faithfulness gate (issue #1576). Entailment-verification of extracted
13978
- // facts against their verified source spans from #1575. Placement: after
13979
- // parse + provenance validation, BEFORE persist/index (rule 44). The
13980
- // substantive batch logic lives in the pure module; this is thin
13981
- // delegation (ground rule 4). off → null map (byte-identical pre-feature
13982
- // pipeline, rule 39); shadow → record only; enforce → pending_review.
13983
- const faithfulnessMode = this.config.extractionFaithfulnessGate;
13984
- const faithfulnessResultsByFactIndex =
13985
- faithfulnessMode === "shadow" || faithfulnessMode === "enforce"
13986
- ? await runFaithfulnessGateBatch(
13987
- facts,
13988
- faithfulnessMode,
13989
- this.config,
13990
- this.localLlm,
13991
- new FallbackLlmClient(
13992
- this.config.gatewayConfig,
13993
- fallbackLlmRuntimeContextFromConfig(this.config),
13994
- ),
13995
- this.faithfulnessCounters,
13996
- sourceText,
13997
- )
13998
- : null;
13999
-
14000
- let factLoopIndex = -1;
14001
- for (const fact of facts) {
14002
- factLoopIndex++;
14003
- if (
14004
- !fact ||
14005
- typeof (fact as any).content !== "string" ||
14006
- !(fact as any).content.trim()
14007
- ) {
14008
- continue;
14009
- }
14010
- if (
14011
- typeof (fact as any).category !== "string" ||
14012
- !(fact as any).category.trim()
14013
- ) {
14014
- continue;
14015
- }
14016
- (fact as any).tags = Array.isArray((fact as any).tags)
14017
- ? (fact as any).tags.filter((t: any) => typeof t === "string")
14018
- : [];
14019
- (fact as any).confidence =
14020
- typeof (fact as any).confidence === "number"
14021
- ? (fact as any).confidence
14022
- : 0.7;
14023
- // #1670 — anchor each fact's event-time to its SOURCE TURN timestamp,
14024
- // not the batch-wide latest. When a buffered conversation spans a date
14025
- // boundary, a relative expression ("yesterday") on an early-turn fact
14026
- // must resolve against that early turn's date. Prefer the fact's
14027
- // explicit sourceTurnTimestamp, then the earliest provenance span's
14028
- // observedAt, then fall back to the batch anchor (legacy extractors).
14029
- const factAnchor = pickFactEventTimeAnchor(fact, sourceContext?.validAt);
14030
- const biTemporal = this.config.temporalBiTemporal && factAnchor ? resolveFactEventTime(fact.eventTime, factAnchor) : undefined;
14031
- // Content-hash dedup check (v6.0)
14032
- //
14033
- // Canonicalize pre-tagged facts before hashing (Codex P2 — issue #369).
14034
- // When a fact already carries an inline citation (e.g. relayed or
14035
- // reprocessed), hashing `fact.content` as-is would produce a different
14036
- // hash than the one stored from the original write (which used the raw,
14037
- // un-cited body as contentHashSource). Strip any citation first so the
14038
- // dedup key matches what the hash index recorded.
14039
- //
14040
- // stripCitationForTemplate handles both the default and custom template
14041
- // formats. For all-placeholder templates it cannot detect citations and
14042
- // returns the text unchanged — dedup may miss in that edge case, which
14043
- // is acceptable (no false-positive suppression).
14044
- //
14045
- // Routing runs before content-hash dedup and scoring so category overrides
14046
- // affect both the dedup fingerprint and importance (issue #519 procedure routing).
14047
- let writeCategory = fact.category;
14048
- let targetStorage = storage;
14049
- const sourceStorageDir = storage.dir; // #1669 thread #2: pre-routing source ns for redaction gate
14050
- // Track the KNOWN target namespace NAME alongside targetStorage (round 6,
14051
- // codex P2 — NCQI0). Re-deriving it from `targetStorage.dir` mangles a raw
14052
- // namespace literally named like a canonical token (e.g. `ns-616c706861`
14053
- // served from its legacy raw dir decodes to `alpha`). We seed it from the
14054
- // EXPLICIT base namespace the caller used to obtain `storage` (NHIdx, codex
14055
- // P2) — `selfNamespace`/`writeNamespaceOverride` — so the catalog write touch
14056
- // records the real namespace, not a guess decoded from the directory. We only
14057
- // fall back to decoding the dir when no base namespace was passed (legacy
14058
- // callers). The EXPLICIT routed name (below) still overrides this verbatim.
14059
- let targetNamespaceName =
14060
- baseNamespace && baseNamespace.length > 0
14061
- ? baseNamespace
14062
- : this.storageDirNamespace(targetStorage.dir);
14063
- let routedRuleId: string | undefined;
14064
- let routedNamespaceExplicit = false;
14065
- if (routeRules.length > 0) {
14066
- try {
14067
- const routeText = `${fact.category} ${fact.tags.join(" ")} ${fact.content}`;
14068
- const selected = selectRouteRule(routeText, routeRules, routeOptions);
14069
- if (selected) {
14070
- routedRuleId = selected.rule.id;
14071
- if (selected.target.category) {
14072
- writeCategory = selected.target.category;
14073
- }
14074
- if (selected.target.namespace) {
14075
- routedNamespaceExplicit = true;
14076
- targetStorage = await this.storageRouter.storageFor(
14077
- selected.target.namespace,
14078
- );
14079
- targetNamespaceName = selected.target.namespace;
14080
- }
14081
- }
14082
- } catch (err) {
14083
- log.warn(
14084
- `routing evaluation failed; fail-open to extracted category/namespace: ${err}`,
14085
- );
14086
- }
14087
- }
14088
-
14089
- // Scope-based namespace routing: when scope classification is enabled
14090
- // and the LLM tagged this fact as "global", route it to the shared
14091
- // namespace so cross-project knowledge is visible everywhere. Only
14092
- // applies when namespaces are enabled and the fact was not already
14093
- // routed to a specific namespace by a routing rule (routing rules
14094
- // that set an explicit namespace take precedence; category-only rules
14095
- // do not block scope routing). Rule 30: gated by
14096
- // extractionScopeClassificationEnabled.
14097
- if (
14098
- lifecycleCaps.extractionScopeClassification &&
14099
- namespacesEnabled &&
14100
- fact.scope === "global" &&
14101
- !routedNamespaceExplicit
14102
- ) {
14103
- const currentNs = this.storageDirNamespace(targetStorage.dir);
14104
- if (currentNs !== this.config.sharedNamespace && profileAllowsSharedWrites) {
14105
- try {
14106
- targetStorage = await this.storageRouter.storageFor(
14107
- this.config.sharedNamespace,
14108
- );
14109
- targetNamespaceName = this.config.sharedNamespace;
14110
- log.debug(
14111
- `scope-routing: fact "${fact.content.slice(0, 60)}…" routed to shared namespace (scope=global)`,
14112
- );
14113
- } catch (scopeRouteErr) {
14114
- log.warn(
14115
- `scope-routing: failed to resolve shared namespace storage; writing to session namespace (fail-open): ${scopeRouteErr}`,
14116
- );
14117
- }
14118
- } else if (currentNs !== this.config.sharedNamespace) {
14119
- log.debug(
14120
- `scope-routing: skipped shared namespace for global fact because active scope profile ${scopeProfileWritePlan?.profileId ?? "none"} does not authorize serverShared writes`,
14121
- );
14122
- }
14123
- }
14124
- // #1669 redaction-rule gate: consult BOTH source and target namespace
14125
- // rules before any write. A never-store pattern registered under the
14126
- // source namespace must survive scope-routing to a different target
14127
- // (review thread #2). Fails open on read error.
14128
- try {
14129
- const redactionRules = await redactionRulesFor(sourceStorageDir, targetStorage.dir);
14130
- const redactionCandidate = fact.content
14131
- + (fact.structuredAttributes ? " " + JSON.stringify(fact.structuredAttributes) : "")
14132
- + (fact.procedureSteps ? " " + fact.procedureSteps.map((s) => `${s.intent} ${s.expectedOutcome ?? ""} ${s.toolCall ? `${s.toolCall.kind} ${s.toolCall.signature}` : ""}`.trim()).join(" ") : "");
14133
- if (redactionRules.length > 0 && contentMatchesRedactionRules(redactionCandidate, redactionRules)) {
14134
- redactionGatedCount++;
14135
- log.debug(`extraction: redaction-rule withheld fact #${redactionGatedCount} in ${targetStorage.dir}`);
14136
- continue;
14137
- }
14138
- } catch (redactionErr) {
14139
- log.warn(`extraction: redaction-rule gate failed open: ${redactionErr}`);
14140
- }
14141
-
14142
- // Procedures: fingerprint the full serialized body (title + steps), not
14143
- // the title alone, so distinct step lists are not collapsed (issue #519).
14144
- const canonicalContentForHash =
14145
- citationEnabled &&
14146
- hasCitationForTemplate(fact.content, citationTemplate)
14147
- ? stripCitationForTemplate(fact.content, citationTemplate)
14148
- : fact.content;
14149
- const contentHashDedupKey =
14150
- writeCategory === "procedure"
14151
- ? buildProcedurePersistBody(fact.content, fact.procedureSteps)
14152
- : canonicalContentForHash;
14153
- // Importance is scored before the dedup short-circuit so #1671 backfill
14154
- // can gate on it (cursor PRRT_OvKnS): a low-value duplicate that the
14155
- // importance write-gate (#372) or the judge pre-filter would drop must
14156
- // not expire an active fact.
14157
- const importance = scoreImportance(
14158
- fact.content,
14159
- writeCategory,
14160
- fact.tags,
14161
- );
14162
- let exactDuplicate = false;
14163
- try {
14164
- exactDuplicate = await this.hasContentHashDedup(
14165
- targetStorage,
14166
- contentHashDedupKey,
14167
- );
14168
- } catch (err) {
14169
- log.warn(
14170
- `content-hash dedup lookup failed for storage ${targetStorage.dir}; writing fact fail-open: ${err}`,
14171
- );
14172
- }
14173
- if (exactDuplicate) {
14174
- // #1671 — before short-circuiting, backfill bi-temporal bounds
14175
- // onto the existing source-namespace copy if it lacks bounds the
14176
- // incoming fact now carries (re-extraction with a resolved invalidAt).
14177
- // Skip when the fact would be rejected/deferred/pending by downstream
14178
- // gates — a non-durable candidate must not expire an active fact
14179
- // (chatgpt-codex P1: faithfulness, requireSpans, extraction judge).
14180
- // #1671 + #1707: backfill bi-temporal bounds onto the existing
14181
- // source-namespace copy if it lacks bounds the incoming fact now
14182
- // carries (re-extraction with a resolved bound). #1707 thread 3:
14183
- // gate on writeCategory === "fact" — the helper only matches facts,
14184
- // so a non-fact duplicate must not reach the fact-only scan.
14185
- // #1707 thread 2: also fire when only a corrected start bound
14186
- // (validFrom) is present, not just an end bound (validUntil). The
14187
- // downstream-gate skip still applies — a non-durable candidate must
14188
- // not expire an active fact (chatgpt-codex P1).
14189
- if (
14190
- biTemporal &&
14191
- writeCategory === "fact" &&
14192
- (biTemporal.validUntil || biTemporal.validFrom)
14193
- ) {
14194
- const fr = faithfulnessResultsByFactIndex?.get(factLoopIndex);
14195
- const faithfulnessWouldPending =
14196
- faithfulnessMode === "enforce" &&
14197
- fr?.ok === true &&
14198
- (fr.verdict === "unsupported" || fr.verdict === "contradicted");
14199
- const requireSpansWouldPending =
14200
- this.config.provenance?.requireSpans === true &&
14201
- fact.requireSpansPending === true;
14202
- const judgeVerdict = judgeVerdictsByFactIndex?.get(factLoopIndex);
14203
- const judgeWouldGate =
14204
- !this.config.extractionJudgeShadow &&
14205
- judgeVerdict !== undefined &&
14206
- !judgeVerdict.durable;
14207
- // Importance gate (cursor PRRT_OvKnS): below-threshold duplicates
14208
- // would never persist (#372) and carry no judge verdict (the
14209
- // pre-filter skips them), so they must not expire an active fact.
14210
- if (
14211
- isAboveImportanceThreshold(importance.level, this.config.extractionMinImportanceLevel) &&
14212
- !faithfulnessWouldPending &&
14213
- !requireSpansWouldPending &&
14214
- !judgeWouldGate
14215
- ) {
14216
- await this.backfillTemporalBoundsOnDedupHit(
14217
- targetStorage,
14218
- contentHashDedupKey,
14219
- {
14220
- invalidAt: biTemporal.validUntil,
14221
- // #1707 thread 2 — carry the corrected start bound too.
14222
- validFrom: biTemporal.validFrom,
14223
- observedAt: biTemporal.observedAt,
14224
- eventTimeSource: biTemporal.eventTimeSource,
14225
- },
14226
- fact.entityRef,
14227
- );
14228
- // #1707 thread 1 — the source branch short-circuits (`continue`
14229
- // below) before the promotion dedup paths run, so promoted
14230
- // shared/profile copies written before the source fact carried
14231
- // resolved bounds stay stale and cross-namespace recall surfaces
14232
- // an expired fact. Backfill the promotion targets too (fail-open,
14233
- // backfill-only — never writes a new promoted copy).
14234
- await backfillTemporalBoundsOnPromotionCopies({
14235
- sourceStorage: targetStorage,
14236
- content: fact.content,
14237
- category: writeCategory,
14238
- confidence: fact.confidence,
14239
- entityRef: fact.entityRef,
14240
- structuredAttributes: fact.structuredAttributes,
14241
- bounds: {
14242
- invalidAt: biTemporal.validUntil,
14243
- validFrom: biTemporal.validFrom,
14244
- observedAt: biTemporal.observedAt,
14245
- eventTimeSource: biTemporal.eventTimeSource,
14246
- },
14247
- });
14248
- }
14249
- }
14250
- log.debug(
14251
- `dedup: skipping duplicate fact "${fact.content.slice(0, 60)}…" in storage ${targetStorage.dir}`,
14252
- );
14253
- dedupedCount++;
14254
- continue;
14255
- }
14256
-
14257
- if (writeCategory === "procedure" && this.config.procedural?.enabled !== true) {
14258
- log.debug("persistExtraction: skip procedure memory (procedural.enabled is false)");
14259
- continue;
14260
- }
14261
-
14262
- // Importance write-gate (issue #372). Drop facts whose locally-scored
14263
- // level falls below the configured minimum BEFORE the semantic dedup
14264
- // lookup so that low-importance facts never incur an embedding search.
14265
- // scoreImportance() already applies category boosts (e.g. corrections
14266
- // +0.15) before deriving the level, so a correction at raw ~0.35
14267
- // still lands at "normal" and passes the default gate. Without this
14268
- // gate, trivial turn-level chatter ("hi", "k", heartbeat pings) gets
14269
- // persisted as a fact memory and dilutes the store.
14270
- if (
14271
- !isAboveImportanceThreshold(
14272
- importance.level,
14273
- this.config.extractionMinImportanceLevel,
14274
- )
14275
- ) {
14276
- importanceGatedCount++;
14277
- const snippet = fact.content.slice(0, 60).replace(/\s+/g, " ").trim();
14278
- log.debug(`extraction: skip trivial "${snippet}"`);
14279
- // Log-based counter (no dedicated metric bus in remnic-core yet).
14280
- // Operators can grep for `metric:importance_gated` in gateway.log
14281
- // to tune extractionMinImportanceLevel.
14282
- log.debug(
14283
- `metric:importance_gated level=${importance.level} threshold=${this.config.extractionMinImportanceLevel} category=${writeCategory} count=${importanceGatedCount}`,
14284
- );
14285
- continue;
14286
- }
14287
-
14288
- // Extraction judge gate (issue #376 + #562 PR 2). After the local
14289
- // importance gate passes, consult the judge verdict (computed before
14290
- // the loop). In active mode, non-durable facts are dropped. In shadow
14291
- // mode, verdicts are logged but all facts proceed to write.
14292
- //
14293
- // Defer verdicts (issue #562): do not persist now, but also do not
14294
- // cache the outcome so the candidate is re-evaluated on a later
14295
- // extraction pass. The judge module tracks how many times the same
14296
- // content has been deferred and converts to reject at the configured
14297
- // cap, so the orchestrator only needs to skip the write here.
14298
- if (judgeVerdictsByFactIndex) {
14299
- const verdict = judgeVerdictsByFactIndex.get(factLoopIndex);
14300
- if (verdict && !verdict.durable) {
14301
- const verdictKind = getVerdictKind(verdict);
14302
- if (this.config.extractionJudgeShadow) {
14303
- log.info(
14304
- `extraction-judge[shadow]: would ${verdictKind} "${fact.content.slice(0, 60)}…" reason="${verdict.reason}"`,
14305
- );
14306
- } else if (verdictKind === "defer") {
14307
- judgeGatedCount++;
14308
- log.debug(
14309
- `extraction-judge: deferred "${fact.content.slice(0, 60)}…" reason="${verdict.reason}"`,
14310
- );
14311
- continue;
14312
- } else {
14313
- judgeGatedCount++;
14314
- log.debug(
14315
- `extraction-judge: rejected "${fact.content.slice(0, 60)}…" reason="${verdict.reason}"`,
14316
- );
14317
- continue;
14318
- }
14319
- }
14320
- }
14321
-
14322
- // Procedure extraction gate (issue #519): ≥2 steps + trigger phrasing.
14323
- // Runs even when extractionJudgeEnabled is false (durability judge is unrelated).
14324
- // Never tied to extractionJudgeShadow — that flag is only for the LLM durability judge.
14325
- if (writeCategory === "procedure") {
14326
- const procGate = validateProcedureExtraction({
14327
- content: fact.content,
14328
- procedureSteps: fact.procedureSteps,
14329
- });
14330
- if (!procGate.durable) {
14331
- log.debug(
14332
- `extraction-procedure-gate: rejected "${fact.content.slice(0, 60)}…" reason="${procGate.reason}"`,
14333
- );
14334
- continue;
14335
- }
14336
- }
14337
-
14338
- // Faithfulness gate verdict application (issue #1576). Look up the
14339
- // pre-computed verdict for this fact and translate it to frontmatter +
14340
- // an optional enforce-mode pending_review status. Logic lives in the
14341
- // pure module; this is thin read-through (ground rule 4).
14342
- const { faithfulness: faithfulnessFm, enforceStatus: faithfulnessGateStatus } =
14343
- applyFaithfulnessVerdict(
14344
- faithfulnessResultsByFactIndex,
14345
- factLoopIndex,
14346
- faithfulnessMode,
14347
- fact.content,
14348
- this.faithfulnessCounters,
14349
- );
14350
-
14351
- // requireSpans enforcement (issue #1575 PR 2): when an operator opts
14352
- // into provenance.requireSpans, a fact whose quote could not be located
14353
- // in any source turn (carried as the transient requireSpansPending
14354
- // signal from the extraction validator) routes to pending_review — the
14355
- // same review queue an unsupported faithfulness verdict uses. This is
14356
- // the persist-path wiring ProvenanceConfig.requireSpans documents.
14357
- // Faithfulness takes precedence when it already routed the fact; both
14358
- // gates agree on pending_review so the merge is a simple coalesce
14359
- // (chatgpt-codex-connector thread 4xB).
14360
- const requireSpansPendingStatus =
14361
- this.config.provenance?.requireSpans === true &&
14362
- fact.requireSpansPending === true
14363
- ? ("pending_review" as const)
14364
- : undefined;
14365
- const faithfulnessEnforceStatus = faithfulnessGateStatus ?? requireSpansPendingStatus;
14366
-
14367
- // Issue #373 — write-time semantic similarity guard. Hook runs after
14368
- // the exact content-hash miss and the importance gate so that:
14369
- // (a) paraphrased near-duplicates never reach writeMemory(), and
14370
- // (b) low-importance facts that will be dropped never trigger an
14371
- // embedding lookup (avoids unnecessary API latency/cost).
14372
- // Fails open when the embedding backend is unavailable.
14373
- //
14374
- // Defense in depth (PR #399 review): decideSemanticDedup already
14375
- // catches lookup errors internally, and the embedding fetch is
14376
- // bounded by a timeout in embedding-fallback.ts. We still wrap the
14377
- // whole call in its own try/catch here so that any unexpected
14378
- // rejection (future refactors, misbehaving custom backends, etc.)
14379
- // can never block the persist loop — a failure in the dedup path
14380
- // must always default to "not a duplicate".
14381
- // Track a pending semantic-skip decision (populated inside the block
14382
- // below). The actual drop happens AFTER contradiction detection so that
14383
- // a high-similarity update/correction is linked as a superseding
14384
- // contradiction rather than silently dropped.
14385
- let pendingSemanticSkip: (SemanticDedupDecision & { action: "skip" }) | null = null;
14386
- if (resolvePipelineProcessingCapabilities(this.config).semanticDedup) {
14387
- let semanticDecision: SemanticDedupDecision;
14388
- // UUI2: skip embedding lookup for the rest of this batch once we know
14389
- // the backend is unavailable. The flag is reset per-batch (set to false
14390
- // at the top of persistExtraction), so a transient hiccup in one call
14391
- // does not permanently disable dedup in subsequent calls.
14392
- if (batchBackendUnavailable) {
14393
- semanticDecision = { action: "keep", reason: "backend_unavailable" };
14394
- } else {
14395
- try {
14396
- // Pass the resolved target storage so the lookup scopes the
14397
- // embedding index to the target namespace (PR #399 P1 fix).
14398
- // Without this, a high-similarity hit in a different namespace
14399
- // would cause the fact to be dropped here — cross-namespace
14400
- // write suppression / data loss.
14401
- const lookupStorage = targetStorage;
14402
- semanticDecision = await decideSemanticDedup(
14403
- fact.content,
14404
- (content, limit) =>
14405
- this.semanticDedupLookup(content, limit, lookupStorage),
14406
- {
14407
- enabled: true,
14408
- threshold: this.config.semanticDedupThreshold,
14409
- candidates: this.config.semanticDedupCandidates,
14410
- },
14411
- );
14412
- } catch (err) {
14413
- log.warn(
14414
- `semantic dedup decision failed; failing open and writing fact: ${err}`,
14415
- );
14416
- semanticDecision = {
14417
- action: "keep",
14418
- reason: "backend_unavailable",
14419
- };
14420
- }
14421
- // UUI2: cache the backend-unavailable signal for the rest of this batch.
14422
- if (semanticDecision.reason === "backend_unavailable") {
14423
- batchBackendUnavailable = true;
14424
- }
14425
- }
14426
- if (semanticDecision.action === "skip") {
14427
- pendingSemanticSkip = semanticDecision;
14428
- }
14429
- }
14430
-
14431
- const inferredIntent = resolveConversationContextCapabilities(this.config).intentRouting
14432
- ? inferIntentFromText(
14433
- `${writeCategory} ${fact.tags.join(" ")} ${fact.content}`,
14434
- )
14435
- : null;
14436
- const extractionWriteSource =
14437
- (fact as any).source === "proactive"
14438
- ? "extraction-proactive"
14439
- : "extraction";
14440
-
14441
- // Check for contradictions before writing (Phase 2B).
14442
- // NOTE: This block was moved above the chunking branch so that the
14443
- // pendingSemanticSkip guard (below) can also protect the chunking path.
14444
- // Previously, contradiction detection only ran on the non-chunked path,
14445
- // meaning chunked facts could be persisted even when semanticDecision was
14446
- // "skip" (the deferred guard was bypassed by the chunking `continue`).
14447
- let supersedes: string | undefined;
14448
- let links: MemoryLink[] = [];
14449
- // True when contradiction detection ran and confirmed a contradiction,
14450
- // regardless of whether auto-resolve is enabled. Used by the
14451
- // semantic-skip guard so that contradictory updates are never silently
14452
- // dropped — even when `contradictionAutoResolve=false` (in which case
14453
- // `supersedes` is intentionally left unset to avoid retiring the old
14454
- // memory without user confirmation).
14455
- let contradictionDetected = false;
14456
- // #1645: hoist the contradiction result so the deferred auto-resolve
14457
- // (post-write, gated on tombstone status) can read its fields.
14458
- let contradiction: {
14459
- supersededId: string;
14460
- confidence: number;
14461
- reason: string;
14462
- supersededPath: string;
14463
- supersededCreated: string;
14464
- supersededTags: string[];
14465
- } | null | undefined;
14466
-
14467
- // Faithfulness gate (#1576, chatgpt P2): skip contradiction detection
14468
- // for a pending_review fact — an unfaithful extraction in the review queue
14469
- // must not trigger auto-resolve and retire an existing active memory.
14470
- if (
14471
- resolveRecallEnhancementCapabilities(this.config).contradictionDetection &&
14472
- this.qmd.isAvailable() &&
14473
- faithfulnessEnforceStatus !== "pending_review"
14474
- ) {
14475
- const targetNamespace = this.storageDirNamespace(targetStorage.dir);
14476
- contradiction = await this.checkForContradiction(
14477
- fact.content,
14478
- writeCategory,
14479
- targetNamespace,
14480
- );
14481
- if (contradiction) {
14482
- contradictionDetected = true;
14483
- // When auto-resolve is enabled the existing memory has already been
14484
- // marked superseded; set `supersedes` so the new write carries the
14485
- // relationship. When auto-resolve is disabled we still record the
14486
- // contradiction link (so the memory is annotated for manual review)
14487
- // but do NOT set `supersedes` on the new write — the old memory
14488
- // remains active until a human resolves it.
14489
- if (this.config.contradictionAutoResolve) {
14490
- supersedes = contradiction.supersededId;
14491
- }
14492
- links.push({
14493
- targetId: contradiction.supersededId,
14494
- linkType: "contradicts",
14495
- strength: contradiction.confidence,
14496
- reason: contradiction.reason,
14497
- });
14498
- // #1645: deindex + supersede are deferred to after writeMemory so the
14499
- // caller can gate them on the new write's tombstone status. A
14500
- // tombstone-blocked write (pending_review) must not deindex or retire
14501
- // the existing active memory — see the post-write guard below.
14502
- }
14503
- }
14504
-
14505
- // Apply the deferred semantic-skip now that contradiction detection has
14506
- // run. If a contradiction was found (contradictionDetected is true), the
14507
- // candidate is a contradictory update and must be written — do not skip
14508
- // it. Only drop it when there is no detected contradiction (true
14509
- // near-duplicate). This check intentionally runs BEFORE the chunking
14510
- // branch so that a fact flagged as a semantic near-duplicate cannot be
14511
- // persisted (with its hash registered) simply because it was long enough
14512
- // to trigger chunking.
14513
- //
14514
- // NOTE: We use `contradictionDetected` rather than `!!supersedes` here
14515
- // so that facts are preserved even when `contradictionAutoResolve=false`.
14516
- // When auto-resolve is disabled `supersedes` is intentionally unset, but
14517
- // the write must still proceed so the user can manually reconcile the
14518
- // two memories later.
14519
- //
14520
- // UUI1: correction category writes are NEVER suppressed by the semantic
14521
- // skip fallback, regardless of whether supersedes is set. When contradiction
14522
- // detection is disabled or QMD is unavailable, supersedes is never set —
14523
- // without this exemption a high-similarity correction would be silently
14524
- // dropped, leaving a stale fact active. writeCategory (not fact.category)
14525
- // is used because routing rules may have overridden the raw category.
14526
- const isCorrection = writeCategory === "correction";
14527
- // Faithfulness gate (#1576, cursor High): a pending_review fact must
14528
- // bypass the semantic-dedup skip so it reaches the review queue — the
14529
- // gate's contract is "persists with status: pending_review, never
14530
- // silently dropped" (issue #1576).
14531
- if (
14532
- pendingSemanticSkip &&
14533
- !contradictionDetected &&
14534
- !isCorrection &&
14535
- faithfulnessEnforceStatus !== "pending_review"
14536
- ) {
14537
- log.debug(
14538
- `dedup: skipping semantic near-duplicate fact "${fact.content
14539
- .slice(0, 60)
14540
- .replace(/\s+/g, " ")}…" score=${pendingSemanticSkip.topScore.toFixed(
14541
- 3,
14542
- )} neighbor=${pendingSemanticSkip.topId}`,
14543
- );
14544
- dedupedCount++;
14545
- // Do NOT add fact.content to contentHashIndex here. No memory was
14546
- // persisted for this fact, so registering a synthetic hash would
14547
- // permanently suppress exact-copy writes once the neighbor memory is
14548
- // archived or deleted (the hash would linger with no backing record).
14549
- continue;
14550
- }
14551
-
14552
- // Check if chunking is enabled and content should be chunked.
14553
- // When semanticChunkingEnabled is true, prefer the embedding-based
14554
- // semantic chunker which produces more coherent topic-aligned segments.
14555
- // Falls back to the recursive sentence-boundary chunker on failure.
14556
- if (resolvePipelineProcessingCapabilities(this.config).chunking && writeCategory !== "procedure") {
14557
- let chunkResult: { chunked: boolean; chunks: { content: string; index: number; tokenCount: number }[] };
14558
-
14559
- if (resolvePipelineProcessingCapabilities(this.config).semanticChunking) {
14560
- try {
14561
- const embedFn = this.embeddingFallback.embedTexts.bind(this.embeddingFallback);
14562
- const semanticResult: SemanticChunkResult = await semanticChunkContent(
14563
- fact.content,
14564
- embedFn,
14565
- this.config.semanticChunkingConfig,
14566
- );
14567
- chunkResult = semanticResult;
14568
- } catch (err) {
14569
- // Honor the fallbackToRecursive contract: when the user explicitly
14570
- // disables fallback, re-throw so extraction fails fast instead of
14571
- // silently using the recursive chunker. semanticChunkContent already
14572
- // throws when fallback is disabled, but this outer catch swallowed
14573
- // that signal. (PR #439 post-merge Finding 1.)
14574
- if (this.config.semanticChunkingConfig?.fallbackToRecursive === false) {
14575
- throw err;
14576
- }
14577
- log.debug(
14578
- `semantic chunking failed, falling back to recursive chunker: ${err}`,
14579
- );
14580
- chunkResult = chunkContent(fact.content, chunkingConfig);
14581
- }
14582
- } else {
14583
- chunkResult = chunkContent(fact.content, chunkingConfig);
14584
- }
14585
-
14586
- if (chunkResult.chunked && chunkResult.chunks.length > 1) {
14587
- // Classify memory kind (v8.0 Phase 2B: HiMem episode/note dual store)
14588
- const memoryKind = resolvePresentationCapabilities(this.config).episodeNoteMode
14589
- ? classifyMemoryKind(fact.content, fact.tags ?? [], writeCategory)
14590
- : undefined;
14591
-
14592
- // Write the parent memory first (with full content for reference).
14593
- //
14594
- // Compute the cited content once so that writeMemory and writeArtifact
14595
- // (when verbatim artifacts are enabled) share the same citation timestamp.
14596
- // See the normal write path comment for the full dedup rationale.
14597
- //
14598
- // Propagate supersedes/links from contradiction detection (round 6
14599
- // fix): contradiction detection now runs BEFORE this branch so the
14600
- // parent must carry the supersession relationship — without it the
14601
- // old memory is deindexed but the new chunked parent has no link
14602
- // back, leaving a dangling deindex with no replacement reference.
14603
- // Child chunks intentionally do NOT carry supersedes; only the
14604
- // parent represents the logical memory unit.
14605
- // Canonicalize contentHashSource before writing (Thread 3 — Codex P2,
14606
- // issue #369). If fact.content already carries an inline citation
14607
- // (e.g. re-processed or relayed fact), strip it so contentHashSource
14608
- // records the raw un-cited body — matching what the dedup check hashes
14609
- // via stripCitationForTemplate before calling hasFactContentHash.
14610
- const rawChunkedContent =
14611
- citationEnabled &&
14612
- hasCitationForTemplate(fact.content, citationTemplate)
14613
- ? stripCitationForTemplate(fact.content, citationTemplate)
14614
- : fact.content;
14615
- const citedChunkedContent = applyInlineCitation(rawChunkedContent);
14616
- const parentWrite = await targetStorage.writeMemory(
14617
- writeCategory,
14618
- citedChunkedContent,
14619
- {
14620
- confidence: fact.confidence,
14621
- tags: [...fact.tags, "chunked"],
14622
- entityRef: fact.entityRef,
14623
- source: extractionWriteSource,
14624
- importance,
14625
- supersedes,
14626
- links: links.length > 0 ? links : undefined,
14627
- intentGoal: inferredIntent?.goal,
14628
- intentActionType: inferredIntent?.actionType,
14629
- intentEntityTypes: inferredIntent?.entityTypes,
14630
- memoryKind,
14631
- structuredAttributes: fact.structuredAttributes,
14632
- validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt,
14633
- ...(biTemporal ? { observedAt: biTemporal.observedAt, eventTimeSource: biTemporal.eventTimeSource, ...(biTemporal.validUntil ? { invalidAt: biTemporal.validUntil } : {}) } : {}),
14634
- contentHashSource: rawChunkedContent,
14635
- // Faithfulness gate (issue #1576).
14636
- ...(faithfulnessFm ? { faithfulness: faithfulnessFm } : {}),
14637
- ...(faithfulnessEnforceStatus ? { status: faithfulnessEnforceStatus } : {}),
14638
- // Claim-level provenance spans (issue #1575 PR 2).
14639
- ...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
14640
- ...(fact.provenance ? { provenance: fact.provenance } : {}),
14641
- },
14642
- );
14643
- const parentId = parentWrite.id;
14644
- // #1645: surface the tombstone block and gate active post-write paths
14645
- // (chunks, supersession, shared promotion, graph/artifact) like #1576.
14646
- const tombstoneBlocked = parentWrite.tombstoneBlocked;
14647
- const postWriteGuard =
14648
- faithfulnessEnforceStatus === "pending_review" || tombstoneBlocked;
14649
- // #1645: defer contradiction auto-resolve until tombstone status is
14650
- // known (see applyDeferredContradictionResolve).
14651
- await this.applyDeferredContradictionResolve(
14652
- contradiction,
14653
- targetStorage,
14654
- parentId,
14655
- postWriteGuard,
14656
- );
14657
- try {
14658
- // Write individual chunks with parent reference
14659
- for (const chunk of chunkResult.chunks) {
14660
- // Score each chunk's importance separately
14661
- const chunkImportance = scoreImportance(
14662
- chunk.content,
14663
- writeCategory,
14664
- fact.tags,
14665
- );
14666
- const chunkWriteSource =
14667
- (fact as any).source === "proactive"
14668
- ? "chunking-proactive"
14669
- : "chunking";
14670
-
14671
- await targetStorage.writeChunk(
14672
- parentId,
14673
- chunk.index,
14674
- chunkResult.chunks.length,
14675
- writeCategory,
14676
- // Each chunk carries its own inline citation so provenance
14677
- // survives when a single chunk is quoted in isolation.
14678
- applyInlineCitation(chunk.content),
14679
- {
14680
- confidence: fact.confidence,
14681
- tags: fact.tags,
14682
- entityRef: fact.entityRef,
14683
- source: chunkWriteSource,
14684
- importance: chunkImportance,
14685
- intentGoal: inferredIntent?.goal,
14686
- intentActionType: inferredIntent?.actionType,
14687
- intentEntityTypes: inferredIntent?.entityTypes,
14688
- memoryKind,
14689
- validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt,
14690
- // #1578: propagate end bound + provenance to chunks (cursor bugbot).
14691
- ...(biTemporal
14692
- ? {
14693
- observedAt: biTemporal.observedAt,
14694
- eventTimeSource: biTemporal.eventTimeSource,
14695
- ...(biTemporal.validUntil
14696
- ? { invalidAt: biTemporal.validUntil }
14697
- : {}),
14698
- }
14699
- : {}),
14700
- // Faithfulness gate (issue #1576): propagate the parent
14701
- // fact's verdict + enforce status so a pending_review fact
14702
- // is not indexed as active through its chunks (chatgpt P2).
14703
- ...(faithfulnessFm ? { faithfulness: faithfulnessFm } : {}),
14704
- // #1645 (OchiE): inherit pending_review + blockedBy so no chunk lands active.
14705
- ...(postWriteGuard
14706
- ? { status: "pending_review" as const }
14707
- : (faithfulnessEnforceStatus ? { status: faithfulnessEnforceStatus } : {})),
14708
- ...(tombstoneBlocked && parentWrite.blockedBy
14709
- ? { blockedBy: parentWrite.blockedBy }
14710
- : {}),
14711
- // Claim-level provenance (issue #1575 PR 2): mirror the
14712
- // parent's spans onto each chunk so a chunk surfaced
14713
- // independently (memory_get/x-ray on a chunk ID) preserves
14714
- // the verified span (chatgpt-codex-connector thread Ocvmo).
14715
- ...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
14716
- ...(fact.provenance ? { provenance: fact.provenance } : {}),
14717
- },
14718
- );
14719
- }
14720
- } finally {
14721
- // The parent memory is durable once writeMemory returns `parentId`.
14722
- // Touch immediately around the chunk-write loop so a later chunk
14723
- // failure still surfaces the partially durable parent/chunk files to
14724
- // catalog-driven `writtenSince` maintenance. The final touch below
14725
- // still refreshes `lastWriteAt` after later durable writes on success.
14726
- }
14727
-
14728
- if (routedRuleId) {
14729
- log.debug(
14730
- `routing applied for chunked memory ${parentId}: rule=${routedRuleId} category=${writeCategory} storage=${targetStorage.dir}`,
14731
- );
14732
- }
14733
- log.debug(
14734
- `chunked memory ${parentId} into ${chunkResult.chunks.length} chunks`,
14735
- );
14736
- trackPersistedId(targetStorage, parentId, {
14737
- pendingReview: postWriteGuard,
14738
- });
14739
- // #1576 (cursor Medium): keep pending_review ids out of threadEpisodeIdsForGraph — else later active facts build thread-predecessor edges to an unfaithful memory.
14740
- if (
14741
- !postWriteGuard &&
14742
- threadEpisodeIdsForGraph &&
14743
- !threadEpisodeIdsForGraph.includes(parentId)
14744
- ) {
14745
- threadEpisodeIdsForGraph.push(parentId);
14746
- }
14747
- // #1645: same gate as the non-chunked path — a blocked chunked parent
14748
- // must not enter the embedding-fallback index (resurrection).
14749
- if (!postWriteGuard) {
14750
- await this.indexPersistedMemory(targetStorage, parentId);
14751
- }
14752
- // PR #402 Thread 1 fix: run source-namespace temporal supersession for
14753
- // chunked writes, matching the non-chunked path. Without this the
14754
- // source namespace retains stale facts that should have been superseded.
14755
- // Faithfulness gate (#1576, cursor High): skip supersession for a
14756
- // pending_review fact — an unfaithful extraction in the review queue
14757
- // must NOT retire older active memories.
14758
- if (!postWriteGuard) {
14759
- try {
14760
- const supersessionEntityRef =
14761
- typeof (fact as any).entityRef === "string"
14762
- ? ((fact as any).entityRef as string)
14763
- : undefined;
14764
- await applyTemporalSupersession({
14765
- storage: targetStorage,
14766
- newMemoryId: parentId,
14767
- entityRef: supersessionEntityRef,
14768
- structuredAttributes: fact.structuredAttributes,
14769
- createdAt: supersessionOrderingAt(biTemporal?.validFrom ?? sourceContext?.validAt),
14770
- // #1578 r3: an extracted end-only bound (validFrom absent) is
14771
- // historical, not a new authoritative state — never let it
14772
- // supersede a later active fact (codex P1 on :15534).
14773
- enabled: lifecycleCaps.temporalSupersession &&
14774
- !(biTemporal && !biTemporal.validFrom),
14775
- });
14776
- } catch (err) {
14777
- log.warn(`temporal-supersession (chunked): unexpected error: ${err}`);
14778
- }
14779
- }
14780
- // Faithfulness gate (#1576, chatgpt P2): do not promote a
14781
- // pending_review fact to shared/profile — it must enter the review
14782
- // queue without active copies that bypass the gate.
14783
- if (!postWriteGuard) await promoteMemoryToShared({
14784
- sourceStorage: targetStorage,
14785
- category: writeCategory,
14786
- content: fact.content,
14787
- confidence: fact.confidence,
14788
- tags: fact.tags,
14789
- entityRef: fact.entityRef,
14790
- structuredAttributes: fact.structuredAttributes,
14791
- sourceMemoryId: parentId,
14792
- importance,
14793
- intentGoal: inferredIntent?.goal,
14794
- intentActionType: inferredIntent?.actionType,
14795
- intentEntityTypes: inferredIntent?.entityTypes,
14796
- memoryKind,
14797
- validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt,
14798
- ...(biTemporal
14799
- ? {
14800
- observedAt: biTemporal.observedAt,
14801
- eventTimeSource: biTemporal.eventTimeSource,
14802
- ...(biTemporal.validUntil ? { invalidAt: biTemporal.validUntil } : {}),
14803
- }
14804
- : {}),
14805
- source: extractionWriteSource,
14806
- ...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
14807
- ...(fact.provenance ? { provenance: fact.provenance } : {}),
14808
- });
14809
- // Register chunked content in the target storage hash index too.
14810
- // Thread 3 fix: canonicalize by stripping any pre-existing citation
14811
- // so the stored hash matches what the dedup check computes.
14812
- try {
14813
- const canonicalChunkedContent =
14814
- citationEnabled &&
14815
- hasCitationForTemplate(fact.content, citationTemplate)
14816
- ? stripCitationForTemplate(fact.content, citationTemplate)
14817
- : fact.content;
14818
- // #1645: do NOT register a tombstone-blocked fact's content in the
14819
- // dedup index — writeMemory already skipped it (rule 44). Re-adding
14820
- // would let the next extraction dedup-skip the tombstone chokepoint
14821
- // and silently ban the retired content (no pending_review row).
14822
- if (!tombstoneBlocked) {
14823
- await this.addContentHashDedup(targetStorage, canonicalChunkedContent);
14824
- }
14825
- } catch (err) {
14826
- log.warn(
14827
- `content-hash dedup registration failed for chunked memory ${parentId}: ${err}`,
14828
- );
14829
- }
14830
-
14831
- for (const chunk of chunkResult.chunks) {
14832
- const chunkId = `${parentId}-chunk-${chunk.index}`;
14833
- // Do NOT push chunkId into persistedIds — chunk IDs must not leak
14834
- // into boxBuilder.onExtraction() or threading.processTurn(), which
14835
- // only expect canonical parent memory IDs. Call indexPersistedMemory
14836
- // directly for embedding-fallback sync of each chunk document.
14837
- // #1645: chunks inherit pending_review under postWriteGuard — don't
14838
- // index them into the embedding fallback (resurrection).
14839
- if (!postWriteGuard) {
14840
- await this.indexPersistedMemory(targetStorage, chunkId);
14841
- }
14842
- }
14843
- try {
14844
- if (
14845
- resolvePresentationCapabilities(this.config).verbatimArtifacts &&
14846
- this.config.verbatimArtifactCategories.includes(writeCategory) &&
14847
- fact.confidence >= this.config.verbatimArtifactsMinConfidence &&
14848
- !postWriteGuard
14849
- ) {
14850
- // Reuse citedChunkedContent so the artifact carries the same citation
14851
- // timestamp as the parent memory write above (Fix #3 — duplicate-citation).
14852
- await targetStorage.writeArtifact(citedChunkedContent, {
14853
- confidence: fact.confidence,
14854
- tags: [...fact.tags, "artifact", "chunked-parent"],
14855
- artifactType: this.artifactTypeForCategory(writeCategory),
14856
- sourceMemoryId: parentId,
14857
- intentGoal: inferredIntent?.goal,
14858
- intentActionType: inferredIntent?.actionType,
14859
- intentEntityTypes: inferredIntent?.entityTypes,
14860
- });
14861
- }
14862
- // v8.2: graph edge building for chunked memories. #1576: skip pending_review.
14863
- if (graphCaps.multiGraphMemory && !postWriteGuard) {
14864
- try {
14865
- const graphContext = await ensureGraphContext(targetStorage);
14866
- const entityRef =
14867
- typeof (fact as any).entityRef === "string"
14868
- ? (fact as any).entityRef
14869
- : undefined;
14870
- const parentRelPath = resolvePersistedMemoryRelativePath({
14871
- memoryId: parentId,
14872
- pathById: graphContext.memoryPathById,
14873
- category: writeCategory,
14874
- });
14875
- graphContext.memoryPathById.set(parentId, parentRelPath);
14876
- appendMemoryToGraphContext({
14877
- allMemsForGraph: graphContext.allMemsForGraph,
14878
- storageDir: targetStorage.dir,
14879
- memoryRelPath: parentRelPath,
14880
- memoryId: parentId,
14881
- category: writeCategory,
14882
- content: fact.content ?? "",
14883
- entityRef,
14884
- });
14885
- await this.buildGraphEdge(
14886
- targetStorage,
14887
- parentRelPath,
14888
- entityRef,
14889
- parentId,
14890
- fact.content ?? "",
14891
- graphContext.allMemsForGraph,
14892
- graphContext.memoryPathById,
14893
- threadIdForExtraction ?? undefined,
14894
- threadEpisodeIdsForGraph,
14895
- graphContext.previousPersistedRelPath,
14896
- graphCaps,
14897
- );
14898
- graphContext.previousPersistedRelPath = parentRelPath;
14899
- } catch {
14900
- /* fail-open */
14901
- }
14902
- }
14903
- } finally {
14904
- // Catalog touch (issue #1499): refresh AFTER later chunked
14905
- // source-namespace durable mutations — temporal supersession, shared
14906
- // promotion, optional artifact writes, and graph-edge writes — so
14907
- // `lastWriteAt` cannot precede later file changes on successful
14908
- // completion. Use the KNOWN routed name, not a dir-decoded guess.
14909
- }
14910
- trackBehaviorSignals(
14911
- targetStorage,
14912
- buildBehaviorSignalsForMemory({
14913
- memoryId: parentId,
14914
- category: writeCategory,
14915
- content: fact.content,
14916
- namespace: this.storageDirNamespace(targetStorage.dir),
14917
- confidence: fact.confidence,
14918
- source: "extraction",
14919
- }),
14920
- );
14921
- continue; // Skip the normal write below
14922
- }
14923
- }
14924
-
14925
- // Suggest links for this memory (Phase 3A)
14926
- if (resolveRecallEnhancementCapabilities(this.config).memoryLinking && this.qmd.isAvailable()) {
14927
- const targetNamespace = this.storageDirNamespace(targetStorage.dir);
14928
- const suggestedLinks = await this.suggestLinksForMemory(
14929
- fact.content,
14930
- writeCategory,
14931
- targetNamespace,
14932
- );
14933
- if (suggestedLinks.length > 0) {
14934
- links.push(...suggestedLinks);
14935
- }
14936
- }
14937
-
14938
- // Classify memory kind (v8.0 Phase 2B: HiMem episode/note dual store)
14939
- const memoryKind =
14940
- writeCategory === "procedure"
14941
- ? undefined
14942
- : resolvePresentationCapabilities(this.config).episodeNoteMode
14943
- ? classifyMemoryKind(fact.content, fact.tags ?? [], writeCategory)
14944
- : undefined;
14945
-
14946
- // Normal write (no chunking)
14947
- // Compute the cited content once so that writeMemory and writeArtifact
14948
- // (when verbatim artifacts are enabled) share the same citation timestamp.
14949
- // Calling applyInlineCitation twice on the same raw content would produce
14950
- // two different timestamps, creating duplicate citations with divergent
14951
- // provenance metadata on the memory and artifact copies of the same fact.
14952
- // Pass the RAW (pre-citation) fact as `contentHashSource` so the
14953
- // fact-content hash index records the hash of the canonical fact text
14954
- // rather than the citation-annotated variant. When inline attribution is
14955
- // enabled, `applyInlineCitation` appends a timestamp-bearing marker, so
14956
- // hashing the persisted body would produce a different hash on every
14957
- // write and defeat cross-session dedup (see `findDuplicateExplicitCapture`
14958
- // in explicit-capture.ts which calls `hasFactContentHash(candidate.content)`
14959
- // on raw content).
14960
- const rawPersistBody =
14961
- writeCategory === "procedure"
14962
- ? buildProcedurePersistBody(fact.content, fact.procedureSteps)
14963
- : fact.content;
14964
- const citedFactContent = applyInlineCitation(rawPersistBody);
14965
- const factWrite = await targetStorage.writeMemory(
14966
- writeCategory,
14967
- citedFactContent,
14968
- {
14969
- confidence: fact.confidence,
14970
- tags: fact.tags,
14971
- entityRef:
14972
- typeof (fact as any).entityRef === "string"
14973
- ? (fact as any).entityRef
14974
- : undefined,
14975
- source: extractionWriteSource,
14976
- importance,
14977
- supersedes,
14978
- links: links.length > 0 ? links : undefined,
14979
- intentGoal: inferredIntent?.goal,
14980
- intentActionType: inferredIntent?.actionType,
14981
- intentEntityTypes: inferredIntent?.entityTypes,
14982
- memoryKind,
14983
- structuredAttributes: fact.structuredAttributes,
14984
- validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt,
14985
- ...(biTemporal ? { observedAt: biTemporal.observedAt, eventTimeSource: biTemporal.eventTimeSource, ...(biTemporal.validUntil ? { invalidAt: biTemporal.validUntil } : {}) } : {}),
14986
- contentHashSource: writeCategory === "fact" ? fact.content : undefined,
14987
- // Faithfulness gate (issue #1576).
14988
- ...(faithfulnessFm ? { faithfulness: faithfulnessFm } : {}),
14989
- ...(faithfulnessEnforceStatus ? { status: faithfulnessEnforceStatus } : {}),
14990
- // Claim-level provenance spans (issue #1575 PR 2). Carry verified
14991
- // sources + the coarse strength tag from the extraction validator
14992
- // through to frontmatter so they survive end-to-end.
14993
- ...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
14994
- ...(fact.provenance ? { provenance: fact.provenance } : {}),
14995
- },
14996
- );
14997
- const memoryId = factWrite.id;
14998
- // #1645: surface the tombstone block; gate active post-write paths like #1576
14999
- // so a blocked fact creates no active shared copy / supersession / graph entry.
15000
- const tombstoneBlocked = factWrite.tombstoneBlocked;
15001
- const postWriteGuard =
15002
- faithfulnessEnforceStatus === "pending_review" || tombstoneBlocked;
15003
- // #1645: defer contradiction auto-resolve until tombstone status is
15004
- // known (see applyDeferredContradictionResolve).
15005
- await this.applyDeferredContradictionResolve(
15006
- contradiction,
15007
- targetStorage,
15008
- memoryId,
15009
- postWriteGuard,
15010
- );
15011
- if (routedRuleId) {
15012
- log.debug(
15013
- `routing applied for memory ${memoryId}: rule=${routedRuleId} category=${writeCategory} storage=${targetStorage.dir}`,
15014
- );
15015
- }
15016
- // Temporal supersession (issue #375): when the new fact has structured
15017
- // attributes, retire any older fact with the same entity + attribute
15018
- // key that has a conflicting value. Faithfulness gate (#1576, cursor
15019
- // High): skip for a pending_review fact — an unfaithful extraction in
15020
- // the review queue must NOT retire older active memories.
15021
- if (!postWriteGuard) {
15022
- try {
15023
- const supersessionEntityRef =
15024
- typeof (fact as any).entityRef === "string"
15025
- ? ((fact as any).entityRef as string)
15026
- : undefined;
15027
- await applyTemporalSupersession({
15028
- storage: targetStorage,
15029
- newMemoryId: memoryId,
15030
- entityRef: supersessionEntityRef,
15031
- structuredAttributes: fact.structuredAttributes,
15032
- createdAt: supersessionOrderingAt(biTemporal?.validFrom ?? sourceContext?.validAt),
15033
- enabled: lifecycleCaps.temporalSupersession &&
15034
- !(biTemporal && !biTemporal.validFrom),
15035
- });
15036
- } catch (err) {
15037
- log.warn(`temporal-supersession: unexpected error: ${err}`);
15038
- }
15039
- }
15040
- try {
15041
- trackBehaviorSignals(
15042
- targetStorage,
15043
- buildBehaviorSignalsForMemory({
15044
- memoryId,
15045
- category: writeCategory,
15046
- content: fact.content,
15047
- namespace: this.storageDirNamespace(targetStorage.dir),
15048
- confidence: fact.confidence,
15049
- source: "extraction",
15050
- }),
15051
- );
15052
- trackPersistedId(targetStorage, memoryId, {
15053
- pendingReview: postWriteGuard,
15054
- });
15055
- if (
15056
- !postWriteGuard &&
15057
- threadEpisodeIdsForGraph &&
15058
- !threadEpisodeIdsForGraph.includes(memoryId)
15059
- ) {
15060
- threadEpisodeIdsForGraph.push(memoryId);
15061
- }
15062
- // #1645: a tombstone-blocked / pending_review fact must NOT enter the
15063
- // embedding-fallback index — otherwise embedding recall surfaces the
15064
- // pending_review row (resurrection). Gate on postWriteGuard like the
15065
- // surrounding supersession / promotion / graph paths.
15066
- if (!postWriteGuard) {
15067
- await this.indexPersistedMemory(targetStorage, memoryId);
15068
- }
15069
- // Faithfulness gate (#1576, chatgpt P2): skip promotion for a
15070
- // pending_review fact so no active shared/profile copy bypasses the gate.
15071
- if (!postWriteGuard) await promoteMemoryToShared({
15072
- sourceStorage: targetStorage,
15073
- category: writeCategory,
15074
- content: fact.content,
15075
- confidence: fact.confidence,
15076
- tags: fact.tags,
15077
- entityRef:
15078
- typeof (fact as any).entityRef === "string"
15079
- ? (fact as any).entityRef
15080
- : undefined,
15081
- structuredAttributes: fact.structuredAttributes,
15082
- sourceMemoryId: memoryId,
15083
- importance,
15084
- intentGoal: inferredIntent?.goal,
15085
- intentActionType: inferredIntent?.actionType,
15086
- intentEntityTypes: inferredIntent?.entityTypes,
15087
- memoryKind,
15088
- validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt,
15089
- ...(biTemporal
15090
- ? {
15091
- observedAt: biTemporal.observedAt,
15092
- eventTimeSource: biTemporal.eventTimeSource,
15093
- ...(biTemporal.validUntil ? { invalidAt: biTemporal.validUntil } : {}),
15094
- }
15095
- : {}),
15096
- source: extractionWriteSource,
15097
- ...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
15098
- ...(fact.provenance ? { provenance: fact.provenance } : {}),
15099
- });
15100
- // v8.2: graph edge building (fail-open). #1576: skip pending_review facts.
15101
- if (graphCaps.multiGraphMemory && !postWriteGuard) {
15102
- try {
15103
- const graphContext = await ensureGraphContext(targetStorage);
15104
- const entityRef =
15105
- typeof (fact as any).entityRef === "string"
15106
- ? (fact as any).entityRef
15107
- : undefined;
15108
- const memoryRelPath = resolvePersistedMemoryRelativePath({
15109
- memoryId,
15110
- pathById: graphContext.memoryPathById,
15111
- category: writeCategory,
15112
- });
15113
- graphContext.memoryPathById.set(memoryId, memoryRelPath);
15114
- appendMemoryToGraphContext({
15115
- allMemsForGraph: graphContext.allMemsForGraph,
15116
- storageDir: targetStorage.dir,
15117
- memoryRelPath: memoryRelPath,
15118
- memoryId,
15119
- category: writeCategory,
15120
- content: fact.content ?? "",
15121
- entityRef,
15122
- });
15123
- await this.buildGraphEdge(
15124
- targetStorage,
15125
- memoryRelPath,
15126
- entityRef,
15127
- memoryId,
15128
- fact.content ?? "",
15129
- graphContext.allMemsForGraph,
15130
- graphContext.memoryPathById,
15131
- threadIdForExtraction ?? undefined,
15132
- threadEpisodeIdsForGraph,
15133
- graphContext.previousPersistedRelPath,
15134
- graphCaps,
15135
- );
15136
- graphContext.previousPersistedRelPath = memoryRelPath;
15137
- } catch {
15138
- /* fail-open */
15139
- }
15140
- }
15141
- if (
15142
- resolvePresentationCapabilities(this.config).verbatimArtifacts &&
15143
- this.config.verbatimArtifactCategories.includes(writeCategory) &&
15144
- fact.confidence >= this.config.verbatimArtifactsMinConfidence &&
15145
- !postWriteGuard
15146
- ) {
15147
- // Reuse citedFactContent so the artifact carries the same citation
15148
- // timestamp as the memory write above (Fix #3 — duplicate-citation).
15149
- await targetStorage.writeArtifact(citedFactContent, {
15150
- confidence: fact.confidence,
15151
- tags: [...fact.tags, "artifact"],
15152
- artifactType: this.artifactTypeForCategory(writeCategory),
15153
- sourceMemoryId: memoryId,
15154
- intentGoal: inferredIntent?.goal,
15155
- intentActionType: inferredIntent?.actionType,
15156
- intentEntityTypes: inferredIntent?.entityTypes,
15157
- });
15158
- }
15159
- // Register in the target storage content-hash index after successful
15160
- // write. Thread 3 fix: canonicalize by stripping any pre-existing
15161
- // citation so the stored hash matches what the dedup check computes.
15162
- try {
15163
- const canonicalFactContent =
15164
- citationEnabled &&
15165
- hasCitationForTemplate(fact.content, citationTemplate)
15166
- ? stripCitationForTemplate(fact.content, citationTemplate)
15167
- : fact.content;
15168
- const hashRegisterKey =
15169
- writeCategory === "procedure"
15170
- ? buildProcedurePersistBody(fact.content, fact.procedureSteps)
15171
- : canonicalFactContent;
15172
- // #1645: do NOT register a tombstone-blocked fact's content in the dedup
15173
- // index (rule 44 defeat) — see chunked path comment.
15174
- if (!tombstoneBlocked) {
15175
- await this.addContentHashDedup(targetStorage, hashRegisterKey);
15176
- }
15177
- } catch (err) {
15178
- log.warn(
15179
- `content-hash dedup registration failed for memory ${memoryId}: ${err}`,
15180
- );
15181
- }
15182
- } finally {
15183
- // Catalog touch (issue #1499): record AFTER every synchronous
15184
- // source-namespace mutation in the non-chunked path: writeMemory,
15185
- // temporal supersession, graph edges, and optional verbatim artifacts.
15186
- // The `finally` preserves the write touch when post-write indexing or
15187
- // promotion fails after the canonical memory is already durable. Use the
15188
- // KNOWN routed name, not a dir-decoded guess (NCQI0).
15189
- }
15190
- }
15191
-
15192
- // Tracks whether THIS extraction persisted any durable, non-fact output to the
15193
- // BASE namespace's storage (entity / relationship / profile / question). The
15194
- // per-fact catalog touch (storage chokepoint #1522) only fires inside the fact write loop, so a
15195
- // fact-less extraction that still persists durable data must record exactly one
15196
- // base-namespace catalog touch after all writes complete (NHZEZ, codex P2).
15197
- let durableNonFactWritten = false;
15198
- let durableNonFactTouchRecorded = false;
15199
- const touchBaseNonFactNamespace = () => {
15200
- const baseTouchNamespace =
15201
- baseNamespace && baseNamespace.length > 0
15202
- ? baseNamespace
15203
- : this.storageDirNamespace(storage.dir);
15204
- };
15205
- const recordDurableNonFactWrite = () => {
15206
- durableNonFactWritten = true;
15207
- if (durableNonFactTouchRecorded) return;
15208
- durableNonFactTouchRecorded = true;
15209
- touchBaseNonFactNamespace();
15210
- };
15211
- for (const entity of entities) {
15212
- try {
15213
- const name = (entity as any)?.name;
15214
- const type = (entity as any)?.type;
15215
- if (
15216
- typeof name !== "string" ||
15217
- !name.trim() ||
15218
- typeof type !== "string" ||
15219
- !type.trim()
15220
- ) {
15221
- continue;
15222
- }
15223
- const safeFacts = Array.isArray((entity as any)?.facts)
15224
- ? (entity as any).facts.filter((f: any) => typeof f === "string")
15225
- : [];
15226
- const id = await storage.writeEntity(name, type, safeFacts, {
15227
- source: typeof (entity as any)?.source === "string" ? (entity as any).source : "extraction",
15228
- timestamp: sourceContext?.validAt,
15229
- sessionKey: sourceContext?.sessionKey,
15230
- principal: sourceContext?.principal,
15231
- structuredSections: Array.isArray((entity as any)?.structuredSections)
15232
- ? (entity as any).structuredSections
15233
- : undefined,
15234
- });
15235
- if (id) {
15236
- trackPersistedId(storage, id);
15237
- recordDurableNonFactWrite();
15238
- }
15239
- } catch (err) {
15240
- log.warn(`persistExtraction: entity write failed: ${err}`);
15241
- }
15242
- }
15243
-
15244
- // Persist entity relationships (v7.0)
15245
- if (
15246
- resolveRecallEnhancementCapabilities(this.config).entityRelationships &&
15247
- Array.isArray(result.relationships)
15248
- ) {
15249
- for (const rel of result.relationships.slice(0, 5)) {
15250
- if (!rel.source || !rel.target || !rel.label) continue;
15251
- try {
15252
- // Add bidirectional relationship
15253
- await storage.addEntityRelationship(rel.source, {
15254
- target: rel.target,
15255
- label: rel.label,
15256
- });
15257
- recordDurableNonFactWrite();
15258
- await storage.addEntityRelationship(rel.target, {
15259
- target: rel.source,
15260
- label: `${rel.label} (reverse)`,
15261
- });
15262
- recordDurableNonFactWrite();
15263
- } catch (err) {
15264
- log.debug(`relationship persist failed: ${err}`);
15265
- }
15266
- }
15267
- }
15268
-
15269
- // Persist entity activity (v7.0)
15270
- if (resolveRecallEnhancementCapabilities(this.config).entityActivityLog) {
15271
- const today = new Date().toISOString().slice(0, 10);
15272
- for (const entity of entities) {
15273
- const name = (entity as any)?.name;
15274
- const type = (entity as any)?.type;
15275
- if (typeof name !== "string" || typeof type !== "string") continue;
15276
- try {
15277
- const normalized = storage.normalizeEntityName(name, type);
15278
- await storage.addEntityActivity(
15279
- normalized,
15280
- { date: today, note: "Mentioned in conversation" },
15281
- this.config.entityActivityLogMaxEntries,
15282
- );
15283
- } catch (err) {
15284
- log.debug(`activity persist failed: ${err}`);
15285
- }
15286
- }
15287
- }
15288
-
15289
- if (profileUpdates.length > 0) {
15290
- await storage.appendToProfile(profileUpdates);
15291
- recordDurableNonFactWrite();
15292
- }
15293
-
15294
- // Persist questions
15295
- for (const q of questions) {
15296
- const id = await storage.writeQuestion(q.question, q.context, q.priority);
15297
- if (id) {
15298
- trackPersistedId(storage, id);
15299
- recordDurableNonFactWrite();
15300
- }
15301
- }
15302
-
15303
- // Persist identity reflection. This writes durable namespace-local state, so
15304
- // an identity-ONLY extraction (no facts/entities/profile/questions) still
15305
- // counts as a durable non-fact write for the catalog touch below (NIIly).
15306
- // Only count it when the write actually succeeds (best-effort write); the
15307
- // touch is recorded AFTER this so a rolled-back/failed write never touches.
15308
- if (resolveRecallEnhancementCapabilities(this.config).identity && result.identityReflection) {
15309
- try {
15310
- await storage.appendIdentityReflection(result.identityReflection);
15311
- recordDurableNonFactWrite();
15312
- } catch (err) {
15313
- log.debug(`identity reflection write failed: ${err}`);
15314
- }
15315
- }
15316
-
15317
- // Catalog touch for durable NON-FACT outputs (NHZEZ / NIIly, codex P2). The
15318
- // per-fact catalog touch (storage chokepoint #1522) above only fires inside the fact write loop, so
15319
- // an extraction that persists ONLY entities, relationships, profile updates,
15320
- // questions, or an identity reflection (no facts) would record durable data to
15321
- // the BASE namespace's storage without ever touching the catalog — leaving that
15322
- // namespace's `lastWriteAt` stale so `listNamespaces({writtenSince})` /
15323
- // write-recency QMD maintenance miss the write. All of these are written to the
15324
- // BASE `storage` (not the per-fact routed `targetStorage`), so we record ONE
15325
- // base-namespace touch here, AFTER every non-fact write completes. Use the
15326
- // KNOWN base namespace name, not a dir-decoded guess (NCQI0). One touch per
15327
- // namespace per extraction — `markWrite` is idempotent, so if the fact path
15328
- // already touched the base namespace this only refreshes `lastWriteAt`.
15329
- // Best-effort and failure-tolerant (storage chokepoint #1522 swallows errors).
15330
- if (durableNonFactWritten) {
15331
- touchBaseNonFactNamespace();
15332
- }
15333
-
15334
- // Save any content-hash indexes touched during the batch.
15335
- await this.saveContentHashIndexes().catch((err) =>
15336
- log.warn(`content-hash index save failed: ${err}`),
15337
- );
15338
-
15339
- for (const {
15340
- storage: targetStorage,
15341
- events,
15342
- } of behaviorSignalsByStorage.values()) {
15343
- const dedupedSignals = dedupeBehaviorSignalsByMemoryAndHash(events);
15344
- if (dedupedSignals.length === 0) continue;
15345
- await targetStorage
15346
- .appendBehaviorSignals(dedupedSignals)
15347
- .catch((err) =>
15348
- log.warn(`appendBehaviorSignals failed (non-fatal): ${err}`),
15349
- );
15350
- }
15351
-
15352
- const dedupSuffix = dedupedCount > 0 ? ` (${dedupedCount} deduped)` : "";
15353
- const gatedSuffix =
15354
- importanceGatedCount > 0 ? ` (${importanceGatedCount} gated)` : "";
15355
- const judgeSuffix =
15356
- judgeGatedCount > 0 ? ` (${judgeGatedCount} judge-rejected)` : "";
15357
- const redactionSuffix =
15358
- redactionGatedCount > 0 ? ` (${redactionGatedCount} redacted)` : "";
15359
- log.info(
15360
- `persisted: ${facts.length - dedupedCount - importanceGatedCount - judgeGatedCount - redactionGatedCount} facts${dedupSuffix}${gatedSuffix}${judgeSuffix}${redactionSuffix}, ${entities.length} entities, ${questions.length} questions, ${profileUpdates.length} profile updates`,
15361
- );
15362
-
15363
- // Update temporal + tag indexes (v8.1) — fire-and-forget, fail-open
15364
- void (async () => {
15365
- if (persistedIdsByStorage.size === 0) {
15366
- await this.updateTemporalTagIndexes(storage, []);
15367
- return;
15368
- }
15369
- for (const entry of persistedIdsByStorage.values()) {
15370
- await this.updateTemporalTagIndexes(entry.storage, entry.ids);
15371
- }
15372
- })().catch((err) =>
15373
- log.debug(`temporal-index update error (non-fatal): ${err}`),
15374
- );
15375
-
15376
- // #1635: surface pending_review ids so the thread episode set excludes them.
15377
- this.lastPersistExtractionPendingReviewIds = pendingReviewPersistedIds;
15378
- // Return the persisted fact IDs for threading
15379
- return persistedIds;
15380
12308
  }
15381
12309
  /**
15382
12310
  * Append persisted ids to the thread episode set, excluding pending_review
@@ -15599,503 +12527,7 @@ export class Orchestrator {
15599
12527
  merged: number;
15600
12528
  invalidated: number;
15601
12529
  }> {
15602
- const lifecycleCaps = resolveMemoryLifecycleCapabilities(this.config);
15603
- log.info("running consolidation pass");
15604
- let merged = 0;
15605
- let invalidated = 0;
15606
- // Tracks whether any consolidation memory-item action (UPDATE / MERGE /
15607
- // INVALIDATE) durably rewrote memory state. A consolidation pass that only
15608
- // mutates memory items (no profile/entity updates) still changes the default
15609
- // namespace's data, so its catalog `lastWriteAt` must refresh too (NIBOi).
15610
- let memoryItemMutated = false;
15611
-
15612
- // Flush access tracking buffer first
15613
- if (this.accessTrackingBuffer.size > 0) {
15614
- await this.flushAccessTracking();
15615
- }
15616
-
15617
- let allMemories = await this.storage.readAllMemories();
15618
- if (allMemories.length < 5) {
15619
- return { memoriesProcessed: allMemories.length, merged, invalidated };
15620
- }
15621
-
15622
- const recent = allMemories
15623
- .sort(
15624
- (a, b) =>
15625
- new Date(b.frontmatter.created).getTime() -
15626
- new Date(a.frontmatter.created).getTime(),
15627
- )
15628
- .slice(0, 20);
15629
-
15630
- const older = allMemories.sort(
15631
- (a, b) =>
15632
- new Date(a.frontmatter.created).getTime() -
15633
- new Date(b.frontmatter.created).getTime(),
15634
- );
15635
-
15636
- const profile = await this.storage.readProfile();
15637
- const result = await this.extraction.consolidate(recent, older, profile);
15638
-
15639
- // Build a lookup map from the already-loaded corpus to avoid repeated
15640
- // readAllMemories() scans inside getMemoryById for pre-action deindex reads.
15641
- const memoryLookup = resolveIndexingCapabilities(this.config).queryAwareIndexing
15642
- ? new Map(allMemories.map((m) => [m.frontmatter.id, m]))
15643
- : null;
15644
-
15645
- for (const item of result.items) {
15646
- switch (item.action) {
15647
- case "INVALIDATE": {
15648
- // Capture path/frontmatter before invalidation for index cleanup
15649
- const toInvalidate = resolveIndexingCapabilities(this.config).queryAwareIndexing
15650
- ? (memoryLookup?.get(item.existingId) ?? null)
15651
- : null;
15652
- if (await this.storage.invalidateMemory(item.existingId)) {
15653
- invalidated += 1;
15654
- memoryItemMutated = true;
15655
- await this.embeddingFallback.removeFromIndex(item.existingId);
15656
- if (toInvalidate?.path && toInvalidate.frontmatter?.created) {
15657
- deindexMemory(
15658
- this.config.memoryDir,
15659
- toInvalidate.path,
15660
- toInvalidate.frontmatter.created,
15661
- toInvalidate.frontmatter.tags ?? [],
15662
- );
15663
- }
15664
- }
15665
- break;
15666
- }
15667
- case "UPDATE":
15668
- if (item.updatedContent) {
15669
- await this.storage.updateMemory(
15670
- item.existingId,
15671
- item.updatedContent,
15672
- {
15673
- lineage: [item.existingId],
15674
- },
15675
- );
15676
- memoryItemMutated = true;
15677
- await this.indexPersistedMemory(this.storage, item.existingId);
15678
- // updateMemory() only changes content/updated/lineage — path, created, and tags
15679
- // are preserved, so the temporal/tag index entry is already correct; no reindex needed.
15680
- }
15681
- break;
15682
- case "MERGE":
15683
- if (item.updatedContent && item.mergeWith) {
15684
- await this.storage.updateMemory(
15685
- item.existingId,
15686
- item.updatedContent,
15687
- {
15688
- supersedes: item.mergeWith,
15689
- lineage: [item.existingId, item.mergeWith],
15690
- },
15691
- );
15692
- memoryItemMutated = true;
15693
- await this.indexPersistedMemory(this.storage, item.existingId);
15694
- // updateMemory() only changes content/updated/supersedes/lineage — path, created, and tags
15695
- // are preserved, so the temporal/tag index entry for the survivor is already correct.
15696
- // Capture before invalidation for index cleanup
15697
- const toMergeInvalidate = resolveIndexingCapabilities(this.config).queryAwareIndexing
15698
- ? (memoryLookup?.get(item.mergeWith) ?? null)
15699
- : null;
15700
- if (await this.storage.invalidateMemory(item.mergeWith)) {
15701
- invalidated += 1;
15702
- merged += 1;
15703
- await this.embeddingFallback.removeFromIndex(item.mergeWith);
15704
- if (
15705
- toMergeInvalidate?.path &&
15706
- toMergeInvalidate.frontmatter?.created
15707
- ) {
15708
- deindexMemory(
15709
- this.config.memoryDir,
15710
- toMergeInvalidate.path,
15711
- toMergeInvalidate.frontmatter.created,
15712
- toMergeInvalidate.frontmatter.tags ?? [],
15713
- );
15714
- }
15715
- }
15716
- }
15717
- break;
15718
- }
15719
- }
15720
-
15721
- if (result.profileUpdates.length > 0) {
15722
- await this.storage.appendToProfile(result.profileUpdates);
15723
- }
15724
-
15725
- for (const entity of result.entityUpdates) {
15726
- const safeFacts = Array.isArray((entity as any)?.facts)
15727
- ? (entity as any).facts.filter((f: any) => typeof f === "string")
15728
- : [];
15729
- await this.storage.writeEntity(entity.name, entity.type, safeFacts, {
15730
- source: "consolidation",
15731
- structuredSections: Array.isArray((entity as any)?.structuredSections)
15732
- ? (entity as any).structuredSections
15733
- : undefined,
15734
- });
15735
- }
15736
-
15737
- // Catalog write touch accounting (issue #1499 sweep): consolidation persists
15738
- // durable mutations directly to the default-namespace `this.storage`, bypassing
15739
- // the extraction write path. We do NOT touch here — later maintenance steps in
15740
- // this same function (entity-file merges, expired-commitment / TTL cleanup,
15741
- // fact archival) can ALSO mutate the namespace on a run with no LLM outputs
15742
- // (NIjwl). So we accumulate every durable mutation into `memoryItemMutated` and
15743
- // record ONE consolidated touch AFTER all mutation-producing steps complete,
15744
- // just before returning (rule #25: touch after the write commits). LLM
15745
- // profile/entity updates and memory-item actions (UPDATE / MERGE / INVALIDATE)
15746
- // count here (NIBOi).
15747
- if (result.profileUpdates.length > 0 || result.entityUpdates.length > 0) {
15748
- memoryItemMutated = true;
15749
- }
15750
-
15751
- // Merge fragmented entity files
15752
- const entitiesMerged = await this.storage.mergeFragmentedEntities();
15753
- if (entitiesMerged > 0) {
15754
- memoryItemMutated = true;
15755
- log.info(`merged ${entitiesMerged} fragmented entity files`);
15756
- }
15757
-
15758
- if (resolvePresentationCapabilities(this.config).entitySummary) {
15759
- try {
15760
- const synthesized = await this.processEntitySynthesisQueue(
15761
- this.config.defaultNamespace,
15762
- 5,
15763
- );
15764
- if (synthesized > 0) {
15765
- // Entity synthesis rewrites entity files — a durable namespace mutation,
15766
- // so record it for the catalog touch even when it is the only change in
15767
- // the pass (codex). Otherwise lastWriteAt goes stale.
15768
- memoryItemMutated = true;
15769
- log.info(`refreshed ${synthesized} entity syntheses`);
15770
- }
15771
- } catch (err) {
15772
- log.debug(`entity synthesis pass failed: ${err}`);
15773
- }
15774
- }
15775
-
15776
- // Clean expired commitments
15777
- const deletedCommitments = await this.storage.cleanExpiredCommitments(
15778
- this.config.commitmentDecayDays,
15779
- );
15780
- if (deletedCommitments.length > 0) {
15781
- memoryItemMutated = true;
15782
- log.info(`cleaned ${deletedCommitments.length} expired commitments`);
15783
- if (resolveIndexingCapabilities(this.config).queryAwareIndexing) {
15784
- for (const m of deletedCommitments) {
15785
- deindexMemory(
15786
- this.config.memoryDir,
15787
- m.path,
15788
- m.frontmatter.created,
15789
- m.frontmatter.tags ?? [],
15790
- );
15791
- }
15792
- }
15793
- }
15794
-
15795
- if (
15796
- resolveCreationMemoryCapabilities(this.config).creationMemory &&
15797
- resolveCreationMemoryCapabilities(this.config).commitmentLedger &&
15798
- resolveCreationMemoryCapabilities(this.config).commitmentLifecycle
15799
- ) {
15800
- try {
15801
- const lifecycle = await applyCommitmentLedgerLifecycle({
15802
- memoryDir: this.config.memoryDir,
15803
- commitmentLedgerDir: this.config.commitmentLedgerDir,
15804
- enabled: true,
15805
- decayDays: this.config.commitmentDecayDays,
15806
- });
15807
- if (
15808
- lifecycle.transitionedToExpired.length > 0 ||
15809
- lifecycle.deletedResolved.length > 0
15810
- ) {
15811
- memoryItemMutated = true;
15812
- log.info(
15813
- `commitment ledger lifecycle: expired ${lifecycle.transitionedToExpired.length}, cleaned ${lifecycle.deletedResolved.length}`,
15814
- );
15815
- }
15816
- } catch (err) {
15817
- log.debug(`commitment ledger lifecycle pass failed: ${err}`);
15818
- }
15819
- }
15820
-
15821
- // Clean memories past their TTL (speculative memories auto-expire)
15822
- const deletedTTL = await this.storage.cleanExpiredTTL();
15823
- if (deletedTTL.length > 0) {
15824
- memoryItemMutated = true;
15825
- log.info(`cleaned ${deletedTTL.length} TTL-expired memories`);
15826
- if (resolveIndexingCapabilities(this.config).queryAwareIndexing) {
15827
- for (const m of deletedTTL) {
15828
- deindexMemory(
15829
- this.config.memoryDir,
15830
- m.path,
15831
- m.frontmatter.created,
15832
- m.frontmatter.tags ?? [],
15833
- );
15834
- }
15835
- }
15836
- }
15837
-
15838
- // v8.3 Lifecycle policy pass — deterministic promotion/decay metadata
15839
- if (lifecycleCaps.lifecyclePolicy) {
15840
- try {
15841
- const lightSleepStartedAt = new Date().toISOString();
15842
- const lifecycleCorpus = await this.storage.readAllMemories();
15843
- // Lifecycle frontmatter writes count as durable mutations for the catalog
15844
- // touch below (codex NR-tS), even when no other consolidation step set
15845
- // memoryItemMutated.
15846
- if ((await this.runLifecyclePolicyPass(lifecycleCorpus)) > 0) {
15847
- memoryItemMutated = true;
15848
- }
15849
- await this.recordScheduledDreamsPhaseRun(
15850
- "lightSleep",
15851
- lifecycleCorpus.length,
15852
- `scheduled lifecycle policy pass assessed ${lifecycleCorpus.length} memories`,
15853
- {
15854
- startedAt: lightSleepStartedAt,
15855
- completedAt: new Date().toISOString(),
15856
- },
15857
- );
15858
- } catch (err) {
15859
- log.warn(`lifecycle policy pass failed (ignored): ${err}`);
15860
- }
15861
- }
15862
-
15863
- // v8.3 Compression guideline learning pass (default off, fail-open).
15864
- await this.runCompressionGuidelineLearningPass();
15865
-
15866
- try {
15867
- const deepSleepStartedAt = new Date().toISOString();
15868
- // Tier migrations move/rewrite memory files; count them as durable
15869
- // mutations for the catalog touch below (codex NThSW).
15870
- const tierMigration = await this.runTierMigrationCycle(this.storage, "maintenance");
15871
- if (tierMigration.migrated > 0) memoryItemMutated = true;
15872
- allMemories = await this.storage.readAllMemories();
15873
-
15874
- // Fact archival pass (v6.0) — move old, low-importance, rarely-accessed facts to archive/
15875
- if (resolveRecallEnhancementCapabilities(this.config).factArchival) {
15876
- const archived = await this.runFactArchival(allMemories);
15877
- if (archived > 0) {
15878
- memoryItemMutated = true;
15879
- log.info(`archived ${archived} old low-importance facts`);
15880
- }
15881
- }
15882
- await this.recordScheduledDreamsPhaseRun(
15883
- "deepSleep",
15884
- allMemories.length,
15885
- `scheduled deep-sleep maintenance assessed ${allMemories.length} memories`,
15886
- {
15887
- startedAt: deepSleepStartedAt,
15888
- completedAt: new Date().toISOString(),
15889
- },
15890
- );
15891
- } catch (err) {
15892
- log.warn(`deep-sleep maintenance pass failed (ignored): ${err}`);
15893
- try {
15894
- allMemories = await this.storage.readAllMemories();
15895
- } catch (readErr) {
15896
- log.warn(`deep-sleep maintenance recovery read failed: ${readErr}`);
15897
- throw err;
15898
- }
15899
- }
15900
-
15901
- // Semantic consolidation pass — find similar memories, synthesize canonical versions
15902
- if (resolveConsolidationCapabilities(this.config).semanticConsolidation) {
15903
- try {
15904
- const stateFilePath = path.join(
15905
- this.config.memoryDir,
15906
- "state",
15907
- "semantic-consolidation-last-run.json",
15908
- );
15909
- let shouldRun = true;
15910
- try {
15911
- const stateRaw = await readFile(stateFilePath, "utf-8");
15912
- const stateData = JSON.parse(stateRaw) as { lastRunAt?: string };
15913
- if (stateData.lastRunAt) {
15914
- const lastRunMs = new Date(stateData.lastRunAt).getTime();
15915
- const intervalMs =
15916
- this.config.semanticConsolidationIntervalHours * 60 * 60 * 1000;
15917
- if (Date.now() - lastRunMs < intervalMs) {
15918
- shouldRun = false;
15919
- log.debug(
15920
- "[semantic-consolidation] skipping — not enough time since last run",
15921
- );
15922
- }
15923
- }
15924
- } catch {
15925
- // No state file yet — first run
15926
- }
15927
-
15928
- if (shouldRun) {
15929
- const remStartedAt = new Date().toISOString();
15930
- const semResult = await this.runSemanticConsolidation();
15931
- let remItemsProcessed = allMemories.length;
15932
- try {
15933
- allMemories = await this.storage.readAllMemories();
15934
- remItemsProcessed = allMemories.length;
15935
- } catch (err) {
15936
- log.warn(
15937
- `[semantic-consolidation] post-run telemetry refresh failed (non-fatal): ${err}`,
15938
- );
15939
- }
15940
- await this.recordScheduledDreamsPhaseRun(
15941
- "rem",
15942
- remItemsProcessed,
15943
- `scheduled REM consolidation found ${semResult.clustersFound} clusters`,
15944
- {
15945
- startedAt: remStartedAt,
15946
- completedAt: new Date().toISOString(),
15947
- },
15948
- );
15949
- if (semResult.memoriesArchived > 0) {
15950
- log.info(
15951
- `[semantic-consolidation] archived ${semResult.memoriesArchived} memories during maintenance`,
15952
- );
15953
- }
15954
- // Only persist last-run timestamp if the run succeeded (had no errors or made progress)
15955
- if (semResult.errors === 0 || semResult.memoriesArchived > 0) {
15956
- const stateDir = path.join(this.config.memoryDir, "state");
15957
- await mkdir(stateDir, { recursive: true });
15958
- await writeFile(
15959
- stateFilePath,
15960
- JSON.stringify({ lastRunAt: new Date().toISOString() }),
15961
- "utf-8",
15962
- );
15963
- }
15964
- }
15965
- } catch (err) {
15966
- log.warn(
15967
- `[semantic-consolidation] maintenance pass failed (non-fatal): ${err}`,
15968
- );
15969
- }
15970
- }
15971
-
15972
- // Auto-consolidate IDENTITY.md if it's getting large
15973
- if (resolveRecallEnhancementCapabilities(this.config).identity) {
15974
- await this.autoConsolidateIdentity();
15975
- }
15976
-
15977
- // Auto-consolidate profile.md if it exceeds max lines
15978
- const profileSection = this.getRecallSectionEntry("profile");
15979
- const profileConsolidationTriggerLines =
15980
- typeof profileSection?.consolidateTriggerLines === "number"
15981
- ? Math.max(0, Math.floor(profileSection.consolidateTriggerLines))
15982
- : undefined;
15983
- const profileConsolidationTargetLines =
15984
- typeof profileSection?.consolidateTargetLines === "number"
15985
- ? Math.max(0, Math.floor(profileSection.consolidateTargetLines))
15986
- : 50;
15987
- if (
15988
- await this.storage.profileNeedsConsolidation(
15989
- profileConsolidationTriggerLines,
15990
- )
15991
- ) {
15992
- log.info("profile.md exceeds max lines — running smart consolidation");
15993
- const currentProfile = await this.storage.readProfile();
15994
- if (currentProfile) {
15995
- const profileResult = await this.extraction.consolidateProfile(
15996
- currentProfile,
15997
- profileConsolidationTargetLines,
15998
- );
15999
- if (profileResult) {
16000
- await this.storage.writeProfile(profileResult.consolidatedProfile);
16001
- // Profile consolidation rewrites profile.md — a durable namespace
16002
- // mutation; record it for the catalog touch even when it is the only
16003
- // change in the pass (codex). Otherwise lastWriteAt goes stale.
16004
- memoryItemMutated = true;
16005
- log.info(
16006
- `profile.md consolidated: removed ${profileResult.removedCount} items — ${profileResult.summary}`,
16007
- );
16008
- }
16009
- }
16010
- }
16011
-
16012
- // Memory Summarization (Phase 4A)
16013
- if (resolvePipelineProcessingCapabilities(this.config).summarization) {
16014
- await this.runSummarization(allMemories);
16015
- }
16016
-
16017
- // Topic Extraction (Phase 4B)
16018
- if (resolvePipelineProcessingCapabilities(this.config).topicExtraction) {
16019
- await this.runTopicExtraction(allMemories);
16020
- }
16021
-
16022
- const meta = await this.storage.loadMeta();
16023
- meta.lastConsolidationAt = new Date().toISOString();
16024
- await this.storage.saveMeta(meta);
16025
-
16026
- // Temporal Memory Tree (v8.2) — rebuild nodes from all memories, fail-open
16027
- if (lifecycleCaps.temporalMemoryTree) {
16028
- try {
16029
- const tmtEntries = allMemories
16030
- .filter(
16031
- (m) =>
16032
- m.frontmatter.status !== "superseded" && m.frontmatter.status !== "archived" &&
16033
- m.frontmatter.status !== "forgotten" && m.frontmatter.status !== "pending_review", // #1576: unfaithful queue items must not feed TMT clusters
16034
- )
16035
- .map((m) => ({
16036
- path: m.path,
16037
- id: m.frontmatter.id,
16038
- created: m.frontmatter.created,
16039
- content: m.content,
16040
- }));
16041
- await this.tmtBuilder.maybeRebuildNodes(
16042
- tmtEntries,
16043
- async (texts, level) => {
16044
- const prompt = `You are a memory archivist. Summarize the following ${level}-level memories into 3–5 sentences, preserving key facts, decisions, and preferences.\n\n${texts.map((t, i) => `[${i + 1}] ${t}`).join("\n\n")}`;
16045
- const response = await this.fastChatCompletion(
16046
- [
16047
- {
16048
- role: "system",
16049
- content:
16050
- "Respond with a 3–5 sentence narrative summary. No JSON, just plain prose.",
16051
- },
16052
- { role: "user", content: prompt },
16053
- ],
16054
- {
16055
- temperature: 0.3,
16056
- maxTokens: this.config.tmtSummaryMaxTokens,
16057
- operation: "tmt_summary",
16058
- priority: "background",
16059
- },
16060
- );
16061
- return response?.content?.trim() || texts.slice(0, 3).join(" ");
16062
- },
16063
- );
16064
- } catch (err) {
16065
- log.warn(`tmt: consolidation hook failed (ignored): ${err}`);
16066
- }
16067
- }
16068
-
16069
- if (this.consolidationObservers.size > 0) {
16070
- const observation: ConsolidationObservation = {
16071
- runAt: new Date().toISOString(),
16072
- recentMemories: recent,
16073
- existingMemories: older.slice(-50),
16074
- profile,
16075
- result,
16076
- merged,
16077
- invalidated,
16078
- };
16079
- for (const observer of this.consolidationObservers) {
16080
- try {
16081
- await observer(observation);
16082
- } catch (err) {
16083
- log.warn(`consolidation observer failed (ignored): ${err}`);
16084
- }
16085
- }
16086
- }
16087
-
16088
- // Consolidated catalog write touch — belt-and-suspenders for cleanup-only
16089
- // passes that mutate the store via delete-only paths (entity merges, TTL
16090
- // cleanup) without triggering the storage chokepoint's post-write hook.
16091
- // Gated on memoryItemMutated (set by every durable mutation including cleanup-only passes)
16092
- // Best-effort and failure-tolerant.
16093
- if (memoryItemMutated) {
16094
- this.storageRouter.recordWrite(this.config.defaultNamespace, this.storage.dir);
16095
- }
16096
-
16097
- log.info("consolidation complete");
16098
- return { memoriesProcessed: allMemories.length, merged, invalidated };
12530
+ return this.consolidationRunCoordinator.run();
16099
12531
  }
16100
12532
 
16101
12533
  async optimizeCompressionGuidelines(options?: {
@@ -16114,36 +12546,10 @@ export class Orchestrator {
16114
12546
 
16115
12547
  // Issue #1526 (seam 4): compression-guideline learning moved to
16116
12548
  // CompressionGuidelineCoordinator. Thin delegation keeps the
16117
- // consolidation-loop hook + recall-section hook call sites stable.
16118
- private async runCompressionGuidelineLearningPass(): Promise<void> {
16119
- return this.compressionGuidelineCoordinator.runCompressionGuidelineLearningPass();
16120
- }
16121
-
16122
12549
  private async buildCompressionGuidelineRecallSection(): Promise<string | null> {
16123
12550
  return this.compressionGuidelineCoordinator.buildCompressionGuidelineRecallSection();
16124
12551
  }
16125
12552
 
16126
- private async recordScheduledDreamsPhaseRun(
16127
- phase: "lightSleep" | "rem" | "deepSleep",
16128
- itemsProcessed: number,
16129
- notes: string,
16130
- timing: { startedAt?: string; completedAt?: string } = {},
16131
- ): Promise<void> {
16132
- try {
16133
- const { recordDreamsPhaseRun } = await import("./maintenance/dreams-ledger.js");
16134
- await recordDreamsPhaseRun({
16135
- memoryDir: this.storage.dir,
16136
- phase,
16137
- trigger: "scheduled",
16138
- itemsProcessed,
16139
- notes,
16140
- startedAt: timing.startedAt,
16141
- completedAt: timing.completedAt,
16142
- });
16143
- } catch (error) {
16144
- log.debug(`dreams ledger scheduled ${phase} write failed (non-fatal): ${error}`);
16145
- }
16146
- }
16147
12553
 
16148
12554
  async runLifecyclePolicyNow(storage: StorageManager = this.storage): Promise<{ memoriesAssessed: number }> {
16149
12555
  const lifecycleCorpus = await storage.readAllMemories();
@@ -16159,21 +12565,6 @@ export class Orchestrator {
16159
12565
  ): Promise<number> {
16160
12566
  return this.lifecyclePolicyCoordinator.runLifecyclePolicyPass(allMemories, storage);
16161
12567
  }
16162
- private async runFactArchival(
16163
- allMemories: import("./types.js").MemoryFile[],
16164
- ): Promise<number> {
16165
- return this.lifecyclePolicyCoordinator.runFactArchival(allMemories);
16166
- }
16167
- private async runSummarization(
16168
- allMemories: import("./types.js").MemoryFile[],
16169
- ): Promise<void> {
16170
- return this.lifecyclePolicyCoordinator.runSummarization(allMemories);
16171
- }
16172
- private async runTopicExtraction(
16173
- allMemories: import("./types.js").MemoryFile[],
16174
- ): Promise<void> {
16175
- return this.lifecyclePolicyCoordinator.runTopicExtraction(allMemories);
16176
- }
16177
12568
  /** Threshold (bytes) at which IDENTITY.md reflections get auto-consolidated */
16178
12569
  private static readonly IDENTITY_CONSOLIDATE_THRESHOLD = 8_000;
16179
12570