@remnic/core 9.3.744 → 9.3.746
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-boundary.d.ts +17 -15
- package/dist/access-boundary.js +14 -15
- package/dist/access-cli.js +92 -33
- package/dist/access-cli.js.map +1 -1
- package/dist/access-http.d.ts +17 -15
- package/dist/access-http.js +20 -21
- package/dist/access-mcp.d.ts +17 -15
- package/dist/access-mcp.js +18 -19
- package/dist/access-operations-batch.js +16 -17
- package/dist/access-operations.d.ts +17 -15
- package/dist/access-operations.js +17 -18
- package/dist/access-schema.js +3 -3
- package/dist/{access-service-OT3Gk79W.d.ts → access-service-Cx16nmCv.d.ts} +3 -30
- package/dist/access-service.d.ts +7 -5
- package/dist/access-service.js +13 -14
- package/dist/access-surface-catalog.d.ts +17 -15
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/{auto-sync-XHSKKQAH.js → auto-sync-IEBKPGAM.js} +2 -2
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +6 -4
- package/dist/briefing.d.ts +1 -1
- package/dist/briefing.js +3 -3
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer.d.ts +1 -1
- package/dist/calibration.d.ts +1 -1
- package/dist/capabilities.d.ts +1 -1
- package/dist/{catalog-CX0CjOaf.d.ts → catalog-D_ZDxu8Y.d.ts} +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-consolidation.js +4 -4
- package/dist/causal-trajectory-graph.d.ts +1 -1
- package/dist/{chunk-WPLRJ4XM.js → chunk-24YDB5QP.js} +2098 -230
- package/dist/chunk-24YDB5QP.js.map +1 -0
- package/dist/{chunk-6XG3NQRC.js → chunk-3IJGF6K2.js} +2 -2
- package/dist/{chunk-U5WVRMCA.js → chunk-5IN26V6W.js} +2 -2
- package/dist/{chunk-XEADGIAA.js → chunk-DIHCFJVV.js} +7 -7
- package/dist/{chunk-C7LDPZJJ.js → chunk-EHYB325U.js} +227 -227
- package/dist/chunk-EHYB325U.js.map +1 -0
- package/dist/{chunk-E7RGJQWP.js → chunk-HIHCB2S6.js} +2 -2
- package/dist/{chunk-JFCS6PQU.js → chunk-JHQUIH3W.js} +5 -5
- package/dist/{chunk-MXDG6SRH.js → chunk-JRDPT6DA.js} +2 -2
- package/dist/{chunk-BYLC4DKI.js → chunk-KMNYXLPL.js} +2 -2
- package/dist/{chunk-TQPSOTC7.js → chunk-L4N7SVNN.js} +8 -10
- package/dist/{chunk-TQPSOTC7.js.map → chunk-L4N7SVNN.js.map} +1 -1
- package/dist/{chunk-DYK27X7V.js → chunk-LMFS64GY.js} +3 -3
- package/dist/{chunk-FWXLKH2H.js → chunk-NVYF5OB6.js} +3 -3
- package/dist/{chunk-G5FP7VSQ.js → chunk-P4FW3ZU5.js} +58 -60
- package/dist/{chunk-G5FP7VSQ.js.map → chunk-P4FW3ZU5.js.map} +1 -1
- package/dist/{chunk-SODQQCAZ.js → chunk-PJE6D3NH.js} +10362 -4909
- package/dist/chunk-PJE6D3NH.js.map +1 -0
- package/dist/{chunk-56F6EAQ6.js → chunk-Q3X7GUAQ.js} +2 -2
- package/dist/{chunk-7N7S2RKF.js → chunk-R2OAN6IP.js} +2 -2
- package/dist/{chunk-CHELXDTO.js → chunk-RB6A7OUN.js} +2 -2
- package/dist/{chunk-3EA5LIFJ.js → chunk-SIV3JZPP.js} +2 -2
- package/dist/{chunk-IP5SANO2.js → chunk-UIKZZ242.js} +2 -2
- package/dist/{chunk-UGFPGWJB.js → chunk-URQLO7BX.js} +2 -2
- package/dist/{chunk-5UNSGRCR.js → chunk-WK3NK6SP.js} +162 -16
- package/dist/chunk-WK3NK6SP.js.map +1 -0
- package/dist/{chunk-VE54E6RP.js → chunk-WWVXXSLF.js} +4 -6
- package/dist/{chunk-VE54E6RP.js.map → chunk-WWVXXSLF.js.map} +1 -1
- package/dist/{chunk-WO22UEM3.js → chunk-XTEWVF3C.js} +2 -2
- package/dist/{chunk-FCQ7ZYCQ.js → chunk-ZGAJB74V.js} +2 -2
- package/dist/{chunk-NBP3YHHS.js → chunk-ZGZNLRKF.js} +2 -2
- package/dist/{chunk-C7C5WIIF.js → chunk-ZPZ4VVLT.js} +2 -2
- package/dist/{cli-BQiTZ7Zs.d.ts → cli-D1y3vooB.d.ts} +3 -3
- package/dist/cli.d.ts +9 -7
- package/dist/cli.js +44 -46
- package/dist/compounding/engine.d.ts +1 -1
- package/dist/compounding/engine.js +3 -3
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.js +3 -3
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/connectors/index.js +3 -3
- package/dist/consolidation-provenance-check.d.ts +1 -1
- package/dist/consolidation-undo.d.ts +1 -1
- package/dist/contradiction/index.d.ts +1 -1
- package/dist/contradiction/index.js +4 -4
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer.d.ts +1 -1
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-retrieval.d.ts +1 -1
- package/dist/entity-retrieval.js +3 -3
- package/dist/entity-schema.d.ts +1 -1
- package/dist/explicit-capture.d.ts +6 -4
- package/dist/extraction-faithfulness.d.ts +1 -1
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction.d.ts +1 -1
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/graph-dashboard-diff.d.ts +1 -1
- package/dist/graph-dashboard-key.d.ts +1 -1
- package/dist/graph-dashboard-parser.d.ts +1 -1
- package/dist/graph-edge-reinforcement.d.ts +1 -1
- package/dist/graph-snapshot.d.ts +1 -1
- package/dist/graph.d.ts +1 -1
- package/dist/identity-continuity.d.ts +1 -1
- package/dist/importance.d.ts +1 -1
- package/dist/index.d.ts +12 -10
- package/dist/index.js +271 -5362
- package/dist/index.js.map +1 -1
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +1 -1
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/local-model-endpoint.d.ts +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/maintenance/memory-governance.js +3 -3
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +3 -3
- package/dist/maintenance/rebuild-memory-projection.js +4 -4
- package/dist/mcp-memory-inspector-app.d.ts +16 -14
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +1 -1
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +2 -2
- package/dist/namespaces/migrate.js +4 -4
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/storage.d.ts +2 -2
- package/dist/namespaces/storage.js +3 -3
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/operator-toolkit.d.ts +1 -1
- package/dist/operator-toolkit.js +10 -11
- package/dist/orchestration/compression-guideline-coordinator.d.ts +1 -1
- package/dist/orchestration/maintenance.d.ts +2 -2
- package/dist/orchestration/maintenance.js +5 -5
- package/dist/{orchestrator-BPQGHpJz.d.ts → orchestrator-CnCE3Hvq.d.ts} +180 -12
- package/dist/orchestrator.d.ts +6 -4
- package/dist/orchestrator.js +119 -16
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/provenance.d.ts +1 -1
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +1 -1
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-state.d.ts +1 -1
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray.d.ts +1 -1
- package/dist/resolve-auth-token.d.ts +1 -1
- package/dist/retrieval-agents.d.ts +1 -1
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/index.d.ts +1 -1
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/{semantic-consolidation-ByzGwC9c.d.ts → semantic-consolidation-D3aSiu9t.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-consolidation.js +4 -4
- package/dist/semantic-rule-promotion.js +3 -3
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/semantic-rule-verifier.js +3 -3
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +1 -1
- package/dist/signal.d.ts +1 -1
- package/dist/storage.d.ts +1 -1
- package/dist/storage.js +2 -2
- package/dist/summarizer.d.ts +1 -1
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/temporal-supersession.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +1 -1
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/transfer/backup.js +2 -2
- package/dist/transfer/capsule-export.js +2 -2
- package/dist/transfer/capsule-import.js +1 -1
- package/dist/trust-score-stage.d.ts +1 -1
- package/dist/trust-score.d.ts +1 -1
- package/dist/{types-5Jik3r4G.d.ts → types-CZm6rx9-.d.ts} +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/utility-runtime.d.ts +1 -1
- package/dist/verified-recall.js +3 -3
- package/package.json +2 -2
- package/src/index.ts +2 -0
- package/src/orchestration/consolidation-run.ts +633 -0
- package/src/orchestration/extraction-persist.ts +2785 -0
- package/src/orchestration/extraction-run.ts +874 -0
- package/src/orchestrator.ts +206 -3815
- package/dist/chunk-5UNSGRCR.js.map +0 -1
- package/dist/chunk-A2KKLCOS.js +0 -1889
- package/dist/chunk-A2KKLCOS.js.map +0 -1
- package/dist/chunk-AVBT5WEW.js +0 -153
- package/dist/chunk-AVBT5WEW.js.map +0 -1
- package/dist/chunk-C7LDPZJJ.js.map +0 -1
- package/dist/chunk-SODQQCAZ.js.map +0 -1
- package/dist/chunk-WPLRJ4XM.js.map +0 -1
- /package/dist/{auto-sync-XHSKKQAH.js.map → auto-sync-IEBKPGAM.js.map} +0 -0
- /package/dist/{chunk-6XG3NQRC.js.map → chunk-3IJGF6K2.js.map} +0 -0
- /package/dist/{chunk-U5WVRMCA.js.map → chunk-5IN26V6W.js.map} +0 -0
- /package/dist/{chunk-XEADGIAA.js.map → chunk-DIHCFJVV.js.map} +0 -0
- /package/dist/{chunk-E7RGJQWP.js.map → chunk-HIHCB2S6.js.map} +0 -0
- /package/dist/{chunk-JFCS6PQU.js.map → chunk-JHQUIH3W.js.map} +0 -0
- /package/dist/{chunk-MXDG6SRH.js.map → chunk-JRDPT6DA.js.map} +0 -0
- /package/dist/{chunk-BYLC4DKI.js.map → chunk-KMNYXLPL.js.map} +0 -0
- /package/dist/{chunk-DYK27X7V.js.map → chunk-LMFS64GY.js.map} +0 -0
- /package/dist/{chunk-FWXLKH2H.js.map → chunk-NVYF5OB6.js.map} +0 -0
- /package/dist/{chunk-56F6EAQ6.js.map → chunk-Q3X7GUAQ.js.map} +0 -0
- /package/dist/{chunk-7N7S2RKF.js.map → chunk-R2OAN6IP.js.map} +0 -0
- /package/dist/{chunk-CHELXDTO.js.map → chunk-RB6A7OUN.js.map} +0 -0
- /package/dist/{chunk-3EA5LIFJ.js.map → chunk-SIV3JZPP.js.map} +0 -0
- /package/dist/{chunk-IP5SANO2.js.map → chunk-UIKZZ242.js.map} +0 -0
- /package/dist/{chunk-UGFPGWJB.js.map → chunk-URQLO7BX.js.map} +0 -0
- /package/dist/{chunk-WO22UEM3.js.map → chunk-XTEWVF3C.js.map} +0 -0
- /package/dist/{chunk-FCQ7ZYCQ.js.map → chunk-ZGAJB74V.js.map} +0 -0
- /package/dist/{chunk-NBP3YHHS.js.map → chunk-ZGZNLRKF.js.map} +0 -0
- /package/dist/{chunk-C7C5WIIF.js.map → chunk-ZPZ4VVLT.js.map} +0 -0
package/src/orchestrator.ts
CHANGED
|
@@ -81,7 +81,6 @@ import { NoopSearchBackend } from "./search/noop-backend.js";
|
|
|
81
81
|
import {
|
|
82
82
|
StorageManager,
|
|
83
83
|
ContentHashIndex,
|
|
84
|
-
normalizeAttributePairs,
|
|
85
84
|
} from "./storage.js";
|
|
86
85
|
import { sanitizeMemoryContent } from "./sanitize.js";
|
|
87
86
|
import { ThreadingManager } from "./threading.js";
|
|
@@ -107,6 +106,9 @@ import { RecallRerankCoordinator } from "./orchestration/recall-rerank-coordinat
|
|
|
107
106
|
import { RecallSectionCoordinator } from "./orchestration/recall-section-coordinator.js";
|
|
108
107
|
import { QmdResultResolver, qmdCollectionPathParts, qmdResultPathCandidates } from "./orchestration/qmd-result-resolver.js";
|
|
109
108
|
import { ContradictionLinkingCoordinator } from "./orchestration/contradiction-linking-coordinator.js";
|
|
109
|
+
import { ExtractionRunCoordinator, type ExtractionRunResult } from "./orchestration/extraction-run.js";
|
|
110
|
+
import { ConsolidationRunCoordinator } from "./orchestration/consolidation-run.js";
|
|
111
|
+
import { ExtractionPersistCoordinator } from "./orchestration/extraction-persist.js";
|
|
110
112
|
export { hasIdentityRecoveryIntent, resolveEffectiveIdentityInjectionMode } from "./orchestration/recall-result-formatter.js";
|
|
111
113
|
import {
|
|
112
114
|
GraphRecallCoordinator,
|
|
@@ -489,13 +491,8 @@ export class BulkImportBatchPartialFailureError extends Error {
|
|
|
489
491
|
}
|
|
490
492
|
}
|
|
491
493
|
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
reason?: string;
|
|
495
|
-
persistedCount: number;
|
|
496
|
-
durableOutputCount: number;
|
|
497
|
-
postPersistMetadataFailed?: boolean;
|
|
498
|
-
}
|
|
494
|
+
// Issue #1526 seam 15: ExtractionRunResult moved to orchestration/extraction-run.ts.
|
|
495
|
+
export type { ExtractionRunResult } from "./orchestration/extraction-run.js";
|
|
499
496
|
|
|
500
497
|
export interface GraphRecallSnapshot {
|
|
501
498
|
recordedAt: string;
|
|
@@ -914,20 +911,6 @@ export function sanitizeSessionKeyForFilename(sessionKey: string): string {
|
|
|
914
911
|
return `${readable}-${hash}`;
|
|
915
912
|
}
|
|
916
913
|
|
|
917
|
-
function latestSourceValidAtFromTurns(turns: readonly BufferTurn[]): string | undefined {
|
|
918
|
-
let latestMs: number | null = null;
|
|
919
|
-
for (const turn of turns) {
|
|
920
|
-
if (turn.extractionContextOnly === true) continue;
|
|
921
|
-
if (typeof turn.sourceValidAt !== "string") continue;
|
|
922
|
-
const parsed = parseFlexibleIsoTimestamp(turn.sourceValidAt.trim());
|
|
923
|
-
if (parsed === null) continue;
|
|
924
|
-
if (latestMs === null || parsed > latestMs) {
|
|
925
|
-
latestMs = parsed;
|
|
926
|
-
}
|
|
927
|
-
}
|
|
928
|
-
return latestMs === null ? undefined : new Date(latestMs).toISOString();
|
|
929
|
-
}
|
|
930
|
-
|
|
931
914
|
function sourceValidAtMs(turn: BufferTurn): number | null {
|
|
932
915
|
if (typeof turn.sourceValidAt !== "string") return null;
|
|
933
916
|
return parseFlexibleIsoTimestamp(turn.sourceValidAt.trim());
|
|
@@ -1044,24 +1027,8 @@ function splitTurnsBySourceValidAt(
|
|
|
1044
1027
|
export function isArtifactMemoryPath(filePath: string): boolean {
|
|
1045
1028
|
return /(?:^|[\\/])artifacts(?:[\\/]|$)/i.test(filePath);
|
|
1046
1029
|
}
|
|
1047
|
-
|
|
1048
|
-
export
|
|
1049
|
-
const topics = new Set<string>();
|
|
1050
|
-
for (const fact of result.facts ?? []) {
|
|
1051
|
-
for (const tag of fact.tags ?? []) {
|
|
1052
|
-
if (tag && tag.length >= 2) topics.add(tag.toLowerCase());
|
|
1053
|
-
}
|
|
1054
|
-
if (fact.entityRef) topics.add(fact.entityRef.toLowerCase());
|
|
1055
|
-
if (fact.category) topics.add(fact.category);
|
|
1056
|
-
}
|
|
1057
|
-
for (const entity of (result as any).entities ?? []) {
|
|
1058
|
-
if (typeof entity.name === "string" && entity.name.length >= 2) {
|
|
1059
|
-
topics.add(entity.name.toLowerCase());
|
|
1060
|
-
}
|
|
1061
|
-
}
|
|
1062
|
-
return [...topics].slice(0, 16);
|
|
1063
|
-
}
|
|
1064
|
-
|
|
1030
|
+
// Issue #1526 seam 15: deriveTopicsFromExtraction moved to orchestration/extraction-run.ts.
|
|
1031
|
+
export { deriveTopicsFromExtraction } from "./orchestration/extraction-run.js";
|
|
1065
1032
|
export function buildCompressionGuidelinesMarkdown(
|
|
1066
1033
|
events: MemoryActionEvent[],
|
|
1067
1034
|
generatedAtIso: string = new Date().toISOString(),
|
|
@@ -1769,6 +1736,128 @@ export class Orchestrator {
|
|
|
1769
1736
|
* hands it to `enqueue`.
|
|
1770
1737
|
*/
|
|
1771
1738
|
readonly extractionQueueCoordinator: ExtractionQueueCoordinator;
|
|
1739
|
+
/**
|
|
1740
|
+
* Extraction-run pipeline coordinator (issue #1526 seam 15). Owns
|
|
1741
|
+
* `runExtraction` + dedupe fingerprint helpers + processed-fingerprint
|
|
1742
|
+
* recording. The orchestrator delegates and injects its own methods.
|
|
1743
|
+
*/
|
|
1744
|
+
private _extractionRunCoordinator: ExtractionRunCoordinator | undefined;
|
|
1745
|
+
|
|
1746
|
+
/**
|
|
1747
|
+
* Lazy getter: creates the ExtractionRunCoordinator on first access using
|
|
1748
|
+
* the orchestrator's live field references. Supports Object.create(prototype)
|
|
1749
|
+
* tests that set fields post-construction without invoking the constructor.
|
|
1750
|
+
*/
|
|
1751
|
+
private get extractionRunCoordinator(): ExtractionRunCoordinator {
|
|
1752
|
+
if (!this._extractionRunCoordinator) {
|
|
1753
|
+
this._extractionRunCoordinator = new ExtractionRunCoordinator({
|
|
1754
|
+
config: this.config,
|
|
1755
|
+
getBuffer: () => this.buffer,
|
|
1756
|
+
getExtraction: () => this.extraction,
|
|
1757
|
+
getStorageRouter: () => this.storageRouter,
|
|
1758
|
+
getThreading: () => this.threading,
|
|
1759
|
+
persistExtraction: (result, storage, threadId, sourceContext, baseNamespace, scopeProfileWritePlan, sourceText, graphCaps, lifecycleCaps) =>
|
|
1760
|
+
this.persistExtraction(result, storage, threadId, sourceContext, baseNamespace, scopeProfileWritePlan, sourceText, graphCaps, lifecycleCaps),
|
|
1761
|
+
maybeCapturePassiveCorrections: (turns, opts) => this.maybeCapturePassiveCorrections(turns, opts),
|
|
1762
|
+
resolveSelfNamespace: (sessionKey) => this.resolveSelfNamespace(sessionKey),
|
|
1763
|
+
getCodingContextForSession: (sessionKey) => this.getCodingContextForSession(sessionKey),
|
|
1764
|
+
applyCodingNamespaceOverlay: (sessionKey, namespace) => this.applyCodingNamespaceOverlay(sessionKey, namespace),
|
|
1765
|
+
boxBuilderFor: (storage) => this.boxBuilderFor(storage),
|
|
1766
|
+
appendPersistedThreadEpisodes: (threadId, ids) => this.appendPersistedThreadEpisodes(threadId, ids),
|
|
1767
|
+
maybeScheduleConsolidation: (nonZero) => this.maybeScheduleConsolidation(nonZero),
|
|
1768
|
+
requestQmdMaintenance: () => this.requestQmdMaintenance(),
|
|
1769
|
+
runTierMigrationCycle: (storage, trigger, options) => this.runTierMigrationCycle(storage, trigger, options),
|
|
1770
|
+
getLastPersistExtractionDeferredCount: () => this.lastPersistExtractionDeferredCount,
|
|
1771
|
+
recordProcessedExtractionFingerprint: (storage, fingerprint, preloadedMeta) =>
|
|
1772
|
+
this.recordProcessedExtractionFingerprint(storage, fingerprint, preloadedMeta),
|
|
1773
|
+
});
|
|
1774
|
+
}
|
|
1775
|
+
return this._extractionRunCoordinator;
|
|
1776
|
+
}
|
|
1777
|
+
/**
|
|
1778
|
+
* Consolidation-run coordinator (issue #1526 seam 17). Owns
|
|
1779
|
+
* the full consolidation maintenance pass (LLM merge/invalidate/update,
|
|
1780
|
+
* entity merge, commitment/TTL cleanup, lifecycle policy, compression
|
|
1781
|
+
* guideline learning, tier migration, fact archival, semantic consolidation,
|
|
1782
|
+
* identity consolidation, profile consolidation, summarization, topic
|
|
1783
|
+
* extraction, TMT rebuild). The orchestrator delegates and injects
|
|
1784
|
+
* its own methods + coordinators.
|
|
1785
|
+
*/
|
|
1786
|
+
private _consolidationRunCoordinator: ConsolidationRunCoordinator | undefined;
|
|
1787
|
+
|
|
1788
|
+
private get consolidationRunCoordinator(): ConsolidationRunCoordinator {
|
|
1789
|
+
if (!this._consolidationRunCoordinator) {
|
|
1790
|
+
this._consolidationRunCoordinator = new ConsolidationRunCoordinator({
|
|
1791
|
+
config: this.config,
|
|
1792
|
+
getStorage: () => this.storage,
|
|
1793
|
+
getStorageRouter: () => this.storageRouter,
|
|
1794
|
+
getExtraction: () => this.extraction,
|
|
1795
|
+
embeddingFallback: this.embeddingFallback,
|
|
1796
|
+
tmtBuilder: this.tmtBuilder,
|
|
1797
|
+
consolidationObservers: this.consolidationObservers,
|
|
1798
|
+
getAccessTrackingBuffer: () => this.accessTrackingBuffer,
|
|
1799
|
+
lifecyclePolicyCoordinator: this.lifecyclePolicyCoordinator,
|
|
1800
|
+
compressionGuidelineCoordinator: this.compressionGuidelineCoordinator,
|
|
1801
|
+
semanticConsolidationCoordinator: this.semanticConsolidationCoordinator,
|
|
1802
|
+
entitySynthesisCoordinator: this.entitySynthesisCoordinator,
|
|
1803
|
+
recallSectionCoordinator: this.recallSectionCoordinator,
|
|
1804
|
+
tierMigrationCoordinator: this.tierMigrationCoordinator,
|
|
1805
|
+
flushAccessTracking: () => this.flushAccessTracking(),
|
|
1806
|
+
indexPersistedMemory: (storage, memoryId) => this.indexPersistedMemory(storage, memoryId),
|
|
1807
|
+
autoConsolidateIdentity: () => this.autoConsolidateIdentity(),
|
|
1808
|
+
fastChatCompletion: (messages, options) => this.fastChatCompletion(messages, options),
|
|
1809
|
+
});
|
|
1810
|
+
}
|
|
1811
|
+
return this._consolidationRunCoordinator;
|
|
1812
|
+
}
|
|
1813
|
+
|
|
1814
|
+
/**
|
|
1815
|
+
* Extraction-persist coordinator (issue #1526 seam 16). Owns the
|
|
1816
|
+
* `persistExtraction` pipeline. Lazy: created on first access so
|
|
1817
|
+
* Object.create(prototype) tests that set fields post-construction work.
|
|
1818
|
+
*/
|
|
1819
|
+
private _extractionPersistCoordinator: ExtractionPersistCoordinator | undefined;
|
|
1820
|
+
|
|
1821
|
+
private get extractionPersistCoordinator(): ExtractionPersistCoordinator {
|
|
1822
|
+
if (!this._extractionPersistCoordinator) {
|
|
1823
|
+
this._extractionPersistCoordinator = new ExtractionPersistCoordinator({
|
|
1824
|
+
config: this.config,
|
|
1825
|
+
getStorageRouter: () => this.storageRouter,
|
|
1826
|
+
getThreading: () => this.threading,
|
|
1827
|
+
getLocalLlm: () => this.localLlm,
|
|
1828
|
+
getQmd: () => this.qmd,
|
|
1829
|
+
getJudgeVerdictCache: () => this.judgeVerdictCache,
|
|
1830
|
+
getJudgeDeferCounts: () => this.judgeDeferCounts,
|
|
1831
|
+
getFaithfulnessCounters: () => this.faithfulnessCounters,
|
|
1832
|
+
getEmbeddingFallback: () => this.embeddingFallback,
|
|
1833
|
+
setLastPersistExtractionDeferredCount: (v) => { this.lastPersistExtractionDeferredCount = v; },
|
|
1834
|
+
setLastPersistExtractionPendingReviewIds: (ids) => { this.lastPersistExtractionPendingReviewIds = ids; },
|
|
1835
|
+
addContentHashDedup: (targetStorage, content) => this.addContentHashDedup(targetStorage, content),
|
|
1836
|
+
hasContentHashDedup: (targetStorage, content) => this.hasContentHashDedup(targetStorage, content),
|
|
1837
|
+
backfillTemporalBoundsOnDedupHit: (targetStorage, dedupContent, bounds, entityRef) =>
|
|
1838
|
+
this.backfillTemporalBoundsOnDedupHit(targetStorage, dedupContent, bounds, entityRef),
|
|
1839
|
+
saveContentHashIndexes: () => this.saveContentHashIndexes(),
|
|
1840
|
+
artifactTypeForCategory: (category) => this.artifactTypeForCategory(category),
|
|
1841
|
+
loadRoutingRules: () => this.loadRoutingRules(),
|
|
1842
|
+
routeEngineOptions: () => this.routeEngineOptions(),
|
|
1843
|
+
semanticDedupLookup: (content, limit, targetStorage) =>
|
|
1844
|
+
this.semanticDedupLookup(content, limit, targetStorage),
|
|
1845
|
+
checkForContradiction: (content, category, namespaceScope) =>
|
|
1846
|
+
this.checkForContradiction(content, category, namespaceScope),
|
|
1847
|
+
applyDeferredContradictionResolve: (contradiction, storage, newMemoryId, postWriteGuard) =>
|
|
1848
|
+
this.applyDeferredContradictionResolve(contradiction, storage, newMemoryId, postWriteGuard),
|
|
1849
|
+
suggestLinksForMemory: (content, category, namespaceScope) =>
|
|
1850
|
+
this.suggestLinksForMemory(content, category, namespaceScope),
|
|
1851
|
+
storageDirNamespace: (storageDir) => this.storageDirNamespace(storageDir),
|
|
1852
|
+
indexPersistedMemory: (storage, memoryId) => this.indexPersistedMemory(storage, memoryId),
|
|
1853
|
+
buildGraphEdge: (storage, memoryRelPath, entityRef, memoryId, factContent, allMemsForGraph, memoryPathById, threadIdForEdge, threadEpisodeIdsForGraph, fallbackCausalPredecessor, graphCaps) =>
|
|
1854
|
+
this.buildGraphEdge(storage, memoryRelPath, entityRef, memoryId, factContent, allMemsForGraph, memoryPathById, threadIdForEdge, threadEpisodeIdsForGraph, fallbackCausalPredecessor, graphCaps),
|
|
1855
|
+
updateTemporalTagIndexes: (storage, persistedIds) =>
|
|
1856
|
+
this.updateTemporalTagIndexes(storage, persistedIds),
|
|
1857
|
+
});
|
|
1858
|
+
}
|
|
1859
|
+
return this._extractionPersistCoordinator;
|
|
1860
|
+
}
|
|
1772
1861
|
/**
|
|
1773
1862
|
* Issue #1526: recall result formatting + identity continuity section moved
|
|
1774
1863
|
* to RecallResultFormatter.
|
|
@@ -2751,6 +2840,7 @@ export class Orchestrator {
|
|
|
2751
2840
|
});
|
|
2752
2841
|
// Issue #1526: background extraction queue lives on its own coordinator.
|
|
2753
2842
|
this.extractionQueueCoordinator = new ExtractionQueueCoordinator();
|
|
2843
|
+
|
|
2754
2844
|
this.recallResultFormatter = new RecallResultFormatter(this.config);
|
|
2755
2845
|
const conversationIndexRuntime = createConversationIndexRuntime(config, {
|
|
2756
2846
|
getQmd: () => this.conversationQmd,
|
|
@@ -11983,67 +12073,18 @@ export class Orchestrator {
|
|
|
11983
12073
|
}
|
|
11984
12074
|
|
|
11985
12075
|
private normalizeExtractionFingerprintTurns(turns: BufferTurn[]): string[] {
|
|
11986
|
-
|
|
11987
|
-
return turns
|
|
11988
|
-
.filter((turn) => turn.role === "user" || turn.role === "assistant")
|
|
11989
|
-
.map((turn) => {
|
|
11990
|
-
if (
|
|
11991
|
-
typeof turn.turnFingerprint === "string" &&
|
|
11992
|
-
turn.turnFingerprint.length > 0
|
|
11993
|
-
) {
|
|
11994
|
-
return `fp:${turn.turnFingerprint}`;
|
|
11995
|
-
}
|
|
11996
|
-
return `${turn.role}:${(turn.content ?? "").replace(/\s+/g, " ").trim().slice(0, this.config.extractionMaxTurnChars)}`;
|
|
11997
|
-
})
|
|
11998
|
-
.filter((value) => value.length > 0);
|
|
12076
|
+
return this.extractionRunCoordinator.normalizeExtractionFingerprintTurns(turns);
|
|
11999
12077
|
}
|
|
12000
12078
|
|
|
12001
|
-
private buildExtractionFingerprint(
|
|
12002
|
-
turns
|
|
12003
|
-
bufferKey: string,
|
|
12004
|
-
): string | null {
|
|
12005
|
-
const normalized = this.normalizeExtractionFingerprintTurns(turns).join("\n");
|
|
12006
|
-
if (!normalized) return null;
|
|
12007
|
-
return createHash("sha256")
|
|
12008
|
-
.update(`${bufferKey}\n${normalized}`)
|
|
12009
|
-
.digest("hex");
|
|
12079
|
+
private buildExtractionFingerprint(turns: BufferTurn[], bufferKey: string): string | null {
|
|
12080
|
+
return this.extractionRunCoordinator.buildExtractionFingerprint(turns, bufferKey);
|
|
12010
12081
|
}
|
|
12011
12082
|
|
|
12012
12083
|
private shouldQueueExtraction(
|
|
12013
12084
|
turns: BufferTurn[],
|
|
12014
12085
|
options: { commit?: boolean; bufferKey?: string } = {},
|
|
12015
12086
|
): boolean {
|
|
12016
|
-
|
|
12017
|
-
if (!lifecycleCaps.extractionDedupe) return true;
|
|
12018
|
-
if (!Array.isArray(turns) || turns.length === 0) return false;
|
|
12019
|
-
|
|
12020
|
-
const bufferKey = options.bufferKey ?? turns[0]?.sessionKey ?? "default";
|
|
12021
|
-
const fingerprint = this.buildExtractionFingerprint(turns, bufferKey);
|
|
12022
|
-
if (!fingerprint) return false;
|
|
12023
|
-
const now = Date.now();
|
|
12024
|
-
const seenAt = this.recentExtractionFingerprints.get(fingerprint);
|
|
12025
|
-
if (seenAt && now - seenAt < this.config.extractionDedupeWindowMs) {
|
|
12026
|
-
log.debug("extraction dedupe: skipped duplicate buffered turn set");
|
|
12027
|
-
return false;
|
|
12028
|
-
}
|
|
12029
|
-
|
|
12030
|
-
if (options.commit !== false) {
|
|
12031
|
-
this.recentExtractionFingerprints.set(fingerprint, now);
|
|
12032
|
-
}
|
|
12033
|
-
// Keep this cache bounded to avoid unbounded growth.
|
|
12034
|
-
if (
|
|
12035
|
-
options.commit !== false &&
|
|
12036
|
-
this.recentExtractionFingerprints.size > 200
|
|
12037
|
-
) {
|
|
12038
|
-
const entries = Array.from(
|
|
12039
|
-
this.recentExtractionFingerprints.entries(),
|
|
12040
|
-
).sort((a, b) => a[1] - b[1]);
|
|
12041
|
-
for (const [key] of entries.slice(0, entries.length - 200)) {
|
|
12042
|
-
this.recentExtractionFingerprints.delete(key);
|
|
12043
|
-
}
|
|
12044
|
-
}
|
|
12045
|
-
|
|
12046
|
-
return true;
|
|
12087
|
+
return this.extractionRunCoordinator.shouldQueueExtraction(turns, options);
|
|
12047
12088
|
}
|
|
12048
12089
|
|
|
12049
12090
|
/**
|
|
@@ -12179,3204 +12220,91 @@ export class Orchestrator {
|
|
|
12179
12220
|
return this._passiveCorrectionService;
|
|
12180
12221
|
}
|
|
12181
12222
|
|
|
12182
|
-
|
|
12183
|
-
|
|
12184
|
-
options: {
|
|
12185
|
-
clearBufferAfterExtraction?: boolean;
|
|
12186
|
-
skipCharThreshold?: boolean;
|
|
12187
|
-
skipUserTurnThreshold?: boolean;
|
|
12188
|
-
deadlineMs?: number;
|
|
12189
|
-
bufferKey?: string;
|
|
12190
|
-
abortSignal?: AbortSignal;
|
|
12191
|
-
failOnExtractionFailure?: boolean;
|
|
12192
|
-
/**
|
|
12193
|
-
* Explicit namespace override for the write path (#460). When set,
|
|
12194
|
-
* extraction writes go to this namespace instead of the one derived
|
|
12195
|
-
* from `defaultNamespaceForPrincipal(resolvePrincipal(sessionKey))`.
|
|
12196
|
-
* The resolved `principal` is still threaded into memory metadata
|
|
12197
|
-
* for provenance; only the storage target is overridden.
|
|
12198
|
-
*/
|
|
12199
|
-
writeNamespaceOverride?: string;
|
|
12200
|
-
/**
|
|
12201
|
-
* Pin the provenance principal instead of deriving it from
|
|
12202
|
-
* `resolvePrincipal(sessionKey)` (#1495 thread 1). When set, this is the
|
|
12203
|
-
* identity an access surface already authenticated; used so observed-turn
|
|
12204
|
-
* provenance is correct even though `turn.sessionKey` is the ORIGINAL
|
|
12205
|
-
* (un-prefixed) key and storage is pinned via `writeNamespaceOverride`.
|
|
12206
|
-
*/
|
|
12207
|
-
principalOverride?: string;
|
|
12208
|
-
} = {},
|
|
12223
|
+
async runExtraction(
|
|
12224
|
+
...args: Parameters<ExtractionRunCoordinator["runExtraction"]>
|
|
12209
12225
|
): Promise<ExtractionRunResult> {
|
|
12210
|
-
|
|
12211
|
-
|
|
12212
|
-
options.clearBufferAfterExtraction ?? true;
|
|
12213
|
-
const skipCharThreshold = options.skipCharThreshold ?? false;
|
|
12214
|
-
const skipUserTurnThreshold = options.skipUserTurnThreshold ?? false;
|
|
12215
|
-
const deadlineMs =
|
|
12216
|
-
typeof options.deadlineMs === "number" &&
|
|
12217
|
-
Number.isFinite(options.deadlineMs)
|
|
12218
|
-
? options.deadlineMs
|
|
12219
|
-
: undefined;
|
|
12220
|
-
const bufferKey = options.bufferKey ?? turns[0]?.sessionKey ?? "default";
|
|
12221
|
-
const throwIfDeadlineExceeded = (stage: string): void => {
|
|
12222
|
-
if (typeof deadlineMs === "number" && Date.now() > deadlineMs) {
|
|
12223
|
-
throw new Error(`replay extraction deadline exceeded (${stage})`);
|
|
12224
|
-
}
|
|
12225
|
-
};
|
|
12226
|
-
const throwIfAborted = (stage: string): void => {
|
|
12227
|
-
throwIfRecallAborted(options.abortSignal, `extraction aborted (${stage})`);
|
|
12228
|
-
};
|
|
12229
|
-
const clearBuffer = async (options?: { ignoreAbort?: boolean }) => {
|
|
12230
|
-
if (options?.ignoreAbort !== true) {
|
|
12231
|
-
throwIfAborted("before_clear_buffer");
|
|
12232
|
-
}
|
|
12233
|
-
if (clearBufferAfterExtraction) {
|
|
12234
|
-
await this.buffer.clearAfterExtraction(bufferKey, turns);
|
|
12235
|
-
}
|
|
12236
|
-
};
|
|
12237
|
-
|
|
12238
|
-
// Skip extraction for cron job sessions - these are system operations, not user conversations
|
|
12239
|
-
const sessionKey = turns[0]?.sessionKey ?? "";
|
|
12240
|
-
if (sessionKey.includes(":cron:")) {
|
|
12241
|
-
log.debug(`skipping extraction for cron session: ${sessionKey}`);
|
|
12242
|
-
await clearBuffer();
|
|
12243
|
-
return {
|
|
12244
|
-
status: "skipped",
|
|
12245
|
-
reason: "cron_session",
|
|
12246
|
-
persistedCount: 0,
|
|
12247
|
-
durableOutputCount: 0,
|
|
12248
|
-
};
|
|
12249
|
-
}
|
|
12226
|
+
return this.extractionRunCoordinator.runExtraction(...args);
|
|
12227
|
+
}
|
|
12250
12228
|
|
|
12251
|
-
|
|
12252
|
-
|
|
12253
|
-
|
|
12254
|
-
|
|
12255
|
-
|
|
12256
|
-
|
|
12257
|
-
|
|
12258
|
-
...t,
|
|
12259
|
-
content: t.content.trim().slice(0, this.config.extractionMaxTurnChars),
|
|
12260
|
-
}))
|
|
12261
|
-
.filter((t) => t.content.length > 0);
|
|
12262
|
-
const targetTurns = normalizedTurns.filter(
|
|
12263
|
-
(turn) => turn.extractionContextOnly !== true,
|
|
12229
|
+
private async recordProcessedExtractionFingerprint(
|
|
12230
|
+
storage: StorageManager,
|
|
12231
|
+
fingerprint: string,
|
|
12232
|
+
preloadedMeta?: Awaited<ReturnType<StorageManager["loadMeta"]>>,
|
|
12233
|
+
): Promise<void> {
|
|
12234
|
+
return this.extractionRunCoordinator.recordProcessedExtractionFingerprint(
|
|
12235
|
+
storage, fingerprint, preloadedMeta,
|
|
12264
12236
|
);
|
|
12265
|
-
|
|
12266
|
-
log.debug("skipping extraction: no non-context turns after normalization");
|
|
12267
|
-
// Context-only turns may still contain corrections (review: "context-only
|
|
12268
|
-
// turns skip capture"). Scan normalizedTurns before clearing the buffer.
|
|
12269
|
-
if (normalizedTurns.length > 0) {
|
|
12270
|
-
await this.maybeCapturePassiveCorrections(normalizedTurns as BufferTurn[], { sessionKey, principal: resolvePrincipal(sessionKey, this.config), namespace: this.resolveSelfNamespace(sessionKey), bufferKey, isLiveSession: clearBufferAfterExtraction });
|
|
12271
|
-
}
|
|
12272
|
-
await clearBuffer();
|
|
12273
|
-
return { status: "skipped", reason: "empty_normalized_turns", persistedCount: 0, durableOutputCount: 0 };
|
|
12274
|
-
}
|
|
12275
|
-
const sourceValidAt = latestSourceValidAtFromTurns(targetTurns);
|
|
12276
|
-
throwIfDeadlineExceeded("before_extract");
|
|
12277
|
-
throwIfAborted("before_extract");
|
|
12237
|
+
}
|
|
12278
12238
|
|
|
12279
|
-
|
|
12280
|
-
|
|
12281
|
-
|
|
12282
|
-
|
|
12283
|
-
|
|
12284
|
-
|
|
12285
|
-
|
|
12286
|
-
|
|
12287
|
-
|
|
12288
|
-
|
|
12289
|
-
|
|
12290
|
-
`skipping extraction: below threshold (totalChars=${totalChars}, userTurns=${userTurns.length})`,
|
|
12291
|
-
);
|
|
12292
|
-
await clearBuffer();
|
|
12293
|
-
// Passive correction capture runs even when extraction is skipped for
|
|
12294
|
-
// being below the char/user-turn threshold (review: "skipped extraction
|
|
12295
|
-
// skips capture"). A short correction like "stop using Vim" is under
|
|
12296
|
-
// extractionMinChars but is exactly the high-value case this feature
|
|
12297
|
-
// exists for. The write namespace resolves through the SAME scope-profile
|
|
12298
|
-
// plan as the full extraction path so corrections target the active
|
|
12299
|
-
// profile's write layer, not just resolveSelfNamespace (review:
|
|
12300
|
-
// "below-threshold wrong namespace"). The ACL in passiveCorrectionService
|
|
12301
|
-
// authorizes the actual plan/apply.
|
|
12302
|
-
{
|
|
12303
|
-
const capturePrincipal =
|
|
12304
|
-
typeof options.principalOverride === "string" && options.principalOverride.length > 0
|
|
12305
|
-
? options.principalOverride
|
|
12306
|
-
: resolvePrincipal(sessionKey, this.config);
|
|
12307
|
-
const captureWO = typeof options.writeNamespaceOverride === "string" && options.writeNamespaceOverride.length > 0
|
|
12308
|
-
? options.writeNamespaceOverride : undefined;
|
|
12309
|
-
const captureCodingCtx = sessionKey ? this.getCodingContextForSession(sessionKey) : null;
|
|
12310
|
-
const captureCodingOv = resolveCodingNamespaceOverlay(captureCodingCtx, this.config.codingMode, this.config.defaultNamespace);
|
|
12311
|
-
const captureScopePlan = resolveScopeProfilePlan({ config: this.config, principal: capturePrincipal, codingContext: captureCodingCtx, codingOverlay: captureCodingOv });
|
|
12312
|
-
const captureNamespace = captureWO ?? captureScopePlan?.writeNamespace ?? this.resolveSelfNamespace(sessionKey);
|
|
12313
|
-
await this.maybeCapturePassiveCorrections(normalizedTurns as BufferTurn[], {
|
|
12314
|
-
sessionKey, principal: capturePrincipal, namespace: captureNamespace, bufferKey, isLiveSession: clearBufferAfterExtraction,
|
|
12315
|
-
});
|
|
12316
|
-
}
|
|
12317
|
-
return {
|
|
12318
|
-
status: "skipped",
|
|
12319
|
-
reason: "below_threshold",
|
|
12320
|
-
persistedCount: 0,
|
|
12321
|
-
durableOutputCount: 0,
|
|
12322
|
-
};
|
|
12323
|
-
}
|
|
12239
|
+
private async runTierMigrationCycle(
|
|
12240
|
+
storage: StorageManager,
|
|
12241
|
+
trigger: "extraction" | "maintenance" | "manual",
|
|
12242
|
+
options?: {
|
|
12243
|
+
dryRun?: boolean;
|
|
12244
|
+
limitOverride?: number;
|
|
12245
|
+
force?: boolean;
|
|
12246
|
+
},
|
|
12247
|
+
): Promise<TierMigrationCycleSummary> {
|
|
12248
|
+
return this.tierMigrationCoordinator.runCycle(storage, trigger, options);
|
|
12249
|
+
}
|
|
12324
12250
|
|
|
12325
|
-
|
|
12326
|
-
|
|
12327
|
-
|
|
12328
|
-
|
|
12329
|
-
|
|
12330
|
-
|
|
12331
|
-
|
|
12332
|
-
|
|
12333
|
-
|
|
12334
|
-
options
|
|
12335
|
-
|
|
12336
|
-
|
|
12337
|
-
// Write path — explicit callers still win. Otherwise, an active hosted
|
|
12338
|
-
// scope profile owns the extraction write target so hook-captured turns land
|
|
12339
|
-
// in the same layer that profile recall searches. Without a profile, preserve
|
|
12340
|
-
// the existing coding-agent overlay behavior (issue #569).
|
|
12341
|
-
const explicitWriteNamespace =
|
|
12342
|
-
typeof options.writeNamespaceOverride === "string" &&
|
|
12343
|
-
options.writeNamespaceOverride.length > 0
|
|
12344
|
-
? options.writeNamespaceOverride
|
|
12345
|
-
: undefined;
|
|
12346
|
-
const codingContextForWrite = sessionKey
|
|
12347
|
-
? this.getCodingContextForSession(sessionKey)
|
|
12348
|
-
: null;
|
|
12349
|
-
const codingOverlayForWrite = resolveCodingNamespaceOverlay(
|
|
12350
|
-
codingContextForWrite,
|
|
12351
|
-
this.config.codingMode,
|
|
12352
|
-
this.config.defaultNamespace,
|
|
12353
|
-
);
|
|
12354
|
-
const scopeProfileGatePlan = resolveScopeProfilePlan({
|
|
12355
|
-
config: this.config,
|
|
12356
|
-
principal,
|
|
12357
|
-
codingContext: codingContextForWrite,
|
|
12358
|
-
codingOverlay: codingOverlayForWrite,
|
|
12251
|
+
async getTierMigrationStatus(): Promise<TierMigrationStatusSnapshot> {
|
|
12252
|
+
return this.tierMigrationStatus.get();
|
|
12253
|
+
}
|
|
12254
|
+
|
|
12255
|
+
async runTierMigrationNow(options?: {
|
|
12256
|
+
dryRun?: boolean;
|
|
12257
|
+
limit?: number;
|
|
12258
|
+
}): Promise<TierMigrationCycleSummary> {
|
|
12259
|
+
return this.runTierMigrationCycle(this.storage, "manual", {
|
|
12260
|
+
dryRun: options?.dryRun === true,
|
|
12261
|
+
limitOverride: options?.limit,
|
|
12262
|
+
force: false,
|
|
12359
12263
|
});
|
|
12360
|
-
|
|
12361
|
-
if (scopeProfileWritePlan) {
|
|
12362
|
-
const selectedLayer = scopeProfileWritePlan.layers.find(
|
|
12363
|
-
(layer) => layer.id === scopeProfileWritePlan.writeLayer,
|
|
12364
|
-
);
|
|
12365
|
-
const writeNamespaceReadable = scopeProfileWritePlan.readNamespaces.includes(
|
|
12366
|
-
scopeProfileWritePlan.writeNamespace,
|
|
12367
|
-
);
|
|
12368
|
-
if (!selectedLayer?.writable || !writeNamespaceReadable) {
|
|
12369
|
-
log.warn(
|
|
12370
|
-
`runExtraction: skipping scope profile ${scopeProfileWritePlan.profileId} because write layer ${scopeProfileWritePlan.writeLayer} is not writable inside the profile read stack`,
|
|
12371
|
-
);
|
|
12372
|
-
await clearBuffer();
|
|
12373
|
-
return {
|
|
12374
|
-
status: "skipped",
|
|
12375
|
-
reason: "scope_profile_no_writable_layer",
|
|
12376
|
-
persistedCount: 0,
|
|
12377
|
-
durableOutputCount: 0,
|
|
12378
|
-
};
|
|
12379
|
-
}
|
|
12380
|
-
}
|
|
12381
|
-
const selfNamespace =
|
|
12382
|
-
explicitWriteNamespace ??
|
|
12383
|
-
scopeProfileWritePlan?.writeNamespace ??
|
|
12384
|
-
this.applyCodingNamespaceOverlay(
|
|
12385
|
-
sessionKey,
|
|
12386
|
-
defaultNamespaceForPrincipal(principal, this.config),
|
|
12387
|
-
);
|
|
12388
|
-
const storage = await this.storageRouter.storageFor(selfNamespace);
|
|
12389
|
-
const shouldPersistProcessedFingerprint = targetTurns.some(
|
|
12390
|
-
(turn) => turn.persistProcessedFingerprint === true,
|
|
12391
|
-
);
|
|
12392
|
-
const extractionFingerprint = this.buildExtractionFingerprint(
|
|
12393
|
-
targetTurns,
|
|
12394
|
-
bufferKey,
|
|
12395
|
-
);
|
|
12396
|
-
let meta =
|
|
12397
|
-
extractionFingerprint && shouldPersistProcessedFingerprint
|
|
12398
|
-
? await storage.loadMeta()
|
|
12399
|
-
: null;
|
|
12400
|
-
if (
|
|
12401
|
-
extractionFingerprint &&
|
|
12402
|
-
shouldPersistProcessedFingerprint &&
|
|
12403
|
-
(meta?.processedExtractionFingerprints ?? []).some(
|
|
12404
|
-
(entry) => entry.fingerprint === extractionFingerprint,
|
|
12405
|
-
)
|
|
12406
|
-
) {
|
|
12407
|
-
log.debug(
|
|
12408
|
-
`runExtraction: skipping already-processed extraction fingerprint for ${bufferKey}`,
|
|
12409
|
-
);
|
|
12410
|
-
await clearBuffer();
|
|
12411
|
-
return {
|
|
12412
|
-
status: "skipped",
|
|
12413
|
-
reason: "processed_fingerprint",
|
|
12414
|
-
persistedCount: 0,
|
|
12415
|
-
durableOutputCount: 0,
|
|
12416
|
-
};
|
|
12417
|
-
}
|
|
12264
|
+
}
|
|
12418
12265
|
|
|
12419
|
-
|
|
12420
|
-
|
|
12421
|
-
|
|
12422
|
-
this.
|
|
12423
|
-
normalizedTurns,
|
|
12424
|
-
existingEntities,
|
|
12425
|
-
),
|
|
12426
|
-
options.abortSignal,
|
|
12427
|
-
"extraction aborted (during_extract)",
|
|
12266
|
+
private maybeScheduleConsolidation(nonZeroExtraction: boolean): void {
|
|
12267
|
+
this.maintenanceScheduler.maybeScheduleConsolidation(
|
|
12268
|
+
nonZeroExtraction,
|
|
12269
|
+
() => this.runConsolidation(),
|
|
12428
12270
|
);
|
|
12429
|
-
|
|
12430
|
-
throwIfAborted("before_persist");
|
|
12431
|
-
|
|
12432
|
-
// Defensive: validate extraction result before processing. Explicit
|
|
12433
|
-
// fail-closed callers, such as flush-plan import, must not observe
|
|
12434
|
-
// malformed extractor output as a successful skip.
|
|
12435
|
-
if (!result) {
|
|
12436
|
-
log.warn("runExtraction: extraction returned null/undefined");
|
|
12437
|
-
if (options.failOnExtractionFailure) {
|
|
12438
|
-
throw new Error("extraction failed: invalid_extraction_result");
|
|
12439
|
-
}
|
|
12440
|
-
await clearBuffer();
|
|
12441
|
-
return {
|
|
12442
|
-
status: "skipped",
|
|
12443
|
-
reason: "invalid_extraction_result",
|
|
12444
|
-
persistedCount: 0,
|
|
12445
|
-
durableOutputCount: 0,
|
|
12446
|
-
};
|
|
12447
|
-
}
|
|
12448
|
-
const invalidExtractionResultFields = [
|
|
12449
|
-
["facts", result.facts],
|
|
12450
|
-
["entities", result.entities],
|
|
12451
|
-
["questions", result.questions],
|
|
12452
|
-
["profileUpdates", result.profileUpdates],
|
|
12453
|
-
]
|
|
12454
|
-
.filter(([, value]) => !Array.isArray(value))
|
|
12455
|
-
.map(([field]) => field);
|
|
12456
|
-
if (invalidExtractionResultFields.length > 0) {
|
|
12457
|
-
log.warn(
|
|
12458
|
-
"runExtraction: extraction returned invalid collection fields",
|
|
12459
|
-
{
|
|
12460
|
-
invalidFields: invalidExtractionResultFields,
|
|
12461
|
-
resultKeys:
|
|
12462
|
-
typeof result === "object" && result !== null
|
|
12463
|
-
? Object.keys(result)
|
|
12464
|
-
: [],
|
|
12465
|
-
},
|
|
12466
|
-
);
|
|
12467
|
-
if (options.failOnExtractionFailure) {
|
|
12468
|
-
throw new Error("extraction failed: invalid_extraction_result");
|
|
12469
|
-
}
|
|
12470
|
-
await clearBuffer();
|
|
12471
|
-
return {
|
|
12472
|
-
status: "skipped",
|
|
12473
|
-
reason: "invalid_extraction_result",
|
|
12474
|
-
persistedCount: 0,
|
|
12475
|
-
durableOutputCount: 0,
|
|
12476
|
-
};
|
|
12477
|
-
}
|
|
12478
|
-
const extractionFailure =
|
|
12479
|
-
typeof result.extractionFailure === "string" &&
|
|
12480
|
-
result.extractionFailure.trim().length > 0
|
|
12481
|
-
? result.extractionFailure
|
|
12482
|
-
: undefined;
|
|
12483
|
-
if (options.failOnExtractionFailure && extractionFailure) {
|
|
12484
|
-
throw new Error(`extraction failed: ${extractionFailure}`);
|
|
12485
|
-
}
|
|
12486
|
-
if (
|
|
12487
|
-
result.facts.length === 0 &&
|
|
12488
|
-
result.entities.length === 0 &&
|
|
12489
|
-
result.questions.length === 0 &&
|
|
12490
|
-
result.profileUpdates.length === 0
|
|
12491
|
-
) {
|
|
12492
|
-
log.debug(
|
|
12493
|
-
"runExtraction: extraction produced no durable outputs; skipping persistence",
|
|
12494
|
-
);
|
|
12495
|
-
if (extractionFailure) {
|
|
12496
|
-
log.warn(
|
|
12497
|
-
"runExtraction: extraction reported failure with no durable outputs; not marking fingerprint processed",
|
|
12498
|
-
{ extractionFailure },
|
|
12499
|
-
);
|
|
12500
|
-
}
|
|
12501
|
-
if (
|
|
12502
|
-
extractionFingerprint &&
|
|
12503
|
-
shouldPersistProcessedFingerprint &&
|
|
12504
|
-
!extractionFailure
|
|
12505
|
-
) {
|
|
12506
|
-
meta ??= await storage.loadMeta();
|
|
12507
|
-
await this.recordProcessedExtractionFingerprint(
|
|
12508
|
-
storage,
|
|
12509
|
-
extractionFingerprint,
|
|
12510
|
-
meta,
|
|
12511
|
-
);
|
|
12512
|
-
meta.extractionCount += 1;
|
|
12513
|
-
meta.lastExtractionAt = new Date().toISOString();
|
|
12514
|
-
await storage.saveMeta(meta);
|
|
12515
|
-
}
|
|
12516
|
-
// Correction-only turns that meet char/user-turn thresholds but yield
|
|
12517
|
-
// zero facts still need passive capture (review: "empty extraction skips
|
|
12518
|
-
// capture"). selfNamespace/principal already resolved above.
|
|
12519
|
-
await this.maybeCapturePassiveCorrections(normalizedTurns as BufferTurn[], { sessionKey, principal, namespace: selfNamespace, bufferKey, isLiveSession: clearBufferAfterExtraction });
|
|
12520
|
-
await clearBuffer();
|
|
12521
|
-
return { status: "skipped", reason: "empty_extraction_result", persistedCount: 0, durableOutputCount: 0 };
|
|
12522
|
-
}
|
|
12271
|
+
}
|
|
12523
12272
|
|
|
12524
|
-
|
|
12525
|
-
|
|
12526
|
-
|
|
12527
|
-
|
|
12528
|
-
|
|
12529
|
-
|
|
12530
|
-
|
|
12531
|
-
|
|
12532
|
-
|
|
12533
|
-
|
|
12534
|
-
|
|
12535
|
-
}
|
|
12536
|
-
}
|
|
12273
|
+
requestQmdMaintenance(): void {
|
|
12274
|
+
this.maintenanceScheduler.requestQmdMaintenanceForTool("__internal__");
|
|
12275
|
+
}
|
|
12276
|
+
|
|
12277
|
+
/**
|
|
12278
|
+
* Public entrypoint for tool-driven QMD maintenance requests.
|
|
12279
|
+
* Delegates to the maintenance scheduler (issue #1526 PR1).
|
|
12280
|
+
*/
|
|
12281
|
+
requestQmdMaintenanceForTool(reason: string): void {
|
|
12282
|
+
this.maintenanceScheduler.requestQmdMaintenanceForTool(reason);
|
|
12283
|
+
}
|
|
12537
12284
|
|
|
12538
|
-
|
|
12285
|
+
private async persistExtraction(
|
|
12286
|
+
result: ExtractionResult,
|
|
12287
|
+
storage: StorageManager,
|
|
12288
|
+
threadIdForExtraction?: string | null,
|
|
12289
|
+
sourceContext?: { sessionKey?: string; principal?: string; validAt?: string },
|
|
12290
|
+
baseNamespace?: string,
|
|
12291
|
+
scopeProfileWritePlan?: ResolvedScopeProfilePlan | null,
|
|
12292
|
+
/** Verbatim source turn text the facts were extracted from (faithfulness gate #1576). */
|
|
12293
|
+
sourceText?: string,
|
|
12294
|
+
graphCaps: GraphConstructionCapabilitySet = resolveGraphConstructionCapabilities(this.config),
|
|
12295
|
+
lifecycleCaps: MemoryLifecycleCapabilitySet = resolveMemoryLifecycleCapabilities(this.config),
|
|
12296
|
+
): Promise<string[]> {
|
|
12297
|
+
return this.extractionPersistCoordinator.persistExtraction(
|
|
12539
12298
|
result,
|
|
12540
12299
|
storage,
|
|
12541
12300
|
threadIdForExtraction,
|
|
12542
|
-
|
|
12543
|
-
|
|
12544
|
-
|
|
12545
|
-
|
|
12546
|
-
|
|
12547
|
-
|
|
12548
|
-
// locate a quote per fact when #1575 spans are absent.
|
|
12549
|
-
normalizedTurns.map((t) => t.content).join("\n\n"),
|
|
12301
|
+
sourceContext,
|
|
12302
|
+
baseNamespace,
|
|
12303
|
+
scopeProfileWritePlan,
|
|
12304
|
+
sourceText,
|
|
12305
|
+
graphCaps,
|
|
12306
|
+
lifecycleCaps,
|
|
12550
12307
|
);
|
|
12551
|
-
let postPersistMetadataFailed = false;
|
|
12552
|
-
meta ??= await storage.loadMeta();
|
|
12553
|
-
if (extractionFingerprint && shouldPersistProcessedFingerprint) {
|
|
12554
|
-
try {
|
|
12555
|
-
await this.recordProcessedExtractionFingerprint(
|
|
12556
|
-
storage,
|
|
12557
|
-
extractionFingerprint,
|
|
12558
|
-
meta,
|
|
12559
|
-
);
|
|
12560
|
-
} catch (error) {
|
|
12561
|
-
log.warn(
|
|
12562
|
-
"runExtraction: failed to persist processed extraction fingerprint; continuing with buffer clear",
|
|
12563
|
-
error,
|
|
12564
|
-
);
|
|
12565
|
-
postPersistMetadataFailed = true;
|
|
12566
|
-
}
|
|
12567
|
-
}
|
|
12568
|
-
// Persist extraction counters and processed fingerprints before running
|
|
12569
|
-
// follow-on helpers so replay dedupe survives any later non-essential
|
|
12570
|
-
// failure. If this aggregate meta write fails, still clear the buffer:
|
|
12571
|
-
// the durable memories are already written and replaying the same turns
|
|
12572
|
-
// would duplicate them.
|
|
12573
|
-
meta.extractionCount += 1;
|
|
12574
|
-
meta.lastExtractionAt = new Date().toISOString();
|
|
12575
|
-
meta.totalMemories += Array.isArray(result?.facts)
|
|
12576
|
-
? result.facts.length
|
|
12577
|
-
: 0;
|
|
12578
|
-
meta.totalEntities += Array.isArray(result?.entities)
|
|
12579
|
-
? result.entities.length
|
|
12580
|
-
: 0;
|
|
12581
|
-
try {
|
|
12582
|
-
await storage.saveMeta(meta);
|
|
12583
|
-
} catch (error) {
|
|
12584
|
-
log.warn(
|
|
12585
|
-
"runExtraction: failed to save extraction metadata after durable persistence; continuing with buffer clear",
|
|
12586
|
-
error,
|
|
12587
|
-
);
|
|
12588
|
-
postPersistMetadataFailed = true;
|
|
12589
|
-
}
|
|
12590
|
-
|
|
12591
|
-
const durableOutputCount =
|
|
12592
|
-
result.facts.length +
|
|
12593
|
-
result.entities.length +
|
|
12594
|
-
result.questions.length +
|
|
12595
|
-
result.profileUpdates.length;
|
|
12596
|
-
|
|
12597
|
-
// Buffer retention for defer verdicts (issue #562, PR 2). When the judge
|
|
12598
|
-
// deferred at least one candidate, retain the tail of the current turn
|
|
12599
|
-
// window so the next extraction pass has the surrounding context that
|
|
12600
|
-
// may disambiguate the deferred fact. Non-defer runs clear the slot.
|
|
12601
|
-
//
|
|
12602
|
-
// Gated on:
|
|
12603
|
-
// - `clearBufferAfterExtraction` — replay / bulk-import paths call
|
|
12604
|
-
// `runExtraction` with this false and do not operate on live buffer
|
|
12605
|
-
// state. Writing retention there would create synthetic buffer
|
|
12606
|
-
// entries and cross-contaminate future live extractions.
|
|
12607
|
-
// - NOT `extractionJudgeShadow` — in shadow mode the judge is only
|
|
12608
|
-
// advisory; facts are still persisted regardless of verdict, so
|
|
12609
|
-
// retaining the turn window on top of a persisted write would both
|
|
12610
|
-
// waste buffer space and cause the same facts to re-enter the
|
|
12611
|
-
// pipeline on the next pass.
|
|
12612
|
-
try {
|
|
12613
|
-
if (
|
|
12614
|
-
clearBufferAfterExtraction &&
|
|
12615
|
-
!this.config.extractionJudgeShadow
|
|
12616
|
-
) {
|
|
12617
|
-
const deferredCount = this.lastPersistExtractionDeferredCount;
|
|
12618
|
-
if (deferredCount > 0 && normalizedTurns.length > 0) {
|
|
12619
|
-
await this.buffer.retainDeferredTurns(
|
|
12620
|
-
bufferKey,
|
|
12621
|
-
normalizedTurns as BufferTurn[],
|
|
12622
|
-
10,
|
|
12623
|
-
);
|
|
12624
|
-
} else {
|
|
12625
|
-
await this.buffer.retainDeferredTurns(bufferKey, [], 0);
|
|
12626
|
-
}
|
|
12627
|
-
}
|
|
12628
|
-
} catch (err) {
|
|
12629
|
-
// Fail-open: retention is a nice-to-have. If it fails the judge will
|
|
12630
|
-
// still cap deferrals and convert to reject on the next pass.
|
|
12631
|
-
log.debug(
|
|
12632
|
-
`extraction-judge: defer retention failed (non-fatal): ${err instanceof Error ? err.message : String(err)}`,
|
|
12633
|
-
);
|
|
12634
|
-
}
|
|
12635
|
-
|
|
12636
|
-
await clearBuffer({ ignoreAbort: true });
|
|
12637
|
-
|
|
12638
|
-
// Passive correction capture (issue #1581) — detect corrections expressed
|
|
12639
|
-
// passively in the extracted turns and route to the Correction Contract.
|
|
12640
|
-
// Runs AFTER persistence + buffer clear so a capture failure never blocks
|
|
12641
|
-
// the extraction return. `clearBufferAfterExtraction` gates live-session
|
|
12642
|
-
// auto-apply (replay/import → queue-only).
|
|
12643
|
-
await this.maybeCapturePassiveCorrections(normalizedTurns as BufferTurn[], {
|
|
12644
|
-
sessionKey,
|
|
12645
|
-
principal,
|
|
12646
|
-
namespace: selfNamespace,
|
|
12647
|
-
bufferKey,
|
|
12648
|
-
isLiveSession: clearBufferAfterExtraction,
|
|
12649
|
-
});
|
|
12650
|
-
|
|
12651
|
-
// Build memory box from this extraction (v8.0 Phase 2A)
|
|
12652
|
-
// Topics are derived from the current extraction's facts and entities only —
|
|
12653
|
-
// not from readAllMemories() — so box topics accurately reflect the current
|
|
12654
|
-
// session window and the call is free of expensive full-corpus I/O.
|
|
12655
|
-
if (resolvePresentationCapabilities(this.config).memoryBoxes && persistedIds.length > 0) {
|
|
12656
|
-
const extractionTopics = deriveTopicsFromExtraction(result);
|
|
12657
|
-
// Derive episodic metadata from buffer turns (REMem-inspired)
|
|
12658
|
-
const firstUserTurn = turns.find((t) => t.role === "user");
|
|
12659
|
-
const boxGoal =
|
|
12660
|
-
firstUserTurn?.content?.slice(0, 100)?.trim() || undefined;
|
|
12661
|
-
await this.boxBuilderFor(storage)
|
|
12662
|
-
.onExtraction({
|
|
12663
|
-
topics: extractionTopics,
|
|
12664
|
-
memoryIds: persistedIds,
|
|
12665
|
-
timestamp: new Date().toISOString(),
|
|
12666
|
-
goal: boxGoal,
|
|
12667
|
-
})
|
|
12668
|
-
.catch((err) =>
|
|
12669
|
-
log.warn("[boxes] onExtraction failed (non-fatal)", err),
|
|
12670
|
-
);
|
|
12671
|
-
}
|
|
12672
|
-
|
|
12673
|
-
// Batch-append persisted IDs so non-fact memories (entities/questions) are
|
|
12674
|
-
// always attached to the thread. The helper excludes pending_review ids (#1635).
|
|
12675
|
-
if (
|
|
12676
|
-
resolvePresentationCapabilities(this.config).threading &&
|
|
12677
|
-
threadIdForExtraction &&
|
|
12678
|
-
persistedIds.length > 0
|
|
12679
|
-
) {
|
|
12680
|
-
await this.appendPersistedThreadEpisodes(
|
|
12681
|
-
threadIdForExtraction,
|
|
12682
|
-
persistedIds,
|
|
12683
|
-
);
|
|
12684
|
-
}
|
|
12685
|
-
|
|
12686
|
-
// Thread title update for the already-established thread context.
|
|
12687
|
-
if (resolvePresentationCapabilities(this.config).threading && threadIdForExtraction) {
|
|
12688
|
-
const conversationContent = turns.map((t) => t.content).join(" ");
|
|
12689
|
-
try {
|
|
12690
|
-
await this.threading.updateThreadTitle(
|
|
12691
|
-
threadIdForExtraction,
|
|
12692
|
-
conversationContent,
|
|
12693
|
-
);
|
|
12694
|
-
} catch (err) {
|
|
12695
|
-
log.warn(
|
|
12696
|
-
"[threading] updateThreadTitle failed after persistence (non-fatal)",
|
|
12697
|
-
err,
|
|
12698
|
-
);
|
|
12699
|
-
}
|
|
12700
|
-
}
|
|
12701
|
-
|
|
12702
|
-
// Check if consolidation is needed (debounced + non-zero gated).
|
|
12703
|
-
const nonZeroExtraction = durableOutputCount > 0;
|
|
12704
|
-
try {
|
|
12705
|
-
// The increment of nonZeroExtractionsSinceConsolidation moved into
|
|
12706
|
-
// the scheduler with the rest of the cadence state (issue #1526 PR1).
|
|
12707
|
-
this.maybeScheduleConsolidation(nonZeroExtraction);
|
|
12708
|
-
} catch (err) {
|
|
12709
|
-
log.warn(
|
|
12710
|
-
"runExtraction: consolidation scheduling failed after persistence (non-fatal)",
|
|
12711
|
-
err,
|
|
12712
|
-
);
|
|
12713
|
-
}
|
|
12714
|
-
|
|
12715
|
-
try {
|
|
12716
|
-
this.requestQmdMaintenance();
|
|
12717
|
-
} catch (err) {
|
|
12718
|
-
log.warn(
|
|
12719
|
-
"runExtraction: QMD maintenance scheduling failed after persistence (non-fatal)",
|
|
12720
|
-
err,
|
|
12721
|
-
);
|
|
12722
|
-
}
|
|
12723
|
-
|
|
12724
|
-
try {
|
|
12725
|
-
await this.runTierMigrationCycle(storage, "extraction");
|
|
12726
|
-
} catch (err) {
|
|
12727
|
-
log.warn(
|
|
12728
|
-
"runExtraction: tier migration failed after persistence (non-fatal)",
|
|
12729
|
-
err,
|
|
12730
|
-
);
|
|
12731
|
-
}
|
|
12732
|
-
|
|
12733
|
-
return {
|
|
12734
|
-
status: "completed",
|
|
12735
|
-
persistedCount: persistedIds.length,
|
|
12736
|
-
durableOutputCount,
|
|
12737
|
-
postPersistMetadataFailed,
|
|
12738
|
-
};
|
|
12739
|
-
}
|
|
12740
|
-
|
|
12741
|
-
private async recordProcessedExtractionFingerprint(
|
|
12742
|
-
storage: StorageManager,
|
|
12743
|
-
fingerprint: string,
|
|
12744
|
-
preloadedMeta?: Awaited<ReturnType<StorageManager["loadMeta"]>>,
|
|
12745
|
-
): Promise<void> {
|
|
12746
|
-
const meta = preloadedMeta ?? (await storage.loadMeta());
|
|
12747
|
-
const observedAt = new Date().toISOString();
|
|
12748
|
-
const seen = new Map(
|
|
12749
|
-
(meta.processedExtractionFingerprints ?? []).map((entry) => [
|
|
12750
|
-
entry.fingerprint,
|
|
12751
|
-
entry.observedAt,
|
|
12752
|
-
]),
|
|
12753
|
-
);
|
|
12754
|
-
seen.set(fingerprint, observedAt);
|
|
12755
|
-
meta.processedExtractionFingerprints = Array.from(seen.entries())
|
|
12756
|
-
.map(([value, at]) => ({ fingerprint: value, observedAt: at }))
|
|
12757
|
-
.sort((left, right) => left.observedAt.localeCompare(right.observedAt))
|
|
12758
|
-
.slice(-500);
|
|
12759
|
-
if (!preloadedMeta) {
|
|
12760
|
-
await storage.saveMeta(meta);
|
|
12761
|
-
}
|
|
12762
|
-
}
|
|
12763
|
-
|
|
12764
|
-
private async runTierMigrationCycle(
|
|
12765
|
-
storage: StorageManager,
|
|
12766
|
-
trigger: "extraction" | "maintenance" | "manual",
|
|
12767
|
-
options?: {
|
|
12768
|
-
dryRun?: boolean;
|
|
12769
|
-
limitOverride?: number;
|
|
12770
|
-
force?: boolean;
|
|
12771
|
-
},
|
|
12772
|
-
): Promise<TierMigrationCycleSummary> {
|
|
12773
|
-
return this.tierMigrationCoordinator.runCycle(storage, trigger, options);
|
|
12774
|
-
}
|
|
12775
|
-
|
|
12776
|
-
async getTierMigrationStatus(): Promise<TierMigrationStatusSnapshot> {
|
|
12777
|
-
return this.tierMigrationStatus.get();
|
|
12778
|
-
}
|
|
12779
|
-
|
|
12780
|
-
async runTierMigrationNow(options?: {
|
|
12781
|
-
dryRun?: boolean;
|
|
12782
|
-
limit?: number;
|
|
12783
|
-
}): Promise<TierMigrationCycleSummary> {
|
|
12784
|
-
return this.runTierMigrationCycle(this.storage, "manual", {
|
|
12785
|
-
dryRun: options?.dryRun === true,
|
|
12786
|
-
limitOverride: options?.limit,
|
|
12787
|
-
force: false,
|
|
12788
|
-
});
|
|
12789
|
-
}
|
|
12790
|
-
|
|
12791
|
-
private maybeScheduleConsolidation(nonZeroExtraction: boolean): void {
|
|
12792
|
-
this.maintenanceScheduler.maybeScheduleConsolidation(
|
|
12793
|
-
nonZeroExtraction,
|
|
12794
|
-
() => this.runConsolidation(),
|
|
12795
|
-
);
|
|
12796
|
-
}
|
|
12797
|
-
|
|
12798
|
-
requestQmdMaintenance(): void {
|
|
12799
|
-
this.maintenanceScheduler.requestQmdMaintenanceForTool("__internal__");
|
|
12800
|
-
}
|
|
12801
|
-
|
|
12802
|
-
/**
|
|
12803
|
-
* Public entrypoint for tool-driven QMD maintenance requests.
|
|
12804
|
-
* Delegates to the maintenance scheduler (issue #1526 PR1).
|
|
12805
|
-
*/
|
|
12806
|
-
requestQmdMaintenanceForTool(reason: string): void {
|
|
12807
|
-
this.maintenanceScheduler.requestQmdMaintenanceForTool(reason);
|
|
12808
|
-
}
|
|
12809
|
-
|
|
12810
|
-
private async persistExtraction(
|
|
12811
|
-
result: ExtractionResult,
|
|
12812
|
-
storage: StorageManager,
|
|
12813
|
-
threadIdForExtraction?: string | null,
|
|
12814
|
-
sourceContext?: { sessionKey?: string; principal?: string; validAt?: string },
|
|
12815
|
-
baseNamespace?: string,
|
|
12816
|
-
scopeProfileWritePlan?: ResolvedScopeProfilePlan | null,
|
|
12817
|
-
/** Verbatim source turn text the facts were extracted from (faithfulness gate #1576). */
|
|
12818
|
-
sourceText?: string,
|
|
12819
|
-
graphCaps: GraphConstructionCapabilitySet = resolveGraphConstructionCapabilities(this.config),
|
|
12820
|
-
lifecycleCaps: MemoryLifecycleCapabilitySet = resolveMemoryLifecycleCapabilities(this.config),
|
|
12821
|
-
): Promise<string[]> {
|
|
12822
|
-
// Inline source attribution (issue #369). When enabled, every extracted
|
|
12823
|
-
// fact is rewritten to carry a compact provenance tag inside its body so
|
|
12824
|
-
// the citation survives hostile memory text, copy/paste, and LLM quoting.
|
|
12825
|
-
// The helper is a no-op when the feature flag is off, so legacy pipelines
|
|
12826
|
-
// see zero behavioral change.
|
|
12827
|
-
const citationEnabled = resolvePipelineProcessingCapabilities(this.config).inlineSourceAttribution === true;
|
|
12828
|
-
const citationTemplate = this.config.inlineSourceAttributionFormat;
|
|
12829
|
-
// The stable fields (agent, session) are computed once; `ts` is intentionally
|
|
12830
|
-
// omitted here and added fresh per invocation so each fact in a large batch
|
|
12831
|
-
// gets its own insertion timestamp rather than sharing a single batch-start time.
|
|
12832
|
-
const citationContextBase: Omit<CitationContext, "ts"> = citationEnabled
|
|
12833
|
-
? {
|
|
12834
|
-
agent: sourceContext?.principal,
|
|
12835
|
-
session: sourceContext?.sessionKey,
|
|
12836
|
-
}
|
|
12837
|
-
: {};
|
|
12838
|
-
const applyInlineCitation = (content: string): string => {
|
|
12839
|
-
if (!citationEnabled) return content;
|
|
12840
|
-
if (typeof content !== "string" || content.length === 0) return content;
|
|
12841
|
-
// Build a fresh CitationContext per call so `ts` reflects the actual
|
|
12842
|
-
// insertion time of each individual fact rather than the batch-start time.
|
|
12843
|
-
const citationContext: CitationContext = {
|
|
12844
|
-
...citationContextBase,
|
|
12845
|
-
ts: new Date().toISOString(),
|
|
12846
|
-
};
|
|
12847
|
-
// `attachCitation` already calls `hasCitationForTemplate` internally and
|
|
12848
|
-
// is a no-op when the content already carries a citation (default or
|
|
12849
|
-
// custom template). The outer check was redundant and has been removed
|
|
12850
|
-
// to avoid a maintenance hazard where the two guard paths could diverge.
|
|
12851
|
-
return attachCitation(content, citationContext, citationTemplate);
|
|
12852
|
-
};
|
|
12853
|
-
const persistedIds: string[] = [];
|
|
12854
|
-
const supersessionOrderingAt = (validAt?: string): string =>
|
|
12855
|
-
validAt && validAt.length > 0 ? validAt : new Date().toISOString();
|
|
12856
|
-
// #1635: pending_review persisted ids, excluded from the thread episode set below.
|
|
12857
|
-
const pendingReviewPersistedIds: string[] = [];
|
|
12858
|
-
const persistedIdsByStorage = new Map<
|
|
12859
|
-
string,
|
|
12860
|
-
{ storage: StorageManager; ids: string[] }
|
|
12861
|
-
>();
|
|
12862
|
-
const trackPersistedId = (
|
|
12863
|
-
targetStorage: StorageManager,
|
|
12864
|
-
id: string,
|
|
12865
|
-
options: {
|
|
12866
|
-
includeReturnedIds?: boolean;
|
|
12867
|
-
/** #1635: keep this id out of the persisted thread episode set. */
|
|
12868
|
-
pendingReview?: boolean;
|
|
12869
|
-
} = {},
|
|
12870
|
-
): void => {
|
|
12871
|
-
if (options.includeReturnedIds !== false) {
|
|
12872
|
-
persistedIds.push(id);
|
|
12873
|
-
}
|
|
12874
|
-
if (options.pendingReview) {
|
|
12875
|
-
pendingReviewPersistedIds.push(id);
|
|
12876
|
-
}
|
|
12877
|
-
const key = targetStorage.dir;
|
|
12878
|
-
const existing = persistedIdsByStorage.get(key);
|
|
12879
|
-
if (existing) {
|
|
12880
|
-
existing.ids.push(id);
|
|
12881
|
-
return;
|
|
12882
|
-
}
|
|
12883
|
-
persistedIdsByStorage.set(key, { storage: targetStorage, ids: [id] });
|
|
12884
|
-
};
|
|
12885
|
-
let dedupedCount = 0;
|
|
12886
|
-
// Counter for facts skipped by the importance write-gate (issue #372).
|
|
12887
|
-
// Emitted via the `importance_gated` metric below and rolled into the
|
|
12888
|
-
// final `persisted:` log line so operators can tune the threshold.
|
|
12889
|
-
let importanceGatedCount = 0;
|
|
12890
|
-
// UUI2: short-circuit semantic dedup after first backend-unavailable signal
|
|
12891
|
-
// within this batch. Once any fact in the batch gets reason="backend_unavailable"
|
|
12892
|
-
// (meaning the embedding backend is degraded), subsequent facts skip the
|
|
12893
|
-
// lookup entirely and proceed directly to write. This prevents N-fact batches
|
|
12894
|
-
// from paying N × timeout when the backend is down. The flag resets per-batch
|
|
12895
|
-
// (declared here, inside persistExtraction) so a transient hiccup in one
|
|
12896
|
-
// batch does not permanently disable dedup in future batches.
|
|
12897
|
-
let batchBackendUnavailable = false;
|
|
12898
|
-
// #1669: per-namespace redaction-rule cache for this persist pass. A
|
|
12899
|
-
// `never_store` / redaction_rule correction persists patterns under each
|
|
12900
|
-
// namespace's state/corrections/redaction-rules/ dir; we consult them
|
|
12901
|
-
// before a fact reaches the storage write chokepoint so matching content
|
|
12902
|
-
// is withheld entirely rather than landing as pending_review. Cached per
|
|
12903
|
-
// dir so a multi-fact batch over one namespace reads the dir once.
|
|
12904
|
-
let redactionGatedCount = 0;
|
|
12905
|
-
const redactionRulesByDir = new Map<string, CompiledRedactionRule[]>();
|
|
12906
|
-
const redactionRulesFor = async (...dirs: string[]): Promise<CompiledRedactionRule[]> => {
|
|
12907
|
-
const out: CompiledRedactionRule[] = [];
|
|
12908
|
-
for (const d of dirs) {
|
|
12909
|
-
let r = redactionRulesByDir.get(d);
|
|
12910
|
-
if (!r) { r = await loadRedactionRules(d); redactionRulesByDir.set(d, r); }
|
|
12911
|
-
out.push(...r);
|
|
12912
|
-
}
|
|
12913
|
-
return out;
|
|
12914
|
-
};
|
|
12915
|
-
const behaviorSignalsByStorage = new Map<
|
|
12916
|
-
string,
|
|
12917
|
-
{ storage: StorageManager; events: BehaviorSignalEvent[] }
|
|
12918
|
-
>();
|
|
12919
|
-
const trackBehaviorSignals = (
|
|
12920
|
-
targetStorage: StorageManager,
|
|
12921
|
-
events: BehaviorSignalEvent[],
|
|
12922
|
-
): void => {
|
|
12923
|
-
if (events.length === 0) return;
|
|
12924
|
-
const key = targetStorage.dir;
|
|
12925
|
-
const existing = behaviorSignalsByStorage.get(key);
|
|
12926
|
-
if (existing) {
|
|
12927
|
-
existing.events.push(...events);
|
|
12928
|
-
return;
|
|
12929
|
-
}
|
|
12930
|
-
behaviorSignalsByStorage.set(key, {
|
|
12931
|
-
storage: targetStorage,
|
|
12932
|
-
events: [...events],
|
|
12933
|
-
});
|
|
12934
|
-
};
|
|
12935
|
-
const confidenceTierOrder = [
|
|
12936
|
-
"explicit",
|
|
12937
|
-
"implied",
|
|
12938
|
-
"inferred",
|
|
12939
|
-
"speculative",
|
|
12940
|
-
] as const;
|
|
12941
|
-
const sharedProfileLayer = scopeProfileWritePlan?.layers.find(
|
|
12942
|
-
(layer) =>
|
|
12943
|
-
layer.id === "serverShared" &&
|
|
12944
|
-
layer.namespace === this.config.sharedNamespace,
|
|
12945
|
-
);
|
|
12946
|
-
const sharedPromotionTarget = scopeProfileWritePlan?.promotionTargets.find(
|
|
12947
|
-
(target) =>
|
|
12948
|
-
target.target === "serverShared" &&
|
|
12949
|
-
target.namespace === this.config.sharedNamespace,
|
|
12950
|
-
);
|
|
12951
|
-
const profileAllowsSharedWrites =
|
|
12952
|
-
!scopeProfileWritePlan ||
|
|
12953
|
-
Boolean(
|
|
12954
|
-
scopeProfileWritePlan.profile.readOrder.includes("serverShared") &&
|
|
12955
|
-
scopeProfileWritePlan.readNamespaces.includes(this.config.sharedNamespace) &&
|
|
12956
|
-
sharedProfileLayer?.readable &&
|
|
12957
|
-
sharedProfileLayer.writable &&
|
|
12958
|
-
sharedPromotionTarget?.authorized,
|
|
12959
|
-
);
|
|
12960
|
-
// #1713: hoist namespaces-enabled once for the scope-routing mirrors
|
|
12961
|
-
// (pre-judge + write-loop) so no new scattered config.*Enabled read is
|
|
12962
|
-
// introduced (ratchet scatteredConfigFlagReads; see #1523).
|
|
12963
|
-
const namespacesEnabled = resolveNamespaceCapabilities(this.config).namespaces;
|
|
12964
|
-
const profileAutoPromotionAllows = (
|
|
12965
|
-
category: string,
|
|
12966
|
-
confidence: number,
|
|
12967
|
-
): boolean => {
|
|
12968
|
-
if (!scopeProfileWritePlan) return false;
|
|
12969
|
-
const actualTier = confidenceTier(confidence);
|
|
12970
|
-
const actualRank = confidenceTierOrder.indexOf(actualTier);
|
|
12971
|
-
if (actualRank === -1) return false;
|
|
12972
|
-
const autoPromote = scopeProfileWritePlan.profile.autoPromote;
|
|
12973
|
-
if (!autoPromote.enabled) return false;
|
|
12974
|
-
if (!autoPromote.categories.includes(category as any)) return false;
|
|
12975
|
-
const minimumRank = confidenceTierOrder.indexOf(autoPromote.minConfidenceTier);
|
|
12976
|
-
return minimumRank !== -1 && actualRank <= minimumRank;
|
|
12977
|
-
};
|
|
12978
|
-
const sharedAutoPromotionAllows = (
|
|
12979
|
-
category: string,
|
|
12980
|
-
confidence: number,
|
|
12981
|
-
): boolean => {
|
|
12982
|
-
if (!scopeProfileWritePlan) {
|
|
12983
|
-
const actualTier = confidenceTier(confidence);
|
|
12984
|
-
const actualRank = confidenceTierOrder.indexOf(actualTier);
|
|
12985
|
-
if (actualRank === -1) return false;
|
|
12986
|
-
if (!resolveRecallEnhancementCapabilities(this.config).autoPromoteToShared) return false;
|
|
12987
|
-
if (!this.config.autoPromoteToSharedCategories.includes(category as any))
|
|
12988
|
-
return false;
|
|
12989
|
-
const minimumRank = confidenceTierOrder.indexOf(
|
|
12990
|
-
this.config.autoPromoteMinConfidenceTier,
|
|
12991
|
-
);
|
|
12992
|
-
return minimumRank !== -1 && actualRank <= minimumRank;
|
|
12993
|
-
}
|
|
12994
|
-
return (
|
|
12995
|
-
scopeProfileWritePlan.profile.autoPromote.targets.includes("serverShared") &&
|
|
12996
|
-
profileAutoPromotionAllows(category, confidence)
|
|
12997
|
-
);
|
|
12998
|
-
};
|
|
12999
|
-
const shouldPromoteToShared = (
|
|
13000
|
-
targetStorage: StorageManager,
|
|
13001
|
-
category: string,
|
|
13002
|
-
confidence: number,
|
|
13003
|
-
): boolean => {
|
|
13004
|
-
if (
|
|
13005
|
-
!resolveNamespaceCapabilities(this.config).namespaces ||
|
|
13006
|
-
!profileAllowsSharedWrites ||
|
|
13007
|
-
!sharedAutoPromotionAllows(category, confidence)
|
|
13008
|
-
)
|
|
13009
|
-
return false;
|
|
13010
|
-
if (
|
|
13011
|
-
this.storageDirNamespace(targetStorage.dir) ===
|
|
13012
|
-
this.config.sharedNamespace
|
|
13013
|
-
)
|
|
13014
|
-
return false;
|
|
13015
|
-
return true;
|
|
13016
|
-
};
|
|
13017
|
-
const promoteMemoryToProfileTargets = async (options: {
|
|
13018
|
-
sourceStorage: StorageManager;
|
|
13019
|
-
category: string;
|
|
13020
|
-
content: string;
|
|
13021
|
-
confidence: number;
|
|
13022
|
-
tags: string[];
|
|
13023
|
-
entityRef?: string;
|
|
13024
|
-
structuredAttributes?: Record<string, string>;
|
|
13025
|
-
sourceMemoryId: string;
|
|
13026
|
-
importance?: ReturnType<typeof scoreImportance>;
|
|
13027
|
-
intentGoal?: string;
|
|
13028
|
-
intentActionType?: string;
|
|
13029
|
-
intentEntityTypes?: string[];
|
|
13030
|
-
memoryKind?: MemoryFrontmatter["memoryKind"];
|
|
13031
|
-
validAt?: string;
|
|
13032
|
-
// #1578 — bi-temporal bounds + ingestion provenance forwarded to profile-
|
|
13033
|
-
// target copies (same defect class as shared promotion; cursor bugbot).
|
|
13034
|
-
invalidAt?: string;
|
|
13035
|
-
observedAt?: string;
|
|
13036
|
-
eventTimeSource?: "extracted" | "assumed";
|
|
13037
|
-
source: string;
|
|
13038
|
-
sources?: ProvenanceSource[];
|
|
13039
|
-
provenance?: "verified" | "unverified" | "none";
|
|
13040
|
-
}): Promise<void> => {
|
|
13041
|
-
if (
|
|
13042
|
-
!scopeProfileWritePlan ||
|
|
13043
|
-
!profileAutoPromotionAllows(options.category, options.confidence)
|
|
13044
|
-
)
|
|
13045
|
-
return;
|
|
13046
|
-
const autoTargets = new Set(scopeProfileWritePlan.profile.autoPromote.targets);
|
|
13047
|
-
const targets = scopeProfileWritePlan.promotionTargets.filter(
|
|
13048
|
-
(target) =>
|
|
13049
|
-
target.target !== "serverShared" &&
|
|
13050
|
-
autoTargets.has(target.target) &&
|
|
13051
|
-
target.authorized &&
|
|
13052
|
-
target.namespace,
|
|
13053
|
-
);
|
|
13054
|
-
if (targets.length === 0) return;
|
|
13055
|
-
const rawContent =
|
|
13056
|
-
citationEnabled && hasCitationForTemplate(options.content, citationTemplate)
|
|
13057
|
-
? stripCitationForTemplate(options.content, citationTemplate)
|
|
13058
|
-
: options.content;
|
|
13059
|
-
const citedContent = applyInlineCitation(rawContent);
|
|
13060
|
-
const sanitizedBase = sanitizeMemoryContent(rawContent);
|
|
13061
|
-
const dedupContent =
|
|
13062
|
-
options.category === "fact" &&
|
|
13063
|
-
options.structuredAttributes &&
|
|
13064
|
-
Object.keys(options.structuredAttributes).length > 0
|
|
13065
|
-
? `${sanitizedBase.text}\n[Attributes: ${normalizeAttributePairs(options.structuredAttributes)}]`
|
|
13066
|
-
: sanitizedBase.text;
|
|
13067
|
-
for (const target of targets) {
|
|
13068
|
-
if (!target.namespace) continue;
|
|
13069
|
-
try {
|
|
13070
|
-
const targetStorage = await this.storageRouter.storageFor(target.namespace);
|
|
13071
|
-
if (targetStorage.dir === options.sourceStorage.dir) continue;
|
|
13072
|
-
if (
|
|
13073
|
-
options.category === "fact" &&
|
|
13074
|
-
(await targetStorage.hasFactContentHash(dedupContent))
|
|
13075
|
-
) {
|
|
13076
|
-
// #1671 — backfill bi-temporal bounds the existing promoted copy
|
|
13077
|
-
// lacks (re-extraction with a now-resolved invalidAt). Best-effort,
|
|
13078
|
-
// fail-open; the helper gates on invalidAt to avoid I/O when no
|
|
13079
|
-
// end bound is present.
|
|
13080
|
-
if (options.invalidAt || options.validAt) {
|
|
13081
|
-
await this.backfillTemporalBoundsOnDedupHit(
|
|
13082
|
-
targetStorage,
|
|
13083
|
-
dedupContent,
|
|
13084
|
-
{
|
|
13085
|
-
invalidAt: options.invalidAt,
|
|
13086
|
-
// #1707 thread 2 — carry the corrected start bound.
|
|
13087
|
-
validFrom: options.validAt,
|
|
13088
|
-
...(options.observedAt ? { observedAt: options.observedAt } : {}),
|
|
13089
|
-
...(options.eventTimeSource ? { eventTimeSource: options.eventTimeSource } : {}),
|
|
13090
|
-
},
|
|
13091
|
-
options.entityRef,
|
|
13092
|
-
);
|
|
13093
|
-
}
|
|
13094
|
-
continue;
|
|
13095
|
-
}
|
|
13096
|
-
const targetPromotion = await targetStorage.writeMemory(
|
|
13097
|
-
options.category as any,
|
|
13098
|
-
citedContent,
|
|
13099
|
-
{
|
|
13100
|
-
confidence: options.confidence,
|
|
13101
|
-
tags: [...options.tags, `${target.target}-promotion`],
|
|
13102
|
-
entityRef: options.entityRef,
|
|
13103
|
-
structuredAttributes: options.structuredAttributes,
|
|
13104
|
-
source: `${options.source}-${target.target}-promotion`,
|
|
13105
|
-
importance: options.importance,
|
|
13106
|
-
lineage: [options.sourceMemoryId],
|
|
13107
|
-
sourceMemoryId: options.sourceMemoryId,
|
|
13108
|
-
intentGoal: options.intentGoal,
|
|
13109
|
-
intentActionType: options.intentActionType,
|
|
13110
|
-
intentEntityTypes: options.intentEntityTypes,
|
|
13111
|
-
memoryKind: options.memoryKind,
|
|
13112
|
-
validAt: options.validAt,
|
|
13113
|
-
// #1578 — forward bi-temporal bounds + ingestion provenance.
|
|
13114
|
-
...(options.invalidAt ? { invalidAt: options.invalidAt } : {}),
|
|
13115
|
-
...(options.observedAt ? { observedAt: options.observedAt } : {}),
|
|
13116
|
-
...(options.eventTimeSource ? { eventTimeSource: options.eventTimeSource } : {}),
|
|
13117
|
-
contentHashSource: options.category === "fact" ? dedupContent : rawContent,
|
|
13118
|
-
...(options.sources && options.sources.length > 0 ? { sources: options.sources } : {}),
|
|
13119
|
-
...(options.provenance ? { provenance: options.provenance } : {}),
|
|
13120
|
-
},
|
|
13121
|
-
);
|
|
13122
|
-
const promotedId = targetPromotion.id;
|
|
13123
|
-
// #1645: if the TARGET namespace's own tombstone blocked this promotion,
|
|
13124
|
-
// the row lands pending_review — do NOT supersede active target memories.
|
|
13125
|
-
if (
|
|
13126
|
-
!targetPromotion.tombstoneBlocked &&
|
|
13127
|
-
lifecycleCaps.temporalSupersession &&
|
|
13128
|
-
options.category === "fact" &&
|
|
13129
|
-
options.entityRef &&
|
|
13130
|
-
options.structuredAttributes &&
|
|
13131
|
-
Object.keys(options.structuredAttributes).length > 0
|
|
13132
|
-
) {
|
|
13133
|
-
try {
|
|
13134
|
-
await applyTemporalSupersession({
|
|
13135
|
-
storage: targetStorage,
|
|
13136
|
-
newMemoryId: promotedId,
|
|
13137
|
-
entityRef: options.entityRef,
|
|
13138
|
-
structuredAttributes: options.structuredAttributes,
|
|
13139
|
-
createdAt: supersessionOrderingAt(options.validAt),
|
|
13140
|
-
enabled: !(options.eventTimeSource === "extracted" && !options.validAt),
|
|
13141
|
-
});
|
|
13142
|
-
} catch (profileSupersessionErr) {
|
|
13143
|
-
log.warn(
|
|
13144
|
-
`persistExtraction: ${target.target} promotion temporal supersession failed open for promoted ${promotedId}: ${profileSupersessionErr}`,
|
|
13145
|
-
);
|
|
13146
|
-
}
|
|
13147
|
-
}
|
|
13148
|
-
// #1645 TV6: a tombstone-blocked promotion is pending_review (no
|
|
13149
|
-
// active copy) — skip catalog/index/behavior like postWriteGuard.
|
|
13150
|
-
if (!targetPromotion.tombstoneBlocked) {
|
|
13151
|
-
trackPersistedId(targetStorage, promotedId, { includeReturnedIds: false });
|
|
13152
|
-
await this.indexPersistedMemory(targetStorage, promotedId);
|
|
13153
|
-
trackBehaviorSignals(
|
|
13154
|
-
targetStorage,
|
|
13155
|
-
buildBehaviorSignalsForMemory({
|
|
13156
|
-
memoryId: promotedId,
|
|
13157
|
-
category: options.category as any,
|
|
13158
|
-
content: options.content,
|
|
13159
|
-
namespace: target.namespace,
|
|
13160
|
-
confidence: options.confidence,
|
|
13161
|
-
source: "extraction",
|
|
13162
|
-
}),
|
|
13163
|
-
);
|
|
13164
|
-
}
|
|
13165
|
-
} catch (err) {
|
|
13166
|
-
log.warn(
|
|
13167
|
-
`persistExtraction: ${target.target} promotion failed open for ${options.sourceMemoryId}: ${err}`,
|
|
13168
|
-
);
|
|
13169
|
-
}
|
|
13170
|
-
}
|
|
13171
|
-
};
|
|
13172
|
-
const promoteMemoryToShared = async (options: {
|
|
13173
|
-
sourceStorage: StorageManager;
|
|
13174
|
-
category: string;
|
|
13175
|
-
content: string;
|
|
13176
|
-
confidence: number;
|
|
13177
|
-
tags: string[];
|
|
13178
|
-
entityRef?: string;
|
|
13179
|
-
structuredAttributes?: Record<string, string>;
|
|
13180
|
-
sourceMemoryId: string;
|
|
13181
|
-
importance?: ReturnType<typeof scoreImportance>;
|
|
13182
|
-
intentGoal?: string;
|
|
13183
|
-
intentActionType?: string;
|
|
13184
|
-
intentEntityTypes?: string[];
|
|
13185
|
-
memoryKind?: MemoryFrontmatter["memoryKind"];
|
|
13186
|
-
validAt?: string;
|
|
13187
|
-
// #1578 — bi-temporal bounds + ingestion provenance forwarded to the
|
|
13188
|
-
// shared-namespace copy so shared recall honours the same invalid_at
|
|
13189
|
-
// window as the source fact (cursor bugbot).
|
|
13190
|
-
invalidAt?: string;
|
|
13191
|
-
observedAt?: string;
|
|
13192
|
-
eventTimeSource?: "extracted" | "assumed";
|
|
13193
|
-
source: string;
|
|
13194
|
-
/** Claim-level provenance spans (issue #1575 PR 2). */
|
|
13195
|
-
sources?: ProvenanceSource[];
|
|
13196
|
-
provenance?: "verified" | "unverified" | "none";
|
|
13197
|
-
}): Promise<void> => {
|
|
13198
|
-
await promoteMemoryToProfileTargets(options);
|
|
13199
|
-
if (
|
|
13200
|
-
!shouldPromoteToShared(
|
|
13201
|
-
options.sourceStorage,
|
|
13202
|
-
options.category,
|
|
13203
|
-
options.confidence,
|
|
13204
|
-
)
|
|
13205
|
-
)
|
|
13206
|
-
return;
|
|
13207
|
-
try {
|
|
13208
|
-
const sharedStorage = await this.storageRouter.storageFor(
|
|
13209
|
-
this.config.sharedNamespace,
|
|
13210
|
-
);
|
|
13211
|
-
// Dedup gate: canonicalize content before hashing.
|
|
13212
|
-
//
|
|
13213
|
-
// Issue #369 (PR #401): When inline attribution is enabled,
|
|
13214
|
-
// `applyInlineCitation` appends a timestamp-bearing marker (e.g.
|
|
13215
|
-
// `[Source: ..., ts=2026-04-11T...]`). Because the timestamp changes
|
|
13216
|
-
// on every call, hashing cited content produces a unique hash each
|
|
13217
|
-
// time — defeating dedup entirely and allowing the same logical fact to
|
|
13218
|
-
// be promoted repeatedly. Also, both promotion call sites pass
|
|
13219
|
-
// `fact.content`, which can already carry an inline citation (e.g. a
|
|
13220
|
-
// relayed or reprocessed fact). Strip any pre-existing citation so the
|
|
13221
|
-
// dedup key matches the hash stored from the original un-cited write.
|
|
13222
|
-
//
|
|
13223
|
-
// PR #402 round-6 (Fix #2 / chatgpt-codex P1 PRRT_kwDORJXyws56U74n):
|
|
13224
|
-
// Compute the enriched content before the hash-dedup check so the
|
|
13225
|
-
// lookup uses the same content that writeMemory will actually store.
|
|
13226
|
-
// When structuredAttributes are present, writeMemory appends an
|
|
13227
|
-
// "[Attributes: ...]" suffix before hashing; hasFactContentHash must
|
|
13228
|
-
// receive the same enriched body or the check is against a different
|
|
13229
|
-
// hash and dedup fails to fire (letting duplicates through) or fires
|
|
13230
|
-
// when it shouldn't (collapsing memories with different enrichments).
|
|
13231
|
-
// Fix #1 (P2 PRRT_kwDORJXyws56VHZc): use normalizeAttributePairs so
|
|
13232
|
-
// key order and casing are canonical — identical to the enrichment
|
|
13233
|
-
// applied by storage.writeMemory — preventing spurious hash misses
|
|
13234
|
-
// when attribute maps arrive with different insertion orders or casing.
|
|
13235
|
-
//
|
|
13236
|
-
// Fix #4 (Low PRRT_kwDORJXyws56VHth): sanitize the base content before
|
|
13237
|
-
// building dedupContent. writeMemory runs sanitizeMemoryContent on the
|
|
13238
|
-
// enriched body before hashing; if sanitization redacts the content to
|
|
13239
|
-
// REDACTED_PLACEHOLDER the stored hash is for the redacted form, not
|
|
13240
|
-
// the raw form. Computing dedupContent from sanitized.text here ensures
|
|
13241
|
-
// the hash lookup and the normalizedIncoming comparison both use the
|
|
13242
|
-
// same content that writeMemory will actually store.
|
|
13243
|
-
//
|
|
13244
|
-
// Combined fix: strip any pre-existing citation FIRST to obtain
|
|
13245
|
-
// rawContent (the canonical body), then sanitize rawContent (not
|
|
13246
|
-
// options.content) when building dedupContent, so that citation
|
|
13247
|
-
// stripping and sanitization are applied in a consistent order.
|
|
13248
|
-
const rawContent =
|
|
13249
|
-
citationEnabled &&
|
|
13250
|
-
hasCitationForTemplate(options.content, citationTemplate)
|
|
13251
|
-
? stripCitationForTemplate(options.content, citationTemplate)
|
|
13252
|
-
: options.content;
|
|
13253
|
-
const citedContent = applyInlineCitation(rawContent);
|
|
13254
|
-
const sanitizedBase = sanitizeMemoryContent(rawContent);
|
|
13255
|
-
const dedupContent =
|
|
13256
|
-
options.category === "fact" &&
|
|
13257
|
-
options.structuredAttributes &&
|
|
13258
|
-
Object.keys(options.structuredAttributes).length > 0
|
|
13259
|
-
? `${sanitizedBase.text}\n[Attributes: ${normalizeAttributePairs(options.structuredAttributes)}]`
|
|
13260
|
-
: sanitizedBase.text;
|
|
13261
|
-
if (
|
|
13262
|
-
options.category === "fact" &&
|
|
13263
|
-
(await sharedStorage.hasFactContentHash(dedupContent))
|
|
13264
|
-
) {
|
|
13265
|
-
// #1671 — backfill bi-temporal bounds onto the existing shared copy
|
|
13266
|
-
// before the supersession short-circuit. Covers all return paths below
|
|
13267
|
-
// (supersession-hit, catch-skip, and the no-supersession short-circuit)
|
|
13268
|
-
// in one shot. Best-effort / fail-open; the helper gates on invalidAt.
|
|
13269
|
-
if (options.invalidAt || options.validAt) {
|
|
13270
|
-
await this.backfillTemporalBoundsOnDedupHit(
|
|
13271
|
-
sharedStorage,
|
|
13272
|
-
dedupContent,
|
|
13273
|
-
{
|
|
13274
|
-
invalidAt: options.invalidAt,
|
|
13275
|
-
// #1707 thread 2 — carry the corrected start bound.
|
|
13276
|
-
validFrom: options.validAt,
|
|
13277
|
-
...(options.observedAt ? { observedAt: options.observedAt } : {}),
|
|
13278
|
-
...(options.eventTimeSource ? { eventTimeSource: options.eventTimeSource } : {}),
|
|
13279
|
-
},
|
|
13280
|
-
options.entityRef,
|
|
13281
|
-
);
|
|
13282
|
-
}
|
|
13283
|
-
// Uj6H fix: shared-namespace temporal supersession must also run when
|
|
13284
|
-
// the hash-dedup short-circuit fires. Without this, an existing shared
|
|
13285
|
-
// fact whose structuredAttributes are stale (or an older conflicting
|
|
13286
|
-
// shared fact that is still active) never gets retired — supersession
|
|
13287
|
-
// only ran in the post-writeMemory block which is unreachable here.
|
|
13288
|
-
//
|
|
13289
|
-
// Strategy: scan the shared namespace for the existing fact whose
|
|
13290
|
-
// normalized content matches the incoming content, then run
|
|
13291
|
-
// applyTemporalSupersession against it using the same logic that
|
|
13292
|
-
// would have run post-writeMemory. This is a best-effort / fail-open
|
|
13293
|
-
// step — if the lookup fails we skip silently (same as the normal path).
|
|
13294
|
-
if (
|
|
13295
|
-
lifecycleCaps.temporalSupersession &&
|
|
13296
|
-
options.entityRef &&
|
|
13297
|
-
options.structuredAttributes &&
|
|
13298
|
-
Object.keys(options.structuredAttributes).length > 0
|
|
13299
|
-
) {
|
|
13300
|
-
// PR #402 round-7 (Fix #2 / Codex P1 PRRT_kwDORJXyws56VALC):
|
|
13301
|
-
// Track whether matchingFact lookup completed before the try block
|
|
13302
|
-
// so the catch block can distinguish an early-lookup failure (where
|
|
13303
|
-
// we don't know if a duplicate exists) from a post-lookup supersession
|
|
13304
|
-
// failure (where we confirmed a duplicate and must skip the write).
|
|
13305
|
-
let hashDedupMatchingFact: MemoryFile | undefined;
|
|
13306
|
-
let hashDedupLookupComplete = false;
|
|
13307
|
-
try {
|
|
13308
|
-
// Fix #2 (P2 PRRT_kwDORJXyws56VHZf): dedupContent is now built
|
|
13309
|
-
// from sanitizedBase.text (see fix #4 above), so normalizedIncoming
|
|
13310
|
-
// uses the same sanitized+normalized content that writeMemory hashes
|
|
13311
|
-
// and that hasFactContentHash just matched. Previously this used the
|
|
13312
|
-
// raw options.content, which diverged from the stored hash when
|
|
13313
|
-
// sanitization redacted the content, causing the candidate lookup to
|
|
13314
|
-
// return undefined and leaving stale facts active.
|
|
13315
|
-
const normalizedIncoming = ContentHashIndex.normalizeContent(dedupContent);
|
|
13316
|
-
const allShared = await sharedStorage.readAllMemories();
|
|
13317
|
-
// PR #402 round-12 (Finding Uybg): restrict hash-dedup matching to
|
|
13318
|
-
// the SAME entity. Content-hash equality alone can collide across
|
|
13319
|
-
// entities when two entities share identical fact text. Using an
|
|
13320
|
-
// unrelated entity's existing fact as `newMemoryId` would anchor
|
|
13321
|
-
// supersession to that entity's record and corrupt its
|
|
13322
|
-
// `supersededBy` links. Only consider facts whose normalized
|
|
13323
|
-
// `entityRef` matches the incoming entity.
|
|
13324
|
-
const incomingEntityNorm = normalizeSupersessionKey(options.entityRef);
|
|
13325
|
-
hashDedupMatchingFact = allShared.find((m) => {
|
|
13326
|
-
if (m.frontmatter.category !== "fact") return false;
|
|
13327
|
-
if ((m.frontmatter.status ?? "active") !== "active") return false;
|
|
13328
|
-
// Same-entity guard: skip if entity doesn't match.
|
|
13329
|
-
if (!m.frontmatter.entityRef) return false;
|
|
13330
|
-
if (normalizeSupersessionKey(m.frontmatter.entityRef) !== incomingEntityNorm) {
|
|
13331
|
-
log.debug(
|
|
13332
|
-
`persistExtraction: hash-dedup skipping cross-entity match (incoming="${incomingEntityNorm}" candidate="${normalizeSupersessionKey(m.frontmatter.entityRef)}")`,
|
|
13333
|
-
);
|
|
13334
|
-
return false;
|
|
13335
|
-
}
|
|
13336
|
-
// PR #402 round-7 (Fix #2): compare stored fact's full body
|
|
13337
|
-
// (including any appended "[Attributes: ...]" suffix) against the
|
|
13338
|
-
// enriched normalizedIncoming so the candidate selected is the one
|
|
13339
|
-
// whose hash actually matched in hasFactContentHash.
|
|
13340
|
-
return ContentHashIndex.normalizeContent(m.content ?? "") === normalizedIncoming;
|
|
13341
|
-
});
|
|
13342
|
-
hashDedupLookupComplete = true;
|
|
13343
|
-
if (hashDedupMatchingFact) {
|
|
13344
|
-
// Finding UvU1 (PR #402 round-11): anchor supersession to the
|
|
13345
|
-
// incoming event's time, not the existing fact's persisted
|
|
13346
|
-
// `created`. For source-dated replay/import, this is the
|
|
13347
|
-
// source valid_at; otherwise it is the current wall-clock. The
|
|
13348
|
-
// matching fact may be an old shared copy whose `created`
|
|
13349
|
-
// predates the incoming promotion event — using it as
|
|
13350
|
-
// `createdAt` would make the new memory appear older than the
|
|
13351
|
-
// existing one, preventing supersession from firing.
|
|
13352
|
-
// PR #402 round-12 (Finding Uyui): the matching fact is an
|
|
13353
|
-
// existing OLD memory — its persisted `frontmatter.created` is
|
|
13354
|
-
// stale relative to the incoming promotion event. Pass
|
|
13355
|
-
// `useCallerTimestamp: true` so the function uses
|
|
13356
|
-
// `createdAt` as the ordering anchor instead of the old fact's
|
|
13357
|
-
// timestamp, ensuring supersession fires correctly even when
|
|
13358
|
-
// the matching fact predates conflicting candidates.
|
|
13359
|
-
const hashDedupSupersession = await applyTemporalSupersession({
|
|
13360
|
-
storage: sharedStorage,
|
|
13361
|
-
newMemoryId: hashDedupMatchingFact.frontmatter.id,
|
|
13362
|
-
entityRef: options.entityRef,
|
|
13363
|
-
structuredAttributes: options.structuredAttributes,
|
|
13364
|
-
createdAt: supersessionOrderingAt(options.validAt),
|
|
13365
|
-
enabled: !(options.eventTimeSource === "extracted" && !options.validAt),
|
|
13366
|
-
useCallerTimestamp: true,
|
|
13367
|
-
});
|
|
13368
|
-
// Catalog touch (issue #1499 — codex P2 NElSf): this dedup branch
|
|
13369
|
-
// returns WITHOUT the post-write catalog touch (now at the storage chokepoint #1522),
|
|
13370
|
-
// but `applyTemporalSupersession` mutated the shared namespace
|
|
13371
|
-
// (it rewrote frontmatter to retire stale shared facts). When any
|
|
13372
|
-
// ids were actually superseded, the shared namespace changed, so we
|
|
13373
|
-
// must record the write — otherwise the shared record's
|
|
13374
|
-
// `lastWriteAt` stays stale and `writtenSince` maintenance / QMD
|
|
13375
|
-
// fanout skips the namespace after a supersession-only update.
|
|
13376
|
-
// Best-effort and failure-tolerant (the storage chokepoint swallows
|
|
13377
|
-
// errors); only touch when work happened to avoid spurious writes.
|
|
13378
|
-
if (hashDedupSupersession.supersededIds.length > 0) {
|
|
13379
|
-
}
|
|
13380
|
-
// Active matching fact exists — normal short-circuit is safe.
|
|
13381
|
-
return;
|
|
13382
|
-
}
|
|
13383
|
-
// No active same-entity shared fact found with this content hash.
|
|
13384
|
-
// This can happen when the previously-written shared fact has since
|
|
13385
|
-
// been superseded (e.g. Austin → NYC → Austin reversion): the hash
|
|
13386
|
-
// index still records the hash but the fact is no longer active.
|
|
13387
|
-
// Fall through to the write path below so a new active shared
|
|
13388
|
-
// memory is created, then supersession fires post-write as usual.
|
|
13389
|
-
log.debug(
|
|
13390
|
-
`persistExtraction: hash-dedup found no active same-entity shared fact for ${options.sourceMemoryId}; falling through to write`,
|
|
13391
|
-
);
|
|
13392
|
-
} catch (hashDedupSupersessionErr) {
|
|
13393
|
-
log.warn(
|
|
13394
|
-
`persistExtraction: shared-namespace supersession on hash-dedup path failed open for ${options.sourceMemoryId}: ${hashDedupSupersessionErr}`,
|
|
13395
|
-
);
|
|
13396
|
-
// PR #402 round-7 (Fix #1 / cursor Medium PRRT_kwDORJXyws56U_ig):
|
|
13397
|
-
// Only skip the write if we CONFIRMED a matching active shared fact
|
|
13398
|
-
// before the error occurred (hashDedupLookupComplete is true AND
|
|
13399
|
-
// hashDedupMatchingFact is set). If the error was thrown before
|
|
13400
|
-
// matchingFact was resolved — e.g. readAllMemories() threw — we
|
|
13401
|
-
// cannot assume a duplicate exists, and unconditionally returning
|
|
13402
|
-
// would permanently lose the shared promotion. Fall through to the
|
|
13403
|
-
// write path so the fact is not silently dropped.
|
|
13404
|
-
if (hashDedupLookupComplete && hashDedupMatchingFact) {
|
|
13405
|
-
// A matching active shared fact was confirmed — skip the write to
|
|
13406
|
-
// avoid duplicating content that is already present. The existing
|
|
13407
|
-
// fact remains active and the supersession failure is logged above.
|
|
13408
|
-
return;
|
|
13409
|
-
}
|
|
13410
|
-
// Lookup did not complete or no candidate was found — we cannot
|
|
13411
|
-
// confirm a duplicate. Fall through to the write + post-write
|
|
13412
|
-
// supersession path so the shared promotion is not lost.
|
|
13413
|
-
log.debug(
|
|
13414
|
-
`persistExtraction: hash-dedup catch: lookup incomplete or no candidate found for ${options.sourceMemoryId}; falling through to write`,
|
|
13415
|
-
);
|
|
13416
|
-
}
|
|
13417
|
-
} else {
|
|
13418
|
-
// temporalSupersessionEnabled is off or no entity/attributes — keep
|
|
13419
|
-
// the original short-circuit behaviour.
|
|
13420
|
-
return;
|
|
13421
|
-
}
|
|
13422
|
-
}
|
|
13423
|
-
const sharedPromotion = await sharedStorage.writeMemory(
|
|
13424
|
-
options.category as any,
|
|
13425
|
-
citedContent,
|
|
13426
|
-
{
|
|
13427
|
-
confidence: options.confidence,
|
|
13428
|
-
tags: [...options.tags, "shared-promotion"],
|
|
13429
|
-
entityRef: options.entityRef,
|
|
13430
|
-
structuredAttributes: options.structuredAttributes,
|
|
13431
|
-
source: `${options.source}-shared-promotion`,
|
|
13432
|
-
importance: options.importance,
|
|
13433
|
-
lineage: [options.sourceMemoryId],
|
|
13434
|
-
sourceMemoryId: options.sourceMemoryId,
|
|
13435
|
-
intentGoal: options.intentGoal,
|
|
13436
|
-
intentActionType: options.intentActionType,
|
|
13437
|
-
intentEntityTypes: options.intentEntityTypes,
|
|
13438
|
-
memoryKind: options.memoryKind,
|
|
13439
|
-
validAt: options.validAt,
|
|
13440
|
-
// #1578 — forward bi-temporal bounds + ingestion provenance.
|
|
13441
|
-
...(options.invalidAt ? { invalidAt: options.invalidAt } : {}),
|
|
13442
|
-
...(options.observedAt ? { observedAt: options.observedAt } : {}),
|
|
13443
|
-
...(options.eventTimeSource ? { eventTimeSource: options.eventTimeSource } : {}),
|
|
13444
|
-
contentHashSource: options.category === "fact" ? dedupContent : rawContent,
|
|
13445
|
-
// Claim-level provenance spans (issue #1575 PR 2).
|
|
13446
|
-
...(options.sources && options.sources.length > 0 ? { sources: options.sources } : {}),
|
|
13447
|
-
...(options.provenance ? { provenance: options.provenance } : {}),
|
|
13448
|
-
},
|
|
13449
|
-
);
|
|
13450
|
-
const promotedId = sharedPromotion.id;
|
|
13451
|
-
// #1645: if the shared namespace's own tombstone blocked this promotion,
|
|
13452
|
-
// leave the row pending_review but do NOT supersede active shared memories.
|
|
13453
|
-
// PR #402 Finding 3 fix: run temporal supersession against the shared
|
|
13454
|
-
// namespace after the promoted write lands so stale shared-namespace
|
|
13455
|
-
// copies of the same entity attribute are retired. Without this,
|
|
13456
|
-
// source-namespace supersession leaves the shared copy active and
|
|
13457
|
-
// shared recall continues returning the stale state. Reuses the same
|
|
13458
|
-
// applyTemporalSupersession helper — no logic duplication.
|
|
13459
|
-
if (
|
|
13460
|
-
!sharedPromotion.tombstoneBlocked &&
|
|
13461
|
-
lifecycleCaps.temporalSupersession &&
|
|
13462
|
-
options.entityRef &&
|
|
13463
|
-
options.structuredAttributes &&
|
|
13464
|
-
Object.keys(options.structuredAttributes).length > 0
|
|
13465
|
-
) {
|
|
13466
|
-
try {
|
|
13467
|
-
await applyTemporalSupersession({
|
|
13468
|
-
storage: sharedStorage,
|
|
13469
|
-
newMemoryId: promotedId,
|
|
13470
|
-
entityRef: options.entityRef,
|
|
13471
|
-
structuredAttributes: options.structuredAttributes,
|
|
13472
|
-
createdAt: supersessionOrderingAt(options.validAt),
|
|
13473
|
-
enabled: !(options.eventTimeSource === "extracted" && !options.validAt),
|
|
13474
|
-
});
|
|
13475
|
-
} catch (sharedSupersessionErr) {
|
|
13476
|
-
log.warn(
|
|
13477
|
-
`persistExtraction: shared-namespace temporal supersession failed open for promoted ${promotedId}: ${sharedSupersessionErr}`,
|
|
13478
|
-
);
|
|
13479
|
-
}
|
|
13480
|
-
}
|
|
13481
|
-
// Catalog touch (issue #1499, Issue B + ordering sweep): a shared-
|
|
13482
|
-
// namespace promotion is the ONLY write the shared namespace receives on
|
|
13483
|
-
// this path, so without this the shared record's lastWriteAt stays stale
|
|
13484
|
-
// and `writtenSince` filters / maintenance fanout skip it. Record AFTER
|
|
13485
|
-
// the promoted write and the shared temporal-supersession attempt so the
|
|
13486
|
-
// catalog timestamp never precedes a later durable frontmatter mutation in
|
|
13487
|
-
// the same promotion pass. The hot-path source-namespace touch uses a
|
|
13488
|
-
// different storage dir, so this does not double-count the source.
|
|
13489
|
-
// Best-effort and failure-tolerant — it must never crash the promotion.
|
|
13490
|
-
// #1645 TV6: same guard as the profile-target promotion above.
|
|
13491
|
-
if (!sharedPromotion.tombstoneBlocked) {
|
|
13492
|
-
trackPersistedId(sharedStorage, promotedId, {
|
|
13493
|
-
includeReturnedIds: false,
|
|
13494
|
-
});
|
|
13495
|
-
await this.indexPersistedMemory(sharedStorage, promotedId);
|
|
13496
|
-
trackBehaviorSignals(
|
|
13497
|
-
sharedStorage,
|
|
13498
|
-
buildBehaviorSignalsForMemory({
|
|
13499
|
-
memoryId: promotedId,
|
|
13500
|
-
category: options.category as any,
|
|
13501
|
-
content: options.content,
|
|
13502
|
-
namespace: this.config.sharedNamespace,
|
|
13503
|
-
confidence: options.confidence,
|
|
13504
|
-
source: "extraction",
|
|
13505
|
-
}),
|
|
13506
|
-
);
|
|
13507
|
-
}
|
|
13508
|
-
} catch (err) {
|
|
13509
|
-
log.warn(
|
|
13510
|
-
`persistExtraction: shared promotion failed open for ${options.sourceMemoryId}: ${err}`,
|
|
13511
|
-
);
|
|
13512
|
-
}
|
|
13513
|
-
};
|
|
13514
|
-
// #1707 thread 1 — backfill temporal bounds onto promotion copies when the
|
|
13515
|
-
// SOURCE-namespace dedup short-circuit fires. That branch patches the
|
|
13516
|
-
// source copy then `continue`s before the promotion dedup paths run, so
|
|
13517
|
-
// promoted shared/profile copies written before the source fact carried
|
|
13518
|
-
// resolved bounds stay stale (cross-namespace recall surfaces an expired
|
|
13519
|
-
// fact). This mirrors the promotion-target resolution used by the two
|
|
13520
|
-
// promote closures above and calls the same fail-open helper against each
|
|
13521
|
-
// target storage. Backfill-only: never writes a new promoted copy.
|
|
13522
|
-
const backfillTemporalBoundsOnPromotionCopies = async (args: {
|
|
13523
|
-
sourceStorage: StorageManager;
|
|
13524
|
-
content: string;
|
|
13525
|
-
category: string;
|
|
13526
|
-
confidence: number;
|
|
13527
|
-
entityRef?: string;
|
|
13528
|
-
structuredAttributes?: Record<string, string>;
|
|
13529
|
-
bounds: {
|
|
13530
|
-
invalidAt?: string;
|
|
13531
|
-
validFrom?: string;
|
|
13532
|
-
observedAt?: string;
|
|
13533
|
-
eventTimeSource?: "extracted" | "assumed";
|
|
13534
|
-
};
|
|
13535
|
-
}): Promise<void> => {
|
|
13536
|
-
// Mirror the helper's I/O gate: only resolve promotion targets when
|
|
13537
|
-
// there is an end bound OR an EXTRACTED start bound. An "assumed"
|
|
13538
|
-
// validFrom is just the ingestion anchor (no recall effect), so
|
|
13539
|
-
// skipping it avoids resolving target storages on every bi-temporal
|
|
13540
|
-
// duplicate (review cursor PRRT_OvHk).
|
|
13541
|
-
const hasExtractedStart =
|
|
13542
|
-
args.bounds.validFrom !== undefined &&
|
|
13543
|
-
args.bounds.eventTimeSource === "extracted";
|
|
13544
|
-
if (!args.bounds.invalidAt && !hasExtractedStart) return;
|
|
13545
|
-
// Build the same dedupContent the promotion functions hash on so the
|
|
13546
|
-
// content-hash lookup in the helper matches what was stored.
|
|
13547
|
-
const rawContent =
|
|
13548
|
-
citationEnabled && hasCitationForTemplate(args.content, citationTemplate)
|
|
13549
|
-
? stripCitationForTemplate(args.content, citationTemplate)
|
|
13550
|
-
: args.content;
|
|
13551
|
-
const sanitizedBase = sanitizeMemoryContent(rawContent);
|
|
13552
|
-
const dedupContent =
|
|
13553
|
-
args.category === "fact" &&
|
|
13554
|
-
args.structuredAttributes &&
|
|
13555
|
-
Object.keys(args.structuredAttributes).length > 0
|
|
13556
|
-
? `${sanitizedBase.text}\n[Attributes: ${normalizeAttributePairs(args.structuredAttributes)}]`
|
|
13557
|
-
: sanitizedBase.text;
|
|
13558
|
-
// Profile targets. NOTE: we do NOT gate on profileAutoPromotionAllows —
|
|
13559
|
-
// a promoted profile copy may exist from an EARLIER extraction with
|
|
13560
|
-
// higher confidence or older auto-promote settings, so the current
|
|
13561
|
-
// duplicate's confidence must not skip backfilling an already-existing
|
|
13562
|
-
// promoted copy (review codex PRRT_Ov7LKF). The helper no-ops when no
|
|
13563
|
-
// matching copy is found, so resolving all configured auto-promote
|
|
13564
|
-
// targets is safe.
|
|
13565
|
-
if (scopeProfileWritePlan) {
|
|
13566
|
-
const autoTargets = new Set(
|
|
13567
|
-
scopeProfileWritePlan.profile.autoPromote.targets,
|
|
13568
|
-
);
|
|
13569
|
-
const profileTargets = scopeProfileWritePlan.promotionTargets.filter(
|
|
13570
|
-
(target) =>
|
|
13571
|
-
target.target !== "serverShared" &&
|
|
13572
|
-
autoTargets.has(target.target) &&
|
|
13573
|
-
target.authorized &&
|
|
13574
|
-
target.namespace,
|
|
13575
|
-
);
|
|
13576
|
-
for (const target of profileTargets) {
|
|
13577
|
-
if (!target.namespace) continue;
|
|
13578
|
-
try {
|
|
13579
|
-
const targetStorage = await this.storageRouter.storageFor(
|
|
13580
|
-
target.namespace,
|
|
13581
|
-
);
|
|
13582
|
-
if (targetStorage.dir === args.sourceStorage.dir) continue;
|
|
13583
|
-
await this.backfillTemporalBoundsOnDedupHit(
|
|
13584
|
-
targetStorage,
|
|
13585
|
-
dedupContent,
|
|
13586
|
-
args.bounds,
|
|
13587
|
-
args.entityRef,
|
|
13588
|
-
);
|
|
13589
|
-
} catch (err) {
|
|
13590
|
-
log.warn(
|
|
13591
|
-
`bitemporal-backfill: profile-target backfill failed open for ${target.target}: ${err}`,
|
|
13592
|
-
);
|
|
13593
|
-
}
|
|
13594
|
-
}
|
|
13595
|
-
}
|
|
13596
|
-
// Shared target. A shared copy may exist from an earlier extraction
|
|
13597
|
-
// regardless of the current confidence, so do not gate on
|
|
13598
|
-
// shouldPromoteToShared (review codex PRRT_Ov7LKF) — BUT still respect
|
|
13599
|
-
// shared-write AUTHORIZATION: a scoped profile that does not authorize
|
|
13600
|
-
// serverShared writes must not have its shared namespace backfilled
|
|
13601
|
-
// (review codex PRRT_Ov7dHR). profileAllowsSharedWrites is true for the
|
|
13602
|
-
// legacy no-scope case and encodes the readable/writable/authorized
|
|
13603
|
-
// checks under a scope profile.
|
|
13604
|
-
if (profileAllowsSharedWrites) {
|
|
13605
|
-
try {
|
|
13606
|
-
const sharedStorage = await this.storageRouter.storageFor(
|
|
13607
|
-
this.config.sharedNamespace,
|
|
13608
|
-
);
|
|
13609
|
-
if (sharedStorage.dir !== args.sourceStorage.dir) {
|
|
13610
|
-
await this.backfillTemporalBoundsOnDedupHit(
|
|
13611
|
-
sharedStorage,
|
|
13612
|
-
dedupContent,
|
|
13613
|
-
args.bounds,
|
|
13614
|
-
args.entityRef,
|
|
13615
|
-
);
|
|
13616
|
-
}
|
|
13617
|
-
} catch (err) {
|
|
13618
|
-
log.warn(
|
|
13619
|
-
`bitemporal-backfill: shared-target backfill failed open: ${err}`,
|
|
13620
|
-
);
|
|
13621
|
-
}
|
|
13622
|
-
}
|
|
13623
|
-
};
|
|
13624
|
-
|
|
13625
|
-
// Defensive: validate result and facts array
|
|
13626
|
-
if (!result || !Array.isArray(result.facts)) {
|
|
13627
|
-
log.warn(
|
|
13628
|
-
"persistExtraction: result or result.facts is invalid, skipping",
|
|
13629
|
-
{ resultType: typeof result, factsType: typeof result?.facts },
|
|
13630
|
-
);
|
|
13631
|
-
return persistedIds;
|
|
13632
|
-
}
|
|
13633
|
-
|
|
13634
|
-
// Chunking config from plugin settings
|
|
13635
|
-
const chunkingConfig: ChunkingConfig = {
|
|
13636
|
-
targetTokens: this.config.chunkingTargetTokens,
|
|
13637
|
-
minTokens: this.config.chunkingMinTokens,
|
|
13638
|
-
overlapSentences: this.config.chunkingOverlapSentences,
|
|
13639
|
-
};
|
|
13640
|
-
|
|
13641
|
-
const rawEntities = Array.isArray((result as any).entities)
|
|
13642
|
-
? (result as any).entities
|
|
13643
|
-
: [];
|
|
13644
|
-
const rawQuestions = Array.isArray((result as any).questions)
|
|
13645
|
-
? (result as any).questions
|
|
13646
|
-
: [];
|
|
13647
|
-
const rawProfileUpdates = Array.isArray((result as any).profileUpdates)
|
|
13648
|
-
? (result as any).profileUpdates
|
|
13649
|
-
: [];
|
|
13650
|
-
|
|
13651
|
-
const facts = result.facts.slice(0, this.config.extractionMaxFactsPerRun);
|
|
13652
|
-
const entities = rawEntities.slice(
|
|
13653
|
-
0,
|
|
13654
|
-
this.config.extractionMaxEntitiesPerRun,
|
|
13655
|
-
);
|
|
13656
|
-
const questions = rawQuestions.slice(
|
|
13657
|
-
0,
|
|
13658
|
-
this.config.extractionMaxQuestionsPerRun,
|
|
13659
|
-
);
|
|
13660
|
-
const profileUpdates = rawProfileUpdates.slice(
|
|
13661
|
-
0,
|
|
13662
|
-
this.config.extractionMaxProfileUpdatesPerRun,
|
|
13663
|
-
);
|
|
13664
|
-
|
|
13665
|
-
if (
|
|
13666
|
-
facts.length < result.facts.length ||
|
|
13667
|
-
entities.length < result.entities.length ||
|
|
13668
|
-
questions.length < result.questions.length ||
|
|
13669
|
-
profileUpdates.length < result.profileUpdates.length
|
|
13670
|
-
) {
|
|
13671
|
-
log.warn(
|
|
13672
|
-
"persistExtraction: capped extraction payload to guardrails " +
|
|
13673
|
-
`(facts ${facts.length}/${result.facts.length}, entities ${entities.length}/${result.entities.length}, ` +
|
|
13674
|
-
`questions ${questions.length}/${result.questions.length}, profile ${profileUpdates.length}/${result.profileUpdates.length})`,
|
|
13675
|
-
);
|
|
13676
|
-
}
|
|
13677
|
-
|
|
13678
|
-
// v8.2: pre-load all memories once for entity-sibling graph edges (avoids per-fact disk scan)
|
|
13679
|
-
type GraphStorageContext = {
|
|
13680
|
-
allMemsForGraph: Awaited<
|
|
13681
|
-
ReturnType<typeof storage.readAllMemories>
|
|
13682
|
-
> | null;
|
|
13683
|
-
memoryPathById: Map<string, string>;
|
|
13684
|
-
previousPersistedRelPath?: string;
|
|
13685
|
-
};
|
|
13686
|
-
const graphContextByStorageDir = new Map<string, GraphStorageContext>();
|
|
13687
|
-
const ensureGraphContext = async (
|
|
13688
|
-
targetStorage: StorageManager,
|
|
13689
|
-
): Promise<GraphStorageContext> => {
|
|
13690
|
-
const existing = graphContextByStorageDir.get(targetStorage.dir);
|
|
13691
|
-
if (existing) return existing;
|
|
13692
|
-
const created: GraphStorageContext = {
|
|
13693
|
-
allMemsForGraph: null,
|
|
13694
|
-
memoryPathById: new Map<string, string>(),
|
|
13695
|
-
};
|
|
13696
|
-
if (graphCaps.multiGraphMemory) {
|
|
13697
|
-
try {
|
|
13698
|
-
created.allMemsForGraph = await targetStorage.readAllMemories();
|
|
13699
|
-
for (const [id, relPath] of buildMemoryPathById(
|
|
13700
|
-
created.allMemsForGraph,
|
|
13701
|
-
targetStorage.dir,
|
|
13702
|
-
)) {
|
|
13703
|
-
created.memoryPathById.set(id, relPath);
|
|
13704
|
-
}
|
|
13705
|
-
} catch {
|
|
13706
|
-
/* fail-open */
|
|
13707
|
-
}
|
|
13708
|
-
}
|
|
13709
|
-
graphContextByStorageDir.set(targetStorage.dir, created);
|
|
13710
|
-
return created;
|
|
13711
|
-
};
|
|
13712
|
-
let threadEpisodeIdsForGraph: string[] | undefined;
|
|
13713
|
-
if (graphCaps.multiGraphMemory && threadIdForExtraction) {
|
|
13714
|
-
try {
|
|
13715
|
-
const thread = await this.threading.loadThread(threadIdForExtraction);
|
|
13716
|
-
threadEpisodeIdsForGraph = thread?.episodeIds
|
|
13717
|
-
? [...thread.episodeIds]
|
|
13718
|
-
: [];
|
|
13719
|
-
} catch {
|
|
13720
|
-
/* fail-open */
|
|
13721
|
-
}
|
|
13722
|
-
}
|
|
13723
|
-
const routeRules = await this.loadRoutingRules();
|
|
13724
|
-
const routeOptions = this.routeEngineOptions();
|
|
13725
|
-
|
|
13726
|
-
// Pre-routing pass: compute the routed category for every fact BEFORE
|
|
13727
|
-
// building judge candidates. Route rules may override f.category (e.g.
|
|
13728
|
-
// via taxonomy remapping), and the judge must evaluate against the
|
|
13729
|
-
// *final* category that will actually be persisted — not the raw
|
|
13730
|
-
// extraction-time category. The per-fact write loop below reuses
|
|
13731
|
-
// these pre-computed results so routing is evaluated exactly once per
|
|
13732
|
-
// fact (no duplicated logic).
|
|
13733
|
-
const preRoutedCategories: Array<string | undefined> = new Array(facts.length);
|
|
13734
|
-
// #1713 Item 1: collect routed target namespaces so the pre-judge redaction
|
|
13735
|
-
// filter can consult rules from cross-namespace targets, not just the source.
|
|
13736
|
-
// #1713 P2 (cursor): track per-fact routed namespace for per-fact pre-judge rules
|
|
13737
|
-
const preRoutedNamespaceByFact: Array<string | undefined> = new Array(facts.length);
|
|
13738
|
-
if (routeRules.length > 0) {
|
|
13739
|
-
for (let fi = 0; fi < facts.length; fi++) {
|
|
13740
|
-
const f = facts[fi];
|
|
13741
|
-
if (
|
|
13742
|
-
!f ||
|
|
13743
|
-
typeof f.content !== "string" ||
|
|
13744
|
-
!f.content.trim() ||
|
|
13745
|
-
typeof f.category !== "string" ||
|
|
13746
|
-
!f.category.trim()
|
|
13747
|
-
) {
|
|
13748
|
-
continue;
|
|
13749
|
-
}
|
|
13750
|
-
try {
|
|
13751
|
-
const tags = Array.isArray(f.tags) ? f.tags : [];
|
|
13752
|
-
const routeText = `${f.category} ${tags.join(" ")} ${f.content}`;
|
|
13753
|
-
const selected = selectRouteRule(routeText, routeRules, routeOptions);
|
|
13754
|
-
if (selected?.target.category) {
|
|
13755
|
-
preRoutedCategories[fi] = selected.target.category;
|
|
13756
|
-
}
|
|
13757
|
-
if (selected?.target.namespace) {
|
|
13758
|
-
preRoutedNamespaceByFact[fi] = selected.target.namespace;
|
|
13759
|
-
}
|
|
13760
|
-
} catch {
|
|
13761
|
-
// Fail-open: routing errors fall through to the extracted category.
|
|
13762
|
-
}
|
|
13763
|
-
}
|
|
13764
|
-
}
|
|
13765
|
-
|
|
13766
|
-
// Extraction judge gate (issue #376). When enabled, batch-evaluate all
|
|
13767
|
-
// candidate facts for durability before the per-fact write loop.
|
|
13768
|
-
// The verdicts map is keyed by candidate index — we maintain a
|
|
13769
|
-
// candidateIndexToFactIndex mapping so the write loop can look up
|
|
13770
|
-
// verdicts by original fact index.
|
|
13771
|
-
//
|
|
13772
|
-
// Candidates are built using the *routed* category (preRoutedCategories)
|
|
13773
|
-
// so the judge evaluates durability against the same category that will
|
|
13774
|
-
// be persisted, not the raw extraction-time category.
|
|
13775
|
-
let judgeVerdictsByFactIndex: Map<number, import("./extraction-judge.js").JudgeVerdict> | null = null;
|
|
13776
|
-
let judgeGatedCount = 0;
|
|
13777
|
-
// Reset the side-channel defer count at the start of every
|
|
13778
|
-
// persistExtraction call so stale state from a prior call cannot leak
|
|
13779
|
-
// into the caller's buffer-retention decision.
|
|
13780
|
-
this.lastPersistExtractionDeferredCount = 0;
|
|
13781
|
-
if (lifecycleCaps.extractionJudge) {
|
|
13782
|
-
// #1669 P1 + #1713 Item 1: pre-filter redacted facts from judge candidates
|
|
13783
|
-
// so never-store content is not persisted as judge training data.
|
|
13784
|
-
// Consult source + shared + routed target namespace rules so a
|
|
13785
|
-
// never-store pattern registered under a cross-namespace target is
|
|
13786
|
-
// caught at the batch pre-filter point, not just at the persist gate.
|
|
13787
|
-
let preJudgeRedactionRules: CompiledRedactionRule[] = [];
|
|
13788
|
-
try {
|
|
13789
|
-
// #1713: base rules are source-only. Per-fact routed target rules
|
|
13790
|
-
// (including shared when a fact is routed there) are checked in the
|
|
13791
|
-
// fact loop, matching the write-time gate's per-fact scoping exactly.
|
|
13792
|
-
preJudgeRedactionRules = await redactionRulesFor(storage.dir);
|
|
13793
|
-
} catch { /* fail open */ }
|
|
13794
|
-
try {
|
|
13795
|
-
const judgeCandidates: JudgeCandidate[] = [];
|
|
13796
|
-
const candidateToFactIndex: number[] = [];
|
|
13797
|
-
for (let fi = 0; fi < facts.length; fi++) {
|
|
13798
|
-
const f = facts[fi];
|
|
13799
|
-
if (
|
|
13800
|
-
!f ||
|
|
13801
|
-
typeof f.content !== "string" ||
|
|
13802
|
-
!f.content.trim() ||
|
|
13803
|
-
typeof f.category !== "string" ||
|
|
13804
|
-
!f.category.trim()
|
|
13805
|
-
) {
|
|
13806
|
-
continue;
|
|
13807
|
-
}
|
|
13808
|
-
// Use the routed category when available so the judge sees the
|
|
13809
|
-
// final persisted category, not the raw extraction-time value.
|
|
13810
|
-
// Cast to MemoryCategory — routing targets are always valid
|
|
13811
|
-
// category slugs defined in the taxonomy; the fallback is the
|
|
13812
|
-
// original ExtractedFact.category which is already typed.
|
|
13813
|
-
const judgeCategory = (preRoutedCategories[fi] ?? f.category) as import("./types.js").MemoryCategory;
|
|
13814
|
-
if (judgeCategory === "procedure") {
|
|
13815
|
-
continue;
|
|
13816
|
-
}
|
|
13817
|
-
const tags = Array.isArray(f.tags) ? f.tags : [];
|
|
13818
|
-
const imp = scoreImportance(
|
|
13819
|
-
f.content,
|
|
13820
|
-
judgeCategory,
|
|
13821
|
-
tags,
|
|
13822
|
-
);
|
|
13823
|
-
// Pre-filter: skip facts below importance threshold to avoid
|
|
13824
|
-
// wasting LLM calls on facts that will be filtered anyway in
|
|
13825
|
-
// the per-fact write loop (issue #376 review finding).
|
|
13826
|
-
if (
|
|
13827
|
-
!isAboveImportanceThreshold(
|
|
13828
|
-
imp.level,
|
|
13829
|
-
this.config.extractionMinImportanceLevel,
|
|
13830
|
-
)
|
|
13831
|
-
) {
|
|
13832
|
-
continue;
|
|
13833
|
-
}
|
|
13834
|
-
// #1713 P2 (cursor): per-fact rules — load this fact's routed target
|
|
13835
|
-
// namespace rules FIRST, then check. This catches target-only rules
|
|
13836
|
-
// even when the base (source+shared) rules are empty (threads acc58c42
|
|
13837
|
-
// + PBJEe/PBKAj).
|
|
13838
|
-
let factRedactionRules = preJudgeRedactionRules;
|
|
13839
|
-
let factNs = preRoutedNamespaceByFact[fi];
|
|
13840
|
-
// #1713 (codex PRRT_kwDORJXyws6PBj5X): scope classification can route
|
|
13841
|
-
// a scope=global fact to the shared namespace independent of routing
|
|
13842
|
-
// rules. If so, the pre-judge filter must consult the shared
|
|
13843
|
-
// namespace's rules too, or a never-store pattern under shared is
|
|
13844
|
-
// missed at pre-judge and only caught at the write gate — letting the
|
|
13845
|
-
// content reach the extraction judge/training path. Mirror the write
|
|
13846
|
-
// loop's scope-routing conditions exactly.
|
|
13847
|
-
if (
|
|
13848
|
-
!factNs &&
|
|
13849
|
-
lifecycleCaps.extractionScopeClassification &&
|
|
13850
|
-
namespacesEnabled &&
|
|
13851
|
-
f.scope === "global" &&
|
|
13852
|
-
profileAllowsSharedWrites &&
|
|
13853
|
-
this.storageDirNamespace(storage.dir) !== this.config.sharedNamespace
|
|
13854
|
-
) {
|
|
13855
|
-
factNs = this.config.sharedNamespace;
|
|
13856
|
-
}
|
|
13857
|
-
if (factNs) {
|
|
13858
|
-
try {
|
|
13859
|
-
const factDir = (await this.storageRouter.storageFor(factNs)).dir;
|
|
13860
|
-
if (factDir !== storage.dir) {
|
|
13861
|
-
const targetRules = await redactionRulesFor(factDir);
|
|
13862
|
-
if (targetRules.length) factRedactionRules = [...preJudgeRedactionRules, ...targetRules];
|
|
13863
|
-
}
|
|
13864
|
-
} catch { /* fail open */ }
|
|
13865
|
-
}
|
|
13866
|
-
if (factRedactionRules.length > 0) {
|
|
13867
|
-
const rc = f.content + (f.structuredAttributes ? " " + JSON.stringify(f.structuredAttributes) : "")
|
|
13868
|
-
+ (f.procedureSteps ? " " + f.procedureSteps.map((s) => `${s.intent} ${s.expectedOutcome ?? ""} ${s.toolCall ? `${s.toolCall.kind} ${s.toolCall.signature}` : ""}`.trim()).join(" ") : "");
|
|
13869
|
-
if (contentMatchesRedactionRules(rc, factRedactionRules)) continue;
|
|
13870
|
-
}
|
|
13871
|
-
judgeCandidates.push({
|
|
13872
|
-
text: f.content,
|
|
13873
|
-
category: judgeCategory,
|
|
13874
|
-
confidence: typeof f.confidence === "number" ? f.confidence : 0.7,
|
|
13875
|
-
tags,
|
|
13876
|
-
importanceLevel: imp.level,
|
|
13877
|
-
});
|
|
13878
|
-
candidateToFactIndex.push(fi);
|
|
13879
|
-
}
|
|
13880
|
-
// Telemetry + training-pair emit (issue #562 PR 3 + PR 4). The
|
|
13881
|
-
// orchestrator wires two fire-and-forget writers behind a single
|
|
13882
|
-
// callback so `judgeFactDurability` does not need to know about
|
|
13883
|
-
// either ledger. Both handlers are skipped when their flags are
|
|
13884
|
-
// off; the combined callback itself is undefined when both are
|
|
13885
|
-
// disabled so there is zero overhead in the default configuration.
|
|
13886
|
-
const judgeTelemetryOpts = {
|
|
13887
|
-
enabled: lifecycleCaps.extractionJudgeTelemetry,
|
|
13888
|
-
memoryDir: this.config.memoryDir,
|
|
13889
|
-
};
|
|
13890
|
-
const judgeTrainingOpts = {
|
|
13891
|
-
enabled: this.config.collectJudgeTrainingPairs === true,
|
|
13892
|
-
...(this.config.judgeTrainingDir
|
|
13893
|
-
? { directory: this.config.judgeTrainingDir }
|
|
13894
|
-
: {}),
|
|
13895
|
-
};
|
|
13896
|
-
const judgeTelemetryHandler =
|
|
13897
|
-
judgeTelemetryOpts.enabled || judgeTrainingOpts.enabled
|
|
13898
|
-
? (obs: import("./extraction-judge.js").JudgeVerdictObservation) => {
|
|
13899
|
-
const ts = new Date().toISOString();
|
|
13900
|
-
const verdictKind = getVerdictKind(obs.verdict);
|
|
13901
|
-
if (judgeTelemetryOpts.enabled) {
|
|
13902
|
-
const event: import("./extraction-judge-telemetry.js").JudgeVerdictEvent = {
|
|
13903
|
-
version: 1,
|
|
13904
|
-
category: EXTRACTION_JUDGE_VERDICT_CATEGORY,
|
|
13905
|
-
ts,
|
|
13906
|
-
verdictKind,
|
|
13907
|
-
reason: obs.verdict.reason,
|
|
13908
|
-
deferrals: obs.priorDeferrals,
|
|
13909
|
-
elapsedMs: obs.elapsedMs,
|
|
13910
|
-
candidateCategory: obs.candidate.category,
|
|
13911
|
-
confidence: obs.candidate.confidence,
|
|
13912
|
-
contentHash: obs.contentHash,
|
|
13913
|
-
fromCache: obs.source === "cache",
|
|
13914
|
-
...(obs.source === "llm-cap-rejected"
|
|
13915
|
-
? { deferCapTriggered: true }
|
|
13916
|
-
: {}),
|
|
13917
|
-
};
|
|
13918
|
-
void recordJudgeVerdict(event, judgeTelemetryOpts);
|
|
13919
|
-
}
|
|
13920
|
-
if (judgeTrainingOpts.enabled) {
|
|
13921
|
-
const pair: import("./extraction-judge-training.js").JudgeTrainingPair = {
|
|
13922
|
-
version: 1,
|
|
13923
|
-
ts,
|
|
13924
|
-
candidateText: obs.candidate.text,
|
|
13925
|
-
candidateCategory: obs.candidate.category,
|
|
13926
|
-
...(typeof obs.candidate.confidence === "number"
|
|
13927
|
-
? { candidateConfidence: obs.candidate.confidence }
|
|
13928
|
-
: {}),
|
|
13929
|
-
verdictKind,
|
|
13930
|
-
reason: obs.verdict.reason,
|
|
13931
|
-
priorDeferrals: obs.priorDeferrals,
|
|
13932
|
-
};
|
|
13933
|
-
void recordJudgeTrainingPair(pair, judgeTrainingOpts);
|
|
13934
|
-
}
|
|
13935
|
-
}
|
|
13936
|
-
: undefined;
|
|
13937
|
-
const judgeResult = await judgeFactDurability(
|
|
13938
|
-
judgeCandidates,
|
|
13939
|
-
this.config,
|
|
13940
|
-
this.localLlm,
|
|
13941
|
-
new FallbackLlmClient(
|
|
13942
|
-
this.config.gatewayConfig,
|
|
13943
|
-
fallbackLlmRuntimeContextFromConfig(this.config),
|
|
13944
|
-
),
|
|
13945
|
-
this.judgeVerdictCache,
|
|
13946
|
-
this.judgeDeferCounts,
|
|
13947
|
-
judgeTelemetryHandler,
|
|
13948
|
-
);
|
|
13949
|
-
// Remap candidate-indexed verdicts to original fact indexes
|
|
13950
|
-
judgeVerdictsByFactIndex = new Map();
|
|
13951
|
-
for (const [candidateIdx, verdict] of judgeResult.verdicts) {
|
|
13952
|
-
const factIdx = candidateToFactIndex[candidateIdx];
|
|
13953
|
-
if (factIdx !== undefined) {
|
|
13954
|
-
judgeVerdictsByFactIndex.set(factIdx, verdict);
|
|
13955
|
-
}
|
|
13956
|
-
}
|
|
13957
|
-
log.info(
|
|
13958
|
-
`extraction-judge: ${judgeResult.verdicts.size}/${judgeCandidates.length} facts evaluated, ` +
|
|
13959
|
-
`${judgeResult.cached} cached, ${judgeResult.judged} judged, ` +
|
|
13960
|
-
`${judgeResult.deferred} deferred` +
|
|
13961
|
-
(judgeResult.deferredCappedToReject > 0
|
|
13962
|
-
? ` (${judgeResult.deferredCappedToReject} cap-rejected)`
|
|
13963
|
-
: "") +
|
|
13964
|
-
`, ${judgeResult.elapsed}ms`,
|
|
13965
|
-
);
|
|
13966
|
-
// Expose defer count to the caller (issue #562 PR 2) so it can decide
|
|
13967
|
-
// whether to retain buffer turns for the next extraction pass.
|
|
13968
|
-
this.lastPersistExtractionDeferredCount = judgeResult.deferred;
|
|
13969
|
-
} catch (err) {
|
|
13970
|
-
// Fail-open: if the entire judge pipeline errors, proceed without filtering
|
|
13971
|
-
log.warn(
|
|
13972
|
-
`extraction-judge: pipeline error, proceeding without filtering (fail-open): ${err instanceof Error ? err.message : String(err)}`,
|
|
13973
|
-
);
|
|
13974
|
-
}
|
|
13975
|
-
}
|
|
13976
|
-
|
|
13977
|
-
// Faithfulness gate (issue #1576). Entailment-verification of extracted
|
|
13978
|
-
// facts against their verified source spans from #1575. Placement: after
|
|
13979
|
-
// parse + provenance validation, BEFORE persist/index (rule 44). The
|
|
13980
|
-
// substantive batch logic lives in the pure module; this is thin
|
|
13981
|
-
// delegation (ground rule 4). off → null map (byte-identical pre-feature
|
|
13982
|
-
// pipeline, rule 39); shadow → record only; enforce → pending_review.
|
|
13983
|
-
const faithfulnessMode = this.config.extractionFaithfulnessGate;
|
|
13984
|
-
const faithfulnessResultsByFactIndex =
|
|
13985
|
-
faithfulnessMode === "shadow" || faithfulnessMode === "enforce"
|
|
13986
|
-
? await runFaithfulnessGateBatch(
|
|
13987
|
-
facts,
|
|
13988
|
-
faithfulnessMode,
|
|
13989
|
-
this.config,
|
|
13990
|
-
this.localLlm,
|
|
13991
|
-
new FallbackLlmClient(
|
|
13992
|
-
this.config.gatewayConfig,
|
|
13993
|
-
fallbackLlmRuntimeContextFromConfig(this.config),
|
|
13994
|
-
),
|
|
13995
|
-
this.faithfulnessCounters,
|
|
13996
|
-
sourceText,
|
|
13997
|
-
)
|
|
13998
|
-
: null;
|
|
13999
|
-
|
|
14000
|
-
let factLoopIndex = -1;
|
|
14001
|
-
for (const fact of facts) {
|
|
14002
|
-
factLoopIndex++;
|
|
14003
|
-
if (
|
|
14004
|
-
!fact ||
|
|
14005
|
-
typeof (fact as any).content !== "string" ||
|
|
14006
|
-
!(fact as any).content.trim()
|
|
14007
|
-
) {
|
|
14008
|
-
continue;
|
|
14009
|
-
}
|
|
14010
|
-
if (
|
|
14011
|
-
typeof (fact as any).category !== "string" ||
|
|
14012
|
-
!(fact as any).category.trim()
|
|
14013
|
-
) {
|
|
14014
|
-
continue;
|
|
14015
|
-
}
|
|
14016
|
-
(fact as any).tags = Array.isArray((fact as any).tags)
|
|
14017
|
-
? (fact as any).tags.filter((t: any) => typeof t === "string")
|
|
14018
|
-
: [];
|
|
14019
|
-
(fact as any).confidence =
|
|
14020
|
-
typeof (fact as any).confidence === "number"
|
|
14021
|
-
? (fact as any).confidence
|
|
14022
|
-
: 0.7;
|
|
14023
|
-
// #1670 — anchor each fact's event-time to its SOURCE TURN timestamp,
|
|
14024
|
-
// not the batch-wide latest. When a buffered conversation spans a date
|
|
14025
|
-
// boundary, a relative expression ("yesterday") on an early-turn fact
|
|
14026
|
-
// must resolve against that early turn's date. Prefer the fact's
|
|
14027
|
-
// explicit sourceTurnTimestamp, then the earliest provenance span's
|
|
14028
|
-
// observedAt, then fall back to the batch anchor (legacy extractors).
|
|
14029
|
-
const factAnchor = pickFactEventTimeAnchor(fact, sourceContext?.validAt);
|
|
14030
|
-
const biTemporal = this.config.temporalBiTemporal && factAnchor ? resolveFactEventTime(fact.eventTime, factAnchor) : undefined;
|
|
14031
|
-
// Content-hash dedup check (v6.0)
|
|
14032
|
-
//
|
|
14033
|
-
// Canonicalize pre-tagged facts before hashing (Codex P2 — issue #369).
|
|
14034
|
-
// When a fact already carries an inline citation (e.g. relayed or
|
|
14035
|
-
// reprocessed), hashing `fact.content` as-is would produce a different
|
|
14036
|
-
// hash than the one stored from the original write (which used the raw,
|
|
14037
|
-
// un-cited body as contentHashSource). Strip any citation first so the
|
|
14038
|
-
// dedup key matches what the hash index recorded.
|
|
14039
|
-
//
|
|
14040
|
-
// stripCitationForTemplate handles both the default and custom template
|
|
14041
|
-
// formats. For all-placeholder templates it cannot detect citations and
|
|
14042
|
-
// returns the text unchanged — dedup may miss in that edge case, which
|
|
14043
|
-
// is acceptable (no false-positive suppression).
|
|
14044
|
-
//
|
|
14045
|
-
// Routing runs before content-hash dedup and scoring so category overrides
|
|
14046
|
-
// affect both the dedup fingerprint and importance (issue #519 procedure routing).
|
|
14047
|
-
let writeCategory = fact.category;
|
|
14048
|
-
let targetStorage = storage;
|
|
14049
|
-
const sourceStorageDir = storage.dir; // #1669 thread #2: pre-routing source ns for redaction gate
|
|
14050
|
-
// Track the KNOWN target namespace NAME alongside targetStorage (round 6,
|
|
14051
|
-
// codex P2 — NCQI0). Re-deriving it from `targetStorage.dir` mangles a raw
|
|
14052
|
-
// namespace literally named like a canonical token (e.g. `ns-616c706861`
|
|
14053
|
-
// served from its legacy raw dir decodes to `alpha`). We seed it from the
|
|
14054
|
-
// EXPLICIT base namespace the caller used to obtain `storage` (NHIdx, codex
|
|
14055
|
-
// P2) — `selfNamespace`/`writeNamespaceOverride` — so the catalog write touch
|
|
14056
|
-
// records the real namespace, not a guess decoded from the directory. We only
|
|
14057
|
-
// fall back to decoding the dir when no base namespace was passed (legacy
|
|
14058
|
-
// callers). The EXPLICIT routed name (below) still overrides this verbatim.
|
|
14059
|
-
let targetNamespaceName =
|
|
14060
|
-
baseNamespace && baseNamespace.length > 0
|
|
14061
|
-
? baseNamespace
|
|
14062
|
-
: this.storageDirNamespace(targetStorage.dir);
|
|
14063
|
-
let routedRuleId: string | undefined;
|
|
14064
|
-
let routedNamespaceExplicit = false;
|
|
14065
|
-
if (routeRules.length > 0) {
|
|
14066
|
-
try {
|
|
14067
|
-
const routeText = `${fact.category} ${fact.tags.join(" ")} ${fact.content}`;
|
|
14068
|
-
const selected = selectRouteRule(routeText, routeRules, routeOptions);
|
|
14069
|
-
if (selected) {
|
|
14070
|
-
routedRuleId = selected.rule.id;
|
|
14071
|
-
if (selected.target.category) {
|
|
14072
|
-
writeCategory = selected.target.category;
|
|
14073
|
-
}
|
|
14074
|
-
if (selected.target.namespace) {
|
|
14075
|
-
routedNamespaceExplicit = true;
|
|
14076
|
-
targetStorage = await this.storageRouter.storageFor(
|
|
14077
|
-
selected.target.namespace,
|
|
14078
|
-
);
|
|
14079
|
-
targetNamespaceName = selected.target.namespace;
|
|
14080
|
-
}
|
|
14081
|
-
}
|
|
14082
|
-
} catch (err) {
|
|
14083
|
-
log.warn(
|
|
14084
|
-
`routing evaluation failed; fail-open to extracted category/namespace: ${err}`,
|
|
14085
|
-
);
|
|
14086
|
-
}
|
|
14087
|
-
}
|
|
14088
|
-
|
|
14089
|
-
// Scope-based namespace routing: when scope classification is enabled
|
|
14090
|
-
// and the LLM tagged this fact as "global", route it to the shared
|
|
14091
|
-
// namespace so cross-project knowledge is visible everywhere. Only
|
|
14092
|
-
// applies when namespaces are enabled and the fact was not already
|
|
14093
|
-
// routed to a specific namespace by a routing rule (routing rules
|
|
14094
|
-
// that set an explicit namespace take precedence; category-only rules
|
|
14095
|
-
// do not block scope routing). Rule 30: gated by
|
|
14096
|
-
// extractionScopeClassificationEnabled.
|
|
14097
|
-
if (
|
|
14098
|
-
lifecycleCaps.extractionScopeClassification &&
|
|
14099
|
-
namespacesEnabled &&
|
|
14100
|
-
fact.scope === "global" &&
|
|
14101
|
-
!routedNamespaceExplicit
|
|
14102
|
-
) {
|
|
14103
|
-
const currentNs = this.storageDirNamespace(targetStorage.dir);
|
|
14104
|
-
if (currentNs !== this.config.sharedNamespace && profileAllowsSharedWrites) {
|
|
14105
|
-
try {
|
|
14106
|
-
targetStorage = await this.storageRouter.storageFor(
|
|
14107
|
-
this.config.sharedNamespace,
|
|
14108
|
-
);
|
|
14109
|
-
targetNamespaceName = this.config.sharedNamespace;
|
|
14110
|
-
log.debug(
|
|
14111
|
-
`scope-routing: fact "${fact.content.slice(0, 60)}…" routed to shared namespace (scope=global)`,
|
|
14112
|
-
);
|
|
14113
|
-
} catch (scopeRouteErr) {
|
|
14114
|
-
log.warn(
|
|
14115
|
-
`scope-routing: failed to resolve shared namespace storage; writing to session namespace (fail-open): ${scopeRouteErr}`,
|
|
14116
|
-
);
|
|
14117
|
-
}
|
|
14118
|
-
} else if (currentNs !== this.config.sharedNamespace) {
|
|
14119
|
-
log.debug(
|
|
14120
|
-
`scope-routing: skipped shared namespace for global fact because active scope profile ${scopeProfileWritePlan?.profileId ?? "none"} does not authorize serverShared writes`,
|
|
14121
|
-
);
|
|
14122
|
-
}
|
|
14123
|
-
}
|
|
14124
|
-
// #1669 redaction-rule gate: consult BOTH source and target namespace
|
|
14125
|
-
// rules before any write. A never-store pattern registered under the
|
|
14126
|
-
// source namespace must survive scope-routing to a different target
|
|
14127
|
-
// (review thread #2). Fails open on read error.
|
|
14128
|
-
try {
|
|
14129
|
-
const redactionRules = await redactionRulesFor(sourceStorageDir, targetStorage.dir);
|
|
14130
|
-
const redactionCandidate = fact.content
|
|
14131
|
-
+ (fact.structuredAttributes ? " " + JSON.stringify(fact.structuredAttributes) : "")
|
|
14132
|
-
+ (fact.procedureSteps ? " " + fact.procedureSteps.map((s) => `${s.intent} ${s.expectedOutcome ?? ""} ${s.toolCall ? `${s.toolCall.kind} ${s.toolCall.signature}` : ""}`.trim()).join(" ") : "");
|
|
14133
|
-
if (redactionRules.length > 0 && contentMatchesRedactionRules(redactionCandidate, redactionRules)) {
|
|
14134
|
-
redactionGatedCount++;
|
|
14135
|
-
log.debug(`extraction: redaction-rule withheld fact #${redactionGatedCount} in ${targetStorage.dir}`);
|
|
14136
|
-
continue;
|
|
14137
|
-
}
|
|
14138
|
-
} catch (redactionErr) {
|
|
14139
|
-
log.warn(`extraction: redaction-rule gate failed open: ${redactionErr}`);
|
|
14140
|
-
}
|
|
14141
|
-
|
|
14142
|
-
// Procedures: fingerprint the full serialized body (title + steps), not
|
|
14143
|
-
// the title alone, so distinct step lists are not collapsed (issue #519).
|
|
14144
|
-
const canonicalContentForHash =
|
|
14145
|
-
citationEnabled &&
|
|
14146
|
-
hasCitationForTemplate(fact.content, citationTemplate)
|
|
14147
|
-
? stripCitationForTemplate(fact.content, citationTemplate)
|
|
14148
|
-
: fact.content;
|
|
14149
|
-
const contentHashDedupKey =
|
|
14150
|
-
writeCategory === "procedure"
|
|
14151
|
-
? buildProcedurePersistBody(fact.content, fact.procedureSteps)
|
|
14152
|
-
: canonicalContentForHash;
|
|
14153
|
-
// Importance is scored before the dedup short-circuit so #1671 backfill
|
|
14154
|
-
// can gate on it (cursor PRRT_OvKnS): a low-value duplicate that the
|
|
14155
|
-
// importance write-gate (#372) or the judge pre-filter would drop must
|
|
14156
|
-
// not expire an active fact.
|
|
14157
|
-
const importance = scoreImportance(
|
|
14158
|
-
fact.content,
|
|
14159
|
-
writeCategory,
|
|
14160
|
-
fact.tags,
|
|
14161
|
-
);
|
|
14162
|
-
let exactDuplicate = false;
|
|
14163
|
-
try {
|
|
14164
|
-
exactDuplicate = await this.hasContentHashDedup(
|
|
14165
|
-
targetStorage,
|
|
14166
|
-
contentHashDedupKey,
|
|
14167
|
-
);
|
|
14168
|
-
} catch (err) {
|
|
14169
|
-
log.warn(
|
|
14170
|
-
`content-hash dedup lookup failed for storage ${targetStorage.dir}; writing fact fail-open: ${err}`,
|
|
14171
|
-
);
|
|
14172
|
-
}
|
|
14173
|
-
if (exactDuplicate) {
|
|
14174
|
-
// #1671 — before short-circuiting, backfill bi-temporal bounds
|
|
14175
|
-
// onto the existing source-namespace copy if it lacks bounds the
|
|
14176
|
-
// incoming fact now carries (re-extraction with a resolved invalidAt).
|
|
14177
|
-
// Skip when the fact would be rejected/deferred/pending by downstream
|
|
14178
|
-
// gates — a non-durable candidate must not expire an active fact
|
|
14179
|
-
// (chatgpt-codex P1: faithfulness, requireSpans, extraction judge).
|
|
14180
|
-
// #1671 + #1707: backfill bi-temporal bounds onto the existing
|
|
14181
|
-
// source-namespace copy if it lacks bounds the incoming fact now
|
|
14182
|
-
// carries (re-extraction with a resolved bound). #1707 thread 3:
|
|
14183
|
-
// gate on writeCategory === "fact" — the helper only matches facts,
|
|
14184
|
-
// so a non-fact duplicate must not reach the fact-only scan.
|
|
14185
|
-
// #1707 thread 2: also fire when only a corrected start bound
|
|
14186
|
-
// (validFrom) is present, not just an end bound (validUntil). The
|
|
14187
|
-
// downstream-gate skip still applies — a non-durable candidate must
|
|
14188
|
-
// not expire an active fact (chatgpt-codex P1).
|
|
14189
|
-
if (
|
|
14190
|
-
biTemporal &&
|
|
14191
|
-
writeCategory === "fact" &&
|
|
14192
|
-
(biTemporal.validUntil || biTemporal.validFrom)
|
|
14193
|
-
) {
|
|
14194
|
-
const fr = faithfulnessResultsByFactIndex?.get(factLoopIndex);
|
|
14195
|
-
const faithfulnessWouldPending =
|
|
14196
|
-
faithfulnessMode === "enforce" &&
|
|
14197
|
-
fr?.ok === true &&
|
|
14198
|
-
(fr.verdict === "unsupported" || fr.verdict === "contradicted");
|
|
14199
|
-
const requireSpansWouldPending =
|
|
14200
|
-
this.config.provenance?.requireSpans === true &&
|
|
14201
|
-
fact.requireSpansPending === true;
|
|
14202
|
-
const judgeVerdict = judgeVerdictsByFactIndex?.get(factLoopIndex);
|
|
14203
|
-
const judgeWouldGate =
|
|
14204
|
-
!this.config.extractionJudgeShadow &&
|
|
14205
|
-
judgeVerdict !== undefined &&
|
|
14206
|
-
!judgeVerdict.durable;
|
|
14207
|
-
// Importance gate (cursor PRRT_OvKnS): below-threshold duplicates
|
|
14208
|
-
// would never persist (#372) and carry no judge verdict (the
|
|
14209
|
-
// pre-filter skips them), so they must not expire an active fact.
|
|
14210
|
-
if (
|
|
14211
|
-
isAboveImportanceThreshold(importance.level, this.config.extractionMinImportanceLevel) &&
|
|
14212
|
-
!faithfulnessWouldPending &&
|
|
14213
|
-
!requireSpansWouldPending &&
|
|
14214
|
-
!judgeWouldGate
|
|
14215
|
-
) {
|
|
14216
|
-
await this.backfillTemporalBoundsOnDedupHit(
|
|
14217
|
-
targetStorage,
|
|
14218
|
-
contentHashDedupKey,
|
|
14219
|
-
{
|
|
14220
|
-
invalidAt: biTemporal.validUntil,
|
|
14221
|
-
// #1707 thread 2 — carry the corrected start bound too.
|
|
14222
|
-
validFrom: biTemporal.validFrom,
|
|
14223
|
-
observedAt: biTemporal.observedAt,
|
|
14224
|
-
eventTimeSource: biTemporal.eventTimeSource,
|
|
14225
|
-
},
|
|
14226
|
-
fact.entityRef,
|
|
14227
|
-
);
|
|
14228
|
-
// #1707 thread 1 — the source branch short-circuits (`continue`
|
|
14229
|
-
// below) before the promotion dedup paths run, so promoted
|
|
14230
|
-
// shared/profile copies written before the source fact carried
|
|
14231
|
-
// resolved bounds stay stale and cross-namespace recall surfaces
|
|
14232
|
-
// an expired fact. Backfill the promotion targets too (fail-open,
|
|
14233
|
-
// backfill-only — never writes a new promoted copy).
|
|
14234
|
-
await backfillTemporalBoundsOnPromotionCopies({
|
|
14235
|
-
sourceStorage: targetStorage,
|
|
14236
|
-
content: fact.content,
|
|
14237
|
-
category: writeCategory,
|
|
14238
|
-
confidence: fact.confidence,
|
|
14239
|
-
entityRef: fact.entityRef,
|
|
14240
|
-
structuredAttributes: fact.structuredAttributes,
|
|
14241
|
-
bounds: {
|
|
14242
|
-
invalidAt: biTemporal.validUntil,
|
|
14243
|
-
validFrom: biTemporal.validFrom,
|
|
14244
|
-
observedAt: biTemporal.observedAt,
|
|
14245
|
-
eventTimeSource: biTemporal.eventTimeSource,
|
|
14246
|
-
},
|
|
14247
|
-
});
|
|
14248
|
-
}
|
|
14249
|
-
}
|
|
14250
|
-
log.debug(
|
|
14251
|
-
`dedup: skipping duplicate fact "${fact.content.slice(0, 60)}…" in storage ${targetStorage.dir}`,
|
|
14252
|
-
);
|
|
14253
|
-
dedupedCount++;
|
|
14254
|
-
continue;
|
|
14255
|
-
}
|
|
14256
|
-
|
|
14257
|
-
if (writeCategory === "procedure" && this.config.procedural?.enabled !== true) {
|
|
14258
|
-
log.debug("persistExtraction: skip procedure memory (procedural.enabled is false)");
|
|
14259
|
-
continue;
|
|
14260
|
-
}
|
|
14261
|
-
|
|
14262
|
-
// Importance write-gate (issue #372). Drop facts whose locally-scored
|
|
14263
|
-
// level falls below the configured minimum BEFORE the semantic dedup
|
|
14264
|
-
// lookup so that low-importance facts never incur an embedding search.
|
|
14265
|
-
// scoreImportance() already applies category boosts (e.g. corrections
|
|
14266
|
-
// +0.15) before deriving the level, so a correction at raw ~0.35
|
|
14267
|
-
// still lands at "normal" and passes the default gate. Without this
|
|
14268
|
-
// gate, trivial turn-level chatter ("hi", "k", heartbeat pings) gets
|
|
14269
|
-
// persisted as a fact memory and dilutes the store.
|
|
14270
|
-
if (
|
|
14271
|
-
!isAboveImportanceThreshold(
|
|
14272
|
-
importance.level,
|
|
14273
|
-
this.config.extractionMinImportanceLevel,
|
|
14274
|
-
)
|
|
14275
|
-
) {
|
|
14276
|
-
importanceGatedCount++;
|
|
14277
|
-
const snippet = fact.content.slice(0, 60).replace(/\s+/g, " ").trim();
|
|
14278
|
-
log.debug(`extraction: skip trivial "${snippet}"`);
|
|
14279
|
-
// Log-based counter (no dedicated metric bus in remnic-core yet).
|
|
14280
|
-
// Operators can grep for `metric:importance_gated` in gateway.log
|
|
14281
|
-
// to tune extractionMinImportanceLevel.
|
|
14282
|
-
log.debug(
|
|
14283
|
-
`metric:importance_gated level=${importance.level} threshold=${this.config.extractionMinImportanceLevel} category=${writeCategory} count=${importanceGatedCount}`,
|
|
14284
|
-
);
|
|
14285
|
-
continue;
|
|
14286
|
-
}
|
|
14287
|
-
|
|
14288
|
-
// Extraction judge gate (issue #376 + #562 PR 2). After the local
|
|
14289
|
-
// importance gate passes, consult the judge verdict (computed before
|
|
14290
|
-
// the loop). In active mode, non-durable facts are dropped. In shadow
|
|
14291
|
-
// mode, verdicts are logged but all facts proceed to write.
|
|
14292
|
-
//
|
|
14293
|
-
// Defer verdicts (issue #562): do not persist now, but also do not
|
|
14294
|
-
// cache the outcome so the candidate is re-evaluated on a later
|
|
14295
|
-
// extraction pass. The judge module tracks how many times the same
|
|
14296
|
-
// content has been deferred and converts to reject at the configured
|
|
14297
|
-
// cap, so the orchestrator only needs to skip the write here.
|
|
14298
|
-
if (judgeVerdictsByFactIndex) {
|
|
14299
|
-
const verdict = judgeVerdictsByFactIndex.get(factLoopIndex);
|
|
14300
|
-
if (verdict && !verdict.durable) {
|
|
14301
|
-
const verdictKind = getVerdictKind(verdict);
|
|
14302
|
-
if (this.config.extractionJudgeShadow) {
|
|
14303
|
-
log.info(
|
|
14304
|
-
`extraction-judge[shadow]: would ${verdictKind} "${fact.content.slice(0, 60)}…" reason="${verdict.reason}"`,
|
|
14305
|
-
);
|
|
14306
|
-
} else if (verdictKind === "defer") {
|
|
14307
|
-
judgeGatedCount++;
|
|
14308
|
-
log.debug(
|
|
14309
|
-
`extraction-judge: deferred "${fact.content.slice(0, 60)}…" reason="${verdict.reason}"`,
|
|
14310
|
-
);
|
|
14311
|
-
continue;
|
|
14312
|
-
} else {
|
|
14313
|
-
judgeGatedCount++;
|
|
14314
|
-
log.debug(
|
|
14315
|
-
`extraction-judge: rejected "${fact.content.slice(0, 60)}…" reason="${verdict.reason}"`,
|
|
14316
|
-
);
|
|
14317
|
-
continue;
|
|
14318
|
-
}
|
|
14319
|
-
}
|
|
14320
|
-
}
|
|
14321
|
-
|
|
14322
|
-
// Procedure extraction gate (issue #519): ≥2 steps + trigger phrasing.
|
|
14323
|
-
// Runs even when extractionJudgeEnabled is false (durability judge is unrelated).
|
|
14324
|
-
// Never tied to extractionJudgeShadow — that flag is only for the LLM durability judge.
|
|
14325
|
-
if (writeCategory === "procedure") {
|
|
14326
|
-
const procGate = validateProcedureExtraction({
|
|
14327
|
-
content: fact.content,
|
|
14328
|
-
procedureSteps: fact.procedureSteps,
|
|
14329
|
-
});
|
|
14330
|
-
if (!procGate.durable) {
|
|
14331
|
-
log.debug(
|
|
14332
|
-
`extraction-procedure-gate: rejected "${fact.content.slice(0, 60)}…" reason="${procGate.reason}"`,
|
|
14333
|
-
);
|
|
14334
|
-
continue;
|
|
14335
|
-
}
|
|
14336
|
-
}
|
|
14337
|
-
|
|
14338
|
-
// Faithfulness gate verdict application (issue #1576). Look up the
|
|
14339
|
-
// pre-computed verdict for this fact and translate it to frontmatter +
|
|
14340
|
-
// an optional enforce-mode pending_review status. Logic lives in the
|
|
14341
|
-
// pure module; this is thin read-through (ground rule 4).
|
|
14342
|
-
const { faithfulness: faithfulnessFm, enforceStatus: faithfulnessGateStatus } =
|
|
14343
|
-
applyFaithfulnessVerdict(
|
|
14344
|
-
faithfulnessResultsByFactIndex,
|
|
14345
|
-
factLoopIndex,
|
|
14346
|
-
faithfulnessMode,
|
|
14347
|
-
fact.content,
|
|
14348
|
-
this.faithfulnessCounters,
|
|
14349
|
-
);
|
|
14350
|
-
|
|
14351
|
-
// requireSpans enforcement (issue #1575 PR 2): when an operator opts
|
|
14352
|
-
// into provenance.requireSpans, a fact whose quote could not be located
|
|
14353
|
-
// in any source turn (carried as the transient requireSpansPending
|
|
14354
|
-
// signal from the extraction validator) routes to pending_review — the
|
|
14355
|
-
// same review queue an unsupported faithfulness verdict uses. This is
|
|
14356
|
-
// the persist-path wiring ProvenanceConfig.requireSpans documents.
|
|
14357
|
-
// Faithfulness takes precedence when it already routed the fact; both
|
|
14358
|
-
// gates agree on pending_review so the merge is a simple coalesce
|
|
14359
|
-
// (chatgpt-codex-connector thread 4xB).
|
|
14360
|
-
const requireSpansPendingStatus =
|
|
14361
|
-
this.config.provenance?.requireSpans === true &&
|
|
14362
|
-
fact.requireSpansPending === true
|
|
14363
|
-
? ("pending_review" as const)
|
|
14364
|
-
: undefined;
|
|
14365
|
-
const faithfulnessEnforceStatus = faithfulnessGateStatus ?? requireSpansPendingStatus;
|
|
14366
|
-
|
|
14367
|
-
// Issue #373 — write-time semantic similarity guard. Hook runs after
|
|
14368
|
-
// the exact content-hash miss and the importance gate so that:
|
|
14369
|
-
// (a) paraphrased near-duplicates never reach writeMemory(), and
|
|
14370
|
-
// (b) low-importance facts that will be dropped never trigger an
|
|
14371
|
-
// embedding lookup (avoids unnecessary API latency/cost).
|
|
14372
|
-
// Fails open when the embedding backend is unavailable.
|
|
14373
|
-
//
|
|
14374
|
-
// Defense in depth (PR #399 review): decideSemanticDedup already
|
|
14375
|
-
// catches lookup errors internally, and the embedding fetch is
|
|
14376
|
-
// bounded by a timeout in embedding-fallback.ts. We still wrap the
|
|
14377
|
-
// whole call in its own try/catch here so that any unexpected
|
|
14378
|
-
// rejection (future refactors, misbehaving custom backends, etc.)
|
|
14379
|
-
// can never block the persist loop — a failure in the dedup path
|
|
14380
|
-
// must always default to "not a duplicate".
|
|
14381
|
-
// Track a pending semantic-skip decision (populated inside the block
|
|
14382
|
-
// below). The actual drop happens AFTER contradiction detection so that
|
|
14383
|
-
// a high-similarity update/correction is linked as a superseding
|
|
14384
|
-
// contradiction rather than silently dropped.
|
|
14385
|
-
let pendingSemanticSkip: (SemanticDedupDecision & { action: "skip" }) | null = null;
|
|
14386
|
-
if (resolvePipelineProcessingCapabilities(this.config).semanticDedup) {
|
|
14387
|
-
let semanticDecision: SemanticDedupDecision;
|
|
14388
|
-
// UUI2: skip embedding lookup for the rest of this batch once we know
|
|
14389
|
-
// the backend is unavailable. The flag is reset per-batch (set to false
|
|
14390
|
-
// at the top of persistExtraction), so a transient hiccup in one call
|
|
14391
|
-
// does not permanently disable dedup in subsequent calls.
|
|
14392
|
-
if (batchBackendUnavailable) {
|
|
14393
|
-
semanticDecision = { action: "keep", reason: "backend_unavailable" };
|
|
14394
|
-
} else {
|
|
14395
|
-
try {
|
|
14396
|
-
// Pass the resolved target storage so the lookup scopes the
|
|
14397
|
-
// embedding index to the target namespace (PR #399 P1 fix).
|
|
14398
|
-
// Without this, a high-similarity hit in a different namespace
|
|
14399
|
-
// would cause the fact to be dropped here — cross-namespace
|
|
14400
|
-
// write suppression / data loss.
|
|
14401
|
-
const lookupStorage = targetStorage;
|
|
14402
|
-
semanticDecision = await decideSemanticDedup(
|
|
14403
|
-
fact.content,
|
|
14404
|
-
(content, limit) =>
|
|
14405
|
-
this.semanticDedupLookup(content, limit, lookupStorage),
|
|
14406
|
-
{
|
|
14407
|
-
enabled: true,
|
|
14408
|
-
threshold: this.config.semanticDedupThreshold,
|
|
14409
|
-
candidates: this.config.semanticDedupCandidates,
|
|
14410
|
-
},
|
|
14411
|
-
);
|
|
14412
|
-
} catch (err) {
|
|
14413
|
-
log.warn(
|
|
14414
|
-
`semantic dedup decision failed; failing open and writing fact: ${err}`,
|
|
14415
|
-
);
|
|
14416
|
-
semanticDecision = {
|
|
14417
|
-
action: "keep",
|
|
14418
|
-
reason: "backend_unavailable",
|
|
14419
|
-
};
|
|
14420
|
-
}
|
|
14421
|
-
// UUI2: cache the backend-unavailable signal for the rest of this batch.
|
|
14422
|
-
if (semanticDecision.reason === "backend_unavailable") {
|
|
14423
|
-
batchBackendUnavailable = true;
|
|
14424
|
-
}
|
|
14425
|
-
}
|
|
14426
|
-
if (semanticDecision.action === "skip") {
|
|
14427
|
-
pendingSemanticSkip = semanticDecision;
|
|
14428
|
-
}
|
|
14429
|
-
}
|
|
14430
|
-
|
|
14431
|
-
const inferredIntent = resolveConversationContextCapabilities(this.config).intentRouting
|
|
14432
|
-
? inferIntentFromText(
|
|
14433
|
-
`${writeCategory} ${fact.tags.join(" ")} ${fact.content}`,
|
|
14434
|
-
)
|
|
14435
|
-
: null;
|
|
14436
|
-
const extractionWriteSource =
|
|
14437
|
-
(fact as any).source === "proactive"
|
|
14438
|
-
? "extraction-proactive"
|
|
14439
|
-
: "extraction";
|
|
14440
|
-
|
|
14441
|
-
// Check for contradictions before writing (Phase 2B).
|
|
14442
|
-
// NOTE: This block was moved above the chunking branch so that the
|
|
14443
|
-
// pendingSemanticSkip guard (below) can also protect the chunking path.
|
|
14444
|
-
// Previously, contradiction detection only ran on the non-chunked path,
|
|
14445
|
-
// meaning chunked facts could be persisted even when semanticDecision was
|
|
14446
|
-
// "skip" (the deferred guard was bypassed by the chunking `continue`).
|
|
14447
|
-
let supersedes: string | undefined;
|
|
14448
|
-
let links: MemoryLink[] = [];
|
|
14449
|
-
// True when contradiction detection ran and confirmed a contradiction,
|
|
14450
|
-
// regardless of whether auto-resolve is enabled. Used by the
|
|
14451
|
-
// semantic-skip guard so that contradictory updates are never silently
|
|
14452
|
-
// dropped — even when `contradictionAutoResolve=false` (in which case
|
|
14453
|
-
// `supersedes` is intentionally left unset to avoid retiring the old
|
|
14454
|
-
// memory without user confirmation).
|
|
14455
|
-
let contradictionDetected = false;
|
|
14456
|
-
// #1645: hoist the contradiction result so the deferred auto-resolve
|
|
14457
|
-
// (post-write, gated on tombstone status) can read its fields.
|
|
14458
|
-
let contradiction: {
|
|
14459
|
-
supersededId: string;
|
|
14460
|
-
confidence: number;
|
|
14461
|
-
reason: string;
|
|
14462
|
-
supersededPath: string;
|
|
14463
|
-
supersededCreated: string;
|
|
14464
|
-
supersededTags: string[];
|
|
14465
|
-
} | null | undefined;
|
|
14466
|
-
|
|
14467
|
-
// Faithfulness gate (#1576, chatgpt P2): skip contradiction detection
|
|
14468
|
-
// for a pending_review fact — an unfaithful extraction in the review queue
|
|
14469
|
-
// must not trigger auto-resolve and retire an existing active memory.
|
|
14470
|
-
if (
|
|
14471
|
-
resolveRecallEnhancementCapabilities(this.config).contradictionDetection &&
|
|
14472
|
-
this.qmd.isAvailable() &&
|
|
14473
|
-
faithfulnessEnforceStatus !== "pending_review"
|
|
14474
|
-
) {
|
|
14475
|
-
const targetNamespace = this.storageDirNamespace(targetStorage.dir);
|
|
14476
|
-
contradiction = await this.checkForContradiction(
|
|
14477
|
-
fact.content,
|
|
14478
|
-
writeCategory,
|
|
14479
|
-
targetNamespace,
|
|
14480
|
-
);
|
|
14481
|
-
if (contradiction) {
|
|
14482
|
-
contradictionDetected = true;
|
|
14483
|
-
// When auto-resolve is enabled the existing memory has already been
|
|
14484
|
-
// marked superseded; set `supersedes` so the new write carries the
|
|
14485
|
-
// relationship. When auto-resolve is disabled we still record the
|
|
14486
|
-
// contradiction link (so the memory is annotated for manual review)
|
|
14487
|
-
// but do NOT set `supersedes` on the new write — the old memory
|
|
14488
|
-
// remains active until a human resolves it.
|
|
14489
|
-
if (this.config.contradictionAutoResolve) {
|
|
14490
|
-
supersedes = contradiction.supersededId;
|
|
14491
|
-
}
|
|
14492
|
-
links.push({
|
|
14493
|
-
targetId: contradiction.supersededId,
|
|
14494
|
-
linkType: "contradicts",
|
|
14495
|
-
strength: contradiction.confidence,
|
|
14496
|
-
reason: contradiction.reason,
|
|
14497
|
-
});
|
|
14498
|
-
// #1645: deindex + supersede are deferred to after writeMemory so the
|
|
14499
|
-
// caller can gate them on the new write's tombstone status. A
|
|
14500
|
-
// tombstone-blocked write (pending_review) must not deindex or retire
|
|
14501
|
-
// the existing active memory — see the post-write guard below.
|
|
14502
|
-
}
|
|
14503
|
-
}
|
|
14504
|
-
|
|
14505
|
-
// Apply the deferred semantic-skip now that contradiction detection has
|
|
14506
|
-
// run. If a contradiction was found (contradictionDetected is true), the
|
|
14507
|
-
// candidate is a contradictory update and must be written — do not skip
|
|
14508
|
-
// it. Only drop it when there is no detected contradiction (true
|
|
14509
|
-
// near-duplicate). This check intentionally runs BEFORE the chunking
|
|
14510
|
-
// branch so that a fact flagged as a semantic near-duplicate cannot be
|
|
14511
|
-
// persisted (with its hash registered) simply because it was long enough
|
|
14512
|
-
// to trigger chunking.
|
|
14513
|
-
//
|
|
14514
|
-
// NOTE: We use `contradictionDetected` rather than `!!supersedes` here
|
|
14515
|
-
// so that facts are preserved even when `contradictionAutoResolve=false`.
|
|
14516
|
-
// When auto-resolve is disabled `supersedes` is intentionally unset, but
|
|
14517
|
-
// the write must still proceed so the user can manually reconcile the
|
|
14518
|
-
// two memories later.
|
|
14519
|
-
//
|
|
14520
|
-
// UUI1: correction category writes are NEVER suppressed by the semantic
|
|
14521
|
-
// skip fallback, regardless of whether supersedes is set. When contradiction
|
|
14522
|
-
// detection is disabled or QMD is unavailable, supersedes is never set —
|
|
14523
|
-
// without this exemption a high-similarity correction would be silently
|
|
14524
|
-
// dropped, leaving a stale fact active. writeCategory (not fact.category)
|
|
14525
|
-
// is used because routing rules may have overridden the raw category.
|
|
14526
|
-
const isCorrection = writeCategory === "correction";
|
|
14527
|
-
// Faithfulness gate (#1576, cursor High): a pending_review fact must
|
|
14528
|
-
// bypass the semantic-dedup skip so it reaches the review queue — the
|
|
14529
|
-
// gate's contract is "persists with status: pending_review, never
|
|
14530
|
-
// silently dropped" (issue #1576).
|
|
14531
|
-
if (
|
|
14532
|
-
pendingSemanticSkip &&
|
|
14533
|
-
!contradictionDetected &&
|
|
14534
|
-
!isCorrection &&
|
|
14535
|
-
faithfulnessEnforceStatus !== "pending_review"
|
|
14536
|
-
) {
|
|
14537
|
-
log.debug(
|
|
14538
|
-
`dedup: skipping semantic near-duplicate fact "${fact.content
|
|
14539
|
-
.slice(0, 60)
|
|
14540
|
-
.replace(/\s+/g, " ")}…" score=${pendingSemanticSkip.topScore.toFixed(
|
|
14541
|
-
3,
|
|
14542
|
-
)} neighbor=${pendingSemanticSkip.topId}`,
|
|
14543
|
-
);
|
|
14544
|
-
dedupedCount++;
|
|
14545
|
-
// Do NOT add fact.content to contentHashIndex here. No memory was
|
|
14546
|
-
// persisted for this fact, so registering a synthetic hash would
|
|
14547
|
-
// permanently suppress exact-copy writes once the neighbor memory is
|
|
14548
|
-
// archived or deleted (the hash would linger with no backing record).
|
|
14549
|
-
continue;
|
|
14550
|
-
}
|
|
14551
|
-
|
|
14552
|
-
// Check if chunking is enabled and content should be chunked.
|
|
14553
|
-
// When semanticChunkingEnabled is true, prefer the embedding-based
|
|
14554
|
-
// semantic chunker which produces more coherent topic-aligned segments.
|
|
14555
|
-
// Falls back to the recursive sentence-boundary chunker on failure.
|
|
14556
|
-
if (resolvePipelineProcessingCapabilities(this.config).chunking && writeCategory !== "procedure") {
|
|
14557
|
-
let chunkResult: { chunked: boolean; chunks: { content: string; index: number; tokenCount: number }[] };
|
|
14558
|
-
|
|
14559
|
-
if (resolvePipelineProcessingCapabilities(this.config).semanticChunking) {
|
|
14560
|
-
try {
|
|
14561
|
-
const embedFn = this.embeddingFallback.embedTexts.bind(this.embeddingFallback);
|
|
14562
|
-
const semanticResult: SemanticChunkResult = await semanticChunkContent(
|
|
14563
|
-
fact.content,
|
|
14564
|
-
embedFn,
|
|
14565
|
-
this.config.semanticChunkingConfig,
|
|
14566
|
-
);
|
|
14567
|
-
chunkResult = semanticResult;
|
|
14568
|
-
} catch (err) {
|
|
14569
|
-
// Honor the fallbackToRecursive contract: when the user explicitly
|
|
14570
|
-
// disables fallback, re-throw so extraction fails fast instead of
|
|
14571
|
-
// silently using the recursive chunker. semanticChunkContent already
|
|
14572
|
-
// throws when fallback is disabled, but this outer catch swallowed
|
|
14573
|
-
// that signal. (PR #439 post-merge Finding 1.)
|
|
14574
|
-
if (this.config.semanticChunkingConfig?.fallbackToRecursive === false) {
|
|
14575
|
-
throw err;
|
|
14576
|
-
}
|
|
14577
|
-
log.debug(
|
|
14578
|
-
`semantic chunking failed, falling back to recursive chunker: ${err}`,
|
|
14579
|
-
);
|
|
14580
|
-
chunkResult = chunkContent(fact.content, chunkingConfig);
|
|
14581
|
-
}
|
|
14582
|
-
} else {
|
|
14583
|
-
chunkResult = chunkContent(fact.content, chunkingConfig);
|
|
14584
|
-
}
|
|
14585
|
-
|
|
14586
|
-
if (chunkResult.chunked && chunkResult.chunks.length > 1) {
|
|
14587
|
-
// Classify memory kind (v8.0 Phase 2B: HiMem episode/note dual store)
|
|
14588
|
-
const memoryKind = resolvePresentationCapabilities(this.config).episodeNoteMode
|
|
14589
|
-
? classifyMemoryKind(fact.content, fact.tags ?? [], writeCategory)
|
|
14590
|
-
: undefined;
|
|
14591
|
-
|
|
14592
|
-
// Write the parent memory first (with full content for reference).
|
|
14593
|
-
//
|
|
14594
|
-
// Compute the cited content once so that writeMemory and writeArtifact
|
|
14595
|
-
// (when verbatim artifacts are enabled) share the same citation timestamp.
|
|
14596
|
-
// See the normal write path comment for the full dedup rationale.
|
|
14597
|
-
//
|
|
14598
|
-
// Propagate supersedes/links from contradiction detection (round 6
|
|
14599
|
-
// fix): contradiction detection now runs BEFORE this branch so the
|
|
14600
|
-
// parent must carry the supersession relationship — without it the
|
|
14601
|
-
// old memory is deindexed but the new chunked parent has no link
|
|
14602
|
-
// back, leaving a dangling deindex with no replacement reference.
|
|
14603
|
-
// Child chunks intentionally do NOT carry supersedes; only the
|
|
14604
|
-
// parent represents the logical memory unit.
|
|
14605
|
-
// Canonicalize contentHashSource before writing (Thread 3 — Codex P2,
|
|
14606
|
-
// issue #369). If fact.content already carries an inline citation
|
|
14607
|
-
// (e.g. re-processed or relayed fact), strip it so contentHashSource
|
|
14608
|
-
// records the raw un-cited body — matching what the dedup check hashes
|
|
14609
|
-
// via stripCitationForTemplate before calling hasFactContentHash.
|
|
14610
|
-
const rawChunkedContent =
|
|
14611
|
-
citationEnabled &&
|
|
14612
|
-
hasCitationForTemplate(fact.content, citationTemplate)
|
|
14613
|
-
? stripCitationForTemplate(fact.content, citationTemplate)
|
|
14614
|
-
: fact.content;
|
|
14615
|
-
const citedChunkedContent = applyInlineCitation(rawChunkedContent);
|
|
14616
|
-
const parentWrite = await targetStorage.writeMemory(
|
|
14617
|
-
writeCategory,
|
|
14618
|
-
citedChunkedContent,
|
|
14619
|
-
{
|
|
14620
|
-
confidence: fact.confidence,
|
|
14621
|
-
tags: [...fact.tags, "chunked"],
|
|
14622
|
-
entityRef: fact.entityRef,
|
|
14623
|
-
source: extractionWriteSource,
|
|
14624
|
-
importance,
|
|
14625
|
-
supersedes,
|
|
14626
|
-
links: links.length > 0 ? links : undefined,
|
|
14627
|
-
intentGoal: inferredIntent?.goal,
|
|
14628
|
-
intentActionType: inferredIntent?.actionType,
|
|
14629
|
-
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14630
|
-
memoryKind,
|
|
14631
|
-
structuredAttributes: fact.structuredAttributes,
|
|
14632
|
-
validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt,
|
|
14633
|
-
...(biTemporal ? { observedAt: biTemporal.observedAt, eventTimeSource: biTemporal.eventTimeSource, ...(biTemporal.validUntil ? { invalidAt: biTemporal.validUntil } : {}) } : {}),
|
|
14634
|
-
contentHashSource: rawChunkedContent,
|
|
14635
|
-
// Faithfulness gate (issue #1576).
|
|
14636
|
-
...(faithfulnessFm ? { faithfulness: faithfulnessFm } : {}),
|
|
14637
|
-
...(faithfulnessEnforceStatus ? { status: faithfulnessEnforceStatus } : {}),
|
|
14638
|
-
// Claim-level provenance spans (issue #1575 PR 2).
|
|
14639
|
-
...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
|
|
14640
|
-
...(fact.provenance ? { provenance: fact.provenance } : {}),
|
|
14641
|
-
},
|
|
14642
|
-
);
|
|
14643
|
-
const parentId = parentWrite.id;
|
|
14644
|
-
// #1645: surface the tombstone block and gate active post-write paths
|
|
14645
|
-
// (chunks, supersession, shared promotion, graph/artifact) like #1576.
|
|
14646
|
-
const tombstoneBlocked = parentWrite.tombstoneBlocked;
|
|
14647
|
-
const postWriteGuard =
|
|
14648
|
-
faithfulnessEnforceStatus === "pending_review" || tombstoneBlocked;
|
|
14649
|
-
// #1645: defer contradiction auto-resolve until tombstone status is
|
|
14650
|
-
// known (see applyDeferredContradictionResolve).
|
|
14651
|
-
await this.applyDeferredContradictionResolve(
|
|
14652
|
-
contradiction,
|
|
14653
|
-
targetStorage,
|
|
14654
|
-
parentId,
|
|
14655
|
-
postWriteGuard,
|
|
14656
|
-
);
|
|
14657
|
-
try {
|
|
14658
|
-
// Write individual chunks with parent reference
|
|
14659
|
-
for (const chunk of chunkResult.chunks) {
|
|
14660
|
-
// Score each chunk's importance separately
|
|
14661
|
-
const chunkImportance = scoreImportance(
|
|
14662
|
-
chunk.content,
|
|
14663
|
-
writeCategory,
|
|
14664
|
-
fact.tags,
|
|
14665
|
-
);
|
|
14666
|
-
const chunkWriteSource =
|
|
14667
|
-
(fact as any).source === "proactive"
|
|
14668
|
-
? "chunking-proactive"
|
|
14669
|
-
: "chunking";
|
|
14670
|
-
|
|
14671
|
-
await targetStorage.writeChunk(
|
|
14672
|
-
parentId,
|
|
14673
|
-
chunk.index,
|
|
14674
|
-
chunkResult.chunks.length,
|
|
14675
|
-
writeCategory,
|
|
14676
|
-
// Each chunk carries its own inline citation so provenance
|
|
14677
|
-
// survives when a single chunk is quoted in isolation.
|
|
14678
|
-
applyInlineCitation(chunk.content),
|
|
14679
|
-
{
|
|
14680
|
-
confidence: fact.confidence,
|
|
14681
|
-
tags: fact.tags,
|
|
14682
|
-
entityRef: fact.entityRef,
|
|
14683
|
-
source: chunkWriteSource,
|
|
14684
|
-
importance: chunkImportance,
|
|
14685
|
-
intentGoal: inferredIntent?.goal,
|
|
14686
|
-
intentActionType: inferredIntent?.actionType,
|
|
14687
|
-
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14688
|
-
memoryKind,
|
|
14689
|
-
validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt,
|
|
14690
|
-
// #1578: propagate end bound + provenance to chunks (cursor bugbot).
|
|
14691
|
-
...(biTemporal
|
|
14692
|
-
? {
|
|
14693
|
-
observedAt: biTemporal.observedAt,
|
|
14694
|
-
eventTimeSource: biTemporal.eventTimeSource,
|
|
14695
|
-
...(biTemporal.validUntil
|
|
14696
|
-
? { invalidAt: biTemporal.validUntil }
|
|
14697
|
-
: {}),
|
|
14698
|
-
}
|
|
14699
|
-
: {}),
|
|
14700
|
-
// Faithfulness gate (issue #1576): propagate the parent
|
|
14701
|
-
// fact's verdict + enforce status so a pending_review fact
|
|
14702
|
-
// is not indexed as active through its chunks (chatgpt P2).
|
|
14703
|
-
...(faithfulnessFm ? { faithfulness: faithfulnessFm } : {}),
|
|
14704
|
-
// #1645 (OchiE): inherit pending_review + blockedBy so no chunk lands active.
|
|
14705
|
-
...(postWriteGuard
|
|
14706
|
-
? { status: "pending_review" as const }
|
|
14707
|
-
: (faithfulnessEnforceStatus ? { status: faithfulnessEnforceStatus } : {})),
|
|
14708
|
-
...(tombstoneBlocked && parentWrite.blockedBy
|
|
14709
|
-
? { blockedBy: parentWrite.blockedBy }
|
|
14710
|
-
: {}),
|
|
14711
|
-
// Claim-level provenance (issue #1575 PR 2): mirror the
|
|
14712
|
-
// parent's spans onto each chunk so a chunk surfaced
|
|
14713
|
-
// independently (memory_get/x-ray on a chunk ID) preserves
|
|
14714
|
-
// the verified span (chatgpt-codex-connector thread Ocvmo).
|
|
14715
|
-
...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
|
|
14716
|
-
...(fact.provenance ? { provenance: fact.provenance } : {}),
|
|
14717
|
-
},
|
|
14718
|
-
);
|
|
14719
|
-
}
|
|
14720
|
-
} finally {
|
|
14721
|
-
// The parent memory is durable once writeMemory returns `parentId`.
|
|
14722
|
-
// Touch immediately around the chunk-write loop so a later chunk
|
|
14723
|
-
// failure still surfaces the partially durable parent/chunk files to
|
|
14724
|
-
// catalog-driven `writtenSince` maintenance. The final touch below
|
|
14725
|
-
// still refreshes `lastWriteAt` after later durable writes on success.
|
|
14726
|
-
}
|
|
14727
|
-
|
|
14728
|
-
if (routedRuleId) {
|
|
14729
|
-
log.debug(
|
|
14730
|
-
`routing applied for chunked memory ${parentId}: rule=${routedRuleId} category=${writeCategory} storage=${targetStorage.dir}`,
|
|
14731
|
-
);
|
|
14732
|
-
}
|
|
14733
|
-
log.debug(
|
|
14734
|
-
`chunked memory ${parentId} into ${chunkResult.chunks.length} chunks`,
|
|
14735
|
-
);
|
|
14736
|
-
trackPersistedId(targetStorage, parentId, {
|
|
14737
|
-
pendingReview: postWriteGuard,
|
|
14738
|
-
});
|
|
14739
|
-
// #1576 (cursor Medium): keep pending_review ids out of threadEpisodeIdsForGraph — else later active facts build thread-predecessor edges to an unfaithful memory.
|
|
14740
|
-
if (
|
|
14741
|
-
!postWriteGuard &&
|
|
14742
|
-
threadEpisodeIdsForGraph &&
|
|
14743
|
-
!threadEpisodeIdsForGraph.includes(parentId)
|
|
14744
|
-
) {
|
|
14745
|
-
threadEpisodeIdsForGraph.push(parentId);
|
|
14746
|
-
}
|
|
14747
|
-
// #1645: same gate as the non-chunked path — a blocked chunked parent
|
|
14748
|
-
// must not enter the embedding-fallback index (resurrection).
|
|
14749
|
-
if (!postWriteGuard) {
|
|
14750
|
-
await this.indexPersistedMemory(targetStorage, parentId);
|
|
14751
|
-
}
|
|
14752
|
-
// PR #402 Thread 1 fix: run source-namespace temporal supersession for
|
|
14753
|
-
// chunked writes, matching the non-chunked path. Without this the
|
|
14754
|
-
// source namespace retains stale facts that should have been superseded.
|
|
14755
|
-
// Faithfulness gate (#1576, cursor High): skip supersession for a
|
|
14756
|
-
// pending_review fact — an unfaithful extraction in the review queue
|
|
14757
|
-
// must NOT retire older active memories.
|
|
14758
|
-
if (!postWriteGuard) {
|
|
14759
|
-
try {
|
|
14760
|
-
const supersessionEntityRef =
|
|
14761
|
-
typeof (fact as any).entityRef === "string"
|
|
14762
|
-
? ((fact as any).entityRef as string)
|
|
14763
|
-
: undefined;
|
|
14764
|
-
await applyTemporalSupersession({
|
|
14765
|
-
storage: targetStorage,
|
|
14766
|
-
newMemoryId: parentId,
|
|
14767
|
-
entityRef: supersessionEntityRef,
|
|
14768
|
-
structuredAttributes: fact.structuredAttributes,
|
|
14769
|
-
createdAt: supersessionOrderingAt(biTemporal?.validFrom ?? sourceContext?.validAt),
|
|
14770
|
-
// #1578 r3: an extracted end-only bound (validFrom absent) is
|
|
14771
|
-
// historical, not a new authoritative state — never let it
|
|
14772
|
-
// supersede a later active fact (codex P1 on :15534).
|
|
14773
|
-
enabled: lifecycleCaps.temporalSupersession &&
|
|
14774
|
-
!(biTemporal && !biTemporal.validFrom),
|
|
14775
|
-
});
|
|
14776
|
-
} catch (err) {
|
|
14777
|
-
log.warn(`temporal-supersession (chunked): unexpected error: ${err}`);
|
|
14778
|
-
}
|
|
14779
|
-
}
|
|
14780
|
-
// Faithfulness gate (#1576, chatgpt P2): do not promote a
|
|
14781
|
-
// pending_review fact to shared/profile — it must enter the review
|
|
14782
|
-
// queue without active copies that bypass the gate.
|
|
14783
|
-
if (!postWriteGuard) await promoteMemoryToShared({
|
|
14784
|
-
sourceStorage: targetStorage,
|
|
14785
|
-
category: writeCategory,
|
|
14786
|
-
content: fact.content,
|
|
14787
|
-
confidence: fact.confidence,
|
|
14788
|
-
tags: fact.tags,
|
|
14789
|
-
entityRef: fact.entityRef,
|
|
14790
|
-
structuredAttributes: fact.structuredAttributes,
|
|
14791
|
-
sourceMemoryId: parentId,
|
|
14792
|
-
importance,
|
|
14793
|
-
intentGoal: inferredIntent?.goal,
|
|
14794
|
-
intentActionType: inferredIntent?.actionType,
|
|
14795
|
-
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14796
|
-
memoryKind,
|
|
14797
|
-
validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt,
|
|
14798
|
-
...(biTemporal
|
|
14799
|
-
? {
|
|
14800
|
-
observedAt: biTemporal.observedAt,
|
|
14801
|
-
eventTimeSource: biTemporal.eventTimeSource,
|
|
14802
|
-
...(biTemporal.validUntil ? { invalidAt: biTemporal.validUntil } : {}),
|
|
14803
|
-
}
|
|
14804
|
-
: {}),
|
|
14805
|
-
source: extractionWriteSource,
|
|
14806
|
-
...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
|
|
14807
|
-
...(fact.provenance ? { provenance: fact.provenance } : {}),
|
|
14808
|
-
});
|
|
14809
|
-
// Register chunked content in the target storage hash index too.
|
|
14810
|
-
// Thread 3 fix: canonicalize by stripping any pre-existing citation
|
|
14811
|
-
// so the stored hash matches what the dedup check computes.
|
|
14812
|
-
try {
|
|
14813
|
-
const canonicalChunkedContent =
|
|
14814
|
-
citationEnabled &&
|
|
14815
|
-
hasCitationForTemplate(fact.content, citationTemplate)
|
|
14816
|
-
? stripCitationForTemplate(fact.content, citationTemplate)
|
|
14817
|
-
: fact.content;
|
|
14818
|
-
// #1645: do NOT register a tombstone-blocked fact's content in the
|
|
14819
|
-
// dedup index — writeMemory already skipped it (rule 44). Re-adding
|
|
14820
|
-
// would let the next extraction dedup-skip the tombstone chokepoint
|
|
14821
|
-
// and silently ban the retired content (no pending_review row).
|
|
14822
|
-
if (!tombstoneBlocked) {
|
|
14823
|
-
await this.addContentHashDedup(targetStorage, canonicalChunkedContent);
|
|
14824
|
-
}
|
|
14825
|
-
} catch (err) {
|
|
14826
|
-
log.warn(
|
|
14827
|
-
`content-hash dedup registration failed for chunked memory ${parentId}: ${err}`,
|
|
14828
|
-
);
|
|
14829
|
-
}
|
|
14830
|
-
|
|
14831
|
-
for (const chunk of chunkResult.chunks) {
|
|
14832
|
-
const chunkId = `${parentId}-chunk-${chunk.index}`;
|
|
14833
|
-
// Do NOT push chunkId into persistedIds — chunk IDs must not leak
|
|
14834
|
-
// into boxBuilder.onExtraction() or threading.processTurn(), which
|
|
14835
|
-
// only expect canonical parent memory IDs. Call indexPersistedMemory
|
|
14836
|
-
// directly for embedding-fallback sync of each chunk document.
|
|
14837
|
-
// #1645: chunks inherit pending_review under postWriteGuard — don't
|
|
14838
|
-
// index them into the embedding fallback (resurrection).
|
|
14839
|
-
if (!postWriteGuard) {
|
|
14840
|
-
await this.indexPersistedMemory(targetStorage, chunkId);
|
|
14841
|
-
}
|
|
14842
|
-
}
|
|
14843
|
-
try {
|
|
14844
|
-
if (
|
|
14845
|
-
resolvePresentationCapabilities(this.config).verbatimArtifacts &&
|
|
14846
|
-
this.config.verbatimArtifactCategories.includes(writeCategory) &&
|
|
14847
|
-
fact.confidence >= this.config.verbatimArtifactsMinConfidence &&
|
|
14848
|
-
!postWriteGuard
|
|
14849
|
-
) {
|
|
14850
|
-
// Reuse citedChunkedContent so the artifact carries the same citation
|
|
14851
|
-
// timestamp as the parent memory write above (Fix #3 — duplicate-citation).
|
|
14852
|
-
await targetStorage.writeArtifact(citedChunkedContent, {
|
|
14853
|
-
confidence: fact.confidence,
|
|
14854
|
-
tags: [...fact.tags, "artifact", "chunked-parent"],
|
|
14855
|
-
artifactType: this.artifactTypeForCategory(writeCategory),
|
|
14856
|
-
sourceMemoryId: parentId,
|
|
14857
|
-
intentGoal: inferredIntent?.goal,
|
|
14858
|
-
intentActionType: inferredIntent?.actionType,
|
|
14859
|
-
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14860
|
-
});
|
|
14861
|
-
}
|
|
14862
|
-
// v8.2: graph edge building for chunked memories. #1576: skip pending_review.
|
|
14863
|
-
if (graphCaps.multiGraphMemory && !postWriteGuard) {
|
|
14864
|
-
try {
|
|
14865
|
-
const graphContext = await ensureGraphContext(targetStorage);
|
|
14866
|
-
const entityRef =
|
|
14867
|
-
typeof (fact as any).entityRef === "string"
|
|
14868
|
-
? (fact as any).entityRef
|
|
14869
|
-
: undefined;
|
|
14870
|
-
const parentRelPath = resolvePersistedMemoryRelativePath({
|
|
14871
|
-
memoryId: parentId,
|
|
14872
|
-
pathById: graphContext.memoryPathById,
|
|
14873
|
-
category: writeCategory,
|
|
14874
|
-
});
|
|
14875
|
-
graphContext.memoryPathById.set(parentId, parentRelPath);
|
|
14876
|
-
appendMemoryToGraphContext({
|
|
14877
|
-
allMemsForGraph: graphContext.allMemsForGraph,
|
|
14878
|
-
storageDir: targetStorage.dir,
|
|
14879
|
-
memoryRelPath: parentRelPath,
|
|
14880
|
-
memoryId: parentId,
|
|
14881
|
-
category: writeCategory,
|
|
14882
|
-
content: fact.content ?? "",
|
|
14883
|
-
entityRef,
|
|
14884
|
-
});
|
|
14885
|
-
await this.buildGraphEdge(
|
|
14886
|
-
targetStorage,
|
|
14887
|
-
parentRelPath,
|
|
14888
|
-
entityRef,
|
|
14889
|
-
parentId,
|
|
14890
|
-
fact.content ?? "",
|
|
14891
|
-
graphContext.allMemsForGraph,
|
|
14892
|
-
graphContext.memoryPathById,
|
|
14893
|
-
threadIdForExtraction ?? undefined,
|
|
14894
|
-
threadEpisodeIdsForGraph,
|
|
14895
|
-
graphContext.previousPersistedRelPath,
|
|
14896
|
-
graphCaps,
|
|
14897
|
-
);
|
|
14898
|
-
graphContext.previousPersistedRelPath = parentRelPath;
|
|
14899
|
-
} catch {
|
|
14900
|
-
/* fail-open */
|
|
14901
|
-
}
|
|
14902
|
-
}
|
|
14903
|
-
} finally {
|
|
14904
|
-
// Catalog touch (issue #1499): refresh AFTER later chunked
|
|
14905
|
-
// source-namespace durable mutations — temporal supersession, shared
|
|
14906
|
-
// promotion, optional artifact writes, and graph-edge writes — so
|
|
14907
|
-
// `lastWriteAt` cannot precede later file changes on successful
|
|
14908
|
-
// completion. Use the KNOWN routed name, not a dir-decoded guess.
|
|
14909
|
-
}
|
|
14910
|
-
trackBehaviorSignals(
|
|
14911
|
-
targetStorage,
|
|
14912
|
-
buildBehaviorSignalsForMemory({
|
|
14913
|
-
memoryId: parentId,
|
|
14914
|
-
category: writeCategory,
|
|
14915
|
-
content: fact.content,
|
|
14916
|
-
namespace: this.storageDirNamespace(targetStorage.dir),
|
|
14917
|
-
confidence: fact.confidence,
|
|
14918
|
-
source: "extraction",
|
|
14919
|
-
}),
|
|
14920
|
-
);
|
|
14921
|
-
continue; // Skip the normal write below
|
|
14922
|
-
}
|
|
14923
|
-
}
|
|
14924
|
-
|
|
14925
|
-
// Suggest links for this memory (Phase 3A)
|
|
14926
|
-
if (resolveRecallEnhancementCapabilities(this.config).memoryLinking && this.qmd.isAvailable()) {
|
|
14927
|
-
const targetNamespace = this.storageDirNamespace(targetStorage.dir);
|
|
14928
|
-
const suggestedLinks = await this.suggestLinksForMemory(
|
|
14929
|
-
fact.content,
|
|
14930
|
-
writeCategory,
|
|
14931
|
-
targetNamespace,
|
|
14932
|
-
);
|
|
14933
|
-
if (suggestedLinks.length > 0) {
|
|
14934
|
-
links.push(...suggestedLinks);
|
|
14935
|
-
}
|
|
14936
|
-
}
|
|
14937
|
-
|
|
14938
|
-
// Classify memory kind (v8.0 Phase 2B: HiMem episode/note dual store)
|
|
14939
|
-
const memoryKind =
|
|
14940
|
-
writeCategory === "procedure"
|
|
14941
|
-
? undefined
|
|
14942
|
-
: resolvePresentationCapabilities(this.config).episodeNoteMode
|
|
14943
|
-
? classifyMemoryKind(fact.content, fact.tags ?? [], writeCategory)
|
|
14944
|
-
: undefined;
|
|
14945
|
-
|
|
14946
|
-
// Normal write (no chunking)
|
|
14947
|
-
// Compute the cited content once so that writeMemory and writeArtifact
|
|
14948
|
-
// (when verbatim artifacts are enabled) share the same citation timestamp.
|
|
14949
|
-
// Calling applyInlineCitation twice on the same raw content would produce
|
|
14950
|
-
// two different timestamps, creating duplicate citations with divergent
|
|
14951
|
-
// provenance metadata on the memory and artifact copies of the same fact.
|
|
14952
|
-
// Pass the RAW (pre-citation) fact as `contentHashSource` so the
|
|
14953
|
-
// fact-content hash index records the hash of the canonical fact text
|
|
14954
|
-
// rather than the citation-annotated variant. When inline attribution is
|
|
14955
|
-
// enabled, `applyInlineCitation` appends a timestamp-bearing marker, so
|
|
14956
|
-
// hashing the persisted body would produce a different hash on every
|
|
14957
|
-
// write and defeat cross-session dedup (see `findDuplicateExplicitCapture`
|
|
14958
|
-
// in explicit-capture.ts which calls `hasFactContentHash(candidate.content)`
|
|
14959
|
-
// on raw content).
|
|
14960
|
-
const rawPersistBody =
|
|
14961
|
-
writeCategory === "procedure"
|
|
14962
|
-
? buildProcedurePersistBody(fact.content, fact.procedureSteps)
|
|
14963
|
-
: fact.content;
|
|
14964
|
-
const citedFactContent = applyInlineCitation(rawPersistBody);
|
|
14965
|
-
const factWrite = await targetStorage.writeMemory(
|
|
14966
|
-
writeCategory,
|
|
14967
|
-
citedFactContent,
|
|
14968
|
-
{
|
|
14969
|
-
confidence: fact.confidence,
|
|
14970
|
-
tags: fact.tags,
|
|
14971
|
-
entityRef:
|
|
14972
|
-
typeof (fact as any).entityRef === "string"
|
|
14973
|
-
? (fact as any).entityRef
|
|
14974
|
-
: undefined,
|
|
14975
|
-
source: extractionWriteSource,
|
|
14976
|
-
importance,
|
|
14977
|
-
supersedes,
|
|
14978
|
-
links: links.length > 0 ? links : undefined,
|
|
14979
|
-
intentGoal: inferredIntent?.goal,
|
|
14980
|
-
intentActionType: inferredIntent?.actionType,
|
|
14981
|
-
intentEntityTypes: inferredIntent?.entityTypes,
|
|
14982
|
-
memoryKind,
|
|
14983
|
-
structuredAttributes: fact.structuredAttributes,
|
|
14984
|
-
validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt,
|
|
14985
|
-
...(biTemporal ? { observedAt: biTemporal.observedAt, eventTimeSource: biTemporal.eventTimeSource, ...(biTemporal.validUntil ? { invalidAt: biTemporal.validUntil } : {}) } : {}),
|
|
14986
|
-
contentHashSource: writeCategory === "fact" ? fact.content : undefined,
|
|
14987
|
-
// Faithfulness gate (issue #1576).
|
|
14988
|
-
...(faithfulnessFm ? { faithfulness: faithfulnessFm } : {}),
|
|
14989
|
-
...(faithfulnessEnforceStatus ? { status: faithfulnessEnforceStatus } : {}),
|
|
14990
|
-
// Claim-level provenance spans (issue #1575 PR 2). Carry verified
|
|
14991
|
-
// sources + the coarse strength tag from the extraction validator
|
|
14992
|
-
// through to frontmatter so they survive end-to-end.
|
|
14993
|
-
...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
|
|
14994
|
-
...(fact.provenance ? { provenance: fact.provenance } : {}),
|
|
14995
|
-
},
|
|
14996
|
-
);
|
|
14997
|
-
const memoryId = factWrite.id;
|
|
14998
|
-
// #1645: surface the tombstone block; gate active post-write paths like #1576
|
|
14999
|
-
// so a blocked fact creates no active shared copy / supersession / graph entry.
|
|
15000
|
-
const tombstoneBlocked = factWrite.tombstoneBlocked;
|
|
15001
|
-
const postWriteGuard =
|
|
15002
|
-
faithfulnessEnforceStatus === "pending_review" || tombstoneBlocked;
|
|
15003
|
-
// #1645: defer contradiction auto-resolve until tombstone status is
|
|
15004
|
-
// known (see applyDeferredContradictionResolve).
|
|
15005
|
-
await this.applyDeferredContradictionResolve(
|
|
15006
|
-
contradiction,
|
|
15007
|
-
targetStorage,
|
|
15008
|
-
memoryId,
|
|
15009
|
-
postWriteGuard,
|
|
15010
|
-
);
|
|
15011
|
-
if (routedRuleId) {
|
|
15012
|
-
log.debug(
|
|
15013
|
-
`routing applied for memory ${memoryId}: rule=${routedRuleId} category=${writeCategory} storage=${targetStorage.dir}`,
|
|
15014
|
-
);
|
|
15015
|
-
}
|
|
15016
|
-
// Temporal supersession (issue #375): when the new fact has structured
|
|
15017
|
-
// attributes, retire any older fact with the same entity + attribute
|
|
15018
|
-
// key that has a conflicting value. Faithfulness gate (#1576, cursor
|
|
15019
|
-
// High): skip for a pending_review fact — an unfaithful extraction in
|
|
15020
|
-
// the review queue must NOT retire older active memories.
|
|
15021
|
-
if (!postWriteGuard) {
|
|
15022
|
-
try {
|
|
15023
|
-
const supersessionEntityRef =
|
|
15024
|
-
typeof (fact as any).entityRef === "string"
|
|
15025
|
-
? ((fact as any).entityRef as string)
|
|
15026
|
-
: undefined;
|
|
15027
|
-
await applyTemporalSupersession({
|
|
15028
|
-
storage: targetStorage,
|
|
15029
|
-
newMemoryId: memoryId,
|
|
15030
|
-
entityRef: supersessionEntityRef,
|
|
15031
|
-
structuredAttributes: fact.structuredAttributes,
|
|
15032
|
-
createdAt: supersessionOrderingAt(biTemporal?.validFrom ?? sourceContext?.validAt),
|
|
15033
|
-
enabled: lifecycleCaps.temporalSupersession &&
|
|
15034
|
-
!(biTemporal && !biTemporal.validFrom),
|
|
15035
|
-
});
|
|
15036
|
-
} catch (err) {
|
|
15037
|
-
log.warn(`temporal-supersession: unexpected error: ${err}`);
|
|
15038
|
-
}
|
|
15039
|
-
}
|
|
15040
|
-
try {
|
|
15041
|
-
trackBehaviorSignals(
|
|
15042
|
-
targetStorage,
|
|
15043
|
-
buildBehaviorSignalsForMemory({
|
|
15044
|
-
memoryId,
|
|
15045
|
-
category: writeCategory,
|
|
15046
|
-
content: fact.content,
|
|
15047
|
-
namespace: this.storageDirNamespace(targetStorage.dir),
|
|
15048
|
-
confidence: fact.confidence,
|
|
15049
|
-
source: "extraction",
|
|
15050
|
-
}),
|
|
15051
|
-
);
|
|
15052
|
-
trackPersistedId(targetStorage, memoryId, {
|
|
15053
|
-
pendingReview: postWriteGuard,
|
|
15054
|
-
});
|
|
15055
|
-
if (
|
|
15056
|
-
!postWriteGuard &&
|
|
15057
|
-
threadEpisodeIdsForGraph &&
|
|
15058
|
-
!threadEpisodeIdsForGraph.includes(memoryId)
|
|
15059
|
-
) {
|
|
15060
|
-
threadEpisodeIdsForGraph.push(memoryId);
|
|
15061
|
-
}
|
|
15062
|
-
// #1645: a tombstone-blocked / pending_review fact must NOT enter the
|
|
15063
|
-
// embedding-fallback index — otherwise embedding recall surfaces the
|
|
15064
|
-
// pending_review row (resurrection). Gate on postWriteGuard like the
|
|
15065
|
-
// surrounding supersession / promotion / graph paths.
|
|
15066
|
-
if (!postWriteGuard) {
|
|
15067
|
-
await this.indexPersistedMemory(targetStorage, memoryId);
|
|
15068
|
-
}
|
|
15069
|
-
// Faithfulness gate (#1576, chatgpt P2): skip promotion for a
|
|
15070
|
-
// pending_review fact so no active shared/profile copy bypasses the gate.
|
|
15071
|
-
if (!postWriteGuard) await promoteMemoryToShared({
|
|
15072
|
-
sourceStorage: targetStorage,
|
|
15073
|
-
category: writeCategory,
|
|
15074
|
-
content: fact.content,
|
|
15075
|
-
confidence: fact.confidence,
|
|
15076
|
-
tags: fact.tags,
|
|
15077
|
-
entityRef:
|
|
15078
|
-
typeof (fact as any).entityRef === "string"
|
|
15079
|
-
? (fact as any).entityRef
|
|
15080
|
-
: undefined,
|
|
15081
|
-
structuredAttributes: fact.structuredAttributes,
|
|
15082
|
-
sourceMemoryId: memoryId,
|
|
15083
|
-
importance,
|
|
15084
|
-
intentGoal: inferredIntent?.goal,
|
|
15085
|
-
intentActionType: inferredIntent?.actionType,
|
|
15086
|
-
intentEntityTypes: inferredIntent?.entityTypes,
|
|
15087
|
-
memoryKind,
|
|
15088
|
-
validAt: biTemporal ? biTemporal.validFrom : sourceContext?.validAt,
|
|
15089
|
-
...(biTemporal
|
|
15090
|
-
? {
|
|
15091
|
-
observedAt: biTemporal.observedAt,
|
|
15092
|
-
eventTimeSource: biTemporal.eventTimeSource,
|
|
15093
|
-
...(biTemporal.validUntil ? { invalidAt: biTemporal.validUntil } : {}),
|
|
15094
|
-
}
|
|
15095
|
-
: {}),
|
|
15096
|
-
source: extractionWriteSource,
|
|
15097
|
-
...(fact.sources && fact.sources.length > 0 ? { sources: fact.sources } : {}),
|
|
15098
|
-
...(fact.provenance ? { provenance: fact.provenance } : {}),
|
|
15099
|
-
});
|
|
15100
|
-
// v8.2: graph edge building (fail-open). #1576: skip pending_review facts.
|
|
15101
|
-
if (graphCaps.multiGraphMemory && !postWriteGuard) {
|
|
15102
|
-
try {
|
|
15103
|
-
const graphContext = await ensureGraphContext(targetStorage);
|
|
15104
|
-
const entityRef =
|
|
15105
|
-
typeof (fact as any).entityRef === "string"
|
|
15106
|
-
? (fact as any).entityRef
|
|
15107
|
-
: undefined;
|
|
15108
|
-
const memoryRelPath = resolvePersistedMemoryRelativePath({
|
|
15109
|
-
memoryId,
|
|
15110
|
-
pathById: graphContext.memoryPathById,
|
|
15111
|
-
category: writeCategory,
|
|
15112
|
-
});
|
|
15113
|
-
graphContext.memoryPathById.set(memoryId, memoryRelPath);
|
|
15114
|
-
appendMemoryToGraphContext({
|
|
15115
|
-
allMemsForGraph: graphContext.allMemsForGraph,
|
|
15116
|
-
storageDir: targetStorage.dir,
|
|
15117
|
-
memoryRelPath: memoryRelPath,
|
|
15118
|
-
memoryId,
|
|
15119
|
-
category: writeCategory,
|
|
15120
|
-
content: fact.content ?? "",
|
|
15121
|
-
entityRef,
|
|
15122
|
-
});
|
|
15123
|
-
await this.buildGraphEdge(
|
|
15124
|
-
targetStorage,
|
|
15125
|
-
memoryRelPath,
|
|
15126
|
-
entityRef,
|
|
15127
|
-
memoryId,
|
|
15128
|
-
fact.content ?? "",
|
|
15129
|
-
graphContext.allMemsForGraph,
|
|
15130
|
-
graphContext.memoryPathById,
|
|
15131
|
-
threadIdForExtraction ?? undefined,
|
|
15132
|
-
threadEpisodeIdsForGraph,
|
|
15133
|
-
graphContext.previousPersistedRelPath,
|
|
15134
|
-
graphCaps,
|
|
15135
|
-
);
|
|
15136
|
-
graphContext.previousPersistedRelPath = memoryRelPath;
|
|
15137
|
-
} catch {
|
|
15138
|
-
/* fail-open */
|
|
15139
|
-
}
|
|
15140
|
-
}
|
|
15141
|
-
if (
|
|
15142
|
-
resolvePresentationCapabilities(this.config).verbatimArtifacts &&
|
|
15143
|
-
this.config.verbatimArtifactCategories.includes(writeCategory) &&
|
|
15144
|
-
fact.confidence >= this.config.verbatimArtifactsMinConfidence &&
|
|
15145
|
-
!postWriteGuard
|
|
15146
|
-
) {
|
|
15147
|
-
// Reuse citedFactContent so the artifact carries the same citation
|
|
15148
|
-
// timestamp as the memory write above (Fix #3 — duplicate-citation).
|
|
15149
|
-
await targetStorage.writeArtifact(citedFactContent, {
|
|
15150
|
-
confidence: fact.confidence,
|
|
15151
|
-
tags: [...fact.tags, "artifact"],
|
|
15152
|
-
artifactType: this.artifactTypeForCategory(writeCategory),
|
|
15153
|
-
sourceMemoryId: memoryId,
|
|
15154
|
-
intentGoal: inferredIntent?.goal,
|
|
15155
|
-
intentActionType: inferredIntent?.actionType,
|
|
15156
|
-
intentEntityTypes: inferredIntent?.entityTypes,
|
|
15157
|
-
});
|
|
15158
|
-
}
|
|
15159
|
-
// Register in the target storage content-hash index after successful
|
|
15160
|
-
// write. Thread 3 fix: canonicalize by stripping any pre-existing
|
|
15161
|
-
// citation so the stored hash matches what the dedup check computes.
|
|
15162
|
-
try {
|
|
15163
|
-
const canonicalFactContent =
|
|
15164
|
-
citationEnabled &&
|
|
15165
|
-
hasCitationForTemplate(fact.content, citationTemplate)
|
|
15166
|
-
? stripCitationForTemplate(fact.content, citationTemplate)
|
|
15167
|
-
: fact.content;
|
|
15168
|
-
const hashRegisterKey =
|
|
15169
|
-
writeCategory === "procedure"
|
|
15170
|
-
? buildProcedurePersistBody(fact.content, fact.procedureSteps)
|
|
15171
|
-
: canonicalFactContent;
|
|
15172
|
-
// #1645: do NOT register a tombstone-blocked fact's content in the dedup
|
|
15173
|
-
// index (rule 44 defeat) — see chunked path comment.
|
|
15174
|
-
if (!tombstoneBlocked) {
|
|
15175
|
-
await this.addContentHashDedup(targetStorage, hashRegisterKey);
|
|
15176
|
-
}
|
|
15177
|
-
} catch (err) {
|
|
15178
|
-
log.warn(
|
|
15179
|
-
`content-hash dedup registration failed for memory ${memoryId}: ${err}`,
|
|
15180
|
-
);
|
|
15181
|
-
}
|
|
15182
|
-
} finally {
|
|
15183
|
-
// Catalog touch (issue #1499): record AFTER every synchronous
|
|
15184
|
-
// source-namespace mutation in the non-chunked path: writeMemory,
|
|
15185
|
-
// temporal supersession, graph edges, and optional verbatim artifacts.
|
|
15186
|
-
// The `finally` preserves the write touch when post-write indexing or
|
|
15187
|
-
// promotion fails after the canonical memory is already durable. Use the
|
|
15188
|
-
// KNOWN routed name, not a dir-decoded guess (NCQI0).
|
|
15189
|
-
}
|
|
15190
|
-
}
|
|
15191
|
-
|
|
15192
|
-
// Tracks whether THIS extraction persisted any durable, non-fact output to the
|
|
15193
|
-
// BASE namespace's storage (entity / relationship / profile / question). The
|
|
15194
|
-
// per-fact catalog touch (storage chokepoint #1522) only fires inside the fact write loop, so a
|
|
15195
|
-
// fact-less extraction that still persists durable data must record exactly one
|
|
15196
|
-
// base-namespace catalog touch after all writes complete (NHZEZ, codex P2).
|
|
15197
|
-
let durableNonFactWritten = false;
|
|
15198
|
-
let durableNonFactTouchRecorded = false;
|
|
15199
|
-
const touchBaseNonFactNamespace = () => {
|
|
15200
|
-
const baseTouchNamespace =
|
|
15201
|
-
baseNamespace && baseNamespace.length > 0
|
|
15202
|
-
? baseNamespace
|
|
15203
|
-
: this.storageDirNamespace(storage.dir);
|
|
15204
|
-
};
|
|
15205
|
-
const recordDurableNonFactWrite = () => {
|
|
15206
|
-
durableNonFactWritten = true;
|
|
15207
|
-
if (durableNonFactTouchRecorded) return;
|
|
15208
|
-
durableNonFactTouchRecorded = true;
|
|
15209
|
-
touchBaseNonFactNamespace();
|
|
15210
|
-
};
|
|
15211
|
-
for (const entity of entities) {
|
|
15212
|
-
try {
|
|
15213
|
-
const name = (entity as any)?.name;
|
|
15214
|
-
const type = (entity as any)?.type;
|
|
15215
|
-
if (
|
|
15216
|
-
typeof name !== "string" ||
|
|
15217
|
-
!name.trim() ||
|
|
15218
|
-
typeof type !== "string" ||
|
|
15219
|
-
!type.trim()
|
|
15220
|
-
) {
|
|
15221
|
-
continue;
|
|
15222
|
-
}
|
|
15223
|
-
const safeFacts = Array.isArray((entity as any)?.facts)
|
|
15224
|
-
? (entity as any).facts.filter((f: any) => typeof f === "string")
|
|
15225
|
-
: [];
|
|
15226
|
-
const id = await storage.writeEntity(name, type, safeFacts, {
|
|
15227
|
-
source: typeof (entity as any)?.source === "string" ? (entity as any).source : "extraction",
|
|
15228
|
-
timestamp: sourceContext?.validAt,
|
|
15229
|
-
sessionKey: sourceContext?.sessionKey,
|
|
15230
|
-
principal: sourceContext?.principal,
|
|
15231
|
-
structuredSections: Array.isArray((entity as any)?.structuredSections)
|
|
15232
|
-
? (entity as any).structuredSections
|
|
15233
|
-
: undefined,
|
|
15234
|
-
});
|
|
15235
|
-
if (id) {
|
|
15236
|
-
trackPersistedId(storage, id);
|
|
15237
|
-
recordDurableNonFactWrite();
|
|
15238
|
-
}
|
|
15239
|
-
} catch (err) {
|
|
15240
|
-
log.warn(`persistExtraction: entity write failed: ${err}`);
|
|
15241
|
-
}
|
|
15242
|
-
}
|
|
15243
|
-
|
|
15244
|
-
// Persist entity relationships (v7.0)
|
|
15245
|
-
if (
|
|
15246
|
-
resolveRecallEnhancementCapabilities(this.config).entityRelationships &&
|
|
15247
|
-
Array.isArray(result.relationships)
|
|
15248
|
-
) {
|
|
15249
|
-
for (const rel of result.relationships.slice(0, 5)) {
|
|
15250
|
-
if (!rel.source || !rel.target || !rel.label) continue;
|
|
15251
|
-
try {
|
|
15252
|
-
// Add bidirectional relationship
|
|
15253
|
-
await storage.addEntityRelationship(rel.source, {
|
|
15254
|
-
target: rel.target,
|
|
15255
|
-
label: rel.label,
|
|
15256
|
-
});
|
|
15257
|
-
recordDurableNonFactWrite();
|
|
15258
|
-
await storage.addEntityRelationship(rel.target, {
|
|
15259
|
-
target: rel.source,
|
|
15260
|
-
label: `${rel.label} (reverse)`,
|
|
15261
|
-
});
|
|
15262
|
-
recordDurableNonFactWrite();
|
|
15263
|
-
} catch (err) {
|
|
15264
|
-
log.debug(`relationship persist failed: ${err}`);
|
|
15265
|
-
}
|
|
15266
|
-
}
|
|
15267
|
-
}
|
|
15268
|
-
|
|
15269
|
-
// Persist entity activity (v7.0)
|
|
15270
|
-
if (resolveRecallEnhancementCapabilities(this.config).entityActivityLog) {
|
|
15271
|
-
const today = new Date().toISOString().slice(0, 10);
|
|
15272
|
-
for (const entity of entities) {
|
|
15273
|
-
const name = (entity as any)?.name;
|
|
15274
|
-
const type = (entity as any)?.type;
|
|
15275
|
-
if (typeof name !== "string" || typeof type !== "string") continue;
|
|
15276
|
-
try {
|
|
15277
|
-
const normalized = storage.normalizeEntityName(name, type);
|
|
15278
|
-
await storage.addEntityActivity(
|
|
15279
|
-
normalized,
|
|
15280
|
-
{ date: today, note: "Mentioned in conversation" },
|
|
15281
|
-
this.config.entityActivityLogMaxEntries,
|
|
15282
|
-
);
|
|
15283
|
-
} catch (err) {
|
|
15284
|
-
log.debug(`activity persist failed: ${err}`);
|
|
15285
|
-
}
|
|
15286
|
-
}
|
|
15287
|
-
}
|
|
15288
|
-
|
|
15289
|
-
if (profileUpdates.length > 0) {
|
|
15290
|
-
await storage.appendToProfile(profileUpdates);
|
|
15291
|
-
recordDurableNonFactWrite();
|
|
15292
|
-
}
|
|
15293
|
-
|
|
15294
|
-
// Persist questions
|
|
15295
|
-
for (const q of questions) {
|
|
15296
|
-
const id = await storage.writeQuestion(q.question, q.context, q.priority);
|
|
15297
|
-
if (id) {
|
|
15298
|
-
trackPersistedId(storage, id);
|
|
15299
|
-
recordDurableNonFactWrite();
|
|
15300
|
-
}
|
|
15301
|
-
}
|
|
15302
|
-
|
|
15303
|
-
// Persist identity reflection. This writes durable namespace-local state, so
|
|
15304
|
-
// an identity-ONLY extraction (no facts/entities/profile/questions) still
|
|
15305
|
-
// counts as a durable non-fact write for the catalog touch below (NIIly).
|
|
15306
|
-
// Only count it when the write actually succeeds (best-effort write); the
|
|
15307
|
-
// touch is recorded AFTER this so a rolled-back/failed write never touches.
|
|
15308
|
-
if (resolveRecallEnhancementCapabilities(this.config).identity && result.identityReflection) {
|
|
15309
|
-
try {
|
|
15310
|
-
await storage.appendIdentityReflection(result.identityReflection);
|
|
15311
|
-
recordDurableNonFactWrite();
|
|
15312
|
-
} catch (err) {
|
|
15313
|
-
log.debug(`identity reflection write failed: ${err}`);
|
|
15314
|
-
}
|
|
15315
|
-
}
|
|
15316
|
-
|
|
15317
|
-
// Catalog touch for durable NON-FACT outputs (NHZEZ / NIIly, codex P2). The
|
|
15318
|
-
// per-fact catalog touch (storage chokepoint #1522) above only fires inside the fact write loop, so
|
|
15319
|
-
// an extraction that persists ONLY entities, relationships, profile updates,
|
|
15320
|
-
// questions, or an identity reflection (no facts) would record durable data to
|
|
15321
|
-
// the BASE namespace's storage without ever touching the catalog — leaving that
|
|
15322
|
-
// namespace's `lastWriteAt` stale so `listNamespaces({writtenSince})` /
|
|
15323
|
-
// write-recency QMD maintenance miss the write. All of these are written to the
|
|
15324
|
-
// BASE `storage` (not the per-fact routed `targetStorage`), so we record ONE
|
|
15325
|
-
// base-namespace touch here, AFTER every non-fact write completes. Use the
|
|
15326
|
-
// KNOWN base namespace name, not a dir-decoded guess (NCQI0). One touch per
|
|
15327
|
-
// namespace per extraction — `markWrite` is idempotent, so if the fact path
|
|
15328
|
-
// already touched the base namespace this only refreshes `lastWriteAt`.
|
|
15329
|
-
// Best-effort and failure-tolerant (storage chokepoint #1522 swallows errors).
|
|
15330
|
-
if (durableNonFactWritten) {
|
|
15331
|
-
touchBaseNonFactNamespace();
|
|
15332
|
-
}
|
|
15333
|
-
|
|
15334
|
-
// Save any content-hash indexes touched during the batch.
|
|
15335
|
-
await this.saveContentHashIndexes().catch((err) =>
|
|
15336
|
-
log.warn(`content-hash index save failed: ${err}`),
|
|
15337
|
-
);
|
|
15338
|
-
|
|
15339
|
-
for (const {
|
|
15340
|
-
storage: targetStorage,
|
|
15341
|
-
events,
|
|
15342
|
-
} of behaviorSignalsByStorage.values()) {
|
|
15343
|
-
const dedupedSignals = dedupeBehaviorSignalsByMemoryAndHash(events);
|
|
15344
|
-
if (dedupedSignals.length === 0) continue;
|
|
15345
|
-
await targetStorage
|
|
15346
|
-
.appendBehaviorSignals(dedupedSignals)
|
|
15347
|
-
.catch((err) =>
|
|
15348
|
-
log.warn(`appendBehaviorSignals failed (non-fatal): ${err}`),
|
|
15349
|
-
);
|
|
15350
|
-
}
|
|
15351
|
-
|
|
15352
|
-
const dedupSuffix = dedupedCount > 0 ? ` (${dedupedCount} deduped)` : "";
|
|
15353
|
-
const gatedSuffix =
|
|
15354
|
-
importanceGatedCount > 0 ? ` (${importanceGatedCount} gated)` : "";
|
|
15355
|
-
const judgeSuffix =
|
|
15356
|
-
judgeGatedCount > 0 ? ` (${judgeGatedCount} judge-rejected)` : "";
|
|
15357
|
-
const redactionSuffix =
|
|
15358
|
-
redactionGatedCount > 0 ? ` (${redactionGatedCount} redacted)` : "";
|
|
15359
|
-
log.info(
|
|
15360
|
-
`persisted: ${facts.length - dedupedCount - importanceGatedCount - judgeGatedCount - redactionGatedCount} facts${dedupSuffix}${gatedSuffix}${judgeSuffix}${redactionSuffix}, ${entities.length} entities, ${questions.length} questions, ${profileUpdates.length} profile updates`,
|
|
15361
|
-
);
|
|
15362
|
-
|
|
15363
|
-
// Update temporal + tag indexes (v8.1) — fire-and-forget, fail-open
|
|
15364
|
-
void (async () => {
|
|
15365
|
-
if (persistedIdsByStorage.size === 0) {
|
|
15366
|
-
await this.updateTemporalTagIndexes(storage, []);
|
|
15367
|
-
return;
|
|
15368
|
-
}
|
|
15369
|
-
for (const entry of persistedIdsByStorage.values()) {
|
|
15370
|
-
await this.updateTemporalTagIndexes(entry.storage, entry.ids);
|
|
15371
|
-
}
|
|
15372
|
-
})().catch((err) =>
|
|
15373
|
-
log.debug(`temporal-index update error (non-fatal): ${err}`),
|
|
15374
|
-
);
|
|
15375
|
-
|
|
15376
|
-
// #1635: surface pending_review ids so the thread episode set excludes them.
|
|
15377
|
-
this.lastPersistExtractionPendingReviewIds = pendingReviewPersistedIds;
|
|
15378
|
-
// Return the persisted fact IDs for threading
|
|
15379
|
-
return persistedIds;
|
|
15380
12308
|
}
|
|
15381
12309
|
/**
|
|
15382
12310
|
* Append persisted ids to the thread episode set, excluding pending_review
|
|
@@ -15599,503 +12527,7 @@ export class Orchestrator {
|
|
|
15599
12527
|
merged: number;
|
|
15600
12528
|
invalidated: number;
|
|
15601
12529
|
}> {
|
|
15602
|
-
|
|
15603
|
-
log.info("running consolidation pass");
|
|
15604
|
-
let merged = 0;
|
|
15605
|
-
let invalidated = 0;
|
|
15606
|
-
// Tracks whether any consolidation memory-item action (UPDATE / MERGE /
|
|
15607
|
-
// INVALIDATE) durably rewrote memory state. A consolidation pass that only
|
|
15608
|
-
// mutates memory items (no profile/entity updates) still changes the default
|
|
15609
|
-
// namespace's data, so its catalog `lastWriteAt` must refresh too (NIBOi).
|
|
15610
|
-
let memoryItemMutated = false;
|
|
15611
|
-
|
|
15612
|
-
// Flush access tracking buffer first
|
|
15613
|
-
if (this.accessTrackingBuffer.size > 0) {
|
|
15614
|
-
await this.flushAccessTracking();
|
|
15615
|
-
}
|
|
15616
|
-
|
|
15617
|
-
let allMemories = await this.storage.readAllMemories();
|
|
15618
|
-
if (allMemories.length < 5) {
|
|
15619
|
-
return { memoriesProcessed: allMemories.length, merged, invalidated };
|
|
15620
|
-
}
|
|
15621
|
-
|
|
15622
|
-
const recent = allMemories
|
|
15623
|
-
.sort(
|
|
15624
|
-
(a, b) =>
|
|
15625
|
-
new Date(b.frontmatter.created).getTime() -
|
|
15626
|
-
new Date(a.frontmatter.created).getTime(),
|
|
15627
|
-
)
|
|
15628
|
-
.slice(0, 20);
|
|
15629
|
-
|
|
15630
|
-
const older = allMemories.sort(
|
|
15631
|
-
(a, b) =>
|
|
15632
|
-
new Date(a.frontmatter.created).getTime() -
|
|
15633
|
-
new Date(b.frontmatter.created).getTime(),
|
|
15634
|
-
);
|
|
15635
|
-
|
|
15636
|
-
const profile = await this.storage.readProfile();
|
|
15637
|
-
const result = await this.extraction.consolidate(recent, older, profile);
|
|
15638
|
-
|
|
15639
|
-
// Build a lookup map from the already-loaded corpus to avoid repeated
|
|
15640
|
-
// readAllMemories() scans inside getMemoryById for pre-action deindex reads.
|
|
15641
|
-
const memoryLookup = resolveIndexingCapabilities(this.config).queryAwareIndexing
|
|
15642
|
-
? new Map(allMemories.map((m) => [m.frontmatter.id, m]))
|
|
15643
|
-
: null;
|
|
15644
|
-
|
|
15645
|
-
for (const item of result.items) {
|
|
15646
|
-
switch (item.action) {
|
|
15647
|
-
case "INVALIDATE": {
|
|
15648
|
-
// Capture path/frontmatter before invalidation for index cleanup
|
|
15649
|
-
const toInvalidate = resolveIndexingCapabilities(this.config).queryAwareIndexing
|
|
15650
|
-
? (memoryLookup?.get(item.existingId) ?? null)
|
|
15651
|
-
: null;
|
|
15652
|
-
if (await this.storage.invalidateMemory(item.existingId)) {
|
|
15653
|
-
invalidated += 1;
|
|
15654
|
-
memoryItemMutated = true;
|
|
15655
|
-
await this.embeddingFallback.removeFromIndex(item.existingId);
|
|
15656
|
-
if (toInvalidate?.path && toInvalidate.frontmatter?.created) {
|
|
15657
|
-
deindexMemory(
|
|
15658
|
-
this.config.memoryDir,
|
|
15659
|
-
toInvalidate.path,
|
|
15660
|
-
toInvalidate.frontmatter.created,
|
|
15661
|
-
toInvalidate.frontmatter.tags ?? [],
|
|
15662
|
-
);
|
|
15663
|
-
}
|
|
15664
|
-
}
|
|
15665
|
-
break;
|
|
15666
|
-
}
|
|
15667
|
-
case "UPDATE":
|
|
15668
|
-
if (item.updatedContent) {
|
|
15669
|
-
await this.storage.updateMemory(
|
|
15670
|
-
item.existingId,
|
|
15671
|
-
item.updatedContent,
|
|
15672
|
-
{
|
|
15673
|
-
lineage: [item.existingId],
|
|
15674
|
-
},
|
|
15675
|
-
);
|
|
15676
|
-
memoryItemMutated = true;
|
|
15677
|
-
await this.indexPersistedMemory(this.storage, item.existingId);
|
|
15678
|
-
// updateMemory() only changes content/updated/lineage — path, created, and tags
|
|
15679
|
-
// are preserved, so the temporal/tag index entry is already correct; no reindex needed.
|
|
15680
|
-
}
|
|
15681
|
-
break;
|
|
15682
|
-
case "MERGE":
|
|
15683
|
-
if (item.updatedContent && item.mergeWith) {
|
|
15684
|
-
await this.storage.updateMemory(
|
|
15685
|
-
item.existingId,
|
|
15686
|
-
item.updatedContent,
|
|
15687
|
-
{
|
|
15688
|
-
supersedes: item.mergeWith,
|
|
15689
|
-
lineage: [item.existingId, item.mergeWith],
|
|
15690
|
-
},
|
|
15691
|
-
);
|
|
15692
|
-
memoryItemMutated = true;
|
|
15693
|
-
await this.indexPersistedMemory(this.storage, item.existingId);
|
|
15694
|
-
// updateMemory() only changes content/updated/supersedes/lineage — path, created, and tags
|
|
15695
|
-
// are preserved, so the temporal/tag index entry for the survivor is already correct.
|
|
15696
|
-
// Capture before invalidation for index cleanup
|
|
15697
|
-
const toMergeInvalidate = resolveIndexingCapabilities(this.config).queryAwareIndexing
|
|
15698
|
-
? (memoryLookup?.get(item.mergeWith) ?? null)
|
|
15699
|
-
: null;
|
|
15700
|
-
if (await this.storage.invalidateMemory(item.mergeWith)) {
|
|
15701
|
-
invalidated += 1;
|
|
15702
|
-
merged += 1;
|
|
15703
|
-
await this.embeddingFallback.removeFromIndex(item.mergeWith);
|
|
15704
|
-
if (
|
|
15705
|
-
toMergeInvalidate?.path &&
|
|
15706
|
-
toMergeInvalidate.frontmatter?.created
|
|
15707
|
-
) {
|
|
15708
|
-
deindexMemory(
|
|
15709
|
-
this.config.memoryDir,
|
|
15710
|
-
toMergeInvalidate.path,
|
|
15711
|
-
toMergeInvalidate.frontmatter.created,
|
|
15712
|
-
toMergeInvalidate.frontmatter.tags ?? [],
|
|
15713
|
-
);
|
|
15714
|
-
}
|
|
15715
|
-
}
|
|
15716
|
-
}
|
|
15717
|
-
break;
|
|
15718
|
-
}
|
|
15719
|
-
}
|
|
15720
|
-
|
|
15721
|
-
if (result.profileUpdates.length > 0) {
|
|
15722
|
-
await this.storage.appendToProfile(result.profileUpdates);
|
|
15723
|
-
}
|
|
15724
|
-
|
|
15725
|
-
for (const entity of result.entityUpdates) {
|
|
15726
|
-
const safeFacts = Array.isArray((entity as any)?.facts)
|
|
15727
|
-
? (entity as any).facts.filter((f: any) => typeof f === "string")
|
|
15728
|
-
: [];
|
|
15729
|
-
await this.storage.writeEntity(entity.name, entity.type, safeFacts, {
|
|
15730
|
-
source: "consolidation",
|
|
15731
|
-
structuredSections: Array.isArray((entity as any)?.structuredSections)
|
|
15732
|
-
? (entity as any).structuredSections
|
|
15733
|
-
: undefined,
|
|
15734
|
-
});
|
|
15735
|
-
}
|
|
15736
|
-
|
|
15737
|
-
// Catalog write touch accounting (issue #1499 sweep): consolidation persists
|
|
15738
|
-
// durable mutations directly to the default-namespace `this.storage`, bypassing
|
|
15739
|
-
// the extraction write path. We do NOT touch here — later maintenance steps in
|
|
15740
|
-
// this same function (entity-file merges, expired-commitment / TTL cleanup,
|
|
15741
|
-
// fact archival) can ALSO mutate the namespace on a run with no LLM outputs
|
|
15742
|
-
// (NIjwl). So we accumulate every durable mutation into `memoryItemMutated` and
|
|
15743
|
-
// record ONE consolidated touch AFTER all mutation-producing steps complete,
|
|
15744
|
-
// just before returning (rule #25: touch after the write commits). LLM
|
|
15745
|
-
// profile/entity updates and memory-item actions (UPDATE / MERGE / INVALIDATE)
|
|
15746
|
-
// count here (NIBOi).
|
|
15747
|
-
if (result.profileUpdates.length > 0 || result.entityUpdates.length > 0) {
|
|
15748
|
-
memoryItemMutated = true;
|
|
15749
|
-
}
|
|
15750
|
-
|
|
15751
|
-
// Merge fragmented entity files
|
|
15752
|
-
const entitiesMerged = await this.storage.mergeFragmentedEntities();
|
|
15753
|
-
if (entitiesMerged > 0) {
|
|
15754
|
-
memoryItemMutated = true;
|
|
15755
|
-
log.info(`merged ${entitiesMerged} fragmented entity files`);
|
|
15756
|
-
}
|
|
15757
|
-
|
|
15758
|
-
if (resolvePresentationCapabilities(this.config).entitySummary) {
|
|
15759
|
-
try {
|
|
15760
|
-
const synthesized = await this.processEntitySynthesisQueue(
|
|
15761
|
-
this.config.defaultNamespace,
|
|
15762
|
-
5,
|
|
15763
|
-
);
|
|
15764
|
-
if (synthesized > 0) {
|
|
15765
|
-
// Entity synthesis rewrites entity files — a durable namespace mutation,
|
|
15766
|
-
// so record it for the catalog touch even when it is the only change in
|
|
15767
|
-
// the pass (codex). Otherwise lastWriteAt goes stale.
|
|
15768
|
-
memoryItemMutated = true;
|
|
15769
|
-
log.info(`refreshed ${synthesized} entity syntheses`);
|
|
15770
|
-
}
|
|
15771
|
-
} catch (err) {
|
|
15772
|
-
log.debug(`entity synthesis pass failed: ${err}`);
|
|
15773
|
-
}
|
|
15774
|
-
}
|
|
15775
|
-
|
|
15776
|
-
// Clean expired commitments
|
|
15777
|
-
const deletedCommitments = await this.storage.cleanExpiredCommitments(
|
|
15778
|
-
this.config.commitmentDecayDays,
|
|
15779
|
-
);
|
|
15780
|
-
if (deletedCommitments.length > 0) {
|
|
15781
|
-
memoryItemMutated = true;
|
|
15782
|
-
log.info(`cleaned ${deletedCommitments.length} expired commitments`);
|
|
15783
|
-
if (resolveIndexingCapabilities(this.config).queryAwareIndexing) {
|
|
15784
|
-
for (const m of deletedCommitments) {
|
|
15785
|
-
deindexMemory(
|
|
15786
|
-
this.config.memoryDir,
|
|
15787
|
-
m.path,
|
|
15788
|
-
m.frontmatter.created,
|
|
15789
|
-
m.frontmatter.tags ?? [],
|
|
15790
|
-
);
|
|
15791
|
-
}
|
|
15792
|
-
}
|
|
15793
|
-
}
|
|
15794
|
-
|
|
15795
|
-
if (
|
|
15796
|
-
resolveCreationMemoryCapabilities(this.config).creationMemory &&
|
|
15797
|
-
resolveCreationMemoryCapabilities(this.config).commitmentLedger &&
|
|
15798
|
-
resolveCreationMemoryCapabilities(this.config).commitmentLifecycle
|
|
15799
|
-
) {
|
|
15800
|
-
try {
|
|
15801
|
-
const lifecycle = await applyCommitmentLedgerLifecycle({
|
|
15802
|
-
memoryDir: this.config.memoryDir,
|
|
15803
|
-
commitmentLedgerDir: this.config.commitmentLedgerDir,
|
|
15804
|
-
enabled: true,
|
|
15805
|
-
decayDays: this.config.commitmentDecayDays,
|
|
15806
|
-
});
|
|
15807
|
-
if (
|
|
15808
|
-
lifecycle.transitionedToExpired.length > 0 ||
|
|
15809
|
-
lifecycle.deletedResolved.length > 0
|
|
15810
|
-
) {
|
|
15811
|
-
memoryItemMutated = true;
|
|
15812
|
-
log.info(
|
|
15813
|
-
`commitment ledger lifecycle: expired ${lifecycle.transitionedToExpired.length}, cleaned ${lifecycle.deletedResolved.length}`,
|
|
15814
|
-
);
|
|
15815
|
-
}
|
|
15816
|
-
} catch (err) {
|
|
15817
|
-
log.debug(`commitment ledger lifecycle pass failed: ${err}`);
|
|
15818
|
-
}
|
|
15819
|
-
}
|
|
15820
|
-
|
|
15821
|
-
// Clean memories past their TTL (speculative memories auto-expire)
|
|
15822
|
-
const deletedTTL = await this.storage.cleanExpiredTTL();
|
|
15823
|
-
if (deletedTTL.length > 0) {
|
|
15824
|
-
memoryItemMutated = true;
|
|
15825
|
-
log.info(`cleaned ${deletedTTL.length} TTL-expired memories`);
|
|
15826
|
-
if (resolveIndexingCapabilities(this.config).queryAwareIndexing) {
|
|
15827
|
-
for (const m of deletedTTL) {
|
|
15828
|
-
deindexMemory(
|
|
15829
|
-
this.config.memoryDir,
|
|
15830
|
-
m.path,
|
|
15831
|
-
m.frontmatter.created,
|
|
15832
|
-
m.frontmatter.tags ?? [],
|
|
15833
|
-
);
|
|
15834
|
-
}
|
|
15835
|
-
}
|
|
15836
|
-
}
|
|
15837
|
-
|
|
15838
|
-
// v8.3 Lifecycle policy pass — deterministic promotion/decay metadata
|
|
15839
|
-
if (lifecycleCaps.lifecyclePolicy) {
|
|
15840
|
-
try {
|
|
15841
|
-
const lightSleepStartedAt = new Date().toISOString();
|
|
15842
|
-
const lifecycleCorpus = await this.storage.readAllMemories();
|
|
15843
|
-
// Lifecycle frontmatter writes count as durable mutations for the catalog
|
|
15844
|
-
// touch below (codex NR-tS), even when no other consolidation step set
|
|
15845
|
-
// memoryItemMutated.
|
|
15846
|
-
if ((await this.runLifecyclePolicyPass(lifecycleCorpus)) > 0) {
|
|
15847
|
-
memoryItemMutated = true;
|
|
15848
|
-
}
|
|
15849
|
-
await this.recordScheduledDreamsPhaseRun(
|
|
15850
|
-
"lightSleep",
|
|
15851
|
-
lifecycleCorpus.length,
|
|
15852
|
-
`scheduled lifecycle policy pass assessed ${lifecycleCorpus.length} memories`,
|
|
15853
|
-
{
|
|
15854
|
-
startedAt: lightSleepStartedAt,
|
|
15855
|
-
completedAt: new Date().toISOString(),
|
|
15856
|
-
},
|
|
15857
|
-
);
|
|
15858
|
-
} catch (err) {
|
|
15859
|
-
log.warn(`lifecycle policy pass failed (ignored): ${err}`);
|
|
15860
|
-
}
|
|
15861
|
-
}
|
|
15862
|
-
|
|
15863
|
-
// v8.3 Compression guideline learning pass (default off, fail-open).
|
|
15864
|
-
await this.runCompressionGuidelineLearningPass();
|
|
15865
|
-
|
|
15866
|
-
try {
|
|
15867
|
-
const deepSleepStartedAt = new Date().toISOString();
|
|
15868
|
-
// Tier migrations move/rewrite memory files; count them as durable
|
|
15869
|
-
// mutations for the catalog touch below (codex NThSW).
|
|
15870
|
-
const tierMigration = await this.runTierMigrationCycle(this.storage, "maintenance");
|
|
15871
|
-
if (tierMigration.migrated > 0) memoryItemMutated = true;
|
|
15872
|
-
allMemories = await this.storage.readAllMemories();
|
|
15873
|
-
|
|
15874
|
-
// Fact archival pass (v6.0) — move old, low-importance, rarely-accessed facts to archive/
|
|
15875
|
-
if (resolveRecallEnhancementCapabilities(this.config).factArchival) {
|
|
15876
|
-
const archived = await this.runFactArchival(allMemories);
|
|
15877
|
-
if (archived > 0) {
|
|
15878
|
-
memoryItemMutated = true;
|
|
15879
|
-
log.info(`archived ${archived} old low-importance facts`);
|
|
15880
|
-
}
|
|
15881
|
-
}
|
|
15882
|
-
await this.recordScheduledDreamsPhaseRun(
|
|
15883
|
-
"deepSleep",
|
|
15884
|
-
allMemories.length,
|
|
15885
|
-
`scheduled deep-sleep maintenance assessed ${allMemories.length} memories`,
|
|
15886
|
-
{
|
|
15887
|
-
startedAt: deepSleepStartedAt,
|
|
15888
|
-
completedAt: new Date().toISOString(),
|
|
15889
|
-
},
|
|
15890
|
-
);
|
|
15891
|
-
} catch (err) {
|
|
15892
|
-
log.warn(`deep-sleep maintenance pass failed (ignored): ${err}`);
|
|
15893
|
-
try {
|
|
15894
|
-
allMemories = await this.storage.readAllMemories();
|
|
15895
|
-
} catch (readErr) {
|
|
15896
|
-
log.warn(`deep-sleep maintenance recovery read failed: ${readErr}`);
|
|
15897
|
-
throw err;
|
|
15898
|
-
}
|
|
15899
|
-
}
|
|
15900
|
-
|
|
15901
|
-
// Semantic consolidation pass — find similar memories, synthesize canonical versions
|
|
15902
|
-
if (resolveConsolidationCapabilities(this.config).semanticConsolidation) {
|
|
15903
|
-
try {
|
|
15904
|
-
const stateFilePath = path.join(
|
|
15905
|
-
this.config.memoryDir,
|
|
15906
|
-
"state",
|
|
15907
|
-
"semantic-consolidation-last-run.json",
|
|
15908
|
-
);
|
|
15909
|
-
let shouldRun = true;
|
|
15910
|
-
try {
|
|
15911
|
-
const stateRaw = await readFile(stateFilePath, "utf-8");
|
|
15912
|
-
const stateData = JSON.parse(stateRaw) as { lastRunAt?: string };
|
|
15913
|
-
if (stateData.lastRunAt) {
|
|
15914
|
-
const lastRunMs = new Date(stateData.lastRunAt).getTime();
|
|
15915
|
-
const intervalMs =
|
|
15916
|
-
this.config.semanticConsolidationIntervalHours * 60 * 60 * 1000;
|
|
15917
|
-
if (Date.now() - lastRunMs < intervalMs) {
|
|
15918
|
-
shouldRun = false;
|
|
15919
|
-
log.debug(
|
|
15920
|
-
"[semantic-consolidation] skipping — not enough time since last run",
|
|
15921
|
-
);
|
|
15922
|
-
}
|
|
15923
|
-
}
|
|
15924
|
-
} catch {
|
|
15925
|
-
// No state file yet — first run
|
|
15926
|
-
}
|
|
15927
|
-
|
|
15928
|
-
if (shouldRun) {
|
|
15929
|
-
const remStartedAt = new Date().toISOString();
|
|
15930
|
-
const semResult = await this.runSemanticConsolidation();
|
|
15931
|
-
let remItemsProcessed = allMemories.length;
|
|
15932
|
-
try {
|
|
15933
|
-
allMemories = await this.storage.readAllMemories();
|
|
15934
|
-
remItemsProcessed = allMemories.length;
|
|
15935
|
-
} catch (err) {
|
|
15936
|
-
log.warn(
|
|
15937
|
-
`[semantic-consolidation] post-run telemetry refresh failed (non-fatal): ${err}`,
|
|
15938
|
-
);
|
|
15939
|
-
}
|
|
15940
|
-
await this.recordScheduledDreamsPhaseRun(
|
|
15941
|
-
"rem",
|
|
15942
|
-
remItemsProcessed,
|
|
15943
|
-
`scheduled REM consolidation found ${semResult.clustersFound} clusters`,
|
|
15944
|
-
{
|
|
15945
|
-
startedAt: remStartedAt,
|
|
15946
|
-
completedAt: new Date().toISOString(),
|
|
15947
|
-
},
|
|
15948
|
-
);
|
|
15949
|
-
if (semResult.memoriesArchived > 0) {
|
|
15950
|
-
log.info(
|
|
15951
|
-
`[semantic-consolidation] archived ${semResult.memoriesArchived} memories during maintenance`,
|
|
15952
|
-
);
|
|
15953
|
-
}
|
|
15954
|
-
// Only persist last-run timestamp if the run succeeded (had no errors or made progress)
|
|
15955
|
-
if (semResult.errors === 0 || semResult.memoriesArchived > 0) {
|
|
15956
|
-
const stateDir = path.join(this.config.memoryDir, "state");
|
|
15957
|
-
await mkdir(stateDir, { recursive: true });
|
|
15958
|
-
await writeFile(
|
|
15959
|
-
stateFilePath,
|
|
15960
|
-
JSON.stringify({ lastRunAt: new Date().toISOString() }),
|
|
15961
|
-
"utf-8",
|
|
15962
|
-
);
|
|
15963
|
-
}
|
|
15964
|
-
}
|
|
15965
|
-
} catch (err) {
|
|
15966
|
-
log.warn(
|
|
15967
|
-
`[semantic-consolidation] maintenance pass failed (non-fatal): ${err}`,
|
|
15968
|
-
);
|
|
15969
|
-
}
|
|
15970
|
-
}
|
|
15971
|
-
|
|
15972
|
-
// Auto-consolidate IDENTITY.md if it's getting large
|
|
15973
|
-
if (resolveRecallEnhancementCapabilities(this.config).identity) {
|
|
15974
|
-
await this.autoConsolidateIdentity();
|
|
15975
|
-
}
|
|
15976
|
-
|
|
15977
|
-
// Auto-consolidate profile.md if it exceeds max lines
|
|
15978
|
-
const profileSection = this.getRecallSectionEntry("profile");
|
|
15979
|
-
const profileConsolidationTriggerLines =
|
|
15980
|
-
typeof profileSection?.consolidateTriggerLines === "number"
|
|
15981
|
-
? Math.max(0, Math.floor(profileSection.consolidateTriggerLines))
|
|
15982
|
-
: undefined;
|
|
15983
|
-
const profileConsolidationTargetLines =
|
|
15984
|
-
typeof profileSection?.consolidateTargetLines === "number"
|
|
15985
|
-
? Math.max(0, Math.floor(profileSection.consolidateTargetLines))
|
|
15986
|
-
: 50;
|
|
15987
|
-
if (
|
|
15988
|
-
await this.storage.profileNeedsConsolidation(
|
|
15989
|
-
profileConsolidationTriggerLines,
|
|
15990
|
-
)
|
|
15991
|
-
) {
|
|
15992
|
-
log.info("profile.md exceeds max lines — running smart consolidation");
|
|
15993
|
-
const currentProfile = await this.storage.readProfile();
|
|
15994
|
-
if (currentProfile) {
|
|
15995
|
-
const profileResult = await this.extraction.consolidateProfile(
|
|
15996
|
-
currentProfile,
|
|
15997
|
-
profileConsolidationTargetLines,
|
|
15998
|
-
);
|
|
15999
|
-
if (profileResult) {
|
|
16000
|
-
await this.storage.writeProfile(profileResult.consolidatedProfile);
|
|
16001
|
-
// Profile consolidation rewrites profile.md — a durable namespace
|
|
16002
|
-
// mutation; record it for the catalog touch even when it is the only
|
|
16003
|
-
// change in the pass (codex). Otherwise lastWriteAt goes stale.
|
|
16004
|
-
memoryItemMutated = true;
|
|
16005
|
-
log.info(
|
|
16006
|
-
`profile.md consolidated: removed ${profileResult.removedCount} items — ${profileResult.summary}`,
|
|
16007
|
-
);
|
|
16008
|
-
}
|
|
16009
|
-
}
|
|
16010
|
-
}
|
|
16011
|
-
|
|
16012
|
-
// Memory Summarization (Phase 4A)
|
|
16013
|
-
if (resolvePipelineProcessingCapabilities(this.config).summarization) {
|
|
16014
|
-
await this.runSummarization(allMemories);
|
|
16015
|
-
}
|
|
16016
|
-
|
|
16017
|
-
// Topic Extraction (Phase 4B)
|
|
16018
|
-
if (resolvePipelineProcessingCapabilities(this.config).topicExtraction) {
|
|
16019
|
-
await this.runTopicExtraction(allMemories);
|
|
16020
|
-
}
|
|
16021
|
-
|
|
16022
|
-
const meta = await this.storage.loadMeta();
|
|
16023
|
-
meta.lastConsolidationAt = new Date().toISOString();
|
|
16024
|
-
await this.storage.saveMeta(meta);
|
|
16025
|
-
|
|
16026
|
-
// Temporal Memory Tree (v8.2) — rebuild nodes from all memories, fail-open
|
|
16027
|
-
if (lifecycleCaps.temporalMemoryTree) {
|
|
16028
|
-
try {
|
|
16029
|
-
const tmtEntries = allMemories
|
|
16030
|
-
.filter(
|
|
16031
|
-
(m) =>
|
|
16032
|
-
m.frontmatter.status !== "superseded" && m.frontmatter.status !== "archived" &&
|
|
16033
|
-
m.frontmatter.status !== "forgotten" && m.frontmatter.status !== "pending_review", // #1576: unfaithful queue items must not feed TMT clusters
|
|
16034
|
-
)
|
|
16035
|
-
.map((m) => ({
|
|
16036
|
-
path: m.path,
|
|
16037
|
-
id: m.frontmatter.id,
|
|
16038
|
-
created: m.frontmatter.created,
|
|
16039
|
-
content: m.content,
|
|
16040
|
-
}));
|
|
16041
|
-
await this.tmtBuilder.maybeRebuildNodes(
|
|
16042
|
-
tmtEntries,
|
|
16043
|
-
async (texts, level) => {
|
|
16044
|
-
const prompt = `You are a memory archivist. Summarize the following ${level}-level memories into 3–5 sentences, preserving key facts, decisions, and preferences.\n\n${texts.map((t, i) => `[${i + 1}] ${t}`).join("\n\n")}`;
|
|
16045
|
-
const response = await this.fastChatCompletion(
|
|
16046
|
-
[
|
|
16047
|
-
{
|
|
16048
|
-
role: "system",
|
|
16049
|
-
content:
|
|
16050
|
-
"Respond with a 3–5 sentence narrative summary. No JSON, just plain prose.",
|
|
16051
|
-
},
|
|
16052
|
-
{ role: "user", content: prompt },
|
|
16053
|
-
],
|
|
16054
|
-
{
|
|
16055
|
-
temperature: 0.3,
|
|
16056
|
-
maxTokens: this.config.tmtSummaryMaxTokens,
|
|
16057
|
-
operation: "tmt_summary",
|
|
16058
|
-
priority: "background",
|
|
16059
|
-
},
|
|
16060
|
-
);
|
|
16061
|
-
return response?.content?.trim() || texts.slice(0, 3).join(" ");
|
|
16062
|
-
},
|
|
16063
|
-
);
|
|
16064
|
-
} catch (err) {
|
|
16065
|
-
log.warn(`tmt: consolidation hook failed (ignored): ${err}`);
|
|
16066
|
-
}
|
|
16067
|
-
}
|
|
16068
|
-
|
|
16069
|
-
if (this.consolidationObservers.size > 0) {
|
|
16070
|
-
const observation: ConsolidationObservation = {
|
|
16071
|
-
runAt: new Date().toISOString(),
|
|
16072
|
-
recentMemories: recent,
|
|
16073
|
-
existingMemories: older.slice(-50),
|
|
16074
|
-
profile,
|
|
16075
|
-
result,
|
|
16076
|
-
merged,
|
|
16077
|
-
invalidated,
|
|
16078
|
-
};
|
|
16079
|
-
for (const observer of this.consolidationObservers) {
|
|
16080
|
-
try {
|
|
16081
|
-
await observer(observation);
|
|
16082
|
-
} catch (err) {
|
|
16083
|
-
log.warn(`consolidation observer failed (ignored): ${err}`);
|
|
16084
|
-
}
|
|
16085
|
-
}
|
|
16086
|
-
}
|
|
16087
|
-
|
|
16088
|
-
// Consolidated catalog write touch — belt-and-suspenders for cleanup-only
|
|
16089
|
-
// passes that mutate the store via delete-only paths (entity merges, TTL
|
|
16090
|
-
// cleanup) without triggering the storage chokepoint's post-write hook.
|
|
16091
|
-
// Gated on memoryItemMutated (set by every durable mutation including cleanup-only passes)
|
|
16092
|
-
// Best-effort and failure-tolerant.
|
|
16093
|
-
if (memoryItemMutated) {
|
|
16094
|
-
this.storageRouter.recordWrite(this.config.defaultNamespace, this.storage.dir);
|
|
16095
|
-
}
|
|
16096
|
-
|
|
16097
|
-
log.info("consolidation complete");
|
|
16098
|
-
return { memoriesProcessed: allMemories.length, merged, invalidated };
|
|
12530
|
+
return this.consolidationRunCoordinator.run();
|
|
16099
12531
|
}
|
|
16100
12532
|
|
|
16101
12533
|
async optimizeCompressionGuidelines(options?: {
|
|
@@ -16114,36 +12546,10 @@ export class Orchestrator {
|
|
|
16114
12546
|
|
|
16115
12547
|
// Issue #1526 (seam 4): compression-guideline learning moved to
|
|
16116
12548
|
// CompressionGuidelineCoordinator. Thin delegation keeps the
|
|
16117
|
-
// consolidation-loop hook + recall-section hook call sites stable.
|
|
16118
|
-
private async runCompressionGuidelineLearningPass(): Promise<void> {
|
|
16119
|
-
return this.compressionGuidelineCoordinator.runCompressionGuidelineLearningPass();
|
|
16120
|
-
}
|
|
16121
|
-
|
|
16122
12549
|
private async buildCompressionGuidelineRecallSection(): Promise<string | null> {
|
|
16123
12550
|
return this.compressionGuidelineCoordinator.buildCompressionGuidelineRecallSection();
|
|
16124
12551
|
}
|
|
16125
12552
|
|
|
16126
|
-
private async recordScheduledDreamsPhaseRun(
|
|
16127
|
-
phase: "lightSleep" | "rem" | "deepSleep",
|
|
16128
|
-
itemsProcessed: number,
|
|
16129
|
-
notes: string,
|
|
16130
|
-
timing: { startedAt?: string; completedAt?: string } = {},
|
|
16131
|
-
): Promise<void> {
|
|
16132
|
-
try {
|
|
16133
|
-
const { recordDreamsPhaseRun } = await import("./maintenance/dreams-ledger.js");
|
|
16134
|
-
await recordDreamsPhaseRun({
|
|
16135
|
-
memoryDir: this.storage.dir,
|
|
16136
|
-
phase,
|
|
16137
|
-
trigger: "scheduled",
|
|
16138
|
-
itemsProcessed,
|
|
16139
|
-
notes,
|
|
16140
|
-
startedAt: timing.startedAt,
|
|
16141
|
-
completedAt: timing.completedAt,
|
|
16142
|
-
});
|
|
16143
|
-
} catch (error) {
|
|
16144
|
-
log.debug(`dreams ledger scheduled ${phase} write failed (non-fatal): ${error}`);
|
|
16145
|
-
}
|
|
16146
|
-
}
|
|
16147
12553
|
|
|
16148
12554
|
async runLifecyclePolicyNow(storage: StorageManager = this.storage): Promise<{ memoriesAssessed: number }> {
|
|
16149
12555
|
const lifecycleCorpus = await storage.readAllMemories();
|
|
@@ -16159,21 +12565,6 @@ export class Orchestrator {
|
|
|
16159
12565
|
): Promise<number> {
|
|
16160
12566
|
return this.lifecyclePolicyCoordinator.runLifecyclePolicyPass(allMemories, storage);
|
|
16161
12567
|
}
|
|
16162
|
-
private async runFactArchival(
|
|
16163
|
-
allMemories: import("./types.js").MemoryFile[],
|
|
16164
|
-
): Promise<number> {
|
|
16165
|
-
return this.lifecyclePolicyCoordinator.runFactArchival(allMemories);
|
|
16166
|
-
}
|
|
16167
|
-
private async runSummarization(
|
|
16168
|
-
allMemories: import("./types.js").MemoryFile[],
|
|
16169
|
-
): Promise<void> {
|
|
16170
|
-
return this.lifecyclePolicyCoordinator.runSummarization(allMemories);
|
|
16171
|
-
}
|
|
16172
|
-
private async runTopicExtraction(
|
|
16173
|
-
allMemories: import("./types.js").MemoryFile[],
|
|
16174
|
-
): Promise<void> {
|
|
16175
|
-
return this.lifecyclePolicyCoordinator.runTopicExtraction(allMemories);
|
|
16176
|
-
}
|
|
16177
12568
|
/** Threshold (bytes) at which IDENTITY.md reflections get auto-consolidated */
|
|
16178
12569
|
private static readonly IDENTITY_CONSOLIDATE_THRESHOLD = 8_000;
|
|
16179
12570
|
|