@remnic/core 9.35.1 → 9.35.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.d.ts +5 -5
- package/dist/access-admin-ops-surface.js +19 -18
- package/dist/access-authorization-probe.d.ts +5 -5
- package/dist/access-authorization-probe.js +20 -19
- package/dist/access-boundary.d.ts +5 -5
- package/dist/access-boundary.js +19 -18
- package/dist/access-cli.js +43 -42
- package/dist/access-cli.js.map +1 -1
- package/dist/access-extraction-force-flush.d.ts +5 -5
- package/dist/access-extraction-force-flush.js +19 -18
- package/dist/access-http-lcm-compaction.d.ts +5 -5
- package/dist/access-http-lifecycle-flush.d.ts +5 -5
- package/dist/access-http.d.ts +5 -5
- package/dist/access-http.js +24 -23
- package/dist/access-identity-continuity-surface.d.ts +4 -4
- package/dist/access-identity-continuity-surface.js +19 -18
- package/dist/access-lcm-surface.d.ts +5 -5
- package/dist/access-lcm-surface.js +19 -18
- package/dist/access-mcp.d.ts +5 -5
- package/dist/access-mcp.js +22 -21
- package/dist/access-namespace-preflight.js +19 -18
- package/dist/access-observe-write-surface.d.ts +5 -5
- package/dist/access-observe-write-surface.js +19 -18
- package/dist/access-operations-batch.js +20 -19
- package/dist/access-operations.d.ts +8 -8
- package/dist/access-operations.js +21 -20
- package/dist/access-recall-concurrency.d.ts +5 -5
- package/dist/access-recall-concurrency.js +19 -18
- package/dist/access-recall-response.d.ts +5 -5
- package/dist/access-recall-response.js +19 -18
- package/dist/access-recall-surface.d.ts +5 -5
- package/dist/access-recall-surface.js +19 -18
- package/dist/access-schema.d.ts +88 -88
- package/dist/{access-service-D9zIBlpp.d.ts → access-service-C2WYT1pO.d.ts} +3 -3
- package/dist/access-service-helpers.d.ts +5 -5
- package/dist/access-service.d.ts +5 -5
- package/dist/access-service.js +19 -18
- package/dist/access-surface-catalog.d.ts +5 -5
- package/dist/access-wearables-meetings-surface.d.ts +2 -2
- package/dist/bootstrap.d.ts +4 -4
- package/dist/briefing.d.ts +1 -1
- package/dist/briefing.js +10 -9
- package/dist/buffer.d.ts +1 -1
- package/dist/{capsule-merge-2H2IUTBT.js → capsule-merge-IDWJPCWK.js} +77 -8
- package/dist/capsule-merge-IDWJPCWK.js.map +1 -0
- package/dist/causal-consolidation.js +11 -10
- package/dist/causal-consolidation.js.map +1 -1
- package/dist/{chunk-65ZPVYTA.js → chunk-3CE77VAP.js} +55 -5
- package/dist/chunk-3CE77VAP.js.map +1 -0
- package/dist/chunk-4J7G5SY4.js +338 -0
- package/dist/chunk-4J7G5SY4.js.map +1 -0
- package/dist/{chunk-EO7X2LDK.js → chunk-4TAD3B56.js} +2 -2
- package/dist/{chunk-QHYB4MJV.js → chunk-5ZGHUWTT.js} +2 -2
- package/dist/{chunk-TCIWHK3N.js → chunk-6D5D5A4M.js} +5 -5
- package/dist/{chunk-BIXWNHIL.js → chunk-6TF4M2LB.js} +5 -5
- package/dist/{chunk-2NUC473W.js → chunk-AKELMJO2.js} +220 -48
- package/dist/{chunk-2NUC473W.js.map → chunk-AKELMJO2.js.map} +1 -1
- package/dist/{chunk-FBUE6PD3.js → chunk-BEQOZOYZ.js} +21 -4
- package/dist/chunk-BEQOZOYZ.js.map +1 -0
- package/dist/{chunk-SA3WJJF2.js → chunk-BGLDKAYY.js} +2 -2
- package/dist/{chunk-OPRP6Q4Z.js → chunk-DKKZ6SO2.js} +3 -3
- package/dist/{chunk-6TAUHL33.js → chunk-DLVPUX7U.js} +10 -2
- package/dist/chunk-DLVPUX7U.js.map +1 -0
- package/dist/{chunk-QNWXCJ76.js → chunk-ENTCVTQR.js} +18 -1
- package/dist/{chunk-QNWXCJ76.js.map → chunk-ENTCVTQR.js.map} +1 -1
- package/dist/{chunk-AEPYA55F.js → chunk-FE7MUX7M.js} +2 -2
- package/dist/{chunk-NQUF7XKR.js → chunk-FT4RF53O.js} +2 -2
- package/dist/{chunk-GHR7WHZX.js → chunk-GORHSN5C.js} +2 -2
- package/dist/{chunk-7TEZUUGM.js → chunk-HUS2XN3T.js} +2 -2
- package/dist/{chunk-XWDZQIYU.js → chunk-IJAQCBGV.js} +2 -2
- package/dist/{chunk-TJLLYV2X.js → chunk-IMEXE4XS.js} +7 -7
- package/dist/{chunk-ZCPP5OW6.js → chunk-JQSBRLJ6.js} +31 -27
- package/dist/chunk-JQSBRLJ6.js.map +1 -0
- package/dist/{chunk-IZXTZ7BF.js → chunk-JYLN6KHH.js} +6 -6
- package/dist/{chunk-HEVWWPTL.js → chunk-LH3KXIBJ.js} +17 -17
- package/dist/{chunk-EH5U2YCX.js → chunk-M3DBB2ND.js} +2 -2
- package/dist/{chunk-XABS455Y.js → chunk-NBAEGFXT.js} +5 -5
- package/dist/{chunk-DP7ZV2II.js → chunk-OA6PIPDE.js} +2 -2
- package/dist/{chunk-T4HH5Q3U.js → chunk-OJRLTZQM.js} +3 -3
- package/dist/{chunk-7ZHGPX56.js → chunk-PRP3M5Y3.js} +2 -2
- package/dist/{chunk-Y2ZD2RVG.js → chunk-PYTUBWAS.js} +2 -2
- package/dist/{chunk-4TGZCYHH.js → chunk-Q3CVY7LV.js} +1954 -1642
- package/dist/chunk-Q3CVY7LV.js.map +1 -0
- package/dist/{chunk-PH3ZWL3W.js → chunk-Q53RLEW7.js} +2 -2
- package/dist/{chunk-Y6OWJ3V5.js → chunk-RY2XZTFN.js} +2 -2
- package/dist/{chunk-2SHIWGY7.js → chunk-RY7CHVGE.js} +2 -2
- package/dist/{chunk-7VUAG236.js → chunk-T3FVAHM7.js} +4 -4
- package/dist/{chunk-2LTDWSR6.js → chunk-TSH56VPU.js} +2 -2
- package/dist/{chunk-J6SLNQRO.js → chunk-UBWOCQ3I.js} +6 -2
- package/dist/chunk-UBWOCQ3I.js.map +1 -0
- package/dist/{chunk-5FHE4JI7.js → chunk-VEUVON5I.js} +2 -2
- package/dist/{chunk-KEWUNL7P.js → chunk-WET4M3R6.js} +2 -2
- package/dist/chunk-WKDFJXW5.js +158 -0
- package/dist/chunk-WKDFJXW5.js.map +1 -0
- package/dist/{chunk-SLPLPGJQ.js → chunk-WL2S2UBB.js} +5 -5
- package/dist/{chunk-AHZYNPLX.js → chunk-YVMVVE4B.js} +2 -2
- package/dist/{chunk-5HTAJCTO.js → chunk-ZB3YODID.js} +15 -15
- package/dist/{cli-mFX-QQvx.d.ts → cli-jkmzdT8P.d.ts} +3 -3
- package/dist/cli.d.ts +6 -6
- package/dist/cli.js +34 -33
- package/dist/compounding/engine.d.ts +1 -1
- package/dist/compounding/engine.js +10 -9
- package/dist/connectors/codex-materialize-runner.js +10 -9
- package/dist/connectors/index.js +10 -9
- package/dist/consolidation-provenance-check.d.ts +1 -1
- package/dist/consolidation-provenance-check.js +8 -2
- package/dist/consolidation-undo.d.ts +1 -1
- package/dist/consolidation-undo.js +10 -1
- package/dist/consolidation-undo.js.map +1 -1
- package/dist/contradiction/index.d.ts +1 -1
- package/dist/corpus-watermark.js +12 -11
- package/dist/entity-retrieval.d.ts +1 -1
- package/dist/entity-retrieval.js +10 -9
- package/dist/explicit-capture.d.ts +4 -4
- package/dist/explicit-capture.js +6 -3
- package/dist/extraction.js +13 -12
- package/dist/{forget-4UY2EOKH.js → forget-DNAJ66ZD.js} +2 -2
- package/dist/index.d.ts +398 -398
- package/dist/index.js +43 -42
- package/dist/local-llm-helpers.d.ts +63 -1
- package/dist/local-llm-helpers.js +10 -1
- package/dist/local-llm.d.ts +4 -1
- package/dist/local-llm.js +2 -2
- package/dist/maintenance/memory-governance.js +10 -9
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.d.ts +1 -1
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +10 -9
- package/dist/maintenance/rebuild-memory-projection.d.ts +1 -1
- package/dist/maintenance/rebuild-memory-projection.js +11 -10
- package/dist/{maintenance-70Wq118b.d.ts → maintenance-Ha9Dg93M.d.ts} +1 -1
- package/dist/mcp-memory-inspector-app.d.ts +5 -5
- package/dist/memory-worth-outcomes.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +1 -1
- package/dist/namespaces/migrate.js +11 -10
- package/dist/namespaces/storage.d.ts +1 -1
- package/dist/namespaces/storage.js +10 -9
- package/dist/offline-sync-impression-drain.js +3 -3
- package/dist/operator-doctor-corpus.js +13 -12
- package/dist/operator-toolkit.d.ts +1 -1
- package/dist/operator-toolkit.js +17 -16
- package/dist/orchestration/compression-guideline-coordinator.d.ts +1 -1
- package/dist/orchestration/maintenance.d.ts +2 -2
- package/dist/orchestration/maintenance.js +15 -14
- package/dist/{orchestrator-BH8LT-NX.d.ts → orchestrator-DV_u9bfx.d.ts} +3 -3
- package/dist/orchestrator.d.ts +4 -4
- package/dist/orchestrator.js +43 -42
- package/dist/page-versioning.js +7 -1
- package/dist/schemas.d.ts +76 -76
- package/dist/semantic-consolidation.js +11 -10
- package/dist/semantic-rule-promotion.js +10 -9
- package/dist/semantic-rule-verifier.js +10 -9
- package/dist/{service-CrxXHz5T.d.ts → service-JxXXY-Eh.d.ts} +1 -1
- package/dist/shared-context/manager.d.ts +8 -8
- package/dist/source-agent-qualifier.js +10 -9
- package/dist/{storage-CyWa1Jm6.d.ts → storage-BbH4HDUS.d.ts} +30 -26
- package/dist/storage.d.ts +1 -1
- package/dist/storage.js +9 -8
- package/dist/summarizer.js +3 -3
- package/dist/temporal-supersession.d.ts +1 -1
- package/dist/tier-migration.d.ts +1 -1
- package/dist/transfer/capsule-import.js +8 -2
- package/dist/transfer/types.d.ts +66 -66
- package/dist/verified-recall.js +10 -9
- package/package.json +2 -2
- package/src/binary-lifecycle/pipeline.ts +52 -1
- package/src/consolidation-undo.ts +4 -0
- package/src/curation/index.ts +41 -5
- package/src/lifecycle/tombstones.ts +40 -0
- package/src/local-llm-helpers.ts +171 -0
- package/src/local-llm.ts +34 -30
- package/src/maintenance/memory-governance.ts +4 -0
- package/src/page-versioning.ts +16 -1
- package/src/review/index.ts +97 -13
- package/src/spaces/index.ts +127 -6
- package/src/storage/citation-hash-source.ts +97 -0
- package/src/storage/entity-canonical-id-lock.ts +33 -0
- package/src/storage/entity-canonical-id-migration-adapter.ts +8 -4
- package/src/storage/entity-canonical-id-migration.ts +129 -77
- package/src/storage/entity-canonical-id-references.ts +557 -0
- package/src/storage/entity-ref-repair.ts +171 -0
- package/src/storage/entity-store.ts +163 -2
- package/src/storage/tombstone-blocked-capture-sync.ts +18 -0
- package/src/storage.ts +363 -396
- package/src/transfer/capsule-import.ts +91 -3
- package/src/transfer/capsule-merge.ts +128 -10
- package/dist/capsule-merge-2H2IUTBT.js.map +0 -1
- package/dist/chunk-4RNZMRAV.js +0 -60
- package/dist/chunk-4RNZMRAV.js.map +0 -1
- package/dist/chunk-4TGZCYHH.js.map +0 -1
- package/dist/chunk-65ZPVYTA.js.map +0 -1
- package/dist/chunk-6TAUHL33.js.map +0 -1
- package/dist/chunk-FBUE6PD3.js.map +0 -1
- package/dist/chunk-J6SLNQRO.js.map +0 -1
- package/dist/chunk-ZCPP5OW6.js.map +0 -1
- /package/dist/{chunk-EO7X2LDK.js.map → chunk-4TAD3B56.js.map} +0 -0
- /package/dist/{chunk-QHYB4MJV.js.map → chunk-5ZGHUWTT.js.map} +0 -0
- /package/dist/{chunk-TCIWHK3N.js.map → chunk-6D5D5A4M.js.map} +0 -0
- /package/dist/{chunk-BIXWNHIL.js.map → chunk-6TF4M2LB.js.map} +0 -0
- /package/dist/{chunk-SA3WJJF2.js.map → chunk-BGLDKAYY.js.map} +0 -0
- /package/dist/{chunk-OPRP6Q4Z.js.map → chunk-DKKZ6SO2.js.map} +0 -0
- /package/dist/{chunk-AEPYA55F.js.map → chunk-FE7MUX7M.js.map} +0 -0
- /package/dist/{chunk-NQUF7XKR.js.map → chunk-FT4RF53O.js.map} +0 -0
- /package/dist/{chunk-GHR7WHZX.js.map → chunk-GORHSN5C.js.map} +0 -0
- /package/dist/{chunk-7TEZUUGM.js.map → chunk-HUS2XN3T.js.map} +0 -0
- /package/dist/{chunk-XWDZQIYU.js.map → chunk-IJAQCBGV.js.map} +0 -0
- /package/dist/{chunk-TJLLYV2X.js.map → chunk-IMEXE4XS.js.map} +0 -0
- /package/dist/{chunk-IZXTZ7BF.js.map → chunk-JYLN6KHH.js.map} +0 -0
- /package/dist/{chunk-HEVWWPTL.js.map → chunk-LH3KXIBJ.js.map} +0 -0
- /package/dist/{chunk-EH5U2YCX.js.map → chunk-M3DBB2ND.js.map} +0 -0
- /package/dist/{chunk-XABS455Y.js.map → chunk-NBAEGFXT.js.map} +0 -0
- /package/dist/{chunk-DP7ZV2II.js.map → chunk-OA6PIPDE.js.map} +0 -0
- /package/dist/{chunk-T4HH5Q3U.js.map → chunk-OJRLTZQM.js.map} +0 -0
- /package/dist/{chunk-7ZHGPX56.js.map → chunk-PRP3M5Y3.js.map} +0 -0
- /package/dist/{chunk-Y2ZD2RVG.js.map → chunk-PYTUBWAS.js.map} +0 -0
- /package/dist/{chunk-PH3ZWL3W.js.map → chunk-Q53RLEW7.js.map} +0 -0
- /package/dist/{chunk-Y6OWJ3V5.js.map → chunk-RY2XZTFN.js.map} +0 -0
- /package/dist/{chunk-2SHIWGY7.js.map → chunk-RY7CHVGE.js.map} +0 -0
- /package/dist/{chunk-7VUAG236.js.map → chunk-T3FVAHM7.js.map} +0 -0
- /package/dist/{chunk-2LTDWSR6.js.map → chunk-TSH56VPU.js.map} +0 -0
- /package/dist/{chunk-5FHE4JI7.js.map → chunk-VEUVON5I.js.map} +0 -0
- /package/dist/{chunk-KEWUNL7P.js.map → chunk-WET4M3R6.js.map} +0 -0
- /package/dist/{chunk-SLPLPGJQ.js.map → chunk-WL2S2UBB.js.map} +0 -0
- /package/dist/{chunk-AHZYNPLX.js.map → chunk-YVMVVE4B.js.map} +0 -0
- /package/dist/{chunk-5HTAJCTO.js.map → chunk-ZB3YODID.js.map} +0 -0
- /package/dist/{forget-4UY2EOKH.js.map → forget-DNAJ66ZD.js.map} +0 -0
package/src/spaces/index.ts
CHANGED
|
@@ -14,6 +14,17 @@ import fs from "node:fs";
|
|
|
14
14
|
import path from "node:path";
|
|
15
15
|
import { readEnvVar, resolveHomeDir } from "../runtime/env.js";
|
|
16
16
|
import { bumpMemoryCorpusVersionForDir } from "../memory-corpus-version.js";
|
|
17
|
+
import {
|
|
18
|
+
HistoricalEntityCanonicalIdCache,
|
|
19
|
+
canonicalizeEntityRefFrontmatter,
|
|
20
|
+
classifyEntityRefWritePath,
|
|
21
|
+
readEntityRefFromFrontmatter,
|
|
22
|
+
repairContentAfterJournalMoveSync,
|
|
23
|
+
requestEntityCanonicalIdReconcileSync,
|
|
24
|
+
} from "../storage/entity-canonical-id-references.js";
|
|
25
|
+
|
|
26
|
+
// Write-boundary journal cache for promoted copies (issue #2213).
|
|
27
|
+
const historicalIdCache = new HistoricalEntityCanonicalIdCache();
|
|
17
28
|
|
|
18
29
|
// ── Types ────────────────────────────────────────────────────────────────────
|
|
19
30
|
|
|
@@ -862,6 +873,29 @@ function copyMemories(
|
|
|
862
873
|
const targetRoot = fs.realpathSync(targetDir);
|
|
863
874
|
|
|
864
875
|
const sourceFiles = walkMd(sourceRoot);
|
|
876
|
+
// Entity pages in THIS copy batch can re-establish collisions the target's
|
|
877
|
+
// journal already resolved (Codex P1, round 19; mirrors capsule-merge):
|
|
878
|
+
// once the next migration parks the mapping, a memory canonicalized onto
|
|
879
|
+
// the target's canonical claimant is permanently misattributed. Memories
|
|
880
|
+
// whose ref targets a batch-supplied entity id copy byte-faithful behind
|
|
881
|
+
// the reconcile marker instead.
|
|
882
|
+
const sourceEntityIds = new Set(
|
|
883
|
+
sourceFiles
|
|
884
|
+
.filter((p) => {
|
|
885
|
+
const rp = safeRealpath(p);
|
|
886
|
+
if (!rp || !isPathInsideRoot(rp, sourceRoot)) return false;
|
|
887
|
+
const rel = path.relative(sourceRoot, rp);
|
|
888
|
+
const mapped = path.resolve(targetRoot, rel);
|
|
889
|
+
if (!isPathInsideRoot(mapped, targetRoot)) return false;
|
|
890
|
+
return classifyEntityRefWritePath(targetRoot, mapped) === "entity";
|
|
891
|
+
})
|
|
892
|
+
.map((p) => path.basename(p, ".md")),
|
|
893
|
+
);
|
|
894
|
+
// The reconcile marker is published ONCE, after every record landed
|
|
895
|
+
// (Codex P1, round 21): a per-record marker could be consumed by a peer
|
|
896
|
+
// between a legacy-preserving memory write and its colliding entity page,
|
|
897
|
+
// canonicalizing the memory onto the wrong claimant irreversibly.
|
|
898
|
+
let reconcileNeeded = false;
|
|
865
899
|
for (const sourcePath of sourceFiles) {
|
|
866
900
|
const sourceRealPath = safeRealpath(sourcePath);
|
|
867
901
|
if (!sourceRealPath || !isPathInsideRoot(sourceRealPath, sourceRoot)) {
|
|
@@ -881,6 +915,21 @@ function copyMemories(
|
|
|
881
915
|
continue;
|
|
882
916
|
}
|
|
883
917
|
|
|
918
|
+
// The target's migration journal is a write boundary for promoted copies
|
|
919
|
+
// too (issue #2213): a pre-migration source memory can carry a legacy
|
|
920
|
+
// entityRef the target already renamed, and corpus bumps no longer
|
|
921
|
+
// re-trigger the migration. Read per record so a mid-batch journal change
|
|
922
|
+
// applies to later copies. Only memory-tier records canonicalize —
|
|
923
|
+
// entities/ pages carry relationship targets (reconcile-marker territory,
|
|
924
|
+
// below) and everything else copies byte-faithful.
|
|
925
|
+
const targetKind = classifyEntityRefWritePath(targetRoot, targetPath);
|
|
926
|
+
const idsAtCompare = historicalIdCache.get(path.join(targetRoot, "state"));
|
|
927
|
+
const sourceRef = targetKind === "memory" ? readEntityRefFromFrontmatter(content) : null;
|
|
928
|
+
const refHasSourceClaimant = sourceRef !== null && sourceEntityIds.has(sourceRef);
|
|
929
|
+
const canonicalContent =
|
|
930
|
+
targetKind === "memory" && !refHasSourceClaimant
|
|
931
|
+
? canonicalizeEntityRefFrontmatter(content, idsAtCompare)
|
|
932
|
+
: content;
|
|
884
933
|
const sourceHash = hashContent(content);
|
|
885
934
|
|
|
886
935
|
// Filter by IDs if specified
|
|
@@ -910,7 +959,21 @@ function copyMemories(
|
|
|
910
959
|
const targetContent = fs.readFileSync(targetPath, "utf8");
|
|
911
960
|
const targetHash = hashContent(targetContent);
|
|
912
961
|
|
|
913
|
-
|
|
962
|
+
// A target already holding the CANONICALIZED form is the same memory —
|
|
963
|
+
// done, and re-promoting after a ref rewrite must not report a conflict.
|
|
964
|
+
// The verdict is only valid for the snapshot it was computed under
|
|
965
|
+
// (Codex P1): a mapping parked/removed mid-compare would leave the
|
|
966
|
+
// target's formerly-canonical ref with nothing left to repair it — a
|
|
967
|
+
// moved journal falls through to the write path, which resolves at the
|
|
968
|
+
// write and repairs after it.
|
|
969
|
+
if (
|
|
970
|
+
hashContent(canonicalContent) === targetHash
|
|
971
|
+
&& historicalIdCache.get(path.join(targetRoot, "state")) === idsAtCompare
|
|
972
|
+
) {
|
|
973
|
+
skipped++;
|
|
974
|
+
continue;
|
|
975
|
+
}
|
|
976
|
+
if (hashContent(canonicalContent) !== targetHash && sourceHash !== targetHash) {
|
|
914
977
|
conflicts.push({
|
|
915
978
|
memoryId: parseSimpleFrontmatter(content)?.id ?? relativePath,
|
|
916
979
|
sourcePath,
|
|
@@ -921,10 +984,10 @@ function copyMemories(
|
|
|
921
984
|
});
|
|
922
985
|
continue;
|
|
923
986
|
}
|
|
924
|
-
|
|
925
|
-
//
|
|
926
|
-
|
|
927
|
-
|
|
987
|
+
// Fall through and rewrite in place: the target matches the RAW source
|
|
988
|
+
// while a mapping applies (a skip would leave the legacy entityRef on
|
|
989
|
+
// disk forever), or it matched the canonical form under a journal that
|
|
990
|
+
// moved mid-compare (the write path re-resolves and repairs) (#2213).
|
|
928
991
|
}
|
|
929
992
|
|
|
930
993
|
// Copy file
|
|
@@ -934,7 +997,59 @@ function copyMemories(
|
|
|
934
997
|
skipped++;
|
|
935
998
|
continue;
|
|
936
999
|
}
|
|
937
|
-
|
|
1000
|
+
const stateDir = path.join(targetRoot, "state");
|
|
1001
|
+
if (targetKind === "memory" && !refHasSourceClaimant) {
|
|
1002
|
+
// Re-resolve at the write and REPAIR afterwards: a peer process can
|
|
1003
|
+
// publish the journal at any point in this loop, and the corpus bump
|
|
1004
|
+
// below no longer re-triggers the migration.
|
|
1005
|
+
const idsAtWrite = historicalIdCache.get(stateDir);
|
|
1006
|
+
const written = canonicalizeEntityRefFrontmatter(content, idsAtWrite);
|
|
1007
|
+
// Snapshot the prior target: a repair failure must restore it (or
|
|
1008
|
+
// remove a newly created file) before propagating — a reported-failed
|
|
1009
|
+
// copy must not replace the prior claimant's bytes (AGENTS.md §14).
|
|
1010
|
+
let priorTarget: string | null = null;
|
|
1011
|
+
try {
|
|
1012
|
+
priorTarget = fs.readFileSync(targetPath, "utf8");
|
|
1013
|
+
} catch {
|
|
1014
|
+
priorTarget = null;
|
|
1015
|
+
}
|
|
1016
|
+
fs.writeFileSync(targetPath, written);
|
|
1017
|
+
try {
|
|
1018
|
+
repairContentAfterJournalMoveSync({
|
|
1019
|
+
stateDir,
|
|
1020
|
+
cache: historicalIdCache,
|
|
1021
|
+
idsAtWrite,
|
|
1022
|
+
rawContent: content,
|
|
1023
|
+
lastWritten: written,
|
|
1024
|
+
rewrite: (next) => fs.writeFileSync(targetPath, next),
|
|
1025
|
+
});
|
|
1026
|
+
} catch (err) {
|
|
1027
|
+
try {
|
|
1028
|
+
if (priorTarget === null) {
|
|
1029
|
+
fs.unlinkSync(targetPath);
|
|
1030
|
+
} else {
|
|
1031
|
+
fs.writeFileSync(targetPath, priorTarget);
|
|
1032
|
+
}
|
|
1033
|
+
} catch {
|
|
1034
|
+
// Best effort — the retryable repair error is the primary signal.
|
|
1035
|
+
}
|
|
1036
|
+
throw err;
|
|
1037
|
+
}
|
|
1038
|
+
} else if (targetKind === "memory") {
|
|
1039
|
+
// Ref targets a batch-supplied entity page (potential collision
|
|
1040
|
+
// claimant): keep the legacy bytes; the DEFERRED marker below lets the
|
|
1041
|
+
// migration settle the collision and rewrite per its verdict (Codex P1).
|
|
1042
|
+
fs.writeFileSync(targetPath, content);
|
|
1043
|
+
reconcileNeeded = true;
|
|
1044
|
+
} else {
|
|
1045
|
+
fs.writeFileSync(targetPath, content);
|
|
1046
|
+
if (targetKind === "entity") {
|
|
1047
|
+
// Entity pages carry relationship TARGETS only rewriteRelationshipTargets
|
|
1048
|
+
// understands — copy bytes faithfully behind the deferred reconcile
|
|
1049
|
+
// pass (issue #2213).
|
|
1050
|
+
reconcileNeeded = true;
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
938
1053
|
merged++;
|
|
939
1054
|
// Bump the target's corpus sentinel per write (out-of-band — not through a
|
|
940
1055
|
// StorageManager mutation), on the CANONICAL root (targetRoot), matching
|
|
@@ -947,6 +1062,12 @@ function copyMemories(
|
|
|
947
1062
|
bumpMemoryCorpusVersionForDir(targetRoot);
|
|
948
1063
|
}
|
|
949
1064
|
|
|
1065
|
+
// Every record — including every collision claimant entity page — is on
|
|
1066
|
+
// disk; NOW the reconcile pass can settle collisions safely.
|
|
1067
|
+
if (reconcileNeeded) {
|
|
1068
|
+
requestEntityCanonicalIdReconcileSync(path.join(targetRoot, "state"));
|
|
1069
|
+
}
|
|
1070
|
+
|
|
950
1071
|
return { merged, conflicts, skipped };
|
|
951
1072
|
}
|
|
952
1073
|
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Citation-marker stripping for dedup-hash sources (extracted from storage.ts
|
|
3
|
+
* under the #1995 structural ratchet; behavior unchanged).
|
|
4
|
+
*
|
|
5
|
+
* Content persisted with an injected citation block must hash to the SAME
|
|
6
|
+
* value as its raw form (AGENTS.md #13/#23 hash-consistency rules), so the
|
|
7
|
+
* hash source strips the configured citation template — literal template
|
|
8
|
+
* parts matched in order, whitespace around the marker normalized — without
|
|
9
|
+
* building a regex from template text (AGENTS.md #34).
|
|
10
|
+
*/
|
|
11
|
+
import { DEFAULT_CITATION_FORMAT } from "../source-attribution.js";
|
|
12
|
+
|
|
13
|
+
function trimTrailingSpacesAndTabs(value: string): string {
|
|
14
|
+
let end = value.length;
|
|
15
|
+
while (end > 0 && (value[end - 1] === " " || value[end - 1] === "\t")) {
|
|
16
|
+
end -= 1;
|
|
17
|
+
}
|
|
18
|
+
return end === value.length ? value : value.slice(0, end);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function trimLeadingSpacesAndTabs(value: string): string {
|
|
22
|
+
let start = 0;
|
|
23
|
+
while (start < value.length && (value[start] === " " || value[start] === "\t")) {
|
|
24
|
+
start += 1;
|
|
25
|
+
}
|
|
26
|
+
return start === 0 ? value : value.slice(start);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export function stripDefaultCitationMarkersWithoutRegex(value: string): string {
|
|
30
|
+
return stripCitationMarkersForHashRemoval(value, DEFAULT_CITATION_FORMAT);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function citationTemplateLiteralParts(template: string): string[] {
|
|
34
|
+
const parts: string[] = [];
|
|
35
|
+
let cursor = 0;
|
|
36
|
+
while (cursor < template.length) {
|
|
37
|
+
const open = template.indexOf("{", cursor);
|
|
38
|
+
if (open === -1) {
|
|
39
|
+
parts.push(template.slice(cursor));
|
|
40
|
+
break;
|
|
41
|
+
}
|
|
42
|
+
parts.push(template.slice(cursor, open));
|
|
43
|
+
const close = template.indexOf("}", open + 1);
|
|
44
|
+
if (close === -1) {
|
|
45
|
+
cursor = open + 1;
|
|
46
|
+
} else {
|
|
47
|
+
cursor = close + 1;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return parts.filter((part) => part.length > 0);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function stripCitationMarkersForHashRemoval(value: string, template: string): string {
|
|
54
|
+
const parts = citationTemplateLiteralParts(template);
|
|
55
|
+
if (parts.length === 0) return value;
|
|
56
|
+
const first = parts[0]!;
|
|
57
|
+
const lowerValue = value.toLowerCase();
|
|
58
|
+
const lowerFirst = first.toLowerCase();
|
|
59
|
+
const lowerParts = parts.map((part) => part.toLowerCase());
|
|
60
|
+
if (!lowerValue.includes(lowerFirst)) return value;
|
|
61
|
+
|
|
62
|
+
let result = "";
|
|
63
|
+
let cursor = 0;
|
|
64
|
+
let removed = false;
|
|
65
|
+
while (cursor < value.length) {
|
|
66
|
+
const markerStart = lowerValue.indexOf(lowerFirst, cursor);
|
|
67
|
+
if (markerStart === -1) {
|
|
68
|
+
result += value.slice(cursor);
|
|
69
|
+
break;
|
|
70
|
+
}
|
|
71
|
+
const boundedEnd = first.startsWith("[") ? value.indexOf("]", markerStart + first.length) : -1;
|
|
72
|
+
if (first.startsWith("[") && boundedEnd === -1) {
|
|
73
|
+
result += value.slice(cursor);
|
|
74
|
+
break;
|
|
75
|
+
}
|
|
76
|
+
const searchLimit = boundedEnd === -1 ? value.length : boundedEnd + 1;
|
|
77
|
+
let markerEnd = markerStart + first.length;
|
|
78
|
+
let matched = true;
|
|
79
|
+
for (let i = 1; i < lowerParts.length; i += 1) {
|
|
80
|
+
const partIndex = lowerValue.indexOf(lowerParts[i]!, markerEnd);
|
|
81
|
+
if (partIndex === -1 || partIndex + parts[i]!.length > searchLimit) {
|
|
82
|
+
matched = false;
|
|
83
|
+
break;
|
|
84
|
+
}
|
|
85
|
+
markerEnd = partIndex + parts[i]!.length;
|
|
86
|
+
}
|
|
87
|
+
if (!matched) {
|
|
88
|
+
result += value.slice(cursor);
|
|
89
|
+
break;
|
|
90
|
+
}
|
|
91
|
+
result += trimTrailingSpacesAndTabs(value.slice(cursor, markerStart));
|
|
92
|
+
cursor = markerEnd;
|
|
93
|
+
removed = true;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return removed ? trimLeadingSpacesAndTabs(result) : value;
|
|
97
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Entity canonical-id MUTATION lock (issue #2213).
|
|
3
|
+
*
|
|
4
|
+
* Serializes every writer whose correctness depends on the canonical-id
|
|
5
|
+
* journal not moving mid-write: entity file mutations (entity-store),
|
|
6
|
+
* per-pair file migration and journal pruning (entity-canonical-id-migration),
|
|
7
|
+
* and the final settle attempt of the post-persist repair helpers
|
|
8
|
+
* (entity-canonical-id-references). Lives in its own module so the reference
|
|
9
|
+
* helpers can take the lock without importing the migration module, which
|
|
10
|
+
* itself imports the reference helpers.
|
|
11
|
+
*/
|
|
12
|
+
import path from "node:path";
|
|
13
|
+
import { withHeldFileLock } from "../utils/serialize-mutations.js";
|
|
14
|
+
|
|
15
|
+
const ENTITY_CANONICAL_ID_MUTATION_LOCK_STALE_MS = 60_000;
|
|
16
|
+
const ENTITY_CANONICAL_ID_MUTATION_LOCK_MAX_WAIT_MS = 300_000;
|
|
17
|
+
|
|
18
|
+
export async function withEntityCanonicalMutationLock<T>(
|
|
19
|
+
stateDir: string,
|
|
20
|
+
task: (refreshLock: () => Promise<boolean>) => Promise<T>,
|
|
21
|
+
): Promise<T> {
|
|
22
|
+
return withHeldFileLock(
|
|
23
|
+
path.join(stateDir, "entity-canonical-id-mutation.lock"),
|
|
24
|
+
{
|
|
25
|
+
staleMs: ENTITY_CANONICAL_ID_MUTATION_LOCK_STALE_MS,
|
|
26
|
+
maxWaitMs: ENTITY_CANONICAL_ID_MUTATION_LOCK_MAX_WAIT_MS,
|
|
27
|
+
},
|
|
28
|
+
async (acquired, lock) => {
|
|
29
|
+
if (!acquired) throw new Error("Timed out waiting for entity mutation lock.");
|
|
30
|
+
return task(() => lock.refresh());
|
|
31
|
+
},
|
|
32
|
+
);
|
|
33
|
+
}
|
|
@@ -29,7 +29,7 @@ export interface EntityCanonicalIdMigrationHost {
|
|
|
29
29
|
invalidateAllMemoriesCache(): void;
|
|
30
30
|
invalidateColdMemoriesCache(): void;
|
|
31
31
|
bumpMemoryStatusVersion(): void;
|
|
32
|
-
|
|
32
|
+
getEntityMutationVersion(): number;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
35
|
export async function runLegacyEntityCanonicalIdMigration(
|
|
@@ -37,9 +37,9 @@ export async function runLegacyEntityCanonicalIdMigration(
|
|
|
37
37
|
parseEntityFile: (content: string) => EntityFile,
|
|
38
38
|
serializeEntityFile: (entity: EntityFile) => string,
|
|
39
39
|
serializeMemoryWithEntityRef: (memory: MemoryFile, entityRef: string) => string,
|
|
40
|
-
): Promise<string> {
|
|
40
|
+
): Promise<string | undefined> {
|
|
41
41
|
const readMigrationFingerprint = () =>
|
|
42
|
-
getFingerprint(host.baseDir, host.entitiesDir, () => host.
|
|
42
|
+
getFingerprint(host.baseDir, host.entitiesDir, () => String(host.getEntityMutationVersion()));
|
|
43
43
|
const completionFingerprint = await migrateLegacyEntityCanonicalIds({
|
|
44
44
|
stateDir: host.stateDir,
|
|
45
45
|
entitiesDir: host.entitiesDir,
|
|
@@ -68,5 +68,9 @@ export async function runLegacyEntityCanonicalIdMigration(
|
|
|
68
68
|
bumpMemoryStatusVersion: host.bumpMemoryStatusVersion.bind(host),
|
|
69
69
|
readMigrationFingerprint,
|
|
70
70
|
} satisfies EntityCanonicalIdMigrationDependencies);
|
|
71
|
-
|
|
71
|
+
// An undefined completion fingerprint is a deliberate signal (issue #2213):
|
|
72
|
+
// a reconcile marker landed while the run scanned, and recomputing the
|
|
73
|
+
// fingerprint here would seal that pending request into the runner's
|
|
74
|
+
// completed state. Propagate it so the runner retries instead.
|
|
75
|
+
return completionFingerprint;
|
|
72
76
|
}
|
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
writeFile,
|
|
12
12
|
} from "node:fs/promises";
|
|
13
13
|
import path from "node:path";
|
|
14
|
-
import { readFileSync } from "node:fs";
|
|
14
|
+
import { lstatSync, readFileSync } from "node:fs";
|
|
15
15
|
import { z } from "zod";
|
|
16
16
|
import { computeSupersessionKey, normalizeSupersessionKey } from "../temporal-supersession.js";
|
|
17
17
|
import type { EntityFile, MemoryFile } from "../types.js";
|
|
@@ -20,52 +20,23 @@ import { isErrnoCode } from "../utils/errno.js";
|
|
|
20
20
|
import { withHeldFileLock } from "../utils/serialize-mutations.js";
|
|
21
21
|
import { RECALL_FALLBACK_DIRS } from "../utils/category-dir.js";
|
|
22
22
|
import { assertPathInsideRoot } from "../utils/path-containment.js";
|
|
23
|
+
import { withEntityCanonicalMutationLock } from "./entity-canonical-id-lock.js";
|
|
24
|
+
|
|
25
|
+
import {
|
|
26
|
+
ENTITY_CANONICAL_ID_MIGRATION_FILE,
|
|
27
|
+
ENTITY_CANONICAL_ID_RECONCILE_CONSUMING_MARKER,
|
|
28
|
+
ENTITY_CANONICAL_ID_RECONCILE_MARKER,
|
|
29
|
+
resolveHistoricalEntityCanonicalId,
|
|
30
|
+
} from "./entity-canonical-id-references.js";
|
|
23
31
|
|
|
24
32
|
const ENTITY_CANONICAL_ID_MIGRATION_VERSION = 1;
|
|
25
|
-
export const ENTITY_CANONICAL_ID_MIGRATION_FILE = "entity-canonical-id-migration-v1.json";
|
|
26
33
|
const ENTITY_CANONICAL_ID_MIGRATION_LOCK_STALE_MS = 60_000;
|
|
27
34
|
const ENTITY_CANONICAL_ID_MIGRATION_LOCK_MAX_WAIT_MS = 300_000;
|
|
28
35
|
const TOMBSTONE_LOCK_STALE_MS = 30_000;
|
|
29
36
|
const TOMBSTONE_LOCK_MAX_WAIT_MS = 5_000;
|
|
30
|
-
const ENTITY_CANONICAL_ID_MUTATION_LOCK_STALE_MS = 60_000;
|
|
31
|
-
const ENTITY_CANONICAL_ID_MUTATION_LOCK_MAX_WAIT_MS = 300_000;
|
|
32
37
|
const MEMORY_REWRITE_MAX_PASSES = 3;
|
|
33
38
|
const ENTITY_MAPPING_RESCAN_MAX_PASSES = 3;
|
|
34
39
|
|
|
35
|
-
export function loadHistoricalEntityCanonicalIds(stateDir: string): Readonly<Record<string, string>> {
|
|
36
|
-
const statePath = path.join(stateDir, ENTITY_CANONICAL_ID_MIGRATION_FILE);
|
|
37
|
-
try {
|
|
38
|
-
const parsed: unknown = JSON.parse(readFileSync(statePath, "utf-8"));
|
|
39
|
-
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return {};
|
|
40
|
-
const mappings = (parsed as { mappings?: unknown }).mappings;
|
|
41
|
-
if (!mappings || typeof mappings !== "object" || Array.isArray(mappings)) return {};
|
|
42
|
-
const cleaned: Record<string, string> = {};
|
|
43
|
-
for (const [legacyId, canonicalId] of Object.entries(mappings)) {
|
|
44
|
-
if (legacyId.length > 0 && typeof canonicalId === "string" && canonicalId.length > 0) {
|
|
45
|
-
cleaned[legacyId] = canonicalId;
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
return cleaned;
|
|
49
|
-
} catch {
|
|
50
|
-
return {};
|
|
51
|
-
}
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
export function resolveHistoricalEntityCanonicalId(
|
|
55
|
-
normalized: string,
|
|
56
|
-
mappings: Readonly<Record<string, string>>,
|
|
57
|
-
): string {
|
|
58
|
-
let current = normalized;
|
|
59
|
-
const seen = new Set<string>();
|
|
60
|
-
while (!seen.has(current)) {
|
|
61
|
-
seen.add(current);
|
|
62
|
-
const next = mappings[current];
|
|
63
|
-
if (!next || next === current) break;
|
|
64
|
-
current = next;
|
|
65
|
-
}
|
|
66
|
-
return current;
|
|
67
|
-
}
|
|
68
|
-
|
|
69
40
|
const EntityCanonicalIdMigrationStateSchema = z.object({
|
|
70
41
|
version: z.literal(ENTITY_CANONICAL_ID_MIGRATION_VERSION),
|
|
71
42
|
complete: z.boolean(),
|
|
@@ -124,16 +95,33 @@ async function fingerprintPath(filePath: string): Promise<string> {
|
|
|
124
95
|
: "missing";
|
|
125
96
|
}
|
|
126
97
|
|
|
98
|
+
/**
|
|
99
|
+
* Completion fingerprint for the migration runner's skip gate. Incorporates
|
|
100
|
+
* only state that can change what a re-run would DO: the entities directory
|
|
101
|
+
* (creates/deletes/renames), the alias table (normalization inputs), the
|
|
102
|
+
* caller-supplied entity-mutation version (store-mediated entity content
|
|
103
|
+
* edits, which touch neither of the first two), and the reconcile-pending
|
|
104
|
+
* marker raw-byte writers touch. It must NOT incorporate the memory-corpus
|
|
105
|
+
* write version: plain memory creates cannot introduce legacy entity
|
|
106
|
+
* references (writes normalize through the historical mapping table), and
|
|
107
|
+
* keying the gate to them re-ran the migration after every fact write — the
|
|
108
|
+
* issue #2213 hot loop.
|
|
109
|
+
*/
|
|
127
110
|
export async function getFingerprint(
|
|
128
111
|
baseDir: string,
|
|
129
112
|
entitiesDir: string,
|
|
130
|
-
|
|
113
|
+
getEntityMutationVersion: () => string,
|
|
131
114
|
): Promise<string> {
|
|
132
|
-
const [entityFingerprint, aliasFingerprint] = await Promise.all([
|
|
115
|
+
const [entityFingerprint, aliasFingerprint, reconcileFingerprint, consumingFingerprint] = await Promise.all([
|
|
133
116
|
fingerprintPath(entitiesDir),
|
|
134
117
|
fingerprintPath(path.join(baseDir, "config", "aliases.json")),
|
|
118
|
+
fingerprintPath(path.join(baseDir, "state", ENTITY_CANONICAL_ID_RECONCILE_MARKER)),
|
|
119
|
+
// A `.consuming` generation stranded by a crashed PEER process must also
|
|
120
|
+
// re-invoke a live runner whose cached fingerprint predates it.
|
|
121
|
+
fingerprintPath(path.join(baseDir, "state", ENTITY_CANONICAL_ID_RECONCILE_CONSUMING_MARKER)),
|
|
135
122
|
]);
|
|
136
|
-
return
|
|
123
|
+
return [entityFingerprint, aliasFingerprint, reconcileFingerprint, consumingFingerprint, getEntityMutationVersion()]
|
|
124
|
+
.join(":");
|
|
137
125
|
}
|
|
138
126
|
|
|
139
127
|
export async function validateRoots(
|
|
@@ -229,22 +217,6 @@ function lockPath(deps: EntityCanonicalIdMigrationDependencies): string {
|
|
|
229
217
|
return path.join(deps.stateDir, "entity-canonical-id-migration.lock");
|
|
230
218
|
}
|
|
231
219
|
|
|
232
|
-
export async function withEntityCanonicalMutationLock<T>(
|
|
233
|
-
stateDir: string,
|
|
234
|
-
task: (refreshLock: () => Promise<boolean>) => Promise<T>,
|
|
235
|
-
): Promise<T> {
|
|
236
|
-
return withHeldFileLock(
|
|
237
|
-
path.join(stateDir, "entity-canonical-id-mutation.lock"),
|
|
238
|
-
{
|
|
239
|
-
staleMs: ENTITY_CANONICAL_ID_MUTATION_LOCK_STALE_MS,
|
|
240
|
-
maxWaitMs: ENTITY_CANONICAL_ID_MUTATION_LOCK_MAX_WAIT_MS,
|
|
241
|
-
},
|
|
242
|
-
async (acquired, lock) => {
|
|
243
|
-
if (!acquired) throw new Error("Timed out waiting for entity mutation lock.");
|
|
244
|
-
return task(() => lock.refresh());
|
|
245
|
-
},
|
|
246
|
-
);
|
|
247
|
-
}
|
|
248
220
|
|
|
249
221
|
async function readState(
|
|
250
222
|
deps: EntityCanonicalIdMigrationDependencies
|
|
@@ -283,6 +255,23 @@ async function writeState(
|
|
|
283
255
|
}
|
|
284
256
|
}
|
|
285
257
|
|
|
258
|
+
/**
|
|
259
|
+
* Every journal publish — additive merges, completion markers, prunes —
|
|
260
|
+
* serializes under the entity MUTATION lock (issue #2213 review): entity
|
|
261
|
+
* writers, post-persist repair settles, and tombstone appends revalidate
|
|
262
|
+
* the journal identity under that lock, so an unlocked publish could land
|
|
263
|
+
* inside their resolve→write window and strand the write on a superseded
|
|
264
|
+
* id space.
|
|
265
|
+
*/
|
|
266
|
+
async function publishState(
|
|
267
|
+
deps: EntityCanonicalIdMigrationDependencies,
|
|
268
|
+
state: EntityCanonicalIdMigrationState
|
|
269
|
+
): Promise<void> {
|
|
270
|
+
await withEntityCanonicalMutationLock(deps.stateDir, async () => {
|
|
271
|
+
await writeState(deps, state);
|
|
272
|
+
});
|
|
273
|
+
}
|
|
274
|
+
|
|
286
275
|
async function fileExists(filePath: string): Promise<boolean> {
|
|
287
276
|
try {
|
|
288
277
|
await access(filePath);
|
|
@@ -830,6 +819,23 @@ export async function migrateLegacyEntityCanonicalIds(
|
|
|
830
819
|
if (!acquired) throw new Error("Timed out waiting for active entity canonical-id migration.");
|
|
831
820
|
await deps.validateMemoryScanRoots?.();
|
|
832
821
|
await readSafeEntityEntries(deps.entitiesDir);
|
|
822
|
+
// A raw-byte writer (offline sync, consolidation-undo, governance
|
|
823
|
+
// restore) may have landed memory bytes this process never vetted; the
|
|
824
|
+
// marker requests ONE reference-reconciliation pass (issue #2213). The
|
|
825
|
+
// observed generation is RENAMED aside before the pass so a request made
|
|
826
|
+
// WHILE this run scans creates a fresh marker that survives finish() —
|
|
827
|
+
// and a `.consuming` file left by a crashed run still owes its pass.
|
|
828
|
+
const reconcileMarkerPath = path.join(deps.stateDir, ENTITY_CANONICAL_ID_RECONCILE_MARKER);
|
|
829
|
+
const consumingMarkerPath = path.join(deps.stateDir, ENTITY_CANONICAL_ID_RECONCILE_CONSUMING_MARKER);
|
|
830
|
+
let reconcileRequested = await fileExists(consumingMarkerPath);
|
|
831
|
+
if (await fileExists(reconcileMarkerPath)) {
|
|
832
|
+
try {
|
|
833
|
+
await rename(reconcileMarkerPath, consumingMarkerPath);
|
|
834
|
+
reconcileRequested = true;
|
|
835
|
+
} catch (error) {
|
|
836
|
+
if (!isErrnoCode(error, "ENOENT")) throw error;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
833
839
|
let state = await readState(deps);
|
|
834
840
|
if (!state) {
|
|
835
841
|
state = {
|
|
@@ -841,7 +847,7 @@ export async function migrateLegacyEntityCanonicalIds(
|
|
|
841
847
|
const blocked: BlockedEntityPairs = new Map();
|
|
842
848
|
// ONE aggregated line per run, not one per pair per restart: this used to
|
|
843
849
|
// abort the daemon, so the operator needs the whole list and the remedy.
|
|
844
|
-
const finish = (): Promise<string | undefined>
|
|
850
|
+
const finish = async (): Promise<string | undefined> => {
|
|
845
851
|
if (blocked.size > 0) {
|
|
846
852
|
const pairs = [...blocked].map(([legacyId, canonicalId]) => `${legacyId} -> ${canonicalId}`).join(", ");
|
|
847
853
|
log.warn(
|
|
@@ -849,7 +855,29 @@ export async function migrateLegacyEntityCanonicalIds(
|
|
|
849
855
|
+ "Both files were kept and the legacy id still resolves; merge or delete one side to migrate it.",
|
|
850
856
|
);
|
|
851
857
|
}
|
|
852
|
-
|
|
858
|
+
// Every finish() path either just rewrote references (main/reconcile
|
|
859
|
+
// paths), has none to rewrite (empty mappings), or observed no marker
|
|
860
|
+
// (stable no-op) — so the CONSUMED generation is removed here. A fresh
|
|
861
|
+
// marker written mid-run is untouched, and a crash before this point
|
|
862
|
+
// leaves `.consuming` set for the next run (the rewrite is idempotent).
|
|
863
|
+
if (reconcileRequested) {
|
|
864
|
+
await unlink(consumingMarkerPath).catch((error: unknown) => {
|
|
865
|
+
if (!isErrnoCode(error, "ENOENT")) throw error;
|
|
866
|
+
});
|
|
867
|
+
}
|
|
868
|
+
// A FRESH marker written while this run scanned means bytes landed the
|
|
869
|
+
// pass may not have covered. Withholding the completion fingerprint
|
|
870
|
+
// makes the runner's stability check fail, so it immediately retries —
|
|
871
|
+
// and the retry consumes the fresh generation. ORDER MATTERS: read the
|
|
872
|
+
// fingerprint BEFORE the marker check. A marker landing after the
|
|
873
|
+
// fingerprint read but before the check is caught by the check; one
|
|
874
|
+
// landing after the check is absent from the returned fingerprint, so
|
|
875
|
+
// the live marker mismatches the runner's cached completion and the
|
|
876
|
+
// next ensure() re-invokes. Checking first left a window where the
|
|
877
|
+
// fingerprint could SEAL a marker the check never saw.
|
|
878
|
+
const completionFingerprint = await deps.readMigrationFingerprint?.();
|
|
879
|
+
if (await fileExists(reconcileMarkerPath)) return undefined;
|
|
880
|
+
return completionFingerprint;
|
|
853
881
|
};
|
|
854
882
|
/**
|
|
855
883
|
* Drop blocked pairs from the working set. Discovery only refuses to
|
|
@@ -905,8 +933,12 @@ export async function migrateLegacyEntityCanonicalIds(
|
|
|
905
933
|
Object.keys(a).length === Object.keys(b).length
|
|
906
934
|
&& Object.entries(a).every(([key, value]) => b[key] === value);
|
|
907
935
|
if (same(collapsed, current.mappings) && same(parked, current.blocked ?? {})) return revived;
|
|
936
|
+
// Parking/removing an active mapping mid-entity-write would land an
|
|
937
|
+
// alias/activity/relationship mutation on a now-unrelated canonical
|
|
938
|
+
// claimant — the reconcile pass rewrites references, it cannot move
|
|
939
|
+
// a misplaced mutation. publishState serializes under the lock.
|
|
908
940
|
state = { ...current, mappings: collapsed, blocked: parked };
|
|
909
|
-
await
|
|
941
|
+
await publishState(deps, state);
|
|
910
942
|
return revived;
|
|
911
943
|
};
|
|
912
944
|
const previousMappings = state.mappings;
|
|
@@ -923,19 +955,39 @@ export async function migrateLegacyEntityCanonicalIds(
|
|
|
923
955
|
state.mappings,
|
|
924
956
|
await discoverMappings(deps, state.mappings, blocked),
|
|
925
957
|
);
|
|
926
|
-
await pruneBlocked();
|
|
958
|
+
const revivedBefore = await pruneBlocked();
|
|
927
959
|
const mergedBefore = collapseMappings(mergeDiscoveredMappings(state.mappings, discoveredBefore));
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
960
|
+
// Stable completed journal → no-op (issue #2213). Retained mappings are
|
|
961
|
+
// read-compat state whose references were already rewritten (to
|
|
962
|
+
// convergence) before `complete: true` was persisted, so re-running
|
|
963
|
+
// `rewriteKnownReferences` here re-read the ENTIRE hot+cold+archived
|
|
964
|
+
// corpus — twice — and invalidated every memory cache on EVERY
|
|
965
|
+
// `ensureDirectories()`. On a write-active daemon that was a
|
|
966
|
+
// self-sustaining hot loop (multi-GB/s of re-reads, caches never warm,
|
|
967
|
+
// recall degraded to the fallback path). A park revived by THIS run's
|
|
968
|
+
// pruneBlocked still owes its reference rewrite: it falls through to
|
|
969
|
+
// the main flow below, which demotes `complete` BEFORE rewriting — so
|
|
970
|
+
// a crash mid-rewrite resumes, which the old in-place rewrite did not.
|
|
971
|
+
if (
|
|
972
|
+
state.complete
|
|
973
|
+
&& Object.keys(discoveredBefore).length === 0
|
|
974
|
+
&& !mappingsChanged
|
|
975
|
+
&& !revivedBefore
|
|
976
|
+
) {
|
|
977
|
+
// Stable journal, but a raw-byte writer requested reconciliation:
|
|
978
|
+
// run ONE bounded rewrite pass over the retained mappings, then let
|
|
979
|
+
// finish() consume the marker.
|
|
980
|
+
if (reconcileRequested && Object.keys(state.mappings).length > 0) {
|
|
981
|
+
const rewroteColdMemory = await rewriteKnownReferences(
|
|
982
|
+
deps,
|
|
983
|
+
state.mappings,
|
|
984
|
+
() => lock.refresh(),
|
|
985
|
+
);
|
|
986
|
+
deps.invalidateKnowledgeIndexCache();
|
|
987
|
+
deps.invalidateAllMemoriesCache();
|
|
988
|
+
if (rewroteColdMemory) deps.invalidateColdMemoriesCache();
|
|
989
|
+
deps.bumpMemoryStatusVersion();
|
|
990
|
+
}
|
|
939
991
|
return finish();
|
|
940
992
|
}
|
|
941
993
|
if (
|
|
@@ -944,7 +996,7 @@ export async function migrateLegacyEntityCanonicalIds(
|
|
|
944
996
|
|| Object.keys(mergedBefore).length !== Object.keys(state.mappings).length
|
|
945
997
|
) {
|
|
946
998
|
state = { ...state, complete: false, mappings: mergedBefore };
|
|
947
|
-
await
|
|
999
|
+
await publishState(deps, state);
|
|
948
1000
|
}
|
|
949
1001
|
if (Object.keys(state.mappings).length === 0) {
|
|
950
1002
|
if (!(await lock.refresh())) throw new Error("Lost entity canonical-id migration lock.");
|
|
@@ -958,7 +1010,7 @@ export async function migrateLegacyEntityCanonicalIds(
|
|
|
958
1010
|
// that already reflects the deletion - so no later run revisits it.
|
|
959
1011
|
const revivedInRescan = await pruneBlocked();
|
|
960
1012
|
if (Object.keys(discoveredAfter).length === 0 && !revivedInRescan) {
|
|
961
|
-
await
|
|
1013
|
+
await publishState(deps, { ...state, complete: true });
|
|
962
1014
|
return finish();
|
|
963
1015
|
}
|
|
964
1016
|
state = {
|
|
@@ -966,7 +1018,7 @@ export async function migrateLegacyEntityCanonicalIds(
|
|
|
966
1018
|
complete: false,
|
|
967
1019
|
mappings: collapseMappings(mergeDiscoveredMappings(state.mappings, discoveredAfter)),
|
|
968
1020
|
};
|
|
969
|
-
await
|
|
1021
|
+
await publishState(deps, state);
|
|
970
1022
|
continue;
|
|
971
1023
|
}
|
|
972
1024
|
let pendingMappings = Object.entries(state.mappings);
|
|
@@ -1016,7 +1068,7 @@ export async function migrateLegacyEntityCanonicalIds(
|
|
|
1016
1068
|
// retries and those references stay stranded.
|
|
1017
1069
|
const revivedAfterRewrite = await pruneBlocked();
|
|
1018
1070
|
if (Object.keys(discoveredAfter).length === 0 && !revivedAfterRewrite) {
|
|
1019
|
-
await
|
|
1071
|
+
await publishState(deps, { ...state, complete: true });
|
|
1020
1072
|
return finish();
|
|
1021
1073
|
}
|
|
1022
1074
|
state = {
|
|
@@ -1024,7 +1076,7 @@ export async function migrateLegacyEntityCanonicalIds(
|
|
|
1024
1076
|
complete: false,
|
|
1025
1077
|
mappings: collapseMappings(mergeDiscoveredMappings(state.mappings, discoveredAfter)),
|
|
1026
1078
|
};
|
|
1027
|
-
await
|
|
1079
|
+
await publishState(deps, state);
|
|
1028
1080
|
}
|
|
1029
1081
|
throw new Error("Entity canonical-id migration mappings changed while migration was running; retry migration.");
|
|
1030
1082
|
}
|