@remnic/core 9.29.1 → 9.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.d.ts +11 -11
- package/dist/access-admin-ops-surface.js +19 -18
- package/dist/access-authorization-probe.d.ts +11 -11
- package/dist/access-authorization-probe.js +20 -19
- package/dist/access-boundary.d.ts +11 -11
- package/dist/access-boundary.js +19 -18
- package/dist/access-citation-namespace.js +2 -2
- package/dist/access-cli.js +45 -44
- package/dist/access-cli.js.map +1 -1
- package/dist/access-health-types.d.ts +1 -1
- package/dist/access-http-lcm-compaction.d.ts +11 -11
- package/dist/access-http.d.ts +11 -11
- package/dist/access-http.js +25 -24
- package/dist/access-identity-continuity-surface.d.ts +8 -8
- package/dist/access-identity-continuity-surface.js +19 -18
- package/dist/access-lcm-surface.d.ts +10 -10
- package/dist/access-lcm-surface.js +19 -18
- package/dist/access-mcp.d.ts +11 -11
- package/dist/access-mcp.js +23 -22
- package/dist/access-memory-search-fanout.d.ts +2 -2
- package/dist/{access-namespace-preflight-DCXBkDp0.d.ts → access-namespace-preflight-VfXssvoI.d.ts} +1 -1
- package/dist/access-namespace-preflight.d.ts +3 -3
- package/dist/access-namespace-preflight.js +19 -18
- package/dist/access-observe-write-surface.d.ts +10 -10
- package/dist/access-observe-write-surface.js +19 -18
- package/dist/access-operations-batch.js +20 -19
- package/dist/access-operations.d.ts +11 -11
- package/dist/access-operations.js +22 -21
- package/dist/access-recall-concurrency.d.ts +11 -11
- package/dist/access-recall-concurrency.js +19 -18
- package/dist/access-recall-response.d.ts +10 -10
- package/dist/access-recall-response.js +19 -18
- package/dist/access-recall-surface.d.ts +10 -10
- package/dist/access-recall-surface.js +19 -18
- package/dist/access-schema.js +3 -3
- package/dist/{access-service-DvPW_M5j.d.ts → access-service-B5RwVsTV.d.ts} +7 -7
- package/dist/access-service.d.ts +11 -11
- package/dist/access-service.js +19 -18
- package/dist/access-surface-catalog.d.ts +11 -11
- package/dist/access-wearables-meetings-surface.d.ts +4 -3
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/active-recall.js +3 -3
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +8 -8
- package/dist/briefing.d.ts +3 -2
- package/dist/briefing.js +10 -9
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer.d.ts +3 -2
- package/dist/bulk-import/index.d.ts +3 -3
- package/dist/calibration.d.ts +1 -1
- package/dist/capabilities.d.ts +1 -1
- package/dist/{catalog-BIVgftWu.d.ts → catalog-V7inUQdp.d.ts} +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-consolidation.js +11 -10
- package/dist/causal-consolidation.js.map +1 -1
- package/dist/causal-trajectory-graph.d.ts +1 -1
- package/dist/{chunk-FZRQZW4H.js → chunk-2EUVEDF5.js} +2 -2
- package/dist/{chunk-PBUZNGMX.js → chunk-4WKA6I65.js} +20 -20
- package/dist/{chunk-PBUZNGMX.js.map → chunk-4WKA6I65.js.map} +1 -1
- package/dist/{chunk-AIK5W3IW.js → chunk-5TFP4CUL.js} +68 -2
- package/dist/chunk-5TFP4CUL.js.map +1 -0
- package/dist/{chunk-3JNDOAQ2.js → chunk-5W4ZLBEZ.js} +9 -9
- package/dist/{chunk-5ZCUDWD4.js → chunk-62U4J2GN.js} +2 -2
- package/dist/{chunk-IGPEJVBY.js → chunk-66772WBP.js} +2 -2
- package/dist/{chunk-EP5BVCRI.js → chunk-6BNE62PM.js} +2 -2
- package/dist/{chunk-AAP5T6QG.js → chunk-6DNMUYEE.js} +2 -2
- package/dist/{chunk-35DL7IOI.js → chunk-6FUPCJUL.js} +7 -7
- package/dist/{chunk-JP3YPDAW.js → chunk-6U7F52DE.js} +19 -19
- package/dist/{chunk-E4OCKU67.js → chunk-7XC2HX75.js} +2 -2
- package/dist/{chunk-OVRQLAYR.js → chunk-AZ5BTEKX.js} +2 -2
- package/dist/{chunk-2AXMQWCN.js → chunk-CZRYLSSW.js} +3 -3
- package/dist/chunk-FBUE6PD3.js +1769 -0
- package/dist/chunk-FBUE6PD3.js.map +1 -0
- package/dist/{chunk-PTSNLEXK.js → chunk-FDAMS4PK.js} +2 -2
- package/dist/{chunk-WL5EBRDH.js → chunk-FFAZ2XFU.js} +9 -8
- package/dist/chunk-FFAZ2XFU.js.map +1 -0
- package/dist/{chunk-2GXM5N5A.js → chunk-GIBMDSJ2.js} +2 -2
- package/dist/{chunk-E5OBR6CJ.js → chunk-H5CPS5JP.js} +3 -3
- package/dist/{chunk-2D7FEUCK.js → chunk-I5NLEENJ.js} +2 -2
- package/dist/{chunk-DRBXHRXE.js → chunk-IB26HVAJ.js} +2 -2
- package/dist/{chunk-QDKK63Y7.js → chunk-IC6CBHCA.js} +3 -3
- package/dist/{chunk-IAEAFLF2.js → chunk-JCBWCYEJ.js} +2 -2
- package/dist/{chunk-3PDOVUHC.js → chunk-KXB4VXHY.js} +2 -2
- package/dist/{chunk-WU7TTVOD.js → chunk-LBOD4U5B.js} +7 -7
- package/dist/{chunk-UBKEQOTD.js → chunk-LGHKJABJ.js} +3 -3
- package/dist/{chunk-VSURVXKY.js → chunk-M3C2V3PV.js} +10 -10
- package/dist/{chunk-OFHFH655.js → chunk-MJCGLM25.js} +8 -8
- package/dist/{chunk-OYJKSZIQ.js → chunk-NHWB6HG2.js} +2 -2
- package/dist/{chunk-TC3VNKGX.js → chunk-NM2WP3B7.js} +4 -4
- package/dist/{chunk-GAWOEQEQ.js → chunk-P5PXWTW7.js} +2 -2
- package/dist/chunk-PH3ZWL3W.js +786 -0
- package/dist/chunk-PH3ZWL3W.js.map +1 -0
- package/dist/{chunk-2QMKMVHH.js → chunk-QNWXCJ76.js} +2 -2
- package/dist/{chunk-JMQUBRV6.js → chunk-RGIWINDN.js} +340 -771
- package/dist/chunk-RGIWINDN.js.map +1 -0
- package/dist/{chunk-WRFBW2TT.js → chunk-SPJXKAVF.js} +3 -3
- package/dist/{chunk-HKJCMOT6.js → chunk-TH2LSRMG.js} +2 -2
- package/dist/{chunk-CW4AGY3G.js → chunk-VI2JQPNQ.js} +2 -2
- package/dist/{chunk-OP6F6L3S.js → chunk-W5EG6LE3.js} +2 -2
- package/dist/{chunk-DEBHXHJR.js → chunk-WIPEXG6R.js} +2 -2
- package/dist/{chunk-SH6YO3LT.js → chunk-WVO4G37W.js} +2 -2
- package/dist/{chunk-FUYEVEX7.js → chunk-YNQBRLC5.js} +24 -22
- package/dist/chunk-YNQBRLC5.js.map +1 -0
- package/dist/{chunk-7L45JFPW.js → chunk-YTQNQ4KB.js} +14 -14
- package/dist/{chunk-PKRIQL3V.js → chunk-YTSNAUXN.js} +2 -2
- package/dist/{chunk-5QDRGWT2.js → chunk-ZPSVBSOF.js} +4 -4
- package/dist/{cli-CxoSOiG0.d.ts → cli-C4GbETPC.d.ts} +5 -5
- package/dist/cli.d.ts +13 -13
- package/dist/cli.js +36 -35
- package/dist/compounding/engine.d.ts +3 -2
- package/dist/compounding/engine.js +10 -9
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.js +3 -3
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.js +10 -9
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/connectors/index.js +10 -9
- package/dist/consolidation-provenance-check.d.ts +3 -2
- package/dist/consolidation-undo.d.ts +3 -2
- package/dist/contradiction/index.d.ts +3 -2
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/corpus-watermark.d.ts +1 -1
- package/dist/corpus-watermark.js +12 -11
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer.d.ts +1 -1
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-retrieval.d.ts +3 -2
- package/dist/entity-retrieval.js +10 -9
- package/dist/entity-schema.d.ts +1 -1
- package/dist/explicit-capture.d.ts +52 -10
- package/dist/explicit-capture.js +13 -1
- package/dist/extraction-error-classification.d.ts +1 -1
- package/dist/extraction-faithfulness.d.ts +1 -1
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction-liveness.d.ts +1 -1
- package/dist/extraction-source-grounding-rules.d.ts +1 -1
- package/dist/extraction-source-grounding.d.ts +1 -1
- package/dist/extraction.d.ts +1 -1
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/{forget-HRPLVB3W.js → forget-4UY2EOKH.js} +3 -3
- package/dist/graph-dashboard-diff.d.ts +1 -1
- package/dist/graph-dashboard-key.d.ts +1 -1
- package/dist/graph-dashboard-parser.d.ts +1 -1
- package/dist/graph-edge-reinforcement.d.ts +1 -1
- package/dist/graph-snapshot.d.ts +1 -1
- package/dist/graph.d.ts +1 -1
- package/dist/importance.d.ts +1 -1
- package/dist/importers/index.d.ts +1 -1
- package/dist/in-flight-reads.d.ts +1 -1
- package/dist/index.d.ts +19 -19
- package/dist/index.js +47 -45
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +1 -1
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/local-model-endpoint.d.ts +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/maintenance/memory-governance.js +10 -9
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.d.ts +2 -2
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +10 -9
- package/dist/maintenance/rebuild-memory-projection.d.ts +3 -2
- package/dist/maintenance/rebuild-memory-projection.js +11 -10
- package/dist/{maintenance-Cb-PjxQd.d.ts → maintenance-CL30NWtc.d.ts} +3 -3
- package/dist/mcp-memory-inspector-app.d.ts +11 -11
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +3 -2
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +4 -3
- package/dist/namespaces/migrate.js +11 -10
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/storage.d.ts +4 -3
- package/dist/namespaces/storage.js +10 -9
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/offline-sync-impression-drain.d.ts +1 -1
- package/dist/offline-sync-impression-drain.js +5 -5
- package/dist/offline-sync.js +2 -2
- package/dist/operator-doctor-corpus.d.ts +1 -1
- package/dist/operator-doctor-corpus.js +13 -12
- package/dist/operator-doctor-replica.d.ts +1 -1
- package/dist/operator-toolkit.d.ts +4 -3
- package/dist/operator-toolkit.js +18 -17
- package/dist/orchestration/compression-guideline-coordinator.d.ts +3 -2
- package/dist/orchestration/maintenance.d.ts +5 -4
- package/dist/orchestration/maintenance.js +15 -14
- package/dist/{orchestrator-5GKqb_HL.d.ts → orchestrator-CKTCRwWV.d.ts} +7 -7
- package/dist/orchestrator.d.ts +8 -8
- package/dist/orchestrator.js +45 -44
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/{pipeline-3Qi1TpRo.d.ts → pipeline-B7wj4rRZ.d.ts} +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/proactive-contention.d.ts +1 -1
- package/dist/provenance.d.ts +1 -1
- package/dist/{purge-YSGX5BOG.js → purge-533FZLS4.js} +16 -9
- package/dist/purge-533FZLS4.js.map +1 -0
- package/dist/qmd-preflight.d.ts +2 -2
- package/dist/{qmd-D9JqzzRX.d.ts → qmd-q9tg4dHp.d.ts} +1 -1
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +2 -2
- package/dist/recall-concurrency-config.d.ts +1 -1
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-memory-map.d.ts +1 -1
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-state.d.ts +1 -1
- package/dist/recall-state.js +2 -2
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-timings.d.ts +1 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray.d.ts +1 -1
- package/dist/replay/normalizers/chatgpt.d.ts +1 -1
- package/dist/replay/normalizers/claude.d.ts +1 -1
- package/dist/replay/normalizers/openclaw.d.ts +1 -1
- package/dist/replay/normalizers/shared.d.ts +1 -1
- package/dist/replay/runner.d.ts +1 -1
- package/dist/replay/types.d.ts +1 -1
- package/dist/replica-divergence.d.ts +1 -1
- package/dist/replica-peers-config.d.ts +1 -1
- package/dist/resolve-auth-token.d.ts +1 -1
- package/dist/resume-bundles.js +4 -4
- package/dist/retrieval-agents.d.ts +2 -2
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/salvage-envelope.d.ts +1 -1
- package/dist/{scope-profiles-C4g6_UdP.d.ts → scope-profiles-CWDyglPM.d.ts} +1 -1
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/index.d.ts +1 -1
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/secure-store/index.js +2 -2
- package/dist/{semantic-consolidation-BrxqnzTW.d.ts → semantic-consolidation-BUE3deLn.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-consolidation.js +11 -10
- package/dist/semantic-rule-promotion.js +10 -9
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/semantic-rule-verifier.js +10 -9
- package/dist/{service-Bi1JgxdP.d.ts → service-CXgM0bQG.d.ts} +2 -2
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +1 -1
- package/dist/signal.d.ts +1 -1
- package/dist/{storage-CcH6yPk2.d.ts → storage-DysJq-XH.d.ts} +163 -27
- package/dist/storage.d.ts +3 -2
- package/dist/storage.js +12 -10
- package/dist/summarizer.d.ts +1 -1
- package/dist/summarizer.js +3 -3
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/summary-snapshot.js +2 -2
- package/dist/temporal-supersession.d.ts +3 -2
- package/dist/temporal-timeline-recall.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +3 -2
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/trust-score-stage.d.ts +1 -1
- package/dist/trust-score.d.ts +1 -1
- package/dist/{types-Dg0w9ADr.d.ts → types-aPPzhnjT.d.ts} +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/utility-runtime.d.ts +1 -1
- package/dist/utils/serialize-mutations.d.ts +11 -7
- package/dist/utils/serialize-mutations.js +1 -1
- package/dist/verified-recall.js +10 -9
- package/dist/write-envelope.d.ts +1 -1
- package/package.json +2 -2
- package/src/access-observe-write-surface.ts +1 -1
- package/src/access-service-coding-write.test.ts +40 -0
- package/src/explicit-capture.test.ts +1110 -1
- package/src/explicit-capture.ts +534 -135
- package/src/maintenance/purge.test.ts +157 -1
- package/src/maintenance/purge.ts +15 -9
- package/src/orchestration/recall-internal-deps.ts +385 -0
- package/src/orchestration/recall-internal.ts +16 -375
- package/src/secure-store/secure-fs.ts +78 -1
- package/src/storage/content-hash-index.ts +43 -17
- package/src/storage/tombstone-blocked-capture-index.ts +562 -0
- package/src/storage/tombstone-blocked-capture-mutation.ts +284 -0
- package/src/storage/tombstone-blocked-capture-sync.ts +872 -0
- package/src/storage-fact-hash-index-defer.test.ts +2900 -57
- package/src/storage.ts +266 -265
- package/src/testing/subjects/tombstone-blocked-capture.test.ts +181 -0
- package/src/utils/serialize-mutations.test.ts +4 -1
- package/src/utils/serialize-mutations.ts +26 -27
- package/dist/chunk-55MDBZLC.js +0 -490
- package/dist/chunk-55MDBZLC.js.map +0 -1
- package/dist/chunk-AIK5W3IW.js.map +0 -1
- package/dist/chunk-FUYEVEX7.js.map +0 -1
- package/dist/chunk-JMQUBRV6.js.map +0 -1
- package/dist/chunk-WL5EBRDH.js.map +0 -1
- package/dist/purge-YSGX5BOG.js.map +0 -1
- /package/dist/{chunk-FZRQZW4H.js.map → chunk-2EUVEDF5.js.map} +0 -0
- /package/dist/{chunk-3JNDOAQ2.js.map → chunk-5W4ZLBEZ.js.map} +0 -0
- /package/dist/{chunk-5ZCUDWD4.js.map → chunk-62U4J2GN.js.map} +0 -0
- /package/dist/{chunk-IGPEJVBY.js.map → chunk-66772WBP.js.map} +0 -0
- /package/dist/{chunk-EP5BVCRI.js.map → chunk-6BNE62PM.js.map} +0 -0
- /package/dist/{chunk-AAP5T6QG.js.map → chunk-6DNMUYEE.js.map} +0 -0
- /package/dist/{chunk-35DL7IOI.js.map → chunk-6FUPCJUL.js.map} +0 -0
- /package/dist/{chunk-JP3YPDAW.js.map → chunk-6U7F52DE.js.map} +0 -0
- /package/dist/{chunk-E4OCKU67.js.map → chunk-7XC2HX75.js.map} +0 -0
- /package/dist/{chunk-OVRQLAYR.js.map → chunk-AZ5BTEKX.js.map} +0 -0
- /package/dist/{chunk-2AXMQWCN.js.map → chunk-CZRYLSSW.js.map} +0 -0
- /package/dist/{chunk-PTSNLEXK.js.map → chunk-FDAMS4PK.js.map} +0 -0
- /package/dist/{chunk-2GXM5N5A.js.map → chunk-GIBMDSJ2.js.map} +0 -0
- /package/dist/{chunk-E5OBR6CJ.js.map → chunk-H5CPS5JP.js.map} +0 -0
- /package/dist/{chunk-2D7FEUCK.js.map → chunk-I5NLEENJ.js.map} +0 -0
- /package/dist/{chunk-DRBXHRXE.js.map → chunk-IB26HVAJ.js.map} +0 -0
- /package/dist/{chunk-QDKK63Y7.js.map → chunk-IC6CBHCA.js.map} +0 -0
- /package/dist/{chunk-IAEAFLF2.js.map → chunk-JCBWCYEJ.js.map} +0 -0
- /package/dist/{chunk-3PDOVUHC.js.map → chunk-KXB4VXHY.js.map} +0 -0
- /package/dist/{chunk-WU7TTVOD.js.map → chunk-LBOD4U5B.js.map} +0 -0
- /package/dist/{chunk-UBKEQOTD.js.map → chunk-LGHKJABJ.js.map} +0 -0
- /package/dist/{chunk-VSURVXKY.js.map → chunk-M3C2V3PV.js.map} +0 -0
- /package/dist/{chunk-OFHFH655.js.map → chunk-MJCGLM25.js.map} +0 -0
- /package/dist/{chunk-OYJKSZIQ.js.map → chunk-NHWB6HG2.js.map} +0 -0
- /package/dist/{chunk-TC3VNKGX.js.map → chunk-NM2WP3B7.js.map} +0 -0
- /package/dist/{chunk-GAWOEQEQ.js.map → chunk-P5PXWTW7.js.map} +0 -0
- /package/dist/{chunk-2QMKMVHH.js.map → chunk-QNWXCJ76.js.map} +0 -0
- /package/dist/{chunk-WRFBW2TT.js.map → chunk-SPJXKAVF.js.map} +0 -0
- /package/dist/{chunk-HKJCMOT6.js.map → chunk-TH2LSRMG.js.map} +0 -0
- /package/dist/{chunk-CW4AGY3G.js.map → chunk-VI2JQPNQ.js.map} +0 -0
- /package/dist/{chunk-OP6F6L3S.js.map → chunk-W5EG6LE3.js.map} +0 -0
- /package/dist/{chunk-DEBHXHJR.js.map → chunk-WIPEXG6R.js.map} +0 -0
- /package/dist/{chunk-SH6YO3LT.js.map → chunk-WVO4G37W.js.map} +0 -0
- /package/dist/{chunk-7L45JFPW.js.map → chunk-YTQNQ4KB.js.map} +0 -0
- /package/dist/{chunk-PKRIQL3V.js.map → chunk-YTSNAUXN.js.map} +0 -0
- /package/dist/{chunk-5QDRGWT2.js.map → chunk-ZPSVBSOF.js.map} +0 -0
- /package/dist/{forget-HRPLVB3W.js.map → forget-4UY2EOKH.js.map} +0 -0
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
import assert from "node:assert/strict";
|
|
2
|
-
import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
|
|
2
|
+
import { mkdtemp, mkdir, readFile, readdir, rename, rm, stat, utimes, writeFile } from "node:fs/promises";
|
|
3
3
|
import { existsSync } from "node:fs";
|
|
4
4
|
import os from "node:os";
|
|
5
5
|
import path from "node:path";
|
|
6
6
|
import test, { mock } from "node:test";
|
|
7
7
|
|
|
8
8
|
import { ContentHashIndex, FactHashIndexNotAuthoritativeError, StorageManager } from "./storage.js";
|
|
9
|
+
import {
|
|
10
|
+
buildExplicitCaptureDedupKey,
|
|
11
|
+
TombstoneBlockedCaptureIndex,
|
|
12
|
+
} from "./storage/tombstone-blocked-capture-index.js";
|
|
13
|
+
import { TombstoneBlockedCaptureWriteLock } from "./storage/tombstone-blocked-capture-mutation.js";
|
|
14
|
+
import { isEncryptedFile } from "./secure-store/secure-fs.js";
|
|
9
15
|
import type { MemoryFile } from "./types.js";
|
|
10
16
|
|
|
11
17
|
// Issue #1909 (Part B): writeMemory("fact") used to rewrite the whole
|
|
@@ -24,6 +30,31 @@ async function withMemoryDir(run: (dir: string) => Promise<void>): Promise<void>
|
|
|
24
30
|
}
|
|
25
31
|
}
|
|
26
32
|
|
|
33
|
+
test("capture write locks combine streamed hashes in canonical order", async () => {
|
|
34
|
+
await withMemoryDir(async (dir) => {
|
|
35
|
+
const acquired: string[] = [];
|
|
36
|
+
const lock = new TombstoneBlockedCaptureWriteLock({
|
|
37
|
+
stateDir: dir,
|
|
38
|
+
withHeldFileLock: (async (lockPath, _options, task) => {
|
|
39
|
+
acquired.push(lockPath);
|
|
40
|
+
return await task(true, { refresh: async () => true });
|
|
41
|
+
}) as typeof import("./utils/serialize-mutations.js").withHeldFileLock,
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
await lock.withCaptureWriteLockAndHashes(async () => {}, "outer identity", ["0".repeat(64)]);
|
|
45
|
+
|
|
46
|
+
assert.equal(acquired.length, 2);
|
|
47
|
+
assert.deepEqual(acquired, [...acquired].sort());
|
|
48
|
+
await assert.rejects(
|
|
49
|
+
lock.withCaptureWriteLock(
|
|
50
|
+
() => lock.withCaptureWriteLockHashes(async () => {}, ["0".repeat(64)]),
|
|
51
|
+
"outer identity"
|
|
52
|
+
),
|
|
53
|
+
/tombstone-blocked capture write lock remained busy/
|
|
54
|
+
);
|
|
55
|
+
});
|
|
56
|
+
});
|
|
57
|
+
|
|
27
58
|
test("deferred fact writes do NO per-fact index saves; a single flush persists all", async () => {
|
|
28
59
|
await withMemoryDir(async (dir) => {
|
|
29
60
|
const saveSpy = mock.method(ContentHashIndex.prototype, "save");
|
|
@@ -41,7 +72,7 @@ test("deferred fact writes do NO per-fact index saves; a single flush persists a
|
|
|
41
72
|
assert.equal(
|
|
42
73
|
saveSpy.mock.callCount(),
|
|
43
74
|
0,
|
|
44
|
-
"deferred fact writes must not flush the index per fact (was 10 flushes)"
|
|
75
|
+
"deferred fact writes must not flush the index per fact (was 10 flushes)"
|
|
45
76
|
);
|
|
46
77
|
|
|
47
78
|
// All 10 hashes are live in the in-memory index (dirty), so same-session
|
|
@@ -53,9 +84,11 @@ test("deferred fact writes do NO per-fact index saves; a single flush persists a
|
|
|
53
84
|
// One authoritative batch save (mirrors saveContentHashIndexes) persists
|
|
54
85
|
// the whole superset in a single rewrite.
|
|
55
86
|
saveSpy.mock.resetCalls();
|
|
56
|
-
const index = await (
|
|
57
|
-
|
|
58
|
-
|
|
87
|
+
const index = await (
|
|
88
|
+
storage as unknown as {
|
|
89
|
+
getFactHashIndex: () => Promise<ContentHashIndex>;
|
|
90
|
+
}
|
|
91
|
+
).getFactHashIndex.call(storage);
|
|
59
92
|
await index.save();
|
|
60
93
|
assert.equal(saveSpy.mock.callCount(), 1, "exactly one batch save, down from 10+");
|
|
61
94
|
} finally {
|
|
@@ -68,7 +101,7 @@ test("deferred fact writes do NO per-fact index saves; a single flush persists a
|
|
|
68
101
|
assert.equal(
|
|
69
102
|
await reopened.hasFactContentHash(`deferred fact number ${i}`),
|
|
70
103
|
true,
|
|
71
|
-
"batch-saved hashes survive into a new session"
|
|
104
|
+
"batch-saved hashes survive into a new session"
|
|
72
105
|
);
|
|
73
106
|
}
|
|
74
107
|
});
|
|
@@ -88,13 +121,9 @@ test("direct writeMemory('fact') without the flag persists the hash via the LOCK
|
|
|
88
121
|
await storage.writeMemory("fact", "single write fact", { source: "manual" });
|
|
89
122
|
assert.ok(
|
|
90
123
|
reconcileSpy.mock.callCount() >= 1,
|
|
91
|
-
"single-write callers flush via the locked reconcile (saveMergingWithDisk)"
|
|
92
|
-
);
|
|
93
|
-
assert.equal(
|
|
94
|
-
saveSpy.mock.callCount(),
|
|
95
|
-
0,
|
|
96
|
-
"the direct write must NOT use the unlocked whole-file save()",
|
|
124
|
+
"single-write callers flush via the locked reconcile (saveMergingWithDisk)"
|
|
97
125
|
);
|
|
126
|
+
assert.equal(saveSpy.mock.callCount(), 0, "the direct write must NOT use the unlocked whole-file save()");
|
|
98
127
|
assert.equal(await storage.hasFactContentHash("single write fact"), true);
|
|
99
128
|
} finally {
|
|
100
129
|
saveSpy.mock.restore();
|
|
@@ -148,7 +177,7 @@ test("#2016 thread SDzOT: a fact hash added during the authoritative rebuild's c
|
|
|
148
177
|
assert.equal(
|
|
149
178
|
s.factOnlyHashes.has(sentinel),
|
|
150
179
|
true,
|
|
151
|
-
"a fact hash added during the rebuild scan must survive publication (lost under reassign)"
|
|
180
|
+
"a fact hash added during the rebuild scan must survive publication (lost under reassign)"
|
|
152
181
|
);
|
|
153
182
|
// The corpus fact is present too — the in-place repopulate did not drop it.
|
|
154
183
|
assert.equal(await storage.hasFactContentHash("alpha established fact"), true);
|
|
@@ -198,7 +227,7 @@ test("#2016 thread PRRT_kwDORJXyws6SEBri: a hash added during rebuildUnderLock's
|
|
|
198
227
|
await fresh.load();
|
|
199
228
|
assert.ok(
|
|
200
229
|
fresh.has(lateContent),
|
|
201
|
-
"a hash added during the locked rebuild save must reach disk (lost under the unconditional clear)"
|
|
230
|
+
"a hash added during the locked rebuild save must reach disk (lost under the unconditional clear)"
|
|
202
231
|
);
|
|
203
232
|
// The rebuilt corpus set survives too — the late-delta handling did not drop it.
|
|
204
233
|
assert.ok(fresh.has("corpus fact one"), "corpus hash one survives the rebuild+reconcile");
|
|
@@ -226,7 +255,7 @@ test("promotion-style immediate write survives restart even with fact-hashes.rea
|
|
|
226
255
|
assert.equal(
|
|
227
256
|
await restarted.hasFactContentHash("promoted profile fact"),
|
|
228
257
|
true,
|
|
229
|
-
"the promoted hash is on disk and found after restart — no duplicate re-creation"
|
|
258
|
+
"the promoted hash is on disk and found after restart — no duplicate re-creation"
|
|
230
259
|
);
|
|
231
260
|
});
|
|
232
261
|
});
|
|
@@ -254,7 +283,7 @@ test("durability: deferred writes with NO batch save are rebuildable from on-dis
|
|
|
254
283
|
assert.equal(
|
|
255
284
|
await recovered.hasFactContentHash(`crash fact ${i}`),
|
|
256
285
|
true,
|
|
257
|
-
"a lost hash is rebuilt from the durable fact corpus — never a data loss"
|
|
286
|
+
"a lost hash is rebuilt from the durable fact corpus — never a data loss"
|
|
258
287
|
);
|
|
259
288
|
}
|
|
260
289
|
});
|
|
@@ -273,9 +302,11 @@ test("concurrent deferred fact writes both land in the index (no lost update)",
|
|
|
273
302
|
deferHashIndexSave: true,
|
|
274
303
|
}),
|
|
275
304
|
]);
|
|
276
|
-
const index = await (
|
|
277
|
-
|
|
278
|
-
|
|
305
|
+
const index = await (
|
|
306
|
+
storage as unknown as {
|
|
307
|
+
getFactHashIndex: () => Promise<ContentHashIndex>;
|
|
308
|
+
}
|
|
309
|
+
).getFactHashIndex.call(storage);
|
|
279
310
|
await index.save();
|
|
280
311
|
|
|
281
312
|
const reopened = new StorageManager(dir);
|
|
@@ -303,7 +334,7 @@ test("#1909 round 11: a deferred write with no batch save is rebuilt from the co
|
|
|
303
334
|
assert.equal(
|
|
304
335
|
await restarted.hasFactContentHash("windowed fact"),
|
|
305
336
|
true,
|
|
306
|
-
"the fact is rebuilt from the corpus — never lost from dedup"
|
|
337
|
+
"the fact is rebuilt from the corpus — never lost from dedup"
|
|
307
338
|
);
|
|
308
339
|
});
|
|
309
340
|
});
|
|
@@ -396,7 +427,7 @@ test("#1909: removing a hash and reconcile-saving drops it from disk (no resurre
|
|
|
396
427
|
assert.equal(
|
|
397
428
|
fresh.has("archived fact body"),
|
|
398
429
|
false,
|
|
399
|
-
"removed hash stays gone → re-extraction of that content is allowed"
|
|
430
|
+
"removed hash stays gone → re-extraction of that content is allowed"
|
|
400
431
|
);
|
|
401
432
|
});
|
|
402
433
|
});
|
|
@@ -482,9 +513,7 @@ test("#1909: rebuild indexes ACTIVE cold-tier facts; a demoted fact's hash survi
|
|
|
482
513
|
await storage.writeMemory("fact", "cold tier fact was demoted", { source: "manual" });
|
|
483
514
|
|
|
484
515
|
storage.invalidateAllMemoriesCacheForDir();
|
|
485
|
-
const demoted = (await storage.readAllMemories()).find(
|
|
486
|
-
(m) => m.content.includes("cold tier fact was demoted"),
|
|
487
|
-
);
|
|
516
|
+
const demoted = (await storage.readAllMemories()).find((m) => m.content.includes("cold tier fact was demoted"));
|
|
488
517
|
assert.ok(demoted, "the fact to demote must be readable in hot before migration");
|
|
489
518
|
await storage.migrateMemoryToTier(demoted!, "cold");
|
|
490
519
|
|
|
@@ -499,12 +528,12 @@ test("#1909: rebuild indexes ACTIVE cold-tier facts; a demoted fact's hash survi
|
|
|
499
528
|
assert.equal(
|
|
500
529
|
await reopened.hasFactContentHash("cold tier fact was demoted"),
|
|
501
530
|
true,
|
|
502
|
-
"a fact demoted to cold must survive the corpus rebuild (hot+cold union)"
|
|
531
|
+
"a fact demoted to cold must survive the corpus rebuild (hot+cold union)"
|
|
503
532
|
);
|
|
504
533
|
assert.equal(
|
|
505
534
|
await reopened.hasFactContentHash("hot tier fact stays put"),
|
|
506
535
|
true,
|
|
507
|
-
"the hot-tier fact must also survive the rebuild"
|
|
536
|
+
"the hot-tier fact must also survive the rebuild"
|
|
508
537
|
);
|
|
509
538
|
});
|
|
510
539
|
});
|
|
@@ -565,7 +594,11 @@ test("#2016: a lock-timed-out append is NOT silently dropped and eventually pers
|
|
|
565
594
|
// Disk is untouched while the lock is held — proves it did not write unlocked.
|
|
566
595
|
const early = new ContentHashIndex(stateDir);
|
|
567
596
|
await early.load();
|
|
568
|
-
assert.equal(
|
|
597
|
+
assert.equal(
|
|
598
|
+
early.has("deferred-under-contended-lock"),
|
|
599
|
+
false,
|
|
600
|
+
"no unlocked publish while the peer holds the lock"
|
|
601
|
+
);
|
|
569
602
|
// In-memory dedup still works this session (dirty addition retained).
|
|
570
603
|
assert.equal(idx.has("deferred-under-contended-lock"), true, "addition stays live in-memory");
|
|
571
604
|
|
|
@@ -578,7 +611,7 @@ test("#2016: a lock-timed-out append is NOT silently dropped and eventually pers
|
|
|
578
611
|
await fresh.load();
|
|
579
612
|
assert.ok(
|
|
580
613
|
fresh.has("deferred-under-contended-lock"),
|
|
581
|
-
"the deferred addition eventually reached disk via the durable retry"
|
|
614
|
+
"the deferred addition eventually reached disk via the durable retry"
|
|
582
615
|
);
|
|
583
616
|
});
|
|
584
617
|
});
|
|
@@ -606,13 +639,21 @@ test("#2016: exhausting retries under a permanently held lock is best-effort, ne
|
|
|
606
639
|
// Let the bounded retry chain run to exhaustion (lock is never released).
|
|
607
640
|
await withRetryLoopAlive(() => idx.whenReconcileRetrySettled());
|
|
608
641
|
|
|
609
|
-
assert.equal(
|
|
642
|
+
assert.equal(
|
|
643
|
+
idx.hasPendingReconcileRetry,
|
|
644
|
+
false,
|
|
645
|
+
"retries are bounded — the chain gives up, it does not spin forever"
|
|
646
|
+
);
|
|
610
647
|
// The addition is still held in-memory (dedup stays correct this session);
|
|
611
648
|
// disk stays empty, so the corpus-rebuild-on-restart safety net covers it.
|
|
612
649
|
assert.equal(idx.has("never-persistable-while-locked"), true, "addition retained in-memory after giving up");
|
|
613
650
|
const fresh = new ContentHashIndex(stateDir);
|
|
614
651
|
await fresh.load();
|
|
615
|
-
assert.equal(
|
|
652
|
+
assert.equal(
|
|
653
|
+
fresh.has("never-persistable-while-locked"),
|
|
654
|
+
false,
|
|
655
|
+
"nothing written unlocked while the lock stayed held"
|
|
656
|
+
);
|
|
616
657
|
});
|
|
617
658
|
});
|
|
618
659
|
|
|
@@ -750,7 +791,7 @@ test("#2016: a rebuild serializes with a concurrent locked writer — the peer's
|
|
|
750
791
|
assert.equal(
|
|
751
792
|
peerBlockedDuringRebuild,
|
|
752
793
|
true,
|
|
753
|
-
"the peer could not acquire the lock while the rebuild held it (serialized)"
|
|
794
|
+
"the peer could not acquire the lock while the rebuild held it (serialized)"
|
|
754
795
|
);
|
|
755
796
|
|
|
756
797
|
// The peer's deferred retry lands after the rebuild releases the lock and
|
|
@@ -762,7 +803,7 @@ test("#2016: a rebuild serializes with a concurrent locked writer — the peer's
|
|
|
762
803
|
assert.ok(fresh.has("rebuilt-corpus-fact"), "the rebuild's set was published");
|
|
763
804
|
assert.ok(
|
|
764
805
|
fresh.has("peer-append-during-rebuild"),
|
|
765
|
-
"the peer's concurrent addition survived — reconciled on top of the rebuild, never clobbered"
|
|
806
|
+
"the peer's concurrent addition survived — reconciled on top of the rebuild, never clobbered"
|
|
766
807
|
);
|
|
767
808
|
});
|
|
768
809
|
});
|
|
@@ -780,7 +821,7 @@ test("#2016: a peer's committed fact survives a fresh-session authoritative rebu
|
|
|
780
821
|
assert.equal(
|
|
781
822
|
await rebuilder.hasFactContentHash("durable multiprocess fact"),
|
|
782
823
|
true,
|
|
783
|
-
"the peer's fact is deduped after a fresh-session locked rebuild"
|
|
824
|
+
"the peer's fact is deduped after a fresh-session locked rebuild"
|
|
784
825
|
);
|
|
785
826
|
});
|
|
786
827
|
});
|
|
@@ -818,17 +859,17 @@ test("#2016 finding 1: a lock-contended miss is verified against the corpus, nev
|
|
|
818
859
|
assert.equal(
|
|
819
860
|
await reader.isFactContentHashAuthoritative(),
|
|
820
861
|
false,
|
|
821
|
-
"index cannot be authoritative while the peer holds the rebuild lock"
|
|
862
|
+
"index cannot be authoritative while the peer holds the rebuild lock"
|
|
822
863
|
);
|
|
823
864
|
assert.equal(
|
|
824
865
|
await reader.hasFactContentHash("durable fact under contention"),
|
|
825
866
|
true,
|
|
826
|
-
"a miss on the empty loaded snapshot is verified against the durable corpus — no false dedup miss"
|
|
867
|
+
"a miss on the empty loaded snapshot is verified against the durable corpus — no false dedup miss"
|
|
827
868
|
);
|
|
828
869
|
assert.equal(
|
|
829
870
|
await reader.hasFactContentHash("was never written anywhere"),
|
|
830
871
|
false,
|
|
831
|
-
"a genuine miss is still a miss under contention (corpus confirms absence)"
|
|
872
|
+
"a genuine miss is still a miss under contention (corpus confirms absence)"
|
|
832
873
|
);
|
|
833
874
|
} finally {
|
|
834
875
|
await rm(lockPath, { force: true });
|
|
@@ -848,12 +889,12 @@ test("#2016 finding 2: getAuthoritativeFactHashIndex fails explicitly instead of
|
|
|
848
889
|
await assert.rejects(
|
|
849
890
|
() => storage.getAuthoritativeFactHashIndex(),
|
|
850
891
|
(err: unknown) => err instanceof FactHashIndexNotAuthoritativeError,
|
|
851
|
-
"must throw rather than return a stale/non-authoritative index"
|
|
892
|
+
"must throw rather than return a stale/non-authoritative index"
|
|
852
893
|
);
|
|
853
894
|
assert.equal(
|
|
854
895
|
await storage.isFactContentHashAuthoritative(),
|
|
855
896
|
false,
|
|
856
|
-
"the non-authoritative state is propagated, not masked as authoritative"
|
|
897
|
+
"the non-authoritative state is propagated, not masked as authoritative"
|
|
857
898
|
);
|
|
858
899
|
} finally {
|
|
859
900
|
await rm(lockPath, { force: true });
|
|
@@ -894,11 +935,38 @@ test("#2016 finding 3: flushReconcileRetry drains a deferred lock-timeout append
|
|
|
894
935
|
await fresh.load();
|
|
895
936
|
assert.ok(
|
|
896
937
|
fresh.has("deferred-drained-on-shutdown"),
|
|
897
|
-
"the deferred hash reached disk via the inline shutdown drain"
|
|
938
|
+
"the deferred hash reached disk via the inline shutdown drain"
|
|
898
939
|
);
|
|
899
940
|
});
|
|
900
941
|
});
|
|
901
942
|
|
|
943
|
+
test("#2016 fingerprint detects same-size same-mtime index replacement", async () => {
|
|
944
|
+
await withMemoryDir(async (dir) => {
|
|
945
|
+
const stateDir = path.join(dir, "state");
|
|
946
|
+
const indexPath = path.join(stateDir, "fact-hashes.txt");
|
|
947
|
+
await mkdir(stateDir, { recursive: true });
|
|
948
|
+
const original = `${"a".repeat(64)}\n`;
|
|
949
|
+
const replacement = `${"b".repeat(64)}\n`;
|
|
950
|
+
const fixedSeconds = 1_700_000_000;
|
|
951
|
+
await writeFile(indexPath, original, "utf8");
|
|
952
|
+
await utimes(indexPath, fixedSeconds, fixedSeconds);
|
|
953
|
+
|
|
954
|
+
const index = new ContentHashIndex(stateDir);
|
|
955
|
+
await index.load();
|
|
956
|
+
assert.equal(await index.isDiskFingerprintCurrent(), true);
|
|
957
|
+
const baseline = await stat(indexPath);
|
|
958
|
+
|
|
959
|
+
const replacementPath = `${indexPath}.replacement`;
|
|
960
|
+
await writeFile(replacementPath, replacement, "utf8");
|
|
961
|
+
await utimes(replacementPath, fixedSeconds, fixedSeconds);
|
|
962
|
+
await rename(replacementPath, indexPath);
|
|
963
|
+
const replaced = await stat(indexPath);
|
|
964
|
+
assert.equal(replaced.size, baseline.size);
|
|
965
|
+
assert.equal(replaced.mtimeMs, baseline.mtimeMs);
|
|
966
|
+
assert.equal(await index.isDiskFingerprintCurrent(), false);
|
|
967
|
+
});
|
|
968
|
+
});
|
|
969
|
+
|
|
902
970
|
// PR #2016 thread SDzOP: the storage-owned removal
|
|
903
971
|
// (removeFactContentHashesForMemories) and the approval re-add
|
|
904
972
|
// (addActiveFactContentHash) used to publish fact-hashes.txt with the unlocked
|
|
@@ -959,13 +1027,9 @@ test("#2016 thread SDzOP: a storage fact-hash removal reconciles under the lock,
|
|
|
959
1027
|
await storage.removeFactContentHashesForMemories([beta!]);
|
|
960
1028
|
assert.ok(
|
|
961
1029
|
reconcileSpy.mock.callCount() >= 1,
|
|
962
|
-
"the removal flushes via the locked reconcile (saveMergingWithDisk)"
|
|
963
|
-
);
|
|
964
|
-
assert.equal(
|
|
965
|
-
saveSpy.mock.callCount(),
|
|
966
|
-
0,
|
|
967
|
-
"the removal must NOT use the unlocked whole-file save()",
|
|
1030
|
+
"the removal flushes via the locked reconcile (saveMergingWithDisk)"
|
|
968
1031
|
);
|
|
1032
|
+
assert.equal(saveSpy.mock.callCount(), 0, "the removal must NOT use the unlocked whole-file save()");
|
|
969
1033
|
} finally {
|
|
970
1034
|
saveSpy.mock.restore();
|
|
971
1035
|
reconcileSpy.mock.restore();
|
|
@@ -974,14 +1038,10 @@ test("#2016 thread SDzOP: a storage fact-hash removal reconciles under the lock,
|
|
|
974
1038
|
const fresh = new ContentHashIndex(stateDir);
|
|
975
1039
|
await fresh.load();
|
|
976
1040
|
assert.ok(fresh.has("alpha stays active"), "the surviving fact's hash is intact");
|
|
977
|
-
assert.equal(
|
|
978
|
-
fresh.has("beta gets archived"),
|
|
979
|
-
false,
|
|
980
|
-
"the archived fact's hash is dropped — no resurrection",
|
|
981
|
-
);
|
|
1041
|
+
assert.equal(fresh.has("beta gets archived"), false, "the archived fact's hash is dropped — no resurrection");
|
|
982
1042
|
assert.ok(
|
|
983
1043
|
fresh.has("peer concurrent append"),
|
|
984
|
-
"the concurrent peer append survives the removal (locked reconcile, not unlocked overwrite)"
|
|
1044
|
+
"the concurrent peer append survives the removal (locked reconcile, not unlocked overwrite)"
|
|
985
1045
|
);
|
|
986
1046
|
});
|
|
987
1047
|
});
|
|
@@ -1039,7 +1099,7 @@ test("#2016 thread SD7Tj: a hash added while a reconcile save awaits disk is not
|
|
|
1039
1099
|
assert.ok(fresh.has("added-before-save"), "pre-save addition still durable after drain");
|
|
1040
1100
|
assert.ok(
|
|
1041
1101
|
fresh.has("added-during-save"),
|
|
1042
|
-
"the mid-save addition reached disk — the mid-flight delta was preserved, not dropped"
|
|
1102
|
+
"the mid-save addition reached disk — the mid-flight delta was preserved, not dropped"
|
|
1043
1103
|
);
|
|
1044
1104
|
});
|
|
1045
1105
|
});
|
|
@@ -1073,7 +1133,7 @@ test("#2016 thread SD7Tk: a direct fact write drains its deferred hash retry bef
|
|
|
1073
1133
|
// called flushReconcileRetry — so this count was 0.
|
|
1074
1134
|
assert.ok(
|
|
1075
1135
|
flushSpy.mock.callCount() >= 1,
|
|
1076
|
-
"direct writeMemory drains the deferred hash retry (flushReconcileRetry) before returning"
|
|
1136
|
+
"direct writeMemory drains the deferred hash retry (flushReconcileRetry) before returning"
|
|
1077
1137
|
);
|
|
1078
1138
|
} finally {
|
|
1079
1139
|
flushSpy.mock.restore();
|
|
@@ -1116,7 +1176,7 @@ test("#2016 thread SD-nG: a direct re-add of a hash the local snapshot still hol
|
|
|
1116
1176
|
assert.equal(
|
|
1117
1177
|
reader.has(CONTENT),
|
|
1118
1178
|
true,
|
|
1119
|
-
"the reintroduced hash is durably republished for peers (stale re-add recorded as a delta)"
|
|
1179
|
+
"the reintroduced hash is durably republished for peers (stale re-add recorded as a delta)"
|
|
1120
1180
|
);
|
|
1121
1181
|
});
|
|
1122
1182
|
});
|
|
@@ -1159,7 +1219,7 @@ test("#2016 thread PRRT_kwDORJXyws6SEHve: addByHash re-add of a hash the local s
|
|
|
1159
1219
|
assert.equal(
|
|
1160
1220
|
reader.has("reactivated-fact-body"),
|
|
1161
1221
|
true,
|
|
1162
|
-
"addByHash on a stale local hit must republish the hash durably for peers"
|
|
1222
|
+
"addByHash on a stale local hit must republish the hash durably for peers"
|
|
1163
1223
|
);
|
|
1164
1224
|
});
|
|
1165
1225
|
});
|
|
@@ -1186,11 +1246,11 @@ test("#2016 thread PRRT_kwDORJXyws6SEHvh: reactivation drains the deferred recon
|
|
|
1186
1246
|
await storage.restoreFactHashAfterApproval(id);
|
|
1187
1247
|
assert.ok(
|
|
1188
1248
|
reconcileSpy.mock.callCount() >= 1,
|
|
1189
|
-
"reactivation publishes via the locked reconcile (saveMergingWithDisk)"
|
|
1249
|
+
"reactivation publishes via the locked reconcile (saveMergingWithDisk)"
|
|
1190
1250
|
);
|
|
1191
1251
|
assert.ok(
|
|
1192
1252
|
flushSpy.mock.callCount() >= 1,
|
|
1193
|
-
"reactivation drains the deferred lock-timeout retry inline (flushReconcileRetry) — same durability guarantee as writeMemory"
|
|
1253
|
+
"reactivation drains the deferred lock-timeout retry inline (flushReconcileRetry) — same durability guarantee as writeMemory"
|
|
1194
1254
|
);
|
|
1195
1255
|
} finally {
|
|
1196
1256
|
reconcileSpy.mock.restore();
|
|
@@ -1198,3 +1258,2786 @@ test("#2016 thread PRRT_kwDORJXyws6SEHvh: reactivation drains the deferred recon
|
|
|
1198
1258
|
}
|
|
1199
1259
|
});
|
|
1200
1260
|
});
|
|
1261
|
+
|
|
1262
|
+
test("tombstone blocked index rebuilds after a failed publish survives restart", async () => {
|
|
1263
|
+
await withMemoryDir(async (dir) => {
|
|
1264
|
+
const blocked = {
|
|
1265
|
+
path: path.join(dir, "facts", "blocked.md"),
|
|
1266
|
+
frontmatter: {
|
|
1267
|
+
id: "blocked-1",
|
|
1268
|
+
category: "fact",
|
|
1269
|
+
created: new Date(0).toISOString(),
|
|
1270
|
+
updated: new Date(0).toISOString(),
|
|
1271
|
+
source: "explicit-inline-review",
|
|
1272
|
+
confidence: 0.2,
|
|
1273
|
+
confidenceTier: "explicit",
|
|
1274
|
+
tags: ["review"],
|
|
1275
|
+
status: "pending_review",
|
|
1276
|
+
blockedBy: "tombstone-1",
|
|
1277
|
+
},
|
|
1278
|
+
content: "A blocked capture must remain deduplicable after an index publish failure.",
|
|
1279
|
+
} as MemoryFile;
|
|
1280
|
+
const options = {
|
|
1281
|
+
stateDir: dir,
|
|
1282
|
+
memoryDir: dir,
|
|
1283
|
+
secureStoreKeyProvider: () => null,
|
|
1284
|
+
secureStoreWriteKeyProvider: () => null,
|
|
1285
|
+
lockOptions: () => ({ retryMaxAttempts: 1, retryBaseMs: 1 }),
|
|
1286
|
+
readAllMemories: async () => [blocked],
|
|
1287
|
+
readAllColdMemories: async () => [],
|
|
1288
|
+
};
|
|
1289
|
+
const index = new TombstoneBlockedCaptureIndex(options);
|
|
1290
|
+
const saveSpy = mock.method(ContentHashIndex.prototype, "saveMergingWithDisk", async () => {});
|
|
1291
|
+
try {
|
|
1292
|
+
await index.add(blocked);
|
|
1293
|
+
assert.equal(
|
|
1294
|
+
existsSync(path.join(dir, "tombstone-blocked-capture", "rebuild-required")),
|
|
1295
|
+
true,
|
|
1296
|
+
"failed publish must leave a durable rebuild marker"
|
|
1297
|
+
);
|
|
1298
|
+
} finally {
|
|
1299
|
+
saveSpy.mock.restore();
|
|
1300
|
+
}
|
|
1301
|
+
|
|
1302
|
+
const restarted = new TombstoneBlockedCaptureIndex(options);
|
|
1303
|
+
assert.equal(
|
|
1304
|
+
await restarted.has(blocked.content, "fact"),
|
|
1305
|
+
true,
|
|
1306
|
+
"restart must rebuild from durable blocked rows instead of trusting stale index data"
|
|
1307
|
+
);
|
|
1308
|
+
assert.deepEqual(
|
|
1309
|
+
await readdir(path.join(dir, "tombstone-blocked-capture", "rebuild-required")),
|
|
1310
|
+
[],
|
|
1311
|
+
"successful rebuild clears the marker owned by the restarted index"
|
|
1312
|
+
);
|
|
1313
|
+
});
|
|
1314
|
+
});
|
|
1315
|
+
|
|
1316
|
+
test("token-specific blocked index markers survive peer interleaving and restart", async () => {
|
|
1317
|
+
await withMemoryDir(async (dir) => {
|
|
1318
|
+
const blocked = {
|
|
1319
|
+
path: path.join(dir, "facts", "blocked.md"),
|
|
1320
|
+
frontmatter: {
|
|
1321
|
+
id: "blocked-peer-1",
|
|
1322
|
+
category: "fact",
|
|
1323
|
+
created: new Date(0).toISOString(),
|
|
1324
|
+
updated: new Date(0).toISOString(),
|
|
1325
|
+
source: "explicit-inline-review",
|
|
1326
|
+
confidence: 0.2,
|
|
1327
|
+
confidenceTier: "explicit",
|
|
1328
|
+
tags: ["review"],
|
|
1329
|
+
status: "pending_review",
|
|
1330
|
+
blockedBy: "tombstone-peer-1",
|
|
1331
|
+
},
|
|
1332
|
+
content: "A peer marker must survive another writer's successful index publish.",
|
|
1333
|
+
} as MemoryFile;
|
|
1334
|
+
const options = {
|
|
1335
|
+
stateDir: dir,
|
|
1336
|
+
memoryDir: dir,
|
|
1337
|
+
secureStoreKeyProvider: () => null,
|
|
1338
|
+
secureStoreWriteKeyProvider: () => null,
|
|
1339
|
+
lockOptions: () => ({ retryMaxAttempts: 1, retryBaseMs: 1 }),
|
|
1340
|
+
readAllMemories: async () => [blocked],
|
|
1341
|
+
readAllColdMemories: async () => [],
|
|
1342
|
+
};
|
|
1343
|
+
const writer = new TombstoneBlockedCaptureIndex(options);
|
|
1344
|
+
const peer = new TombstoneBlockedCaptureIndex(options);
|
|
1345
|
+
const peerInternals = peer as unknown as {
|
|
1346
|
+
markRebuildRequired: () => Promise<string>;
|
|
1347
|
+
markRebuildCommitted: (markerPath: string) => Promise<void>;
|
|
1348
|
+
};
|
|
1349
|
+
const originalSave = ContentHashIndex.prototype.saveMergingWithDisk;
|
|
1350
|
+
let peerMarker: string | undefined;
|
|
1351
|
+
const saveSpy = mock.method(
|
|
1352
|
+
ContentHashIndex.prototype,
|
|
1353
|
+
"saveMergingWithDisk",
|
|
1354
|
+
async function (this: ContentHashIndex) {
|
|
1355
|
+
if (!peerMarker) {
|
|
1356
|
+
peerMarker = await peerInternals.markRebuildRequired();
|
|
1357
|
+
}
|
|
1358
|
+
return originalSave.call(this);
|
|
1359
|
+
}
|
|
1360
|
+
);
|
|
1361
|
+
try {
|
|
1362
|
+
await writer.add(blocked);
|
|
1363
|
+
assert.ok(peerMarker);
|
|
1364
|
+
assert.equal(
|
|
1365
|
+
(await readdir(path.join(dir, "tombstone-blocked-capture", "rebuild-required"))).length,
|
|
1366
|
+
1,
|
|
1367
|
+
"writer success must clear only its marker and preserve the peer marker"
|
|
1368
|
+
);
|
|
1369
|
+
} finally {
|
|
1370
|
+
saveSpy.mock.restore();
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
if (!peerMarker) throw new Error("peer marker was not created");
|
|
1374
|
+
await peerInternals.markRebuildCommitted(peerMarker);
|
|
1375
|
+
|
|
1376
|
+
const restarted = new TombstoneBlockedCaptureIndex(options);
|
|
1377
|
+
assert.equal(await restarted.has(blocked.content, "fact"), true);
|
|
1378
|
+
assert.deepEqual(
|
|
1379
|
+
await readdir(path.join(dir, "tombstone-blocked-capture", "rebuild-required")),
|
|
1380
|
+
[],
|
|
1381
|
+
"restart rebuild clears the peer marker after incorporating durable rows"
|
|
1382
|
+
);
|
|
1383
|
+
});
|
|
1384
|
+
});
|
|
1385
|
+
|
|
1386
|
+
test("tombstone blocked index sync failure persists a rebuild marker across restart", async () => {
|
|
1387
|
+
await withMemoryDir(async (dir) => {
|
|
1388
|
+
const before = {
|
|
1389
|
+
path: path.join(dir, "facts", "before.md"),
|
|
1390
|
+
frontmatter: {
|
|
1391
|
+
id: "sync-before",
|
|
1392
|
+
category: "fact",
|
|
1393
|
+
created: new Date(0).toISOString(),
|
|
1394
|
+
updated: new Date(0).toISOString(),
|
|
1395
|
+
source: "explicit-inline-review",
|
|
1396
|
+
confidence: 0.2,
|
|
1397
|
+
confidenceTier: "explicit",
|
|
1398
|
+
tags: ["review"],
|
|
1399
|
+
status: "pending_review",
|
|
1400
|
+
blockedBy: "tombstone-sync",
|
|
1401
|
+
sourceConnector: "provider-a",
|
|
1402
|
+
},
|
|
1403
|
+
content: "A blocked capture changes provider identity during an unavailable index rebuild.",
|
|
1404
|
+
} as MemoryFile;
|
|
1405
|
+
const after = {
|
|
1406
|
+
...before,
|
|
1407
|
+
path: path.join(dir, "facts", "after.md"),
|
|
1408
|
+
frontmatter: {
|
|
1409
|
+
...before.frontmatter,
|
|
1410
|
+
id: "sync-after",
|
|
1411
|
+
sourceConnector: "provider-b",
|
|
1412
|
+
},
|
|
1413
|
+
};
|
|
1414
|
+
const memories = [before];
|
|
1415
|
+
const options = {
|
|
1416
|
+
stateDir: dir,
|
|
1417
|
+
memoryDir: dir,
|
|
1418
|
+
secureStoreKeyProvider: () => null,
|
|
1419
|
+
secureStoreWriteKeyProvider: () => null,
|
|
1420
|
+
lockOptions: () => ({ retryMaxAttempts: 1, retryBaseMs: 1 }),
|
|
1421
|
+
readAllMemories: async () => memories,
|
|
1422
|
+
readAllColdMemories: async () => [],
|
|
1423
|
+
};
|
|
1424
|
+
const index = new TombstoneBlockedCaptureIndex(options);
|
|
1425
|
+
await index.add(before);
|
|
1426
|
+
memories[0] = after;
|
|
1427
|
+
|
|
1428
|
+
const rebuildSpy = mock.method(ContentHashIndex.prototype, "rebuildUnderLock", async () => false);
|
|
1429
|
+
try {
|
|
1430
|
+
await index.sync(before, after);
|
|
1431
|
+
assert.equal(
|
|
1432
|
+
(await readdir(path.join(dir, "tombstone-blocked-capture", "rebuild-required"))).length > 0,
|
|
1433
|
+
true,
|
|
1434
|
+
"a failed identity rebuild must leave durable rebuild intent"
|
|
1435
|
+
);
|
|
1436
|
+
} finally {
|
|
1437
|
+
rebuildSpy.mock.restore();
|
|
1438
|
+
}
|
|
1439
|
+
|
|
1440
|
+
const restarted = new TombstoneBlockedCaptureIndex(options);
|
|
1441
|
+
assert.equal(await restarted.has(after.content, "fact", "provider-b"), true);
|
|
1442
|
+
assert.equal(await restarted.has(before.content, "fact", "provider-a"), false);
|
|
1443
|
+
assert.deepEqual(await readdir(path.join(dir, "tombstone-blocked-capture", "rebuild-required")), []);
|
|
1444
|
+
});
|
|
1445
|
+
});
|
|
1446
|
+
|
|
1447
|
+
test("tombstone-blocked writes reserve rebuild intent before post-write index work", async () => {
|
|
1448
|
+
await withMemoryDir(async (dir) => {
|
|
1449
|
+
const storage = new StorageManager(dir);
|
|
1450
|
+
await storage.ensureDirectories();
|
|
1451
|
+
storage.setTombstonesConfig({
|
|
1452
|
+
enabled: true,
|
|
1453
|
+
semanticMatch: false,
|
|
1454
|
+
semanticThreshold: 0.9,
|
|
1455
|
+
namespace: "default",
|
|
1456
|
+
});
|
|
1457
|
+
const content = "A blocked write must remain deduplicable if its index hook fails.";
|
|
1458
|
+
const tombstoneId = await storage.appendTombstone({
|
|
1459
|
+
reason: "correction",
|
|
1460
|
+
createdBy: "user_correction",
|
|
1461
|
+
sourceMemoryId: "retired-blocked-write",
|
|
1462
|
+
rawContent: content,
|
|
1463
|
+
});
|
|
1464
|
+
assert.ok(tombstoneId, "test tombstone must persist");
|
|
1465
|
+
|
|
1466
|
+
const markerDir = path.join(dir, "state", "tombstone-blocked-capture", "rebuild-required");
|
|
1467
|
+
const addSpy = mock.method(TombstoneBlockedCaptureIndex.prototype, "addWrittenMemory", async () => {
|
|
1468
|
+
throw new Error("simulated post-write index failure");
|
|
1469
|
+
});
|
|
1470
|
+
try {
|
|
1471
|
+
await assert.rejects(
|
|
1472
|
+
storage.writeMemory("fact", content, { source: "test", sourceConnector: "provider-a" }),
|
|
1473
|
+
/simulated post-write index failure/
|
|
1474
|
+
);
|
|
1475
|
+
assert.equal(
|
|
1476
|
+
(await readdir(markerDir)).length > 0,
|
|
1477
|
+
true,
|
|
1478
|
+
"blocked memory persistence must leave a durable marker before its index hook"
|
|
1479
|
+
);
|
|
1480
|
+
} finally {
|
|
1481
|
+
addSpy.mock.restore();
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
const restarted = new StorageManager(dir);
|
|
1485
|
+
restarted.setTombstonesConfig({
|
|
1486
|
+
enabled: true,
|
|
1487
|
+
semanticMatch: false,
|
|
1488
|
+
semanticThreshold: 0.9,
|
|
1489
|
+
namespace: "default",
|
|
1490
|
+
});
|
|
1491
|
+
assert.equal(
|
|
1492
|
+
await restarted.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"),
|
|
1493
|
+
true,
|
|
1494
|
+
"restart must rebuild the blocked identity from the durable memory"
|
|
1495
|
+
);
|
|
1496
|
+
assert.deepEqual(await readdir(markerDir), [], "successful restart rebuild clears the marker");
|
|
1497
|
+
});
|
|
1498
|
+
});
|
|
1499
|
+
|
|
1500
|
+
test("blocked rewrites reserve markers before durable mutation index hooks", async () => {
|
|
1501
|
+
await withMemoryDir(async (dir) => {
|
|
1502
|
+
const storage = new StorageManager(dir);
|
|
1503
|
+
await storage.ensureDirectories();
|
|
1504
|
+
storage.setTombstonesConfig({
|
|
1505
|
+
enabled: true,
|
|
1506
|
+
semanticMatch: false,
|
|
1507
|
+
semanticThreshold: 0.9,
|
|
1508
|
+
namespace: "default",
|
|
1509
|
+
});
|
|
1510
|
+
const content = "A blocked rewrite must remain deduplicable across an index hook failure.";
|
|
1511
|
+
const tombstoneId = await storage.appendTombstone({
|
|
1512
|
+
reason: "correction",
|
|
1513
|
+
createdBy: "user_correction",
|
|
1514
|
+
sourceMemoryId: "retired-rewrite",
|
|
1515
|
+
rawContent: content,
|
|
1516
|
+
});
|
|
1517
|
+
assert.ok(tombstoneId, "test tombstone must persist");
|
|
1518
|
+
const result = await storage.writeMemory("fact", content, {
|
|
1519
|
+
source: "test",
|
|
1520
|
+
sourceConnector: "provider-a",
|
|
1521
|
+
});
|
|
1522
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
1523
|
+
const markerDir = path.join(dir, "state", "tombstone-blocked-capture", "rebuild-required");
|
|
1524
|
+
const syncMemorySpy = mock.method(TombstoneBlockedCaptureIndex.prototype, "syncUpdatedMemory", async () => {});
|
|
1525
|
+
try {
|
|
1526
|
+
assert.equal(await storage.updateMemory(result.id, `${content} changed`), true);
|
|
1527
|
+
assert.equal((await readdir(markerDir)).length, 1, "blocked update must leave a committed marker for its hook");
|
|
1528
|
+
} finally {
|
|
1529
|
+
syncMemorySpy.mock.restore();
|
|
1530
|
+
}
|
|
1531
|
+
|
|
1532
|
+
const restarted = new StorageManager(dir);
|
|
1533
|
+
restarted.setTombstonesConfig({
|
|
1534
|
+
enabled: true,
|
|
1535
|
+
semanticMatch: false,
|
|
1536
|
+
semanticThreshold: 0.9,
|
|
1537
|
+
namespace: "default",
|
|
1538
|
+
});
|
|
1539
|
+
assert.equal(await restarted.hasTombstoneBlockedExplicitCapture(`${content} changed`, "fact", "provider-a"), true);
|
|
1540
|
+
const memory = await restarted.getMemoryById(result.id);
|
|
1541
|
+
assert.ok(memory, "updated blocked memory must be readable");
|
|
1542
|
+
const syncFrontmatterSpy = mock.method(
|
|
1543
|
+
TombstoneBlockedCaptureIndex.prototype,
|
|
1544
|
+
"syncUpdatedFrontmatter",
|
|
1545
|
+
async () => {}
|
|
1546
|
+
);
|
|
1547
|
+
try {
|
|
1548
|
+
assert.equal(await restarted.writeMemoryFrontmatter(memory, { sourceConnector: "provider-b" }), true);
|
|
1549
|
+
assert.equal((await readdir(markerDir)).length, 1, "blocked frontmatter rewrite must reserve a marker");
|
|
1550
|
+
} finally {
|
|
1551
|
+
syncFrontmatterSpy.mock.restore();
|
|
1552
|
+
}
|
|
1553
|
+
});
|
|
1554
|
+
});
|
|
1555
|
+
test("blocked rewrites rebuild the index from post-write memory cache state", async () => {
|
|
1556
|
+
await withMemoryDir(async (dir) => {
|
|
1557
|
+
const storage = new StorageManager(dir);
|
|
1558
|
+
await storage.ensureDirectories();
|
|
1559
|
+
storage.setTombstonesConfig({
|
|
1560
|
+
enabled: true,
|
|
1561
|
+
semanticMatch: false,
|
|
1562
|
+
semanticThreshold: 0.9,
|
|
1563
|
+
namespace: "default",
|
|
1564
|
+
});
|
|
1565
|
+
const content = "A blocked rewrite must rebuild from the post-write cache state.";
|
|
1566
|
+
const tombstoneId = await storage.appendTombstone({
|
|
1567
|
+
reason: "correction",
|
|
1568
|
+
createdBy: "user_correction",
|
|
1569
|
+
sourceMemoryId: "post-write-cache-order",
|
|
1570
|
+
rawContent: content,
|
|
1571
|
+
});
|
|
1572
|
+
assert.ok(tombstoneId, "test tombstone must persist");
|
|
1573
|
+
const result = await storage.writeMemory("fact", content, {
|
|
1574
|
+
source: "test",
|
|
1575
|
+
sourceConnector: "provider-a",
|
|
1576
|
+
});
|
|
1577
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
1578
|
+
|
|
1579
|
+
await storage.readAllMemories();
|
|
1580
|
+
const originalReadAllMemories = storage.readAllMemories.bind(storage);
|
|
1581
|
+
const rebuiltContents: string[] = [];
|
|
1582
|
+
storage.readAllMemories = async () => {
|
|
1583
|
+
const memories = await originalReadAllMemories();
|
|
1584
|
+
const memory = memories.find((candidate) => candidate.frontmatter.id === result.id);
|
|
1585
|
+
if (memory) rebuiltContents.push(memory.content);
|
|
1586
|
+
return memories;
|
|
1587
|
+
};
|
|
1588
|
+
|
|
1589
|
+
const updatedContent = `${content} changed`;
|
|
1590
|
+
assert.equal(await storage.updateMemory(result.id, updatedContent), true);
|
|
1591
|
+
assert.ok(
|
|
1592
|
+
rebuiltContents.includes(updatedContent),
|
|
1593
|
+
"the blocked-index rebuild must read the durable post-write content, not the pre-write hot cache"
|
|
1594
|
+
);
|
|
1595
|
+
});
|
|
1596
|
+
});
|
|
1597
|
+
|
|
1598
|
+
test("blocked chunk writes enter the targeted dedupe index", async () => {
|
|
1599
|
+
await withMemoryDir(async (dir) => {
|
|
1600
|
+
const storage = new StorageManager(dir);
|
|
1601
|
+
await storage.ensureDirectories();
|
|
1602
|
+
const content = "A blocked chunk must remain visible to explicit-capture dedupe.";
|
|
1603
|
+
await storage.writeChunk("chunk-parent", 0, 1, "fact", content, {
|
|
1604
|
+
status: "pending_review",
|
|
1605
|
+
blockedBy: "tombstone-chunk",
|
|
1606
|
+
sourceConnector: "provider-a",
|
|
1607
|
+
});
|
|
1608
|
+
assert.equal(
|
|
1609
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"),
|
|
1610
|
+
true,
|
|
1611
|
+
"blocked chunk content must be indexed with its connector identity"
|
|
1612
|
+
);
|
|
1613
|
+
});
|
|
1614
|
+
});
|
|
1615
|
+
|
|
1616
|
+
test("blocked chunk writes recheck identity after waiting on the capture lock", async () => {
|
|
1617
|
+
await withMemoryDir(async (dir) => {
|
|
1618
|
+
const storage = new StorageManager(dir);
|
|
1619
|
+
await storage.ensureDirectories();
|
|
1620
|
+
storage.setTombstonesConfig({
|
|
1621
|
+
enabled: true,
|
|
1622
|
+
semanticMatch: false,
|
|
1623
|
+
semanticThreshold: 0.9,
|
|
1624
|
+
namespace: "default",
|
|
1625
|
+
});
|
|
1626
|
+
const content = "A blocked chunk must reuse a capture committed while it waited.";
|
|
1627
|
+
const tombstoneId = await storage.appendTombstone({
|
|
1628
|
+
reason: "correction",
|
|
1629
|
+
createdBy: "user_correction",
|
|
1630
|
+
sourceMemoryId: "blocked-chunk-lock-recheck",
|
|
1631
|
+
rawContent: content,
|
|
1632
|
+
});
|
|
1633
|
+
assert.ok(tombstoneId);
|
|
1634
|
+
const captureResult = await storage.writeMemory("fact", content, {
|
|
1635
|
+
source: "test",
|
|
1636
|
+
sourceConnector: "provider-a",
|
|
1637
|
+
});
|
|
1638
|
+
assert.equal(captureResult.tombstoneBlocked, true);
|
|
1639
|
+
const identity = buildExplicitCaptureDedupKey(content, "fact", "provider-a");
|
|
1640
|
+
const entered = Promise.withResolvers<void>();
|
|
1641
|
+
const release = Promise.withResolvers<void>();
|
|
1642
|
+
const held = storage.withTombstoneBlockedCaptureWriteLock(async () => {
|
|
1643
|
+
entered.resolve();
|
|
1644
|
+
await release.promise;
|
|
1645
|
+
}, identity);
|
|
1646
|
+
await entered.promise;
|
|
1647
|
+
|
|
1648
|
+
const chunk = storage.writeChunk("chunk-parent", 0, 1, "fact", content, {
|
|
1649
|
+
source: "chunking",
|
|
1650
|
+
sourceConnector: "provider-a",
|
|
1651
|
+
status: "pending_review",
|
|
1652
|
+
blockedBy: tombstoneId,
|
|
1653
|
+
});
|
|
1654
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
1655
|
+
release.resolve();
|
|
1656
|
+
|
|
1657
|
+
const chunkId = await chunk;
|
|
1658
|
+
await held;
|
|
1659
|
+
assert.equal(chunkId, captureResult.id);
|
|
1660
|
+
const matches = (await storage.readAllMemories()).filter((memory) => memory.content === content);
|
|
1661
|
+
assert.equal(matches.length, 1, "the capture and waiting chunk must share one durable row");
|
|
1662
|
+
});
|
|
1663
|
+
});
|
|
1664
|
+
|
|
1665
|
+
test("offline sync rechecks blocked identity after waiting on the capture lock", async () => {
|
|
1666
|
+
await withMemoryDir(async (dir) => {
|
|
1667
|
+
const storage = new StorageManager(dir);
|
|
1668
|
+
await storage.ensureDirectories();
|
|
1669
|
+
storage.setTombstonesConfig({
|
|
1670
|
+
enabled: true,
|
|
1671
|
+
semanticMatch: false,
|
|
1672
|
+
semanticThreshold: 0.9,
|
|
1673
|
+
namespace: "default",
|
|
1674
|
+
});
|
|
1675
|
+
const content = "A buffer sync must not publish a duplicate after waiting on capture.";
|
|
1676
|
+
const tombstoneId = await storage.appendTombstone({
|
|
1677
|
+
reason: "correction",
|
|
1678
|
+
createdBy: "user_correction",
|
|
1679
|
+
sourceMemoryId: "offline-buffer-lock-recheck",
|
|
1680
|
+
rawContent: content,
|
|
1681
|
+
});
|
|
1682
|
+
assert.ok(tombstoneId);
|
|
1683
|
+
const captureResult = await storage.writeMemory("fact", content, {
|
|
1684
|
+
source: "test",
|
|
1685
|
+
sourceConnector: "provider-a",
|
|
1686
|
+
});
|
|
1687
|
+
assert.equal(captureResult.tombstoneBlocked, true);
|
|
1688
|
+
const identity = buildExplicitCaptureDedupKey(content, "fact", "provider-a");
|
|
1689
|
+
const entered = Promise.withResolvers<void>();
|
|
1690
|
+
const release = Promise.withResolvers<void>();
|
|
1691
|
+
const held = storage.withTombstoneBlockedCaptureWriteLock(async () => {
|
|
1692
|
+
entered.resolve();
|
|
1693
|
+
await release.promise;
|
|
1694
|
+
}, identity);
|
|
1695
|
+
await entered.promise;
|
|
1696
|
+
|
|
1697
|
+
const incomingFile = [
|
|
1698
|
+
"---",
|
|
1699
|
+
"id: incoming-buffer-sync",
|
|
1700
|
+
"category: fact",
|
|
1701
|
+
"created: 2026-01-01T00:00:00.000Z",
|
|
1702
|
+
"updated: 2026-01-01T00:00:00.000Z",
|
|
1703
|
+
"source: offline-sync",
|
|
1704
|
+
"confidence: 0.8",
|
|
1705
|
+
"confidenceTier: implied",
|
|
1706
|
+
"tags: []",
|
|
1707
|
+
"sourceConnector: provider-a",
|
|
1708
|
+
"status: pending_review",
|
|
1709
|
+
`blockedBy: ${tombstoneId}`,
|
|
1710
|
+
"---",
|
|
1711
|
+
"",
|
|
1712
|
+
content,
|
|
1713
|
+
"",
|
|
1714
|
+
].join("\n");
|
|
1715
|
+
const target = path.join(dir, "facts", "2026-01-01", "incoming-buffer.md");
|
|
1716
|
+
const pending = storage.writeOfflineSyncFile(target, Buffer.from(incomingFile, "utf8"));
|
|
1717
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
1718
|
+
release.resolve();
|
|
1719
|
+
|
|
1720
|
+
await pending;
|
|
1721
|
+
await held;
|
|
1722
|
+
assert.equal(existsSync(target), false, "sync must not publish a duplicate blocked row");
|
|
1723
|
+
const matches = (await storage.readAllMemories()).filter((memory) => memory.content === content);
|
|
1724
|
+
assert.equal(matches.length, 1);
|
|
1725
|
+
assert.equal(matches[0]?.frontmatter.id, captureResult.id);
|
|
1726
|
+
});
|
|
1727
|
+
});
|
|
1728
|
+
test("offline sync reacquires a rewritten blocked target identity", async () => {
|
|
1729
|
+
await withMemoryDir(async (dir) => {
|
|
1730
|
+
const storage = new StorageManager(dir);
|
|
1731
|
+
const peer = new StorageManager(dir);
|
|
1732
|
+
await storage.ensureDirectories();
|
|
1733
|
+
await peer.ensureDirectories();
|
|
1734
|
+
const tombstoneConfig = {
|
|
1735
|
+
enabled: true,
|
|
1736
|
+
semanticMatch: false,
|
|
1737
|
+
semanticThreshold: 0.9,
|
|
1738
|
+
namespace: "default",
|
|
1739
|
+
};
|
|
1740
|
+
storage.setTombstonesConfig(tombstoneConfig);
|
|
1741
|
+
peer.setTombstonesConfig(tombstoneConfig);
|
|
1742
|
+
const contentA = "Offline sync incoming identity must replace the target.";
|
|
1743
|
+
const contentB = "The target identity is stale before the sync lock is acquired.";
|
|
1744
|
+
const contentC = "A peer rewrote the target identity while sync waited.";
|
|
1745
|
+
const tombstones = await Promise.all(
|
|
1746
|
+
[contentA, contentB, contentC].map((rawContent, index) =>
|
|
1747
|
+
storage.appendTombstone({
|
|
1748
|
+
reason: "correction",
|
|
1749
|
+
createdBy: "user_correction",
|
|
1750
|
+
sourceMemoryId: `offline-sync-reloaded-target-${index}`,
|
|
1751
|
+
rawContent,
|
|
1752
|
+
})
|
|
1753
|
+
)
|
|
1754
|
+
);
|
|
1755
|
+
assert.ok(tombstones.every(Boolean));
|
|
1756
|
+
const target = await storage.writeMemory("fact", contentB, {
|
|
1757
|
+
source: "test",
|
|
1758
|
+
sourceConnector: "provider-a",
|
|
1759
|
+
});
|
|
1760
|
+
assert.equal(target.tombstoneBlocked, true);
|
|
1761
|
+
const incomingFile = [
|
|
1762
|
+
"---",
|
|
1763
|
+
"id: incoming-reloaded-target",
|
|
1764
|
+
"category: fact",
|
|
1765
|
+
"created: 2026-01-01T00:00:00.000Z",
|
|
1766
|
+
"updated: 2026-01-01T00:00:00.000Z",
|
|
1767
|
+
"source: offline-sync",
|
|
1768
|
+
"confidence: 0.8",
|
|
1769
|
+
"confidenceTier: implied",
|
|
1770
|
+
"tags: []",
|
|
1771
|
+
"sourceConnector: provider-a",
|
|
1772
|
+
"status: pending_review",
|
|
1773
|
+
`blockedBy: ${tombstones[0]}`,
|
|
1774
|
+
"---",
|
|
1775
|
+
"",
|
|
1776
|
+
contentA,
|
|
1777
|
+
"",
|
|
1778
|
+
].join("\n");
|
|
1779
|
+
const targetMemory = (await storage.readAllMemories()).find(
|
|
1780
|
+
(memory) => memory.frontmatter.id === target.id
|
|
1781
|
+
);
|
|
1782
|
+
assert.ok(targetMemory);
|
|
1783
|
+
const identityA = buildExplicitCaptureDedupKey(contentA, "fact", "provider-a");
|
|
1784
|
+
const identityB = buildExplicitCaptureDedupKey(contentB, "fact", "provider-a");
|
|
1785
|
+
const identityC = buildExplicitCaptureDedupKey(contentC, "fact", "provider-a");
|
|
1786
|
+
const storageWithOverride = storage as unknown as {
|
|
1787
|
+
withTombstoneBlockedCaptureWriteLock: (
|
|
1788
|
+
task: () => Promise<unknown>,
|
|
1789
|
+
identity?: string | readonly string[],
|
|
1790
|
+
) => Promise<unknown>;
|
|
1791
|
+
};
|
|
1792
|
+
const originalLock = storageWithOverride.withTombstoneBlockedCaptureWriteLock.bind(storage);
|
|
1793
|
+
const capturedIdentities: Array<string | readonly string[] | undefined> = [];
|
|
1794
|
+
let firstLock = true;
|
|
1795
|
+
let pauseNextPrepare = false;
|
|
1796
|
+
const markerEntered = Promise.withResolvers<void>();
|
|
1797
|
+
const releaseMarker = Promise.withResolvers<void>();
|
|
1798
|
+
storageWithOverride.withTombstoneBlockedCaptureWriteLock = async (task, identity) => {
|
|
1799
|
+
capturedIdentities.push(identity);
|
|
1800
|
+
if (firstLock) {
|
|
1801
|
+
firstLock = false;
|
|
1802
|
+
await peer.updateMemory(target.id, contentC, { sourceConnector: "provider-a" });
|
|
1803
|
+
pauseNextPrepare = true;
|
|
1804
|
+
}
|
|
1805
|
+
return await originalLock(task, identity);
|
|
1806
|
+
};
|
|
1807
|
+
const originalPrepare = TombstoneBlockedCaptureIndex.prototype.prepareWrite;
|
|
1808
|
+
const prepareSpy = mock.method(
|
|
1809
|
+
TombstoneBlockedCaptureIndex.prototype,
|
|
1810
|
+
"prepareWrite",
|
|
1811
|
+
async function (this: TombstoneBlockedCaptureIndex) {
|
|
1812
|
+
const marker = await originalPrepare.call(this);
|
|
1813
|
+
if (pauseNextPrepare) {
|
|
1814
|
+
pauseNextPrepare = false;
|
|
1815
|
+
markerEntered.resolve();
|
|
1816
|
+
await releaseMarker.promise;
|
|
1817
|
+
}
|
|
1818
|
+
return marker;
|
|
1819
|
+
},
|
|
1820
|
+
);
|
|
1821
|
+
|
|
1822
|
+
const sync = storage.writeOfflineSyncFile(targetMemory.path, Buffer.from(incomingFile, "utf8"));
|
|
1823
|
+
try {
|
|
1824
|
+
await markerEntered.promise;
|
|
1825
|
+
const firstKeys = capturedIdentities[0];
|
|
1826
|
+
const secondKeys = capturedIdentities[1];
|
|
1827
|
+
assert.ok(
|
|
1828
|
+
(Array.isArray(firstKeys) ? firstKeys : [firstKeys]).includes(identityB),
|
|
1829
|
+
"sync must first observe the original target identity",
|
|
1830
|
+
);
|
|
1831
|
+
assert.ok(
|
|
1832
|
+
(Array.isArray(secondKeys) ? secondKeys : [secondKeys]).includes(identityC),
|
|
1833
|
+
"sync must reacquire the rewritten target identity",
|
|
1834
|
+
);
|
|
1835
|
+
let recaptureCompleted = false;
|
|
1836
|
+
const recapture = peer
|
|
1837
|
+
.writeMemory("fact", contentC, {
|
|
1838
|
+
source: "test",
|
|
1839
|
+
sourceConnector: "provider-a",
|
|
1840
|
+
})
|
|
1841
|
+
.then((result) => {
|
|
1842
|
+
recaptureCompleted = true;
|
|
1843
|
+
return result;
|
|
1844
|
+
});
|
|
1845
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
1846
|
+
assert.equal(recaptureCompleted, false, "recapture must wait for the current target identity lock");
|
|
1847
|
+
releaseMarker.resolve();
|
|
1848
|
+
await sync;
|
|
1849
|
+
const recaptured = await recapture;
|
|
1850
|
+
assert.equal(recaptured.tombstoneBlocked, true);
|
|
1851
|
+
const after = await peer.readAllMemories();
|
|
1852
|
+
assert.equal(
|
|
1853
|
+
after.find((memory) => memory.path === targetMemory.path)?.content,
|
|
1854
|
+
contentA,
|
|
1855
|
+
"offline sync must replace the target after holding its current identity lock",
|
|
1856
|
+
);
|
|
1857
|
+
assert.equal(after.filter((memory) => memory.content === contentC).length, 1);
|
|
1858
|
+
} finally {
|
|
1859
|
+
releaseMarker.resolve();
|
|
1860
|
+
prepareSpy.mock.restore();
|
|
1861
|
+
storageWithOverride.withTombstoneBlockedCaptureWriteLock = originalLock;
|
|
1862
|
+
await sync.catch(() => undefined);
|
|
1863
|
+
}
|
|
1864
|
+
});
|
|
1865
|
+
});
|
|
1866
|
+
|
|
1867
|
+
test("offline sync verifies blocked-index hits against exact identities", async () => {
|
|
1868
|
+
await withMemoryDir(async (dir) => {
|
|
1869
|
+
const storage = new StorageManager(dir);
|
|
1870
|
+
await storage.ensureDirectories();
|
|
1871
|
+
storage.setTombstonesConfig({
|
|
1872
|
+
enabled: true,
|
|
1873
|
+
semanticMatch: false,
|
|
1874
|
+
semanticThreshold: 0.9,
|
|
1875
|
+
namespace: "default",
|
|
1876
|
+
});
|
|
1877
|
+
const existingContent = "Remember cats.";
|
|
1878
|
+
const incomingContent = "Remember cats!";
|
|
1879
|
+
const existingTombstone = await storage.appendTombstone({
|
|
1880
|
+
reason: "correction",
|
|
1881
|
+
createdBy: "user_correction",
|
|
1882
|
+
sourceMemoryId: "exact-identity-existing",
|
|
1883
|
+
rawContent: existingContent,
|
|
1884
|
+
});
|
|
1885
|
+
const incomingTombstone = await storage.appendTombstone({
|
|
1886
|
+
reason: "correction",
|
|
1887
|
+
createdBy: "user_correction",
|
|
1888
|
+
sourceMemoryId: "exact-identity-incoming",
|
|
1889
|
+
rawContent: incomingContent,
|
|
1890
|
+
});
|
|
1891
|
+
assert.ok(existingTombstone);
|
|
1892
|
+
assert.ok(incomingTombstone);
|
|
1893
|
+
const existing = await storage.writeMemory("fact", existingContent, {
|
|
1894
|
+
source: "test",
|
|
1895
|
+
sourceConnector: "provider-a",
|
|
1896
|
+
});
|
|
1897
|
+
assert.equal(existing.tombstoneBlocked, true);
|
|
1898
|
+
const incomingFile = [
|
|
1899
|
+
"---",
|
|
1900
|
+
"id: exact-identity-incoming",
|
|
1901
|
+
"category: fact",
|
|
1902
|
+
"created: 2026-01-01T00:00:00.000Z",
|
|
1903
|
+
"updated: 2026-01-01T00:00:00.000Z",
|
|
1904
|
+
"source: offline-sync",
|
|
1905
|
+
"confidence: 0.8",
|
|
1906
|
+
"confidenceTier: implied",
|
|
1907
|
+
"tags: []",
|
|
1908
|
+
"sourceConnector: provider-a",
|
|
1909
|
+
"status: pending_review",
|
|
1910
|
+
`blockedBy: ${incomingTombstone}`,
|
|
1911
|
+
"---",
|
|
1912
|
+
"",
|
|
1913
|
+
incomingContent,
|
|
1914
|
+
"",
|
|
1915
|
+
].join("\n");
|
|
1916
|
+
const target = path.join(dir, "facts", "2026-01-01", "exact-identity-incoming.md");
|
|
1917
|
+
await storage.writeOfflineSyncFile(target, Buffer.from(incomingFile, "utf8"));
|
|
1918
|
+
assert.equal(existsSync(target), true);
|
|
1919
|
+
assert.equal((await storage.readAllMemories()).filter((memory) => memory.content === incomingContent).length, 1);
|
|
1920
|
+
});
|
|
1921
|
+
});
|
|
1922
|
+
|
|
1923
|
+
test("offline sync replaces a blocked target when the incoming identity exists elsewhere", async () => {
|
|
1924
|
+
await withMemoryDir(async (dir) => {
|
|
1925
|
+
const storage = new StorageManager(dir);
|
|
1926
|
+
await storage.ensureDirectories();
|
|
1927
|
+
storage.setTombstonesConfig({
|
|
1928
|
+
enabled: true,
|
|
1929
|
+
semanticMatch: false,
|
|
1930
|
+
semanticThreshold: 0.9,
|
|
1931
|
+
namespace: "default",
|
|
1932
|
+
});
|
|
1933
|
+
const incomingContent = "Replica canonical content.";
|
|
1934
|
+
const targetContent = "Replica stale content.";
|
|
1935
|
+
const incomingTombstone = await storage.appendTombstone({
|
|
1936
|
+
reason: "correction",
|
|
1937
|
+
createdBy: "user_correction",
|
|
1938
|
+
sourceMemoryId: "target-replacement-incoming",
|
|
1939
|
+
rawContent: incomingContent,
|
|
1940
|
+
});
|
|
1941
|
+
const targetTombstone = await storage.appendTombstone({
|
|
1942
|
+
reason: "correction",
|
|
1943
|
+
createdBy: "user_correction",
|
|
1944
|
+
sourceMemoryId: "target-replacement-target",
|
|
1945
|
+
rawContent: targetContent,
|
|
1946
|
+
});
|
|
1947
|
+
assert.ok(incomingTombstone);
|
|
1948
|
+
assert.ok(targetTombstone);
|
|
1949
|
+
const incoming = await storage.writeMemory("fact", incomingContent, {
|
|
1950
|
+
source: "test",
|
|
1951
|
+
sourceConnector: "provider-a",
|
|
1952
|
+
});
|
|
1953
|
+
const target = await storage.writeMemory("fact", targetContent, {
|
|
1954
|
+
source: "test",
|
|
1955
|
+
sourceConnector: "provider-a",
|
|
1956
|
+
});
|
|
1957
|
+
assert.equal(incoming.tombstoneBlocked, true);
|
|
1958
|
+
assert.equal(target.tombstoneBlocked, true);
|
|
1959
|
+
const incomingFile = [
|
|
1960
|
+
"---",
|
|
1961
|
+
"id: target-replacement-incoming",
|
|
1962
|
+
"category: fact",
|
|
1963
|
+
"created: 2026-01-01T00:00:00.000Z",
|
|
1964
|
+
"updated: 2026-01-01T00:00:00.000Z",
|
|
1965
|
+
"source: offline-sync",
|
|
1966
|
+
"confidence: 0.8",
|
|
1967
|
+
"confidenceTier: implied",
|
|
1968
|
+
"tags: []",
|
|
1969
|
+
"sourceConnector: provider-a",
|
|
1970
|
+
"status: pending_review",
|
|
1971
|
+
`blockedBy: ${incomingTombstone}`,
|
|
1972
|
+
"---",
|
|
1973
|
+
"",
|
|
1974
|
+
incomingContent,
|
|
1975
|
+
"",
|
|
1976
|
+
].join("\n");
|
|
1977
|
+
const targetMemory = (await storage.readAllMemories()).find((memory) => memory.frontmatter.id === target.id);
|
|
1978
|
+
assert.ok(targetMemory);
|
|
1979
|
+
await storage.writeOfflineSyncFile(targetMemory.path, Buffer.from(incomingFile, "utf8"));
|
|
1980
|
+
const replaced = (await storage.readAllMemories()).find((memory) => memory.path === targetMemory.path);
|
|
1981
|
+
assert.ok(replaced);
|
|
1982
|
+
assert.equal(replaced.content, incomingContent);
|
|
1983
|
+
assert.equal(replaced.frontmatter.id, "target-replacement-incoming");
|
|
1984
|
+
});
|
|
1985
|
+
});
|
|
1986
|
+
|
|
1987
|
+
test("offline sync replaces a non-blocked target when the incoming identity exists elsewhere", async () => {
|
|
1988
|
+
await withMemoryDir(async (dir) => {
|
|
1989
|
+
const storage = new StorageManager(dir);
|
|
1990
|
+
await storage.ensureDirectories();
|
|
1991
|
+
storage.setTombstonesConfig({
|
|
1992
|
+
enabled: true,
|
|
1993
|
+
semanticMatch: false,
|
|
1994
|
+
semanticThreshold: 0.9,
|
|
1995
|
+
namespace: "default",
|
|
1996
|
+
});
|
|
1997
|
+
const incomingContent = "Replica canonical content for an active target.";
|
|
1998
|
+
const targetContent = "Replica stale content for an active target.";
|
|
1999
|
+
const incomingTombstone = await storage.appendTombstone({
|
|
2000
|
+
reason: "correction",
|
|
2001
|
+
createdBy: "user_correction",
|
|
2002
|
+
sourceMemoryId: "nonblocked-target-incoming",
|
|
2003
|
+
rawContent: incomingContent,
|
|
2004
|
+
});
|
|
2005
|
+
assert.ok(incomingTombstone);
|
|
2006
|
+
const incoming = await storage.writeMemory("fact", incomingContent, {
|
|
2007
|
+
source: "test",
|
|
2008
|
+
sourceConnector: "provider-a",
|
|
2009
|
+
});
|
|
2010
|
+
const target = await storage.writeMemory("fact", targetContent, {
|
|
2011
|
+
source: "test",
|
|
2012
|
+
sourceConnector: "provider-a",
|
|
2013
|
+
});
|
|
2014
|
+
assert.equal(incoming.tombstoneBlocked, true);
|
|
2015
|
+
assert.equal(target.tombstoneBlocked, false);
|
|
2016
|
+
const targetMemory = await storage.getMemoryById(target.id);
|
|
2017
|
+
assert.ok(targetMemory);
|
|
2018
|
+
const incomingFile = [
|
|
2019
|
+
"---",
|
|
2020
|
+
"id: nonblocked-target-incoming",
|
|
2021
|
+
"category: fact",
|
|
2022
|
+
"created: 2026-01-01T00:00:00.000Z",
|
|
2023
|
+
"updated: 2026-01-01T00:00:00.000Z",
|
|
2024
|
+
"source: offline-sync",
|
|
2025
|
+
"confidence: 0.8",
|
|
2026
|
+
"confidenceTier: implied",
|
|
2027
|
+
"tags: []",
|
|
2028
|
+
"sourceConnector: provider-a",
|
|
2029
|
+
"status: pending_review",
|
|
2030
|
+
`blockedBy: ${incomingTombstone}`,
|
|
2031
|
+
"---",
|
|
2032
|
+
"",
|
|
2033
|
+
incomingContent,
|
|
2034
|
+
"",
|
|
2035
|
+
].join("\n");
|
|
2036
|
+
await storage.writeOfflineSyncFile(targetMemory.path, Buffer.from(incomingFile, "utf8"));
|
|
2037
|
+
const replaced = (await storage.readAllMemories()).find((memory) => memory.path === targetMemory.path);
|
|
2038
|
+
assert.ok(replaced);
|
|
2039
|
+
assert.equal(replaced.content, incomingContent);
|
|
2040
|
+
assert.equal(replaced.frontmatter.id, "nonblocked-target-incoming");
|
|
2041
|
+
});
|
|
2042
|
+
});
|
|
2043
|
+
|
|
2044
|
+
test("offline sync applies same-identity durable metadata updates", async () => {
|
|
2045
|
+
await withMemoryDir(async (dir) => {
|
|
2046
|
+
const storage = new StorageManager(dir);
|
|
2047
|
+
await storage.ensureDirectories();
|
|
2048
|
+
storage.setTombstonesConfig({
|
|
2049
|
+
enabled: true,
|
|
2050
|
+
semanticMatch: false,
|
|
2051
|
+
semanticThreshold: 0.9,
|
|
2052
|
+
namespace: "default",
|
|
2053
|
+
});
|
|
2054
|
+
const content = "Replica metadata needs to stay synchronized.";
|
|
2055
|
+
const firstTombstone = await storage.appendTombstone({
|
|
2056
|
+
reason: "correction",
|
|
2057
|
+
createdBy: "user_correction",
|
|
2058
|
+
sourceMemoryId: "same-identity-before",
|
|
2059
|
+
rawContent: content,
|
|
2060
|
+
});
|
|
2061
|
+
const secondTombstone = await storage.appendTombstone({
|
|
2062
|
+
reason: "correction",
|
|
2063
|
+
createdBy: "user_correction",
|
|
2064
|
+
sourceMemoryId: "same-identity-after",
|
|
2065
|
+
rawContent: content,
|
|
2066
|
+
});
|
|
2067
|
+
assert.ok(firstTombstone);
|
|
2068
|
+
assert.ok(secondTombstone);
|
|
2069
|
+
const initial = await storage.writeMemory("fact", content, {
|
|
2070
|
+
source: "test",
|
|
2071
|
+
sourceConnector: "provider-a",
|
|
2072
|
+
});
|
|
2073
|
+
assert.equal(initial.tombstoneBlocked, true);
|
|
2074
|
+
const before = await storage.getMemoryById(initial.id);
|
|
2075
|
+
assert.ok(before);
|
|
2076
|
+
const incomingFile = [
|
|
2077
|
+
"---",
|
|
2078
|
+
"id: same-identity-after",
|
|
2079
|
+
"category: fact",
|
|
2080
|
+
"created: 2026-01-01T00:00:00.000Z",
|
|
2081
|
+
"updated: 2026-02-01T00:00:00.000Z",
|
|
2082
|
+
"source: offline-sync",
|
|
2083
|
+
"confidence: 0.95",
|
|
2084
|
+
"confidenceTier: explicit",
|
|
2085
|
+
'tags: ["synced"]',
|
|
2086
|
+
"sourceConnector: provider-a",
|
|
2087
|
+
"status: pending_review",
|
|
2088
|
+
`blockedBy: ${secondTombstone}`,
|
|
2089
|
+
"---",
|
|
2090
|
+
"",
|
|
2091
|
+
content,
|
|
2092
|
+
"",
|
|
2093
|
+
].join("\n");
|
|
2094
|
+
await storage.writeOfflineSyncFile(before.path, Buffer.from(incomingFile, "utf8"));
|
|
2095
|
+
const after = await storage.getMemoryById("same-identity-after");
|
|
2096
|
+
assert.ok(after);
|
|
2097
|
+
assert.equal(after.content, content);
|
|
2098
|
+
assert.equal(after.frontmatter.tags.join(","), "synced");
|
|
2099
|
+
assert.equal(after.frontmatter.confidence, 0.95);
|
|
2100
|
+
assert.equal(after.frontmatter.confidenceTier, "explicit");
|
|
2101
|
+
assert.equal(after.frontmatter.updated, "2026-02-01T00:00:00.000Z");
|
|
2102
|
+
assert.equal(after.frontmatter.blockedBy, secondTombstone);
|
|
2103
|
+
});
|
|
2104
|
+
});
|
|
2105
|
+
|
|
2106
|
+
test("offline sync mutation rebuilds a loaded tombstone-blocked index", async () => {
|
|
2107
|
+
await withMemoryDir(async (dir) => {
|
|
2108
|
+
const storage = new StorageManager(dir);
|
|
2109
|
+
await storage.ensureDirectories();
|
|
2110
|
+
storage.setTombstonesConfig({
|
|
2111
|
+
enabled: true,
|
|
2112
|
+
semanticMatch: false,
|
|
2113
|
+
semanticThreshold: 0.9,
|
|
2114
|
+
namespace: "default",
|
|
2115
|
+
});
|
|
2116
|
+
const content = "An offline sync update changes the blocked provider identity.";
|
|
2117
|
+
const tombstoneId = await storage.appendTombstone({
|
|
2118
|
+
reason: "correction",
|
|
2119
|
+
createdBy: "user_correction",
|
|
2120
|
+
sourceMemoryId: "retired-offline-sync",
|
|
2121
|
+
rawContent: content,
|
|
2122
|
+
});
|
|
2123
|
+
assert.ok(tombstoneId, "test tombstone must persist");
|
|
2124
|
+
const result = await storage.writeMemory("fact", content, {
|
|
2125
|
+
source: "test",
|
|
2126
|
+
sourceConnector: "provider-a",
|
|
2127
|
+
});
|
|
2128
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
2129
|
+
assert.equal(await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"), true);
|
|
2130
|
+
|
|
2131
|
+
const memory = (await storage.readAllMemories()).find((candidate) => candidate.frontmatter.id === result.id);
|
|
2132
|
+
assert.ok(memory, "blocked memory must be readable before offline sync");
|
|
2133
|
+
const updatedAt = new Date(Date.now() + 1_000).toISOString();
|
|
2134
|
+
const updatedFile = [
|
|
2135
|
+
"---",
|
|
2136
|
+
`id: ${memory.frontmatter.id}`,
|
|
2137
|
+
"category: fact",
|
|
2138
|
+
`created: ${memory.frontmatter.created}`,
|
|
2139
|
+
`updated: ${updatedAt}`,
|
|
2140
|
+
`source: ${memory.frontmatter.source}`,
|
|
2141
|
+
`confidence: ${memory.frontmatter.confidence}`,
|
|
2142
|
+
`confidenceTier: ${memory.frontmatter.confidenceTier}`,
|
|
2143
|
+
"tags: []",
|
|
2144
|
+
"sourceConnector: provider-b",
|
|
2145
|
+
"status: pending_review",
|
|
2146
|
+
`blockedBy: ${memory.frontmatter.blockedBy}`,
|
|
2147
|
+
"---",
|
|
2148
|
+
"",
|
|
2149
|
+
memory.content,
|
|
2150
|
+
"",
|
|
2151
|
+
].join("\n");
|
|
2152
|
+
await storage.writeOfflineSyncFile(memory.path, Buffer.from(updatedFile, "utf8"));
|
|
2153
|
+
|
|
2154
|
+
assert.equal(
|
|
2155
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"),
|
|
2156
|
+
false,
|
|
2157
|
+
"offline sync must remove the stale provider identity from the loaded index"
|
|
2158
|
+
);
|
|
2159
|
+
assert.equal(
|
|
2160
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-b"),
|
|
2161
|
+
true,
|
|
2162
|
+
"offline sync must add the updated provider identity"
|
|
2163
|
+
);
|
|
2164
|
+
});
|
|
2165
|
+
});
|
|
2166
|
+
|
|
2167
|
+
test("chunked offline sync reserves the incoming blocked identity before streaming", async () => {
|
|
2168
|
+
await withMemoryDir(async (dir) => {
|
|
2169
|
+
const storage = new StorageManager(dir);
|
|
2170
|
+
await storage.ensureDirectories();
|
|
2171
|
+
storage.setTombstonesConfig({
|
|
2172
|
+
enabled: true,
|
|
2173
|
+
semanticMatch: false,
|
|
2174
|
+
semanticThreshold: 0.9,
|
|
2175
|
+
namespace: "default",
|
|
2176
|
+
});
|
|
2177
|
+
const content = "Chunked offline sync must coordinate its incoming blocked identity.";
|
|
2178
|
+
const tombstoneId = await storage.appendTombstone({
|
|
2179
|
+
reason: "correction",
|
|
2180
|
+
createdBy: "user_correction",
|
|
2181
|
+
sourceMemoryId: "chunked-incoming-identity",
|
|
2182
|
+
rawContent: content,
|
|
2183
|
+
});
|
|
2184
|
+
assert.ok(tombstoneId);
|
|
2185
|
+
const result = await storage.writeMemory("fact", content, {
|
|
2186
|
+
source: "test",
|
|
2187
|
+
sourceConnector: "provider-a",
|
|
2188
|
+
});
|
|
2189
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
2190
|
+
const memory = (await storage.readAllMemories()).find((candidate) => candidate.frontmatter.id === result.id);
|
|
2191
|
+
assert.ok(memory);
|
|
2192
|
+
|
|
2193
|
+
const updatedFile = [
|
|
2194
|
+
"---",
|
|
2195
|
+
`id: ${memory.frontmatter.id}`,
|
|
2196
|
+
"category: fact",
|
|
2197
|
+
`created: ${memory.frontmatter.created}`,
|
|
2198
|
+
`updated: ${new Date(Date.now() + 1_000).toISOString()}`,
|
|
2199
|
+
`source: ${memory.frontmatter.source}`,
|
|
2200
|
+
`confidence: ${memory.frontmatter.confidence}`,
|
|
2201
|
+
`confidenceTier: ${memory.frontmatter.confidenceTier}`,
|
|
2202
|
+
"tags: []",
|
|
2203
|
+
"sourceConnector: provider-b",
|
|
2204
|
+
"status: pending_review",
|
|
2205
|
+
`blockedBy: ${memory.frontmatter.blockedBy}`,
|
|
2206
|
+
"---",
|
|
2207
|
+
"",
|
|
2208
|
+
memory.content,
|
|
2209
|
+
"",
|
|
2210
|
+
].join("\n");
|
|
2211
|
+
const chunks = (async function* (): AsyncIterable<Buffer> {
|
|
2212
|
+
for (let offset = 0; offset < updatedFile.length; offset += 31) {
|
|
2213
|
+
yield Buffer.from(updatedFile.slice(offset, offset + 31), "utf8");
|
|
2214
|
+
}
|
|
2215
|
+
})();
|
|
2216
|
+
const identity = buildExplicitCaptureDedupKey(memory.content, "fact", "provider-b");
|
|
2217
|
+
const releaseState = Promise.withResolvers<void>();
|
|
2218
|
+
const enteredState = Promise.withResolvers<void>();
|
|
2219
|
+
const held = storage.withTombstoneBlockedCaptureWriteLock(async () => {
|
|
2220
|
+
enteredState.resolve();
|
|
2221
|
+
await releaseState.promise;
|
|
2222
|
+
}, identity);
|
|
2223
|
+
await enteredState.promise;
|
|
2224
|
+
|
|
2225
|
+
let completed = false;
|
|
2226
|
+
const pending = storage.writeOfflineSyncFileChunks(memory.path, chunks).then(() => {
|
|
2227
|
+
completed = true;
|
|
2228
|
+
});
|
|
2229
|
+
const delayState = Promise.withResolvers<void>();
|
|
2230
|
+
setTimeout(delayState.resolve, 50);
|
|
2231
|
+
await delayState.promise;
|
|
2232
|
+
assert.equal(completed, false, "chunked sync must wait for the incoming identity lock");
|
|
2233
|
+
releaseState.resolve();
|
|
2234
|
+
await held;
|
|
2235
|
+
await pending;
|
|
2236
|
+
assert.equal(await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-b"), true);
|
|
2237
|
+
});
|
|
2238
|
+
});
|
|
2239
|
+
|
|
2240
|
+
test("chunked offline sync scans oversized frontmatter before locking its identity", async () => {
|
|
2241
|
+
await withMemoryDir(async (dir) => {
|
|
2242
|
+
const storage = new StorageManager(dir);
|
|
2243
|
+
await storage.ensureDirectories();
|
|
2244
|
+
storage.setTombstonesConfig({
|
|
2245
|
+
enabled: true,
|
|
2246
|
+
semanticMatch: false,
|
|
2247
|
+
semanticThreshold: 0.9,
|
|
2248
|
+
namespace: "default",
|
|
2249
|
+
});
|
|
2250
|
+
const content = `Oversized frontmatter sync ${"x".repeat(900_000)}`;
|
|
2251
|
+
const tombstoneId = await storage.appendTombstone({
|
|
2252
|
+
reason: "correction",
|
|
2253
|
+
createdBy: "user_correction",
|
|
2254
|
+
sourceMemoryId: "oversized-frontmatter-sync",
|
|
2255
|
+
rawContent: content,
|
|
2256
|
+
});
|
|
2257
|
+
assert.ok(tombstoneId);
|
|
2258
|
+
const result = await storage.writeMemory("fact", content, {
|
|
2259
|
+
source: "test",
|
|
2260
|
+
sourceConnector: "provider-a",
|
|
2261
|
+
});
|
|
2262
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
2263
|
+
const memory = (await storage.readAllMemories()).find((candidate) => candidate.frontmatter.id === result.id);
|
|
2264
|
+
assert.ok(memory);
|
|
2265
|
+
|
|
2266
|
+
const largeTags = Array.from({ length: 22_000 }, (_, index) => `"tag-${index}"`).join(", ");
|
|
2267
|
+
const updatedFile = [
|
|
2268
|
+
"---",
|
|
2269
|
+
`id: ${memory.frontmatter.id}`,
|
|
2270
|
+
"category: fact",
|
|
2271
|
+
`created: ${memory.frontmatter.created}`,
|
|
2272
|
+
`updated: ${new Date(Date.now() + 1_000).toISOString()}`,
|
|
2273
|
+
`source: ${memory.frontmatter.source}`,
|
|
2274
|
+
`confidence: ${memory.frontmatter.confidence}`,
|
|
2275
|
+
`confidenceTier: ${memory.frontmatter.confidenceTier}`,
|
|
2276
|
+
`tags: [${largeTags}]`,
|
|
2277
|
+
"sourceConnector: provider-b",
|
|
2278
|
+
"status: pending_review",
|
|
2279
|
+
`blockedBy: ${memory.frontmatter.blockedBy}`,
|
|
2280
|
+
"---",
|
|
2281
|
+
"",
|
|
2282
|
+
content,
|
|
2283
|
+
"",
|
|
2284
|
+
].join("\n");
|
|
2285
|
+
const chunks = (async function* (): AsyncIterable<Buffer> {
|
|
2286
|
+
for (let offset = 0; offset < updatedFile.length; offset += 32 * 1024) {
|
|
2287
|
+
yield Buffer.from(updatedFile.slice(offset, offset + 32 * 1024), "utf8");
|
|
2288
|
+
}
|
|
2289
|
+
})();
|
|
2290
|
+
const incomingIdentity = buildExplicitCaptureDedupKey(content, "fact", "provider-b");
|
|
2291
|
+
const releaseState = Promise.withResolvers<void>();
|
|
2292
|
+
const enteredState = Promise.withResolvers<void>();
|
|
2293
|
+
const held = storage.withTombstoneBlockedCaptureWriteLock(async () => {
|
|
2294
|
+
enteredState.resolve();
|
|
2295
|
+
await releaseState.promise;
|
|
2296
|
+
}, incomingIdentity);
|
|
2297
|
+
await enteredState.promise;
|
|
2298
|
+
|
|
2299
|
+
let completed = false;
|
|
2300
|
+
const pending = storage.writeOfflineSyncFileChunks(memory.path, chunks).then(() => {
|
|
2301
|
+
completed = true;
|
|
2302
|
+
});
|
|
2303
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
2304
|
+
assert.equal(completed, false, "oversized frontmatter sync must wait for its incoming identity lock");
|
|
2305
|
+
releaseState.resolve();
|
|
2306
|
+
await held;
|
|
2307
|
+
await pending;
|
|
2308
|
+
assert.equal(await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-b"), true);
|
|
2309
|
+
});
|
|
2310
|
+
});
|
|
2311
|
+
|
|
2312
|
+
test("chunked offline sync encrypts oversized staging files when secure storage is enabled", async () => {
|
|
2313
|
+
await withMemoryDir(async (dir) => {
|
|
2314
|
+
const storage = new StorageManager(dir);
|
|
2315
|
+
await storage.ensureDirectories();
|
|
2316
|
+
storage.setSecureStoreKey(Buffer.alloc(32, 7));
|
|
2317
|
+
storage.setTombstonesConfig({
|
|
2318
|
+
enabled: true,
|
|
2319
|
+
semanticMatch: false,
|
|
2320
|
+
semanticThreshold: 0.9,
|
|
2321
|
+
namespace: "default",
|
|
2322
|
+
});
|
|
2323
|
+
const content = `Oversized encrypted staging content ${"x".repeat(1_048_576)}`;
|
|
2324
|
+
const tombstoneId = await storage.appendTombstone({
|
|
2325
|
+
reason: "correction",
|
|
2326
|
+
createdBy: "user_correction",
|
|
2327
|
+
sourceMemoryId: "encrypted-staging-memory",
|
|
2328
|
+
rawContent: content,
|
|
2329
|
+
});
|
|
2330
|
+
assert.ok(tombstoneId);
|
|
2331
|
+
const result = await storage.writeMemory("fact", content, {
|
|
2332
|
+
source: "test",
|
|
2333
|
+
sourceConnector: "provider-a",
|
|
2334
|
+
});
|
|
2335
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
2336
|
+
const memory = (await storage.readAllMemories()).find((candidate) => candidate.frontmatter.id === result.id);
|
|
2337
|
+
assert.ok(memory);
|
|
2338
|
+
|
|
2339
|
+
const incomingFile = [
|
|
2340
|
+
"---",
|
|
2341
|
+
`id: ${memory.frontmatter.id}`,
|
|
2342
|
+
"category: fact",
|
|
2343
|
+
`created: ${memory.frontmatter.created}`,
|
|
2344
|
+
`updated: ${new Date(Date.now() + 1_000).toISOString()}`,
|
|
2345
|
+
`source: ${memory.frontmatter.source}`,
|
|
2346
|
+
`confidence: ${memory.frontmatter.confidence}`,
|
|
2347
|
+
`confidenceTier: ${memory.frontmatter.confidenceTier}`,
|
|
2348
|
+
"tags: []",
|
|
2349
|
+
"sourceConnector: provider-a",
|
|
2350
|
+
"status: pending_review",
|
|
2351
|
+
`blockedBy: ${memory.frontmatter.blockedBy}`,
|
|
2352
|
+
"---",
|
|
2353
|
+
"",
|
|
2354
|
+
content,
|
|
2355
|
+
"",
|
|
2356
|
+
].join("\n");
|
|
2357
|
+
const chunks = (async function* (): AsyncIterable<Buffer> {
|
|
2358
|
+
for (let offset = 0; offset < incomingFile.length; offset += 32 * 1024) {
|
|
2359
|
+
yield Buffer.from(incomingFile.slice(offset, offset + 32 * 1024), "utf8");
|
|
2360
|
+
}
|
|
2361
|
+
})();
|
|
2362
|
+
const identity = buildExplicitCaptureDedupKey(content, "fact", "provider-a");
|
|
2363
|
+
const releaseState = Promise.withResolvers<void>();
|
|
2364
|
+
const enteredState = Promise.withResolvers<void>();
|
|
2365
|
+
const held = storage.withTombstoneBlockedCaptureWriteLock(async () => {
|
|
2366
|
+
enteredState.resolve();
|
|
2367
|
+
await releaseState.promise;
|
|
2368
|
+
}, identity);
|
|
2369
|
+
await enteredState.promise;
|
|
2370
|
+
|
|
2371
|
+
const pending = storage.writeOfflineSyncFileChunks(memory.path, chunks);
|
|
2372
|
+
const stagingDir = path.join(dir, "state", "tombstone-blocked-capture", "offline-sync-staging");
|
|
2373
|
+
let stagePath: string | undefined;
|
|
2374
|
+
for (let attempt = 0; attempt < 20 && stagePath === undefined; attempt += 1) {
|
|
2375
|
+
const entries = await readdir(stagingDir).catch(() => []);
|
|
2376
|
+
if (entries.length > 0) stagePath = path.join(stagingDir, entries[0]);
|
|
2377
|
+
else await new Promise((resolve) => setTimeout(resolve, 10));
|
|
2378
|
+
}
|
|
2379
|
+
assert.ok(stagePath, "oversized sync must create a staging file before waiting for its lock");
|
|
2380
|
+
assert.equal(isEncryptedFile(await readFile(stagePath)), true);
|
|
2381
|
+
releaseState.resolve();
|
|
2382
|
+
await held;
|
|
2383
|
+
await pending;
|
|
2384
|
+
});
|
|
2385
|
+
});
|
|
2386
|
+
|
|
2387
|
+
test("chunked offline sync rechecks blocked identity after buffering", async () => {
|
|
2388
|
+
await withMemoryDir(async (dir) => {
|
|
2389
|
+
const storage = new StorageManager(dir);
|
|
2390
|
+
await storage.ensureDirectories();
|
|
2391
|
+
storage.setTombstonesConfig({
|
|
2392
|
+
enabled: true,
|
|
2393
|
+
semanticMatch: false,
|
|
2394
|
+
semanticThreshold: 0.9,
|
|
2395
|
+
namespace: "default",
|
|
2396
|
+
});
|
|
2397
|
+
const content = "Chunked offline sync must recheck after acquiring the identity lock.";
|
|
2398
|
+
const tombstoneId = await storage.appendTombstone({
|
|
2399
|
+
reason: "correction",
|
|
2400
|
+
createdBy: "user_correction",
|
|
2401
|
+
sourceMemoryId: "chunked-recheck",
|
|
2402
|
+
rawContent: content,
|
|
2403
|
+
});
|
|
2404
|
+
assert.ok(tombstoneId);
|
|
2405
|
+
const incomingFile = [
|
|
2406
|
+
"---",
|
|
2407
|
+
"id: incoming-sync",
|
|
2408
|
+
"category: fact",
|
|
2409
|
+
"created: 2026-01-01T00:00:00.000Z",
|
|
2410
|
+
"updated: 2026-01-01T00:00:00.000Z",
|
|
2411
|
+
"source: offline-sync",
|
|
2412
|
+
"confidence: 0.8",
|
|
2413
|
+
"confidenceTier: implied",
|
|
2414
|
+
"tags: []",
|
|
2415
|
+
"sourceConnector: provider-a",
|
|
2416
|
+
"status: pending_review",
|
|
2417
|
+
`blockedBy: ${tombstoneId}`,
|
|
2418
|
+
"---",
|
|
2419
|
+
"",
|
|
2420
|
+
content,
|
|
2421
|
+
"",
|
|
2422
|
+
].join("\n");
|
|
2423
|
+
const entered = Promise.withResolvers<void>();
|
|
2424
|
+
const release = Promise.withResolvers<void>();
|
|
2425
|
+
const chunks = (async function* (): AsyncIterable<Buffer> {
|
|
2426
|
+
const split = Math.floor(incomingFile.length / 2);
|
|
2427
|
+
yield Buffer.from(incomingFile.slice(0, split), "utf8");
|
|
2428
|
+
entered.resolve();
|
|
2429
|
+
await release.promise;
|
|
2430
|
+
yield Buffer.from(incomingFile.slice(split), "utf8");
|
|
2431
|
+
})();
|
|
2432
|
+
const target = path.join(dir, "facts", "2026-01-01", "incoming.md");
|
|
2433
|
+
const pending = storage.writeOfflineSyncFileChunks(target, chunks);
|
|
2434
|
+
await entered.promise;
|
|
2435
|
+
|
|
2436
|
+
const duplicate = await storage.writeMemory("fact", content, {
|
|
2437
|
+
source: "test",
|
|
2438
|
+
sourceConnector: "provider-a",
|
|
2439
|
+
});
|
|
2440
|
+
assert.equal(duplicate.tombstoneBlocked, true);
|
|
2441
|
+
release.resolve();
|
|
2442
|
+
await pending;
|
|
2443
|
+
|
|
2444
|
+
assert.equal(existsSync(target), false, "sync must not publish a duplicate blocked row");
|
|
2445
|
+
const matches = (await storage.readAllMemories()).filter((memory) => memory.content === content);
|
|
2446
|
+
assert.equal(matches.length, 1, "the explicit capture remains the sole blocked row");
|
|
2447
|
+
});
|
|
2448
|
+
});
|
|
2449
|
+
test("chunked offline sync acquires a target identity discovered in a nested recheck", async () => {
|
|
2450
|
+
await withMemoryDir(async (dir) => {
|
|
2451
|
+
const storage = new StorageManager(dir);
|
|
2452
|
+
const peer = new StorageManager(dir);
|
|
2453
|
+
await storage.ensureDirectories();
|
|
2454
|
+
await peer.ensureDirectories();
|
|
2455
|
+
const tombstoneConfig = {
|
|
2456
|
+
enabled: true,
|
|
2457
|
+
semanticMatch: false,
|
|
2458
|
+
semanticThreshold: 0.9,
|
|
2459
|
+
namespace: "default",
|
|
2460
|
+
};
|
|
2461
|
+
storage.setTombstonesConfig(tombstoneConfig);
|
|
2462
|
+
peer.setTombstonesConfig(tombstoneConfig);
|
|
2463
|
+
const targetContent = "The chunk target becomes blocked after its outer lock recheck.";
|
|
2464
|
+
const incomingContent = "The incoming chunk replaces the newly blocked target.";
|
|
2465
|
+
const incomingTombstone = await storage.appendTombstone({
|
|
2466
|
+
reason: "correction",
|
|
2467
|
+
createdBy: "user_correction",
|
|
2468
|
+
sourceMemoryId: "chunk-nested-incoming",
|
|
2469
|
+
rawContent: incomingContent,
|
|
2470
|
+
});
|
|
2471
|
+
assert.ok(incomingTombstone);
|
|
2472
|
+
const target = await storage.writeMemory("fact", targetContent, {
|
|
2473
|
+
source: "test",
|
|
2474
|
+
sourceConnector: "provider-a",
|
|
2475
|
+
});
|
|
2476
|
+
const targetTombstone = await storage.appendTombstone({
|
|
2477
|
+
reason: "correction",
|
|
2478
|
+
createdBy: "user_correction",
|
|
2479
|
+
sourceMemoryId: "chunk-nested-target",
|
|
2480
|
+
rawContent: targetContent,
|
|
2481
|
+
});
|
|
2482
|
+
assert.ok(targetTombstone);
|
|
2483
|
+
assert.equal(target.tombstoneBlocked, false);
|
|
2484
|
+
const targetMemory = (await storage.readAllMemories()).find(
|
|
2485
|
+
(memory) => memory.frontmatter.id === target.id
|
|
2486
|
+
);
|
|
2487
|
+
assert.ok(targetMemory);
|
|
2488
|
+
const incomingFile = [
|
|
2489
|
+
"---",
|
|
2490
|
+
"id: chunk-nested-incoming",
|
|
2491
|
+
"category: fact",
|
|
2492
|
+
"created: 2026-01-01T00:00:00.000Z",
|
|
2493
|
+
"updated: 2026-01-01T00:00:00.000Z",
|
|
2494
|
+
"source: offline-sync",
|
|
2495
|
+
"confidence: 0.8",
|
|
2496
|
+
"confidenceTier: implied",
|
|
2497
|
+
"tags: []",
|
|
2498
|
+
"sourceConnector: provider-a",
|
|
2499
|
+
"status: pending_review",
|
|
2500
|
+
`blockedBy: ${incomingTombstone}`,
|
|
2501
|
+
"---",
|
|
2502
|
+
"",
|
|
2503
|
+
incomingContent,
|
|
2504
|
+
"",
|
|
2505
|
+
].join("\n");
|
|
2506
|
+
const chunks = (async function* (): AsyncIterable<Buffer> {
|
|
2507
|
+
for (let offset = 0; offset < incomingFile.length; offset += 23) {
|
|
2508
|
+
yield Buffer.from(incomingFile.slice(offset, offset + 23), "utf8");
|
|
2509
|
+
}
|
|
2510
|
+
})();
|
|
2511
|
+
const identityC = buildExplicitCaptureDedupKey(targetContent, "fact", "provider-a");
|
|
2512
|
+
const storageWithOverrides = storage as unknown as {
|
|
2513
|
+
withTombstoneBlockedCaptureWriteLock: (
|
|
2514
|
+
task: () => Promise<unknown>,
|
|
2515
|
+
identity?: string | readonly string[],
|
|
2516
|
+
) => Promise<unknown>;
|
|
2517
|
+
runTombstoneBlockedOfflineSyncMutation: (
|
|
2518
|
+
target: string,
|
|
2519
|
+
after: MemoryFile | null,
|
|
2520
|
+
write: () => Promise<void>,
|
|
2521
|
+
coordinate?: boolean,
|
|
2522
|
+
) => Promise<void>;
|
|
2523
|
+
};
|
|
2524
|
+
const originalLock = storageWithOverrides.withTombstoneBlockedCaptureWriteLock.bind(storage);
|
|
2525
|
+
const originalRun = storageWithOverrides.runTombstoneBlockedOfflineSyncMutation.bind(storage);
|
|
2526
|
+
const capturedIdentities: Array<string | readonly string[] | undefined> = [];
|
|
2527
|
+
storageWithOverrides.withTombstoneBlockedCaptureWriteLock = async (task, identity) => {
|
|
2528
|
+
capturedIdentities.push(identity);
|
|
2529
|
+
return await originalLock(task, identity);
|
|
2530
|
+
};
|
|
2531
|
+
let rewroteTarget = false;
|
|
2532
|
+
let pauseNextPrepare = false;
|
|
2533
|
+
const markerEntered = Promise.withResolvers<void>();
|
|
2534
|
+
const releaseMarker = Promise.withResolvers<void>();
|
|
2535
|
+
storageWithOverrides.runTombstoneBlockedOfflineSyncMutation = async (
|
|
2536
|
+
syncTarget,
|
|
2537
|
+
after,
|
|
2538
|
+
write,
|
|
2539
|
+
coordinate,
|
|
2540
|
+
) => {
|
|
2541
|
+
if (!rewroteTarget) {
|
|
2542
|
+
rewroteTarget = true;
|
|
2543
|
+
const current = (await peer.readAllMemories()).find(
|
|
2544
|
+
(memory) => memory.frontmatter.id === target.id
|
|
2545
|
+
);
|
|
2546
|
+
assert.ok(current);
|
|
2547
|
+
await peer.writeMemoryFrontmatter(current, {
|
|
2548
|
+
status: "pending_review",
|
|
2549
|
+
blockedBy: targetTombstone,
|
|
2550
|
+
});
|
|
2551
|
+
pauseNextPrepare = true;
|
|
2552
|
+
}
|
|
2553
|
+
return await originalRun(syncTarget, after, write, coordinate);
|
|
2554
|
+
};
|
|
2555
|
+
const originalPrepare = TombstoneBlockedCaptureIndex.prototype.prepareWrite;
|
|
2556
|
+
const prepareSpy = mock.method(
|
|
2557
|
+
TombstoneBlockedCaptureIndex.prototype,
|
|
2558
|
+
"prepareWrite",
|
|
2559
|
+
async function (this: TombstoneBlockedCaptureIndex) {
|
|
2560
|
+
const marker = await originalPrepare.call(this);
|
|
2561
|
+
if (pauseNextPrepare) {
|
|
2562
|
+
pauseNextPrepare = false;
|
|
2563
|
+
markerEntered.resolve();
|
|
2564
|
+
await releaseMarker.promise;
|
|
2565
|
+
}
|
|
2566
|
+
return marker;
|
|
2567
|
+
},
|
|
2568
|
+
);
|
|
2569
|
+
|
|
2570
|
+
const sync = storage.writeOfflineSyncFileChunks(targetMemory.path, chunks);
|
|
2571
|
+
try {
|
|
2572
|
+
await markerEntered.promise;
|
|
2573
|
+
assert.ok(
|
|
2574
|
+
capturedIdentities.some((identity) =>
|
|
2575
|
+
(Array.isArray(identity) ? identity : [identity]).includes(identityC)
|
|
2576
|
+
),
|
|
2577
|
+
"nested chunk sync must acquire the identity discovered after the outer recheck",
|
|
2578
|
+
);
|
|
2579
|
+
let recaptureCompleted = false;
|
|
2580
|
+
const recapture = peer
|
|
2581
|
+
.writeMemory("fact", targetContent, {
|
|
2582
|
+
source: "test",
|
|
2583
|
+
sourceConnector: "provider-a",
|
|
2584
|
+
})
|
|
2585
|
+
.then((result) => {
|
|
2586
|
+
recaptureCompleted = true;
|
|
2587
|
+
return result;
|
|
2588
|
+
});
|
|
2589
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
2590
|
+
assert.equal(recaptureCompleted, false, "recapture must wait for the nested target identity lock");
|
|
2591
|
+
releaseMarker.resolve();
|
|
2592
|
+
await sync;
|
|
2593
|
+
const recaptured = await recapture;
|
|
2594
|
+
assert.equal(recaptured.tombstoneBlocked, true);
|
|
2595
|
+
const after = await peer.readAllMemories();
|
|
2596
|
+
assert.equal(
|
|
2597
|
+
after.find((memory) => memory.path === targetMemory.path)?.content,
|
|
2598
|
+
incomingContent,
|
|
2599
|
+
"chunk sync must replace the target only while holding its current identity lock",
|
|
2600
|
+
);
|
|
2601
|
+
assert.equal(after.filter((memory) => memory.content === targetContent).length, 1);
|
|
2602
|
+
} finally {
|
|
2603
|
+
releaseMarker.resolve();
|
|
2604
|
+
prepareSpy.mock.restore();
|
|
2605
|
+
storageWithOverrides.withTombstoneBlockedCaptureWriteLock = originalLock;
|
|
2606
|
+
storageWithOverrides.runTombstoneBlockedOfflineSyncMutation = originalRun;
|
|
2607
|
+
await sync.catch(() => undefined);
|
|
2608
|
+
}
|
|
2609
|
+
});
|
|
2610
|
+
});
|
|
2611
|
+
|
|
2612
|
+
test("chunked offline sync streams non-memory files through the host writer", async () => {
|
|
2613
|
+
await withMemoryDir(async (dir) => {
|
|
2614
|
+
const storage = new StorageManager(dir);
|
|
2615
|
+
await storage.ensureDirectories();
|
|
2616
|
+
const target = path.join(dir, "state", "streamed.bin");
|
|
2617
|
+
const input = ["first chunk\n", "second chunk\n"];
|
|
2618
|
+
let consumed = 0;
|
|
2619
|
+
const chunks = (async function* (): AsyncIterable<Buffer> {
|
|
2620
|
+
for (const value of input) {
|
|
2621
|
+
consumed += 1;
|
|
2622
|
+
yield Buffer.from(value, "utf8");
|
|
2623
|
+
}
|
|
2624
|
+
})();
|
|
2625
|
+
|
|
2626
|
+
await storage.writeOfflineSyncFileChunks(target, chunks);
|
|
2627
|
+
|
|
2628
|
+
assert.equal(consumed, input.length);
|
|
2629
|
+
assert.equal(await readFile(target, "utf8"), input.join(""));
|
|
2630
|
+
});
|
|
2631
|
+
});
|
|
2632
|
+
|
|
2633
|
+
test("large blocked offline sync streams through bounded staging and deduplicates", async () => {
|
|
2634
|
+
await withMemoryDir(async (dir) => {
|
|
2635
|
+
const storage = new StorageManager(dir);
|
|
2636
|
+
await storage.ensureDirectories();
|
|
2637
|
+
storage.setTombstonesConfig({
|
|
2638
|
+
enabled: true,
|
|
2639
|
+
semanticMatch: false,
|
|
2640
|
+
semanticThreshold: 0.9,
|
|
2641
|
+
namespace: "default",
|
|
2642
|
+
});
|
|
2643
|
+
const largeContent = "large stream duplicate ".repeat(60_000);
|
|
2644
|
+
const tombstoneId = await storage.appendTombstone({
|
|
2645
|
+
reason: "correction",
|
|
2646
|
+
createdBy: "user_correction",
|
|
2647
|
+
sourceMemoryId: "large-stream-duplicate",
|
|
2648
|
+
rawContent: largeContent,
|
|
2649
|
+
});
|
|
2650
|
+
assert.ok(tombstoneId);
|
|
2651
|
+
const existing = await storage.writeMemory("fact", largeContent, {
|
|
2652
|
+
source: "test",
|
|
2653
|
+
sourceConnector: "provider-a",
|
|
2654
|
+
});
|
|
2655
|
+
assert.equal(existing.tombstoneBlocked, true);
|
|
2656
|
+
const incomingFile = Buffer.from(
|
|
2657
|
+
[
|
|
2658
|
+
"---",
|
|
2659
|
+
"id: large-stream-incoming",
|
|
2660
|
+
"category: fact",
|
|
2661
|
+
"created: 2026-01-01T00:00:00.000Z",
|
|
2662
|
+
"updated: 2026-01-01T00:00:00.000Z",
|
|
2663
|
+
"source: offline-sync",
|
|
2664
|
+
"confidence: 0.8",
|
|
2665
|
+
"confidenceTier: implied",
|
|
2666
|
+
"tags: []",
|
|
2667
|
+
"sourceConnector: provider-a",
|
|
2668
|
+
"status: pending_review",
|
|
2669
|
+
`blockedBy: ${tombstoneId}`,
|
|
2670
|
+
"---",
|
|
2671
|
+
"",
|
|
2672
|
+
largeContent,
|
|
2673
|
+
"",
|
|
2674
|
+
].join("\n"),
|
|
2675
|
+
"utf8",
|
|
2676
|
+
);
|
|
2677
|
+
assert.ok(incomingFile.byteLength > 1_048_576);
|
|
2678
|
+
const target = path.join(dir, "cold", "large-stream-incoming.md");
|
|
2679
|
+
let consumedBytes = 0;
|
|
2680
|
+
const chunks = (async function* (): AsyncIterable<Buffer> {
|
|
2681
|
+
for (let offset = 0; offset < incomingFile.byteLength; offset += 32 * 1024) {
|
|
2682
|
+
const chunk = incomingFile.subarray(offset, offset + 32 * 1024);
|
|
2683
|
+
consumedBytes += chunk.byteLength;
|
|
2684
|
+
yield chunk;
|
|
2685
|
+
}
|
|
2686
|
+
})();
|
|
2687
|
+
|
|
2688
|
+
await storage.writeOfflineSyncFileChunks(target, chunks);
|
|
2689
|
+
|
|
2690
|
+
assert.equal(consumedBytes, incomingFile.byteLength);
|
|
2691
|
+
assert.equal(existsSync(target), false, "a streamed blocked duplicate must not publish a new row");
|
|
2692
|
+
const matches = [
|
|
2693
|
+
...(await storage.readAllMemories()),
|
|
2694
|
+
...(await storage.readAllColdMemories()),
|
|
2695
|
+
].filter((memory) => memory.frontmatter.id === existing.id);
|
|
2696
|
+
assert.equal(matches.length, 1, "the existing blocked row remains the sole durable duplicate");
|
|
2697
|
+
});
|
|
2698
|
+
});
|
|
2699
|
+
|
|
2700
|
+
test("offline sync mutation rebuilds blocked index for every recall category", async () => {
|
|
2701
|
+
await withMemoryDir(async (dir) => {
|
|
2702
|
+
const storage = new StorageManager(dir);
|
|
2703
|
+
await storage.ensureDirectories();
|
|
2704
|
+
storage.setTombstonesConfig({
|
|
2705
|
+
enabled: true,
|
|
2706
|
+
semanticMatch: false,
|
|
2707
|
+
semanticThreshold: 0.9,
|
|
2708
|
+
namespace: "default",
|
|
2709
|
+
});
|
|
2710
|
+
const content = "A decision-category sync update must refresh blocked capture identity.";
|
|
2711
|
+
const tombstoneId = await storage.appendTombstone({
|
|
2712
|
+
reason: "correction",
|
|
2713
|
+
createdBy: "user_correction",
|
|
2714
|
+
sourceMemoryId: "retired-decision-sync",
|
|
2715
|
+
rawContent: content,
|
|
2716
|
+
});
|
|
2717
|
+
assert.ok(tombstoneId, "test tombstone must persist");
|
|
2718
|
+
const result = await storage.writeMemory("fact", content, {
|
|
2719
|
+
source: "test",
|
|
2720
|
+
sourceConnector: "provider-a",
|
|
2721
|
+
});
|
|
2722
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
2723
|
+
assert.equal(await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"), true);
|
|
2724
|
+
|
|
2725
|
+
const memory = (await storage.readAllMemories()).find((candidate) => candidate.frontmatter.id === result.id);
|
|
2726
|
+
assert.ok(memory, "blocked memory must be readable before offline sync");
|
|
2727
|
+
const movedPath = path.join(dir, "decisions", "2026-01-01", "moved.md");
|
|
2728
|
+
await mkdir(path.dirname(movedPath), { recursive: true });
|
|
2729
|
+
await rename(memory.path, movedPath);
|
|
2730
|
+
const updatedFile = (await readFile(movedPath, "utf8")).replace(
|
|
2731
|
+
"sourceConnector: provider-a",
|
|
2732
|
+
"sourceConnector: provider-b"
|
|
2733
|
+
);
|
|
2734
|
+
await storage.writeOfflineSyncFile(movedPath, Buffer.from(updatedFile, "utf8"));
|
|
2735
|
+
|
|
2736
|
+
assert.equal(
|
|
2737
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"),
|
|
2738
|
+
false,
|
|
2739
|
+
"offline sync must remove the stale provider identity from every recall category"
|
|
2740
|
+
);
|
|
2741
|
+
assert.equal(
|
|
2742
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-b"),
|
|
2743
|
+
true,
|
|
2744
|
+
"offline sync must add the updated provider identity in every recall category"
|
|
2745
|
+
);
|
|
2746
|
+
});
|
|
2747
|
+
});
|
|
2748
|
+
|
|
2749
|
+
test("offline sync invalidates cold cache before blocked index rebuild", async () => {
|
|
2750
|
+
await withMemoryDir(async (dir) => {
|
|
2751
|
+
const storage = new StorageManager(dir);
|
|
2752
|
+
await storage.ensureDirectories();
|
|
2753
|
+
storage.setTombstonesConfig({
|
|
2754
|
+
enabled: true,
|
|
2755
|
+
semanticMatch: false,
|
|
2756
|
+
semanticThreshold: 0.9,
|
|
2757
|
+
namespace: "default",
|
|
2758
|
+
});
|
|
2759
|
+
const content = "A cold blocked rewrite must rebuild from the post-sync cache.";
|
|
2760
|
+
const tombstoneId = await storage.appendTombstone({
|
|
2761
|
+
reason: "correction",
|
|
2762
|
+
createdBy: "user_correction",
|
|
2763
|
+
sourceMemoryId: "retired-cold-sync",
|
|
2764
|
+
rawContent: content,
|
|
2765
|
+
});
|
|
2766
|
+
assert.ok(tombstoneId, "test tombstone must persist");
|
|
2767
|
+
const result = await storage.writeMemory("fact", content, {
|
|
2768
|
+
source: "test",
|
|
2769
|
+
sourceConnector: "provider-a",
|
|
2770
|
+
});
|
|
2771
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
2772
|
+
assert.equal(await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"), true);
|
|
2773
|
+
|
|
2774
|
+
const memory = await storage.getMemoryById(result.id);
|
|
2775
|
+
assert.ok(memory, "blocked memory must be readable before tier migration");
|
|
2776
|
+
const moved = await storage.migrateMemoryToTier(memory, "cold");
|
|
2777
|
+
assert.equal(moved.changed, true);
|
|
2778
|
+
await storage.readAllColdMemories();
|
|
2779
|
+
const updatedFile = (await readFile(moved.targetPath, "utf8")).replace(
|
|
2780
|
+
"sourceConnector: provider-a",
|
|
2781
|
+
"sourceConnector: provider-b"
|
|
2782
|
+
);
|
|
2783
|
+
await storage.writeOfflineSyncFile(moved.targetPath, Buffer.from(updatedFile, "utf8"));
|
|
2784
|
+
|
|
2785
|
+
assert.equal(
|
|
2786
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"),
|
|
2787
|
+
false,
|
|
2788
|
+
"cold sync must remove the stale provider identity"
|
|
2789
|
+
);
|
|
2790
|
+
assert.equal(
|
|
2791
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-b"),
|
|
2792
|
+
true,
|
|
2793
|
+
"cold sync must index the post-mutation provider identity"
|
|
2794
|
+
);
|
|
2795
|
+
});
|
|
2796
|
+
});
|
|
2797
|
+
|
|
2798
|
+
test("frontmatter rewrites invalidate cold cache before blocked index sync", async () => {
|
|
2799
|
+
await withMemoryDir(async (dir) => {
|
|
2800
|
+
const storage = new StorageManager(dir);
|
|
2801
|
+
await storage.ensureDirectories();
|
|
2802
|
+
storage.setTombstonesConfig({
|
|
2803
|
+
enabled: true,
|
|
2804
|
+
semanticMatch: false,
|
|
2805
|
+
semanticThreshold: 0.9,
|
|
2806
|
+
namespace: "default",
|
|
2807
|
+
});
|
|
2808
|
+
const content = "A cold frontmatter rewrite must publish its new blocked identity.";
|
|
2809
|
+
const tombstoneId = await storage.appendTombstone({
|
|
2810
|
+
reason: "correction",
|
|
2811
|
+
createdBy: "user_correction",
|
|
2812
|
+
sourceMemoryId: "retired-cold-frontmatter",
|
|
2813
|
+
rawContent: content,
|
|
2814
|
+
});
|
|
2815
|
+
assert.ok(tombstoneId, "test tombstone must persist");
|
|
2816
|
+
const result = await storage.writeMemory("fact", content, {
|
|
2817
|
+
source: "test",
|
|
2818
|
+
sourceConnector: "provider-a",
|
|
2819
|
+
});
|
|
2820
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
2821
|
+
const memory = await storage.getMemoryById(result.id);
|
|
2822
|
+
assert.ok(memory, "blocked memory must be readable before tier migration");
|
|
2823
|
+
const moved = await storage.migrateMemoryToTier(memory, "cold");
|
|
2824
|
+
assert.equal(moved.changed, true);
|
|
2825
|
+
const coldMemory = (await storage.readAllColdMemories()).find(
|
|
2826
|
+
(candidate) => candidate.frontmatter.id === result.id
|
|
2827
|
+
);
|
|
2828
|
+
assert.ok(coldMemory, "migrated blocked memory must be readable from cold tier");
|
|
2829
|
+
|
|
2830
|
+
assert.equal(
|
|
2831
|
+
await storage.writeMemoryFrontmatter(coldMemory, { sourceConnector: "provider-b" }),
|
|
2832
|
+
true
|
|
2833
|
+
);
|
|
2834
|
+
assert.equal(
|
|
2835
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"),
|
|
2836
|
+
false,
|
|
2837
|
+
"frontmatter rewrite must remove the stale cold identity"
|
|
2838
|
+
);
|
|
2839
|
+
assert.equal(
|
|
2840
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-b"),
|
|
2841
|
+
true,
|
|
2842
|
+
"frontmatter rewrite must index the updated cold identity"
|
|
2843
|
+
);
|
|
2844
|
+
});
|
|
2845
|
+
});
|
|
2846
|
+
|
|
2847
|
+
test("memory invalidation rebuilds an unloaded persisted blocked index", async () => {
|
|
2848
|
+
await withMemoryDir(async (dir) => {
|
|
2849
|
+
const storage = new StorageManager(dir);
|
|
2850
|
+
await storage.ensureDirectories();
|
|
2851
|
+
storage.setTombstonesConfig({
|
|
2852
|
+
enabled: true,
|
|
2853
|
+
semanticMatch: false,
|
|
2854
|
+
semanticThreshold: 0.9,
|
|
2855
|
+
namespace: "default",
|
|
2856
|
+
});
|
|
2857
|
+
const content = "An invalidated blocked row must leave the persisted index.";
|
|
2858
|
+
const tombstoneId = await storage.appendTombstone({
|
|
2859
|
+
reason: "correction",
|
|
2860
|
+
createdBy: "user_correction",
|
|
2861
|
+
sourceMemoryId: "retired-invalidation",
|
|
2862
|
+
rawContent: content,
|
|
2863
|
+
});
|
|
2864
|
+
assert.ok(tombstoneId, "test tombstone must persist");
|
|
2865
|
+
const result = await storage.writeMemory("fact", content, {
|
|
2866
|
+
source: "test",
|
|
2867
|
+
sourceConnector: "provider-a",
|
|
2868
|
+
});
|
|
2869
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
2870
|
+
assert.equal(await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"), true);
|
|
2871
|
+
|
|
2872
|
+
const restarted = new StorageManager(dir);
|
|
2873
|
+
restarted.setTombstonesConfig({
|
|
2874
|
+
enabled: true,
|
|
2875
|
+
semanticMatch: false,
|
|
2876
|
+
semanticThreshold: 0.9,
|
|
2877
|
+
namespace: "default",
|
|
2878
|
+
});
|
|
2879
|
+
assert.equal(await restarted.invalidateMemory(result.id), true);
|
|
2880
|
+
assert.equal(
|
|
2881
|
+
await restarted.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"),
|
|
2882
|
+
false,
|
|
2883
|
+
"memory invalidation must rebuild a persisted index before its first lookup"
|
|
2884
|
+
);
|
|
2885
|
+
});
|
|
2886
|
+
});
|
|
2887
|
+
|
|
2888
|
+
test("permanent capture lock failures stop retrying", async () => {
|
|
2889
|
+
await withMemoryDir(async (dir) => {
|
|
2890
|
+
let attempts = 0;
|
|
2891
|
+
const index = new TombstoneBlockedCaptureIndex({
|
|
2892
|
+
stateDir: dir,
|
|
2893
|
+
memoryDir: dir,
|
|
2894
|
+
secureStoreKeyProvider: () => null,
|
|
2895
|
+
secureStoreWriteKeyProvider: () => null,
|
|
2896
|
+
lockOptions: () => ({ retryMaxAttempts: 2, retryBaseMs: 1 }),
|
|
2897
|
+
readAllMemories: async () => [],
|
|
2898
|
+
readAllColdMemories: async () => [],
|
|
2899
|
+
withHeldFileLock: async (_lockPath, _options, task) => {
|
|
2900
|
+
attempts += 1;
|
|
2901
|
+
return await task(false, {
|
|
2902
|
+
failure: "error",
|
|
2903
|
+
refresh: async () => false,
|
|
2904
|
+
});
|
|
2905
|
+
},
|
|
2906
|
+
});
|
|
2907
|
+
|
|
2908
|
+
await assert.rejects(
|
|
2909
|
+
index.withCaptureWriteLock(async () => "unreachable"),
|
|
2910
|
+
/capture write lock acquisition failed/
|
|
2911
|
+
);
|
|
2912
|
+
assert.equal(attempts, 1, "permanent filesystem failures must not retry as contention");
|
|
2913
|
+
});
|
|
2914
|
+
});
|
|
2915
|
+
|
|
2916
|
+
test("pre-load blocked index invalidation rebuilds a persisted index", async () => {
|
|
2917
|
+
await withMemoryDir(async (dir) => {
|
|
2918
|
+
const content = "A pre-load invalidation must refresh blocked capture identity.";
|
|
2919
|
+
const before: MemoryFile = {
|
|
2920
|
+
path: path.join(dir, "facts", "blocked.md"),
|
|
2921
|
+
frontmatter: {
|
|
2922
|
+
id: "preload-blocked",
|
|
2923
|
+
category: "fact",
|
|
2924
|
+
created: new Date(0).toISOString(),
|
|
2925
|
+
updated: new Date(0).toISOString(),
|
|
2926
|
+
source: "explicit-inline-review",
|
|
2927
|
+
confidence: 0.2,
|
|
2928
|
+
confidenceTier: "explicit",
|
|
2929
|
+
tags: [],
|
|
2930
|
+
status: "pending_review",
|
|
2931
|
+
blockedBy: "preload-tombstone",
|
|
2932
|
+
sourceConnector: "provider-a",
|
|
2933
|
+
},
|
|
2934
|
+
content,
|
|
2935
|
+
};
|
|
2936
|
+
const after: MemoryFile = {
|
|
2937
|
+
...before,
|
|
2938
|
+
frontmatter: {
|
|
2939
|
+
...before.frontmatter,
|
|
2940
|
+
sourceConnector: "provider-b",
|
|
2941
|
+
},
|
|
2942
|
+
};
|
|
2943
|
+
const memories = [before];
|
|
2944
|
+
const options = {
|
|
2945
|
+
stateDir: dir,
|
|
2946
|
+
memoryDir: dir,
|
|
2947
|
+
secureStoreKeyProvider: () => null,
|
|
2948
|
+
secureStoreWriteKeyProvider: () => null,
|
|
2949
|
+
lockOptions: () => ({ retryMaxAttempts: 2, retryBaseMs: 1 }),
|
|
2950
|
+
readAllMemories: async () => memories,
|
|
2951
|
+
readAllColdMemories: async () => [],
|
|
2952
|
+
};
|
|
2953
|
+
|
|
2954
|
+
const initial = new TombstoneBlockedCaptureIndex(options);
|
|
2955
|
+
await initial.add(before);
|
|
2956
|
+
memories[0] = after;
|
|
2957
|
+
|
|
2958
|
+
const restarted = new TombstoneBlockedCaptureIndex(options);
|
|
2959
|
+
await restarted.rebuildIfLoaded();
|
|
2960
|
+
assert.equal(await restarted.has(content, "fact", "provider-a"), false);
|
|
2961
|
+
assert.equal(await restarted.has(content, "fact", "provider-b"), true);
|
|
2962
|
+
});
|
|
2963
|
+
});
|
|
2964
|
+
|
|
2965
|
+
test("abandoned pending blocked-index markers rebuild and are reaped", async () => {
|
|
2966
|
+
await withMemoryDir(async (dir) => {
|
|
2967
|
+
const content = "An abandoned blocked-index writer must not suppress dedupe forever.";
|
|
2968
|
+
const memory: MemoryFile = {
|
|
2969
|
+
path: path.join(dir, "facts", "abandoned.md"),
|
|
2970
|
+
frontmatter: {
|
|
2971
|
+
id: "abandoned-blocked",
|
|
2972
|
+
category: "fact",
|
|
2973
|
+
created: new Date(0).toISOString(),
|
|
2974
|
+
updated: new Date(0).toISOString(),
|
|
2975
|
+
source: "explicit-inline-review",
|
|
2976
|
+
confidence: 0.2,
|
|
2977
|
+
confidenceTier: "explicit",
|
|
2978
|
+
tags: [],
|
|
2979
|
+
status: "pending_review",
|
|
2980
|
+
blockedBy: "abandoned-tombstone",
|
|
2981
|
+
sourceConnector: "provider-a",
|
|
2982
|
+
},
|
|
2983
|
+
content,
|
|
2984
|
+
};
|
|
2985
|
+
const options = {
|
|
2986
|
+
stateDir: dir,
|
|
2987
|
+
memoryDir: dir,
|
|
2988
|
+
secureStoreKeyProvider: () => null,
|
|
2989
|
+
secureStoreWriteKeyProvider: () => null,
|
|
2990
|
+
lockOptions: () => ({ retryMaxAttempts: 2, retryBaseMs: 1 }),
|
|
2991
|
+
readAllMemories: async () => [memory],
|
|
2992
|
+
readAllColdMemories: async () => [],
|
|
2993
|
+
};
|
|
2994
|
+
const markerDir = path.join(dir, "tombstone-blocked-capture", "rebuild-required");
|
|
2995
|
+
await mkdir(markerDir, { recursive: true });
|
|
2996
|
+
await writeFile(
|
|
2997
|
+
path.join(markerDir, "abandoned-writer"),
|
|
2998
|
+
`${JSON.stringify({
|
|
2999
|
+
state: "pending",
|
|
3000
|
+
pid: process.pid,
|
|
3001
|
+
ownerId: "abandoned-writer",
|
|
3002
|
+
createdAt: Date.now() - 120_000,
|
|
3003
|
+
processStartedAtMs: 1,
|
|
3004
|
+
})}\n`,
|
|
3005
|
+
"utf8"
|
|
3006
|
+
);
|
|
3007
|
+
|
|
3008
|
+
const index = new TombstoneBlockedCaptureIndex(options);
|
|
3009
|
+
assert.equal(await index.has(content, "fact", "provider-a"), true);
|
|
3010
|
+
assert.deepEqual(await readdir(markerDir), []);
|
|
3011
|
+
});
|
|
3012
|
+
});
|
|
3013
|
+
|
|
3014
|
+
test("truncated pending blocked-index markers retain filesystem age and are reaped", async () => {
|
|
3015
|
+
await withMemoryDir(async (dir) => {
|
|
3016
|
+
const content = "A truncated blocked-index marker must not suppress dedupe forever.";
|
|
3017
|
+
const memory: MemoryFile = {
|
|
3018
|
+
path: path.join(dir, "facts", "truncated.md"),
|
|
3019
|
+
frontmatter: {
|
|
3020
|
+
id: "truncated-blocked",
|
|
3021
|
+
category: "fact",
|
|
3022
|
+
created: new Date(0).toISOString(),
|
|
3023
|
+
updated: new Date(0).toISOString(),
|
|
3024
|
+
source: "explicit-inline-review",
|
|
3025
|
+
confidence: 0.2,
|
|
3026
|
+
confidenceTier: "explicit",
|
|
3027
|
+
tags: [],
|
|
3028
|
+
status: "pending_review",
|
|
3029
|
+
blockedBy: "truncated-tombstone",
|
|
3030
|
+
},
|
|
3031
|
+
content,
|
|
3032
|
+
};
|
|
3033
|
+
const options = {
|
|
3034
|
+
stateDir: dir,
|
|
3035
|
+
memoryDir: dir,
|
|
3036
|
+
secureStoreKeyProvider: () => null,
|
|
3037
|
+
secureStoreWriteKeyProvider: () => null,
|
|
3038
|
+
lockOptions: () => ({ retryMaxAttempts: 2, retryBaseMs: 1 }),
|
|
3039
|
+
readAllMemories: async () => [memory],
|
|
3040
|
+
readAllColdMemories: async () => [],
|
|
3041
|
+
};
|
|
3042
|
+
const markerDir = path.join(dir, "tombstone-blocked-capture", "rebuild-required");
|
|
3043
|
+
await mkdir(markerDir, { recursive: true });
|
|
3044
|
+
const markerPath = path.join(markerDir, "truncated-writer");
|
|
3045
|
+
await writeFile(markerPath, "{", "utf8");
|
|
3046
|
+
const staleAt = new Date(Date.now() - 120_000);
|
|
3047
|
+
await utimes(markerPath, staleAt, staleAt);
|
|
3048
|
+
|
|
3049
|
+
const index = new TombstoneBlockedCaptureIndex(options);
|
|
3050
|
+
assert.equal(await index.has(content, "fact"), true);
|
|
3051
|
+
assert.deepEqual(await readdir(markerDir), []);
|
|
3052
|
+
});
|
|
3053
|
+
});
|
|
3054
|
+
|
|
3055
|
+
test("explicit capture write locks serialize identities without head-of-line blocking", async () => {
|
|
3056
|
+
await withMemoryDir(async (dir) => {
|
|
3057
|
+
const options = {
|
|
3058
|
+
stateDir: dir,
|
|
3059
|
+
memoryDir: dir,
|
|
3060
|
+
secureStoreKeyProvider: () => null,
|
|
3061
|
+
secureStoreWriteKeyProvider: () => null,
|
|
3062
|
+
lockOptions: () => ({ retryMaxAttempts: 2, retryBaseMs: 1 }),
|
|
3063
|
+
readAllMemories: async () => [],
|
|
3064
|
+
readAllColdMemories: async () => [],
|
|
3065
|
+
};
|
|
3066
|
+
const index = new TombstoneBlockedCaptureIndex(options);
|
|
3067
|
+
let inFlight = 0;
|
|
3068
|
+
let maxInFlight = 0;
|
|
3069
|
+
const firstReleaseState = Promise.withResolvers<void>();
|
|
3070
|
+
const firstRelease = firstReleaseState.promise;
|
|
3071
|
+
const releaseFirst = firstReleaseState.resolve;
|
|
3072
|
+
const firstEnteredState = Promise.withResolvers<void>();
|
|
3073
|
+
const firstEnteredSignal = firstEnteredState.promise;
|
|
3074
|
+
const firstEntered = firstEnteredState.resolve;
|
|
3075
|
+
const otherEnteredState = Promise.withResolvers<void>();
|
|
3076
|
+
const otherEnteredSignal = otherEnteredState.promise;
|
|
3077
|
+
const otherEntered = otherEnteredState.resolve;
|
|
3078
|
+
const first = index.withCaptureWriteLock(async () => {
|
|
3079
|
+
inFlight += 1;
|
|
3080
|
+
maxInFlight = Math.max(maxInFlight, inFlight);
|
|
3081
|
+
firstEntered();
|
|
3082
|
+
await firstRelease;
|
|
3083
|
+
inFlight -= 1;
|
|
3084
|
+
}, "capture-a");
|
|
3085
|
+
await firstEnteredSignal;
|
|
3086
|
+
const queued = Promise.all(
|
|
3087
|
+
Array.from({ length: 3 }, () =>
|
|
3088
|
+
index.withCaptureWriteLock(async () => {
|
|
3089
|
+
inFlight += 1;
|
|
3090
|
+
maxInFlight = Math.max(maxInFlight, inFlight);
|
|
3091
|
+
inFlight -= 1;
|
|
3092
|
+
}, "capture-a")
|
|
3093
|
+
)
|
|
3094
|
+
);
|
|
3095
|
+
const other = index.withCaptureWriteLock(async () => {
|
|
3096
|
+
inFlight += 1;
|
|
3097
|
+
maxInFlight = Math.max(maxInFlight, inFlight);
|
|
3098
|
+
otherEntered();
|
|
3099
|
+
inFlight -= 1;
|
|
3100
|
+
}, "capture-b");
|
|
3101
|
+
const all = Promise.all([first, queued, other]);
|
|
3102
|
+
const timeoutState = Promise.withResolvers<void>();
|
|
3103
|
+
setTimeout(timeoutState.resolve, 100);
|
|
3104
|
+
try {
|
|
3105
|
+
await Promise.race([otherEnteredSignal, timeoutState.promise]);
|
|
3106
|
+
assert.equal(maxInFlight, 2, "an unrelated capture identity must not wait behind a slow capture");
|
|
3107
|
+
} finally {
|
|
3108
|
+
releaseFirst();
|
|
3109
|
+
await all;
|
|
3110
|
+
}
|
|
3111
|
+
});
|
|
3112
|
+
});
|
|
3113
|
+
test("blocked generic writes share their explicit-capture identity lock", async () => {
|
|
3114
|
+
await withMemoryDir(async (dir) => {
|
|
3115
|
+
const storage = new StorageManager(dir);
|
|
3116
|
+
await storage.ensureDirectories();
|
|
3117
|
+
storage.setTombstonesConfig({
|
|
3118
|
+
enabled: true,
|
|
3119
|
+
semanticMatch: false,
|
|
3120
|
+
semanticThreshold: 0.9,
|
|
3121
|
+
namespace: "default",
|
|
3122
|
+
});
|
|
3123
|
+
const content = "Generic blocked writes must coordinate with explicit captures.";
|
|
3124
|
+
await storage.appendTombstone({
|
|
3125
|
+
reason: "correction",
|
|
3126
|
+
createdBy: "user_correction",
|
|
3127
|
+
sourceMemoryId: "generic-lock-coordination",
|
|
3128
|
+
rawContent: content,
|
|
3129
|
+
});
|
|
3130
|
+
const identity = buildExplicitCaptureDedupKey(content, "fact", "provider-a");
|
|
3131
|
+
const firstResult = await storage.writeMemory("fact", content, {
|
|
3132
|
+
source: "test",
|
|
3133
|
+
sourceConnector: "provider-a",
|
|
3134
|
+
});
|
|
3135
|
+
const releaseHeldState = Promise.withResolvers<void>();
|
|
3136
|
+
const releaseHeld = releaseHeldState.promise;
|
|
3137
|
+
const release = releaseHeldState.resolve;
|
|
3138
|
+
const enteredState = Promise.withResolvers<void>();
|
|
3139
|
+
const enteredSignal = enteredState.promise;
|
|
3140
|
+
const entered = enteredState.resolve;
|
|
3141
|
+
const held = storage.withTombstoneBlockedCaptureWriteLock(async () => {
|
|
3142
|
+
entered();
|
|
3143
|
+
await releaseHeld;
|
|
3144
|
+
}, identity);
|
|
3145
|
+
await enteredSignal;
|
|
3146
|
+
let completed = false;
|
|
3147
|
+
const pending = storage
|
|
3148
|
+
.writeMemory("fact", content, {
|
|
3149
|
+
source: "test",
|
|
3150
|
+
sourceConnector: "provider-a",
|
|
3151
|
+
})
|
|
3152
|
+
.then((result) => {
|
|
3153
|
+
completed = true;
|
|
3154
|
+
return result;
|
|
3155
|
+
});
|
|
3156
|
+
const delayState = Promise.withResolvers<void>();
|
|
3157
|
+
setTimeout(delayState.resolve, 50);
|
|
3158
|
+
await delayState.promise;
|
|
3159
|
+
assert.equal(completed, false, "a matching generic write must wait for the capture lock");
|
|
3160
|
+
release();
|
|
3161
|
+
await held;
|
|
3162
|
+
const result = await pending;
|
|
3163
|
+
assert.ok(firstResult);
|
|
3164
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
3165
|
+
assert.equal(result.id, firstResult.id, "the waiting writer must recheck identity after lock acquisition");
|
|
3166
|
+
assert.equal(result.duplicateOf, firstResult.id, "the coalesced blocked write must identify its duplicate");
|
|
3167
|
+
assert.equal((await storage.readAllMemories()).length, 1, "the duplicate row must not be persisted");
|
|
3168
|
+
});
|
|
3169
|
+
});
|
|
3170
|
+
|
|
3171
|
+
test("blocked invalidation reserves the index marker before deleting under the identity lock", async () => {
|
|
3172
|
+
await withMemoryDir(async (dir) => {
|
|
3173
|
+
const storage = new StorageManager(dir);
|
|
3174
|
+
await storage.ensureDirectories();
|
|
3175
|
+
storage.setTombstonesConfig({
|
|
3176
|
+
enabled: true,
|
|
3177
|
+
semanticMatch: false,
|
|
3178
|
+
semanticThreshold: 0.9,
|
|
3179
|
+
namespace: "default",
|
|
3180
|
+
});
|
|
3181
|
+
const peer = new StorageManager(dir);
|
|
3182
|
+
await peer.ensureDirectories();
|
|
3183
|
+
peer.setTombstonesConfig({
|
|
3184
|
+
enabled: true,
|
|
3185
|
+
semanticMatch: false,
|
|
3186
|
+
semanticThreshold: 0.9,
|
|
3187
|
+
namespace: "default",
|
|
3188
|
+
});
|
|
3189
|
+
const content = "Invalidating a blocked row must serialize with recapture.";
|
|
3190
|
+
await storage.appendTombstone({
|
|
3191
|
+
reason: "correction",
|
|
3192
|
+
createdBy: "user_correction",
|
|
3193
|
+
sourceMemoryId: "blocked-invalidation-lock",
|
|
3194
|
+
rawContent: content,
|
|
3195
|
+
});
|
|
3196
|
+
const first = await storage.writeMemory("fact", content, {
|
|
3197
|
+
source: "test",
|
|
3198
|
+
sourceConnector: "provider-a",
|
|
3199
|
+
});
|
|
3200
|
+
assert.equal(first.tombstoneBlocked, true);
|
|
3201
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a");
|
|
3202
|
+
await peer.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a");
|
|
3203
|
+
|
|
3204
|
+
const identity = buildExplicitCaptureDedupKey(content, "fact", "provider-a");
|
|
3205
|
+
const entered = Promise.withResolvers<void>();
|
|
3206
|
+
const release = Promise.withResolvers<void>();
|
|
3207
|
+
const storageWithOverride = storage as unknown as {
|
|
3208
|
+
withTombstoneBlockedCaptureWriteLock: (
|
|
3209
|
+
task: () => Promise<boolean>,
|
|
3210
|
+
identity?: string | readonly string[],
|
|
3211
|
+
) => Promise<boolean>;
|
|
3212
|
+
};
|
|
3213
|
+
let capturedIdentity: string | readonly string[] | undefined;
|
|
3214
|
+
const originalLock = storageWithOverride.withTombstoneBlockedCaptureWriteLock.bind(storage);
|
|
3215
|
+
storageWithOverride.withTombstoneBlockedCaptureWriteLock = async (task, lockIdentity) => {
|
|
3216
|
+
capturedIdentity = lockIdentity;
|
|
3217
|
+
return originalLock(task, lockIdentity);
|
|
3218
|
+
};
|
|
3219
|
+
const originalPrepare = TombstoneBlockedCaptureIndex.prototype.prepareWrite;
|
|
3220
|
+
let paused = false;
|
|
3221
|
+
const prepareSpy = mock.method(
|
|
3222
|
+
TombstoneBlockedCaptureIndex.prototype,
|
|
3223
|
+
"prepareWrite",
|
|
3224
|
+
async function (this: TombstoneBlockedCaptureIndex) {
|
|
3225
|
+
const marker = await originalPrepare.call(this);
|
|
3226
|
+
if (!paused) {
|
|
3227
|
+
paused = true;
|
|
3228
|
+
entered.resolve();
|
|
3229
|
+
await release.promise;
|
|
3230
|
+
}
|
|
3231
|
+
return marker;
|
|
3232
|
+
},
|
|
3233
|
+
);
|
|
3234
|
+
|
|
3235
|
+
const invalidation = storage.invalidateMemory(first.id);
|
|
3236
|
+
try {
|
|
3237
|
+
const enteredBeforeTimeout = await Promise.race([
|
|
3238
|
+
entered.promise.then(() => true),
|
|
3239
|
+
new Promise<boolean>((resolve) => setTimeout(() => resolve(false), 100)),
|
|
3240
|
+
]);
|
|
3241
|
+
assert.equal(
|
|
3242
|
+
enteredBeforeTimeout,
|
|
3243
|
+
true,
|
|
3244
|
+
"blocked invalidation must reserve its rebuild marker before unlinking",
|
|
3245
|
+
);
|
|
3246
|
+
assert.equal(capturedIdentity, identity);
|
|
3247
|
+
assert.ok(
|
|
3248
|
+
(await readdir(path.join(dir, "state", "tombstone-blocked-capture", "rebuild-required"))).length > 0,
|
|
3249
|
+
"blocked invalidation must publish its rebuild marker before unlinking",
|
|
3250
|
+
);
|
|
3251
|
+
|
|
3252
|
+
let peerCompleted = false;
|
|
3253
|
+
const recapture = peer
|
|
3254
|
+
.writeMemory("fact", content, {
|
|
3255
|
+
source: "test",
|
|
3256
|
+
sourceConnector: "provider-a",
|
|
3257
|
+
})
|
|
3258
|
+
.then((result) => {
|
|
3259
|
+
peerCompleted = true;
|
|
3260
|
+
return result;
|
|
3261
|
+
});
|
|
3262
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
3263
|
+
assert.equal(peerCompleted, false, "recapture must wait while invalidation owns the identity lock");
|
|
3264
|
+
release.resolve();
|
|
3265
|
+
assert.equal(await invalidation, true);
|
|
3266
|
+
const second = await recapture;
|
|
3267
|
+
assert.equal(second.tombstoneBlocked, true);
|
|
3268
|
+
assert.notEqual(second.id, first.id);
|
|
3269
|
+
assert.equal(
|
|
3270
|
+
(await peer.readAllMemories()).filter((memory) => memory.content === content).length,
|
|
3271
|
+
1,
|
|
3272
|
+
);
|
|
3273
|
+
} finally {
|
|
3274
|
+
release.resolve();
|
|
3275
|
+
prepareSpy.mock.restore();
|
|
3276
|
+
await invalidation.catch(() => false);
|
|
3277
|
+
}
|
|
3278
|
+
});
|
|
3279
|
+
});
|
|
3280
|
+
|
|
3281
|
+
test("blocked archive serializes capture scans with the archive transition", async () => {
|
|
3282
|
+
await withMemoryDir(async (dir) => {
|
|
3283
|
+
const storage = new StorageManager(dir);
|
|
3284
|
+
const peer = new StorageManager(dir);
|
|
3285
|
+
await storage.ensureDirectories();
|
|
3286
|
+
await peer.ensureDirectories();
|
|
3287
|
+
const tombstoneConfig = {
|
|
3288
|
+
enabled: true,
|
|
3289
|
+
semanticMatch: false,
|
|
3290
|
+
semanticThreshold: 0.9,
|
|
3291
|
+
namespace: "default",
|
|
3292
|
+
};
|
|
3293
|
+
storage.setTombstonesConfig(tombstoneConfig);
|
|
3294
|
+
peer.setTombstonesConfig(tombstoneConfig);
|
|
3295
|
+
const content = "Archiving a blocked row must serialize with recapture.";
|
|
3296
|
+
await storage.appendTombstone({
|
|
3297
|
+
reason: "correction",
|
|
3298
|
+
createdBy: "user_correction",
|
|
3299
|
+
sourceMemoryId: "blocked-archive-lock",
|
|
3300
|
+
rawContent: content,
|
|
3301
|
+
});
|
|
3302
|
+
const first = await storage.writeMemory("fact", content, {
|
|
3303
|
+
source: "test",
|
|
3304
|
+
sourceConnector: "provider-a",
|
|
3305
|
+
});
|
|
3306
|
+
assert.equal(first.tombstoneBlocked, true);
|
|
3307
|
+
const firstMemory = (await storage.readAllMemories()).find((memory) => memory.frontmatter.id === first.id);
|
|
3308
|
+
assert.ok(firstMemory);
|
|
3309
|
+
const identity = buildExplicitCaptureDedupKey(content, "fact", "provider-a");
|
|
3310
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a");
|
|
3311
|
+
await peer.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a");
|
|
3312
|
+
|
|
3313
|
+
const storageWithOverride = storage as unknown as {
|
|
3314
|
+
withTombstoneBlockedCaptureWriteLock: (
|
|
3315
|
+
task: () => Promise<unknown>,
|
|
3316
|
+
identity?: string | readonly string[],
|
|
3317
|
+
) => Promise<unknown>;
|
|
3318
|
+
};
|
|
3319
|
+
const originalLock = storageWithOverride.withTombstoneBlockedCaptureWriteLock.bind(storage);
|
|
3320
|
+
let capturedIdentity: string | readonly string[] | undefined;
|
|
3321
|
+
storageWithOverride.withTombstoneBlockedCaptureWriteLock = async (task, lockIdentity) => {
|
|
3322
|
+
capturedIdentity = lockIdentity;
|
|
3323
|
+
return await originalLock(task, lockIdentity);
|
|
3324
|
+
};
|
|
3325
|
+
|
|
3326
|
+
const markerEntered = Promise.withResolvers<void>();
|
|
3327
|
+
const releaseMarker = Promise.withResolvers<void>();
|
|
3328
|
+
const originalPrepare = TombstoneBlockedCaptureIndex.prototype.prepareWrite;
|
|
3329
|
+
let paused = false;
|
|
3330
|
+
const prepareSpy = mock.method(
|
|
3331
|
+
TombstoneBlockedCaptureIndex.prototype,
|
|
3332
|
+
"prepareWrite",
|
|
3333
|
+
async function (this: TombstoneBlockedCaptureIndex) {
|
|
3334
|
+
const marker = await originalPrepare.call(this);
|
|
3335
|
+
if (!paused) {
|
|
3336
|
+
paused = true;
|
|
3337
|
+
markerEntered.resolve();
|
|
3338
|
+
await releaseMarker.promise;
|
|
3339
|
+
}
|
|
3340
|
+
return marker;
|
|
3341
|
+
},
|
|
3342
|
+
);
|
|
3343
|
+
|
|
3344
|
+
const archive = storage.archiveMemory(firstMemory);
|
|
3345
|
+
try {
|
|
3346
|
+
const enteredBeforeTimeout = await Promise.race([
|
|
3347
|
+
markerEntered.promise.then(() => true),
|
|
3348
|
+
new Promise<boolean>((resolve) => setTimeout(() => resolve(false), 100)),
|
|
3349
|
+
]);
|
|
3350
|
+
assert.equal(enteredBeforeTimeout, true, "archive must reserve its marker under the capture lock");
|
|
3351
|
+
assert.equal(capturedIdentity, identity);
|
|
3352
|
+
|
|
3353
|
+
let recaptureCompleted = false;
|
|
3354
|
+
const recapture = peer
|
|
3355
|
+
.writeMemory("fact", content, {
|
|
3356
|
+
source: "test",
|
|
3357
|
+
sourceConnector: "provider-a",
|
|
3358
|
+
})
|
|
3359
|
+
.then((result) => {
|
|
3360
|
+
recaptureCompleted = true;
|
|
3361
|
+
return result;
|
|
3362
|
+
});
|
|
3363
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
3364
|
+
assert.equal(recaptureCompleted, false, "recapture must wait while archive owns the identity lock");
|
|
3365
|
+
|
|
3366
|
+
releaseMarker.resolve();
|
|
3367
|
+
const archivePath = await archive;
|
|
3368
|
+
assert.ok(archivePath);
|
|
3369
|
+
const second = await recapture;
|
|
3370
|
+
assert.equal(second.tombstoneBlocked, true);
|
|
3371
|
+
assert.notEqual(second.id, first.id);
|
|
3372
|
+
assert.equal(
|
|
3373
|
+
(await peer.readAllMemories()).filter((memory) => memory.content === content).length,
|
|
3374
|
+
1,
|
|
3375
|
+
"recapture must recreate the review row after archive removes the old row",
|
|
3376
|
+
);
|
|
3377
|
+
assert.equal(
|
|
3378
|
+
await peer.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"),
|
|
3379
|
+
true,
|
|
3380
|
+
"the recreated review row must remain indexed",
|
|
3381
|
+
);
|
|
3382
|
+
} finally {
|
|
3383
|
+
releaseMarker.resolve();
|
|
3384
|
+
prepareSpy.mock.restore();
|
|
3385
|
+
storageWithOverride.withTombstoneBlockedCaptureWriteLock = originalLock;
|
|
3386
|
+
await archive.catch(() => null);
|
|
3387
|
+
}
|
|
3388
|
+
});
|
|
3389
|
+
});
|
|
3390
|
+
|
|
3391
|
+
test("blocked archive failures discard their uncommitted rebuild marker", async () => {
|
|
3392
|
+
await withMemoryDir(async (dir) => {
|
|
3393
|
+
class FailingArchiveStorageManager extends StorageManager {
|
|
3394
|
+
failArchiveWrites = false;
|
|
3395
|
+
|
|
3396
|
+
protected override writeStorageSecureFile(
|
|
3397
|
+
filePath: string,
|
|
3398
|
+
content: string | Buffer,
|
|
3399
|
+
forceEncrypt = false,
|
|
3400
|
+
): Promise<void> {
|
|
3401
|
+
if (this.failArchiveWrites && filePath.includes(`${path.sep}archive${path.sep}`)) {
|
|
3402
|
+
return Promise.reject(new Error("simulated archive write failure"));
|
|
3403
|
+
}
|
|
3404
|
+
return super.writeStorageSecureFile(filePath, content, forceEncrypt);
|
|
3405
|
+
}
|
|
3406
|
+
}
|
|
3407
|
+
|
|
3408
|
+
const storage = new FailingArchiveStorageManager(dir);
|
|
3409
|
+
await storage.ensureDirectories();
|
|
3410
|
+
storage.setTombstonesConfig({
|
|
3411
|
+
enabled: true,
|
|
3412
|
+
semanticMatch: false,
|
|
3413
|
+
semanticThreshold: 0.9,
|
|
3414
|
+
namespace: "default",
|
|
3415
|
+
});
|
|
3416
|
+
const content = "A failed blocked archive must not poison rebuild state.";
|
|
3417
|
+
await storage.appendTombstone({
|
|
3418
|
+
reason: "correction",
|
|
3419
|
+
createdBy: "user_correction",
|
|
3420
|
+
sourceMemoryId: "failed-blocked-archive",
|
|
3421
|
+
rawContent: content,
|
|
3422
|
+
});
|
|
3423
|
+
const result = await storage.writeMemory("fact", content, {
|
|
3424
|
+
source: "test",
|
|
3425
|
+
sourceConnector: "provider-a",
|
|
3426
|
+
});
|
|
3427
|
+
const memory = (await storage.readAllMemories()).find((candidate) => candidate.frontmatter.id === result.id);
|
|
3428
|
+
assert.ok(memory);
|
|
3429
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a");
|
|
3430
|
+
const markerDir = path.join(dir, "state", "tombstone-blocked-capture", "rebuild-required");
|
|
3431
|
+
storage.failArchiveWrites = true;
|
|
3432
|
+
|
|
3433
|
+
assert.equal(await storage.archiveMemory(memory), null);
|
|
3434
|
+
assert.deepEqual(await readdir(markerDir), []);
|
|
3435
|
+
assert.equal(await storage.isTombstoneBlockedExplicitCaptureIndexAuthoritative(), true);
|
|
3436
|
+
});
|
|
3437
|
+
});
|
|
3438
|
+
test("blocked invalidation reacquires the reloaded identity lock", async () => {
|
|
3439
|
+
await withMemoryDir(async (dir) => {
|
|
3440
|
+
const storage = new StorageManager(dir);
|
|
3441
|
+
const peer = new StorageManager(dir);
|
|
3442
|
+
await storage.ensureDirectories();
|
|
3443
|
+
await peer.ensureDirectories();
|
|
3444
|
+
const tombstoneConfig = {
|
|
3445
|
+
enabled: true,
|
|
3446
|
+
semanticMatch: false,
|
|
3447
|
+
semanticThreshold: 0.9,
|
|
3448
|
+
namespace: "default",
|
|
3449
|
+
};
|
|
3450
|
+
storage.setTombstonesConfig(tombstoneConfig);
|
|
3451
|
+
peer.setTombstonesConfig(tombstoneConfig);
|
|
3452
|
+
const contentA = "Blocked invalidation must follow a rewritten identity.";
|
|
3453
|
+
const contentB = "The peer rewrites the blocked row before invalidation acquires its lock.";
|
|
3454
|
+
await storage.appendTombstone({
|
|
3455
|
+
reason: "correction",
|
|
3456
|
+
createdBy: "user_correction",
|
|
3457
|
+
sourceMemoryId: "reloaded-identity-lock",
|
|
3458
|
+
rawContent: contentA,
|
|
3459
|
+
});
|
|
3460
|
+
await storage.appendTombstone({
|
|
3461
|
+
reason: "correction",
|
|
3462
|
+
createdBy: "user_correction",
|
|
3463
|
+
sourceMemoryId: "reloaded-identity-lock-b",
|
|
3464
|
+
rawContent: contentB,
|
|
3465
|
+
});
|
|
3466
|
+
const first = await storage.writeMemory("fact", contentA, {
|
|
3467
|
+
source: "test",
|
|
3468
|
+
sourceConnector: "provider-a",
|
|
3469
|
+
});
|
|
3470
|
+
assert.equal(first.tombstoneBlocked, true);
|
|
3471
|
+
const identityA = buildExplicitCaptureDedupKey(contentA, "fact", "provider-a");
|
|
3472
|
+
const identityB = buildExplicitCaptureDedupKey(contentB, "fact", "provider-a");
|
|
3473
|
+
await storage.hasTombstoneBlockedExplicitCapture(contentA, "fact", "provider-a");
|
|
3474
|
+
await peer.hasTombstoneBlockedExplicitCapture(contentA, "fact", "provider-a");
|
|
3475
|
+
|
|
3476
|
+
const storageWithOverride = storage as unknown as {
|
|
3477
|
+
withTombstoneBlockedCaptureWriteLock: (
|
|
3478
|
+
task: () => Promise<unknown>,
|
|
3479
|
+
identity?: string | readonly string[],
|
|
3480
|
+
) => Promise<unknown>;
|
|
3481
|
+
};
|
|
3482
|
+
const originalLock = storageWithOverride.withTombstoneBlockedCaptureWriteLock.bind(storage);
|
|
3483
|
+
const capturedIdentities: Array<string | readonly string[] | undefined> = [];
|
|
3484
|
+
let firstLock = true;
|
|
3485
|
+
let pauseNextPrepare = false;
|
|
3486
|
+
const markerEntered = Promise.withResolvers<void>();
|
|
3487
|
+
const releaseMarker = Promise.withResolvers<void>();
|
|
3488
|
+
storageWithOverride.withTombstoneBlockedCaptureWriteLock = async (task, identity) => {
|
|
3489
|
+
capturedIdentities.push(identity);
|
|
3490
|
+
if (firstLock) {
|
|
3491
|
+
firstLock = false;
|
|
3492
|
+
await peer.updateMemory(first.id, contentB, { sourceConnector: "provider-a" });
|
|
3493
|
+
pauseNextPrepare = true;
|
|
3494
|
+
}
|
|
3495
|
+
return await originalLock(task, identity);
|
|
3496
|
+
};
|
|
3497
|
+
const originalPrepare = TombstoneBlockedCaptureIndex.prototype.prepareWrite;
|
|
3498
|
+
const prepareSpy = mock.method(
|
|
3499
|
+
TombstoneBlockedCaptureIndex.prototype,
|
|
3500
|
+
"prepareWrite",
|
|
3501
|
+
async function (this: TombstoneBlockedCaptureIndex) {
|
|
3502
|
+
const marker = await originalPrepare.call(this);
|
|
3503
|
+
if (pauseNextPrepare) {
|
|
3504
|
+
pauseNextPrepare = false;
|
|
3505
|
+
markerEntered.resolve();
|
|
3506
|
+
await releaseMarker.promise;
|
|
3507
|
+
}
|
|
3508
|
+
return marker;
|
|
3509
|
+
},
|
|
3510
|
+
);
|
|
3511
|
+
|
|
3512
|
+
const invalidation = storage.invalidateMemory(first.id);
|
|
3513
|
+
try {
|
|
3514
|
+
await markerEntered.promise;
|
|
3515
|
+
assert.deepEqual(capturedIdentities, [identityA, identityB]);
|
|
3516
|
+
let recaptureCompleted = false;
|
|
3517
|
+
const recapture = peer
|
|
3518
|
+
.writeMemory("fact", contentB, {
|
|
3519
|
+
source: "test",
|
|
3520
|
+
sourceConnector: "provider-a",
|
|
3521
|
+
})
|
|
3522
|
+
.then((result) => {
|
|
3523
|
+
recaptureCompleted = true;
|
|
3524
|
+
return result;
|
|
3525
|
+
});
|
|
3526
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
3527
|
+
assert.equal(recaptureCompleted, false, "recapture must wait for the reloaded identity lock");
|
|
3528
|
+
releaseMarker.resolve();
|
|
3529
|
+
assert.equal(await invalidation, true);
|
|
3530
|
+
const second = await recapture;
|
|
3531
|
+
assert.equal(second.tombstoneBlocked, true);
|
|
3532
|
+
assert.notEqual(second.id, first.id);
|
|
3533
|
+
assert.equal(
|
|
3534
|
+
(await peer.readAllMemories()).filter((memory) => memory.content === contentB).length,
|
|
3535
|
+
1,
|
|
3536
|
+
);
|
|
3537
|
+
} finally {
|
|
3538
|
+
releaseMarker.resolve();
|
|
3539
|
+
prepareSpy.mock.restore();
|
|
3540
|
+
storageWithOverride.withTombstoneBlockedCaptureWriteLock = originalLock;
|
|
3541
|
+
await invalidation.catch(() => false);
|
|
3542
|
+
}
|
|
3543
|
+
});
|
|
3544
|
+
});
|
|
3545
|
+
test("blocked rewrites reacquire a target identity changed before update", async () => {
|
|
3546
|
+
await withMemoryDir(async (dir) => {
|
|
3547
|
+
const storage = new StorageManager(dir);
|
|
3548
|
+
const peer = new StorageManager(dir);
|
|
3549
|
+
await storage.ensureDirectories();
|
|
3550
|
+
await peer.ensureDirectories();
|
|
3551
|
+
const tombstoneConfig = {
|
|
3552
|
+
enabled: true,
|
|
3553
|
+
semanticMatch: false,
|
|
3554
|
+
semanticThreshold: 0.9,
|
|
3555
|
+
namespace: "default",
|
|
3556
|
+
};
|
|
3557
|
+
storage.setTombstonesConfig(tombstoneConfig);
|
|
3558
|
+
peer.setTombstonesConfig(tombstoneConfig);
|
|
3559
|
+
const contentA = "Blocked update starts with identity A.";
|
|
3560
|
+
const contentB = "A peer rewrites the target to identity B before update locks.";
|
|
3561
|
+
const contentC = "The update itself must publish identity C.";
|
|
3562
|
+
for (const [sourceMemoryId, rawContent] of [
|
|
3563
|
+
["blocked-update-a", contentA],
|
|
3564
|
+
["blocked-update-b", contentB],
|
|
3565
|
+
["blocked-update-c", contentC],
|
|
3566
|
+
] as const) {
|
|
3567
|
+
assert.ok(
|
|
3568
|
+
await storage.appendTombstone({
|
|
3569
|
+
reason: "correction",
|
|
3570
|
+
createdBy: "user_correction",
|
|
3571
|
+
sourceMemoryId,
|
|
3572
|
+
rawContent,
|
|
3573
|
+
}),
|
|
3574
|
+
);
|
|
3575
|
+
}
|
|
3576
|
+
const target = await storage.writeMemory("fact", contentA, {
|
|
3577
|
+
source: "test",
|
|
3578
|
+
sourceConnector: "provider-a",
|
|
3579
|
+
});
|
|
3580
|
+
assert.equal(target.tombstoneBlocked, true);
|
|
3581
|
+
const targetMemory = (await storage.readAllMemories()).find(
|
|
3582
|
+
(memory) => memory.frontmatter.id === target.id
|
|
3583
|
+
);
|
|
3584
|
+
assert.ok(targetMemory);
|
|
3585
|
+
const identityA = buildExplicitCaptureDedupKey(contentA, "fact", "provider-a");
|
|
3586
|
+
const identityB = buildExplicitCaptureDedupKey(contentB, "fact", "provider-a");
|
|
3587
|
+
const identityC = buildExplicitCaptureDedupKey(contentC, "fact", "provider-a");
|
|
3588
|
+
const storageWithOverride = storage as unknown as {
|
|
3589
|
+
withTombstoneBlockedCaptureWriteLock: (
|
|
3590
|
+
task: () => Promise<unknown>,
|
|
3591
|
+
identity?: string | readonly string[],
|
|
3592
|
+
) => Promise<unknown>;
|
|
3593
|
+
};
|
|
3594
|
+
const originalLock = storageWithOverride.withTombstoneBlockedCaptureWriteLock.bind(storage);
|
|
3595
|
+
const capturedIdentities: Array<string | readonly string[] | undefined> = [];
|
|
3596
|
+
let firstLock = true;
|
|
3597
|
+
let pauseNextPrepare = false;
|
|
3598
|
+
const markerEntered = Promise.withResolvers<void>();
|
|
3599
|
+
const releaseMarker = Promise.withResolvers<void>();
|
|
3600
|
+
storageWithOverride.withTombstoneBlockedCaptureWriteLock = async (task, identity) => {
|
|
3601
|
+
capturedIdentities.push(identity);
|
|
3602
|
+
if (firstLock) {
|
|
3603
|
+
firstLock = false;
|
|
3604
|
+
await peer.updateMemory(target.id, contentB, { sourceConnector: "provider-a" });
|
|
3605
|
+
pauseNextPrepare = true;
|
|
3606
|
+
}
|
|
3607
|
+
return await originalLock(task, identity);
|
|
3608
|
+
};
|
|
3609
|
+
const originalPrepare = TombstoneBlockedCaptureIndex.prototype.prepareWrite;
|
|
3610
|
+
const prepareSpy = mock.method(
|
|
3611
|
+
TombstoneBlockedCaptureIndex.prototype,
|
|
3612
|
+
"prepareWrite",
|
|
3613
|
+
async function (this: TombstoneBlockedCaptureIndex) {
|
|
3614
|
+
const marker = await originalPrepare.call(this);
|
|
3615
|
+
if (pauseNextPrepare) {
|
|
3616
|
+
pauseNextPrepare = false;
|
|
3617
|
+
markerEntered.resolve();
|
|
3618
|
+
await releaseMarker.promise;
|
|
3619
|
+
}
|
|
3620
|
+
return marker;
|
|
3621
|
+
},
|
|
3622
|
+
);
|
|
3623
|
+
|
|
3624
|
+
const update = storage.updateMemory(target.id, contentC, { sourceConnector: "provider-a" });
|
|
3625
|
+
try {
|
|
3626
|
+
await markerEntered.promise;
|
|
3627
|
+
assert.equal(capturedIdentities.length, 2);
|
|
3628
|
+
const firstKeys = capturedIdentities[0];
|
|
3629
|
+
const secondKeys = capturedIdentities[1];
|
|
3630
|
+
assert.ok(
|
|
3631
|
+
(Array.isArray(firstKeys) ? firstKeys : [firstKeys]).includes(identityA),
|
|
3632
|
+
"update must first lock the identity read before the peer rewrite",
|
|
3633
|
+
);
|
|
3634
|
+
assert.ok(
|
|
3635
|
+
(Array.isArray(secondKeys) ? secondKeys : [secondKeys]).includes(identityB),
|
|
3636
|
+
"update must reacquire the rewritten target identity",
|
|
3637
|
+
);
|
|
3638
|
+
let recaptureCompleted = false;
|
|
3639
|
+
const recapture = peer
|
|
3640
|
+
.writeMemory("fact", contentB, {
|
|
3641
|
+
source: "test",
|
|
3642
|
+
sourceConnector: "provider-a",
|
|
3643
|
+
})
|
|
3644
|
+
.then((result) => {
|
|
3645
|
+
recaptureCompleted = true;
|
|
3646
|
+
return result;
|
|
3647
|
+
});
|
|
3648
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
3649
|
+
assert.equal(recaptureCompleted, false, "recapture must wait for the rewritten target identity lock");
|
|
3650
|
+
releaseMarker.resolve();
|
|
3651
|
+
assert.equal(await update, true);
|
|
3652
|
+
const recaptured = await recapture;
|
|
3653
|
+
assert.equal(recaptured.tombstoneBlocked, true);
|
|
3654
|
+
const after = await peer.readAllMemories();
|
|
3655
|
+
assert.equal(
|
|
3656
|
+
after.find((memory) => memory.path === targetMemory.path)?.content,
|
|
3657
|
+
contentC,
|
|
3658
|
+
"update must write only after holding the current target identity lock",
|
|
3659
|
+
);
|
|
3660
|
+
assert.equal(after.filter((memory) => memory.content === contentB).length, 1);
|
|
3661
|
+
} finally {
|
|
3662
|
+
releaseMarker.resolve();
|
|
3663
|
+
prepareSpy.mock.restore();
|
|
3664
|
+
storageWithOverride.withTombstoneBlockedCaptureWriteLock = originalLock;
|
|
3665
|
+
await update.catch(() => false);
|
|
3666
|
+
}
|
|
3667
|
+
});
|
|
3668
|
+
});
|
|
3669
|
+
|
|
3670
|
+
test("blocked explicit writes reuse the active identity lock", async () => {
|
|
3671
|
+
await withMemoryDir(async (dir) => {
|
|
3672
|
+
const storage = new StorageManager(dir);
|
|
3673
|
+
await storage.ensureDirectories();
|
|
3674
|
+
storage.setTombstonesConfig({
|
|
3675
|
+
enabled: true,
|
|
3676
|
+
semanticMatch: false,
|
|
3677
|
+
semanticThreshold: 0.9,
|
|
3678
|
+
namespace: "default",
|
|
3679
|
+
});
|
|
3680
|
+
const content = "A blocked explicit write must not reacquire its identity lock.";
|
|
3681
|
+
await storage.appendTombstone({
|
|
3682
|
+
reason: "correction",
|
|
3683
|
+
createdBy: "user_correction",
|
|
3684
|
+
sourceMemoryId: "reentrant-explicit-lock",
|
|
3685
|
+
rawContent: content,
|
|
3686
|
+
});
|
|
3687
|
+
const identity = buildExplicitCaptureDedupKey(content, "fact", "provider-a");
|
|
3688
|
+
|
|
3689
|
+
const result = await storage.withTombstoneBlockedCaptureWriteLock(
|
|
3690
|
+
() =>
|
|
3691
|
+
storage.writeMemory("fact", content, {
|
|
3692
|
+
source: "explicit",
|
|
3693
|
+
sourceConnector: "provider-a",
|
|
3694
|
+
}),
|
|
3695
|
+
identity
|
|
3696
|
+
);
|
|
3697
|
+
|
|
3698
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
3699
|
+
assert.equal((await storage.readAllMemories()).length, 1);
|
|
3700
|
+
});
|
|
3701
|
+
});
|
|
3702
|
+
|
|
3703
|
+
test("blocked dedupe preserves punctuation and connector identity", async () => {
|
|
3704
|
+
await withMemoryDir(async (dir) => {
|
|
3705
|
+
const storage = new StorageManager(dir);
|
|
3706
|
+
await storage.ensureDirectories();
|
|
3707
|
+
storage.setTombstonesConfig({
|
|
3708
|
+
enabled: true,
|
|
3709
|
+
semanticMatch: false,
|
|
3710
|
+
semanticThreshold: 0.9,
|
|
3711
|
+
namespace: "default",
|
|
3712
|
+
});
|
|
3713
|
+
await storage.appendTombstone({
|
|
3714
|
+
reason: "correction",
|
|
3715
|
+
createdBy: "user_correction",
|
|
3716
|
+
sourceMemoryId: "identity-preservation",
|
|
3717
|
+
rawContent: "Remember cats.",
|
|
3718
|
+
});
|
|
3719
|
+
await storage.appendTombstone({
|
|
3720
|
+
reason: "correction",
|
|
3721
|
+
createdBy: "user_correction",
|
|
3722
|
+
sourceMemoryId: "connector-preservation",
|
|
3723
|
+
rawContent: "Remember dogs.",
|
|
3724
|
+
});
|
|
3725
|
+
|
|
3726
|
+
const punctuationA = await storage.writeMemory("fact", "Remember cats.", {
|
|
3727
|
+
source: "test",
|
|
3728
|
+
sourceConnector: "provider-a",
|
|
3729
|
+
});
|
|
3730
|
+
const punctuationB = await storage.writeMemory("fact", "Remember cats!", {
|
|
3731
|
+
source: "test",
|
|
3732
|
+
sourceConnector: "provider-a",
|
|
3733
|
+
});
|
|
3734
|
+
const connectorA = await storage.writeMemory("fact", "Remember dogs.", {
|
|
3735
|
+
source: "test",
|
|
3736
|
+
sourceConnector: "provider-a",
|
|
3737
|
+
});
|
|
3738
|
+
const connectorB = await storage.writeMemory("fact", "Remember dogs.", {
|
|
3739
|
+
source: "test",
|
|
3740
|
+
sourceConnector: "provider.a",
|
|
3741
|
+
});
|
|
3742
|
+
|
|
3743
|
+
assert.equal(punctuationA.tombstoneBlocked, true);
|
|
3744
|
+
assert.equal(punctuationB.tombstoneBlocked, true);
|
|
3745
|
+
assert.equal(connectorA.tombstoneBlocked, true);
|
|
3746
|
+
assert.equal(connectorB.tombstoneBlocked, true);
|
|
3747
|
+
assert.notEqual(punctuationA.id, punctuationB.id);
|
|
3748
|
+
assert.notEqual(connectorA.id, connectorB.id);
|
|
3749
|
+
assert.equal((await storage.readAllMemories()).length, 4);
|
|
3750
|
+
});
|
|
3751
|
+
});
|
|
3752
|
+
|
|
3753
|
+
test("blocked write failures clear their uncommitted rebuild marker", async () => {
|
|
3754
|
+
await withMemoryDir(async (dir) => {
|
|
3755
|
+
class FailingStorageManager extends StorageManager {
|
|
3756
|
+
failWrites = false;
|
|
3757
|
+
|
|
3758
|
+
protected override writeStorageSecureFile(
|
|
3759
|
+
filePath: string,
|
|
3760
|
+
content: string | Buffer,
|
|
3761
|
+
forceEncrypt = false
|
|
3762
|
+
): Promise<void> {
|
|
3763
|
+
if (this.failWrites) return Promise.reject(new Error("simulated durable write failure"));
|
|
3764
|
+
return super.writeStorageSecureFile(filePath, content, forceEncrypt);
|
|
3765
|
+
}
|
|
3766
|
+
}
|
|
3767
|
+
|
|
3768
|
+
const storage = new FailingStorageManager(dir);
|
|
3769
|
+
await storage.ensureDirectories();
|
|
3770
|
+
storage.setTombstonesConfig({
|
|
3771
|
+
enabled: true,
|
|
3772
|
+
semanticMatch: false,
|
|
3773
|
+
semanticThreshold: 0.9,
|
|
3774
|
+
namespace: "default",
|
|
3775
|
+
});
|
|
3776
|
+
const content = "A failed blocked write must not poison the rebuild state.";
|
|
3777
|
+
await storage.appendTombstone({
|
|
3778
|
+
reason: "correction",
|
|
3779
|
+
createdBy: "user_correction",
|
|
3780
|
+
sourceMemoryId: "failed-blocked-write",
|
|
3781
|
+
rawContent: content,
|
|
3782
|
+
});
|
|
3783
|
+
const markerDir = path.join(dir, "state", "tombstone-blocked-capture", "rebuild-required");
|
|
3784
|
+
storage.failWrites = true;
|
|
3785
|
+
await assert.rejects(
|
|
3786
|
+
storage.writeMemory("fact", content, {
|
|
3787
|
+
source: "test",
|
|
3788
|
+
sourceConnector: "provider-a",
|
|
3789
|
+
}),
|
|
3790
|
+
/simulated durable write failure/
|
|
3791
|
+
);
|
|
3792
|
+
assert.deepEqual(await readdir(markerDir), []);
|
|
3793
|
+
});
|
|
3794
|
+
});
|
|
3795
|
+
|
|
3796
|
+
test("blocked writes stay successful when post-commit marker publication fails", async () => {
|
|
3797
|
+
await withMemoryDir(async (dir) => {
|
|
3798
|
+
const storage = new StorageManager(dir);
|
|
3799
|
+
await storage.ensureDirectories();
|
|
3800
|
+
storage.setTombstonesConfig({
|
|
3801
|
+
enabled: true,
|
|
3802
|
+
semanticMatch: false,
|
|
3803
|
+
semanticThreshold: 0.9,
|
|
3804
|
+
namespace: "default",
|
|
3805
|
+
});
|
|
3806
|
+
const content = "A durable blocked write must survive marker publication failure.";
|
|
3807
|
+
await storage.appendTombstone({
|
|
3808
|
+
reason: "correction",
|
|
3809
|
+
createdBy: "user_correction",
|
|
3810
|
+
sourceMemoryId: "marker-publication-failure",
|
|
3811
|
+
rawContent: content,
|
|
3812
|
+
});
|
|
3813
|
+
const markerDir = path.join(dir, "state", "tombstone-blocked-capture", "rebuild-required");
|
|
3814
|
+
const commitSpy = mock.method(TombstoneBlockedCaptureIndex.prototype, "commitWrite", async () => {
|
|
3815
|
+
throw new Error("simulated marker publication failure");
|
|
3816
|
+
});
|
|
3817
|
+
try {
|
|
3818
|
+
const result = await storage.writeMemory("fact", content, {
|
|
3819
|
+
source: "test",
|
|
3820
|
+
sourceConnector: "provider-a",
|
|
3821
|
+
});
|
|
3822
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
3823
|
+
assert.ok(await storage.getMemoryById(result.id), "the durable memory must remain readable");
|
|
3824
|
+
assert.deepEqual(await readdir(markerDir), [], "the index hook should clear the retried marker");
|
|
3825
|
+
assert.equal(
|
|
3826
|
+
await storage.hasTombstoneBlockedExplicitCapture(content, "fact", "provider-a"),
|
|
3827
|
+
true,
|
|
3828
|
+
"the retried marker publication must leave the blocked identity indexed"
|
|
3829
|
+
);
|
|
3830
|
+
assert.equal(await storage.isTombstoneBlockedExplicitCaptureIndexAuthoritative(), true);
|
|
3831
|
+
} finally {
|
|
3832
|
+
commitSpy.mock.restore();
|
|
3833
|
+
}
|
|
3834
|
+
});
|
|
3835
|
+
});
|
|
3836
|
+
|
|
3837
|
+
test("blocked additions exclude their own marker from rebuild checks", async () => {
|
|
3838
|
+
await withMemoryDir(async (dir) => {
|
|
3839
|
+
const storage = new StorageManager(dir);
|
|
3840
|
+
await storage.ensureDirectories();
|
|
3841
|
+
storage.setTombstonesConfig({
|
|
3842
|
+
enabled: true,
|
|
3843
|
+
semanticMatch: false,
|
|
3844
|
+
semanticThreshold: 0.9,
|
|
3845
|
+
namespace: "default",
|
|
3846
|
+
});
|
|
3847
|
+
const content = "A blocked addition should update the index incrementally.";
|
|
3848
|
+
await storage.appendTombstone({
|
|
3849
|
+
reason: "correction",
|
|
3850
|
+
createdBy: "user_correction",
|
|
3851
|
+
sourceMemoryId: "incremental-blocked-add",
|
|
3852
|
+
rawContent: content,
|
|
3853
|
+
});
|
|
3854
|
+
await storage.hasTombstoneBlockedExplicitCapture("unrelated", "fact", "provider-a");
|
|
3855
|
+
let rebuilds = 0;
|
|
3856
|
+
const rebuildSpy = mock.method(ContentHashIndex.prototype, "rebuildUnderLock", async () => {
|
|
3857
|
+
rebuilds += 1;
|
|
3858
|
+
return true;
|
|
3859
|
+
});
|
|
3860
|
+
try {
|
|
3861
|
+
const result = await storage.writeMemory("fact", content, {
|
|
3862
|
+
source: "test",
|
|
3863
|
+
sourceConnector: "provider-a",
|
|
3864
|
+
});
|
|
3865
|
+
assert.equal(result.tombstoneBlocked, true);
|
|
3866
|
+
assert.equal(rebuilds, 0);
|
|
3867
|
+
} finally {
|
|
3868
|
+
rebuildSpy.mock.restore();
|
|
3869
|
+
}
|
|
3870
|
+
});
|
|
3871
|
+
});
|
|
3872
|
+
|
|
3873
|
+
test("queued-review dispositions coordinate with explicit capture dedupe", async () => {
|
|
3874
|
+
await withMemoryDir(async (dir) => {
|
|
3875
|
+
const storage = new StorageManager(dir);
|
|
3876
|
+
await storage.ensureDirectories();
|
|
3877
|
+
const content = "A queued-review disposition must hold its capture identity lock.";
|
|
3878
|
+
const result = await storage.writeMemory("fact", content, {
|
|
3879
|
+
source: "explicit-inline-review",
|
|
3880
|
+
sourceConnector: "provider-a",
|
|
3881
|
+
status: "pending_review",
|
|
3882
|
+
tags: ["explicit-capture", "queued-review"],
|
|
3883
|
+
});
|
|
3884
|
+
const memory = await storage.getMemoryById(result.id);
|
|
3885
|
+
assert.ok(memory);
|
|
3886
|
+
const identity = buildExplicitCaptureDedupKey(content, "fact", "provider-a");
|
|
3887
|
+
const entered = Promise.withResolvers<void>();
|
|
3888
|
+
const release = Promise.withResolvers<void>();
|
|
3889
|
+
const held = storage.withTombstoneBlockedCaptureWriteLock(async () => {
|
|
3890
|
+
entered.resolve();
|
|
3891
|
+
await release.promise;
|
|
3892
|
+
}, identity);
|
|
3893
|
+
const disposition = (async () => {
|
|
3894
|
+
await entered.promise;
|
|
3895
|
+
let completed = false;
|
|
3896
|
+
const update = storage.writeMemoryFrontmatter(memory, { status: "rejected" }).then((value) => {
|
|
3897
|
+
completed = true;
|
|
3898
|
+
return value;
|
|
3899
|
+
});
|
|
3900
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
3901
|
+
assert.equal(completed, false, "queued-review disposition must wait for the capture identity lock");
|
|
3902
|
+
release.resolve();
|
|
3903
|
+
assert.equal(await update, true);
|
|
3904
|
+
})();
|
|
3905
|
+
try {
|
|
3906
|
+
await disposition;
|
|
3907
|
+
await held;
|
|
3908
|
+
assert.equal((await storage.getMemoryById(result.id))?.frontmatter.status, "rejected");
|
|
3909
|
+
} finally {
|
|
3910
|
+
release.resolve();
|
|
3911
|
+
await held.catch(() => undefined);
|
|
3912
|
+
await disposition.catch(() => undefined);
|
|
3913
|
+
}
|
|
3914
|
+
});
|
|
3915
|
+
});
|
|
3916
|
+
|
|
3917
|
+
test("active rewrites recheck a target that becomes blocked before the unlocked write", async () => {
|
|
3918
|
+
await withMemoryDir(async (dir) => {
|
|
3919
|
+
const storage = new StorageManager(dir);
|
|
3920
|
+
const peer = new StorageManager(dir);
|
|
3921
|
+
await storage.ensureDirectories();
|
|
3922
|
+
await peer.ensureDirectories();
|
|
3923
|
+
const tombstoneConfig = {
|
|
3924
|
+
enabled: true,
|
|
3925
|
+
semanticMatch: false,
|
|
3926
|
+
semanticThreshold: 0.9,
|
|
3927
|
+
namespace: "default",
|
|
3928
|
+
};
|
|
3929
|
+
storage.setTombstonesConfig(tombstoneConfig);
|
|
3930
|
+
peer.setTombstonesConfig(tombstoneConfig);
|
|
3931
|
+
const contentA = "An active target becomes blocked before an update writes.";
|
|
3932
|
+
const contentB = "The rewritten blocked target identity must be locked.";
|
|
3933
|
+
const contentC = "The stale update must publish only after locking identity B.";
|
|
3934
|
+
const tombstoneId = await peer.appendTombstone({
|
|
3935
|
+
reason: "correction",
|
|
3936
|
+
createdBy: "user_correction",
|
|
3937
|
+
sourceMemoryId: "active-rewrite-recheck",
|
|
3938
|
+
rawContent: contentB,
|
|
3939
|
+
});
|
|
3940
|
+
assert.ok(tombstoneId);
|
|
3941
|
+
const target = await storage.writeMemory("fact", contentA, {
|
|
3942
|
+
source: "test",
|
|
3943
|
+
sourceConnector: "provider-a",
|
|
3944
|
+
});
|
|
3945
|
+
const targetMemory = await storage.getMemoryById(target.id);
|
|
3946
|
+
assert.ok(targetMemory);
|
|
3947
|
+
assert.equal(targetMemory.frontmatter.status, "active");
|
|
3948
|
+
const identityB = buildExplicitCaptureDedupKey(contentB, "fact", "provider-a");
|
|
3949
|
+
await storage.hasTombstoneBlockedExplicitCapture(contentB, "fact", "provider-a");
|
|
3950
|
+
|
|
3951
|
+
const storageWithOverrides = storage as unknown as {
|
|
3952
|
+
readMemoryByPath: (filePath: string) => Promise<MemoryFile | null>;
|
|
3953
|
+
withTombstoneBlockedCaptureWriteLock: (
|
|
3954
|
+
task: () => Promise<unknown>,
|
|
3955
|
+
identity?: string | readonly string[],
|
|
3956
|
+
) => Promise<unknown>;
|
|
3957
|
+
};
|
|
3958
|
+
const originalRead = storageWithOverrides.readMemoryByPath.bind(storage);
|
|
3959
|
+
let rewriteOnRead = true;
|
|
3960
|
+
storageWithOverrides.readMemoryByPath = async (filePath) => {
|
|
3961
|
+
const current = await originalRead(filePath);
|
|
3962
|
+
if (rewriteOnRead) {
|
|
3963
|
+
rewriteOnRead = false;
|
|
3964
|
+
assert.ok(current);
|
|
3965
|
+
assert.equal(await peer.updateMemory(target.id, contentB, { sourceConnector: "provider-a" }), true);
|
|
3966
|
+
const rewritten = await peer.getMemoryById(target.id);
|
|
3967
|
+
assert.ok(rewritten);
|
|
3968
|
+
assert.equal(
|
|
3969
|
+
await peer.writeMemoryFrontmatter(rewritten, {
|
|
3970
|
+
status: "pending_review",
|
|
3971
|
+
blockedBy: tombstoneId,
|
|
3972
|
+
}),
|
|
3973
|
+
true,
|
|
3974
|
+
);
|
|
3975
|
+
pauseNextPrepare = true;
|
|
3976
|
+
return await originalRead(filePath);
|
|
3977
|
+
}
|
|
3978
|
+
return current;
|
|
3979
|
+
};
|
|
3980
|
+
const originalLock = storageWithOverrides.withTombstoneBlockedCaptureWriteLock.bind(storage);
|
|
3981
|
+
const capturedIdentities: Array<string | readonly string[] | undefined> = [];
|
|
3982
|
+
storageWithOverrides.withTombstoneBlockedCaptureWriteLock = async (task, identity) => {
|
|
3983
|
+
capturedIdentities.push(identity);
|
|
3984
|
+
return await originalLock(task, identity);
|
|
3985
|
+
};
|
|
3986
|
+
const releaseMarker = Promise.withResolvers<void>();
|
|
3987
|
+
const markerEntered = Promise.withResolvers<void>();
|
|
3988
|
+
let pauseNextPrepare = false;
|
|
3989
|
+
const originalPrepare = TombstoneBlockedCaptureIndex.prototype.prepareWrite;
|
|
3990
|
+
const prepareSpy = mock.method(
|
|
3991
|
+
TombstoneBlockedCaptureIndex.prototype,
|
|
3992
|
+
"prepareWrite",
|
|
3993
|
+
async function (this: TombstoneBlockedCaptureIndex) {
|
|
3994
|
+
const marker = await originalPrepare.call(this);
|
|
3995
|
+
if (pauseNextPrepare) {
|
|
3996
|
+
pauseNextPrepare = false;
|
|
3997
|
+
markerEntered.resolve();
|
|
3998
|
+
await releaseMarker.promise;
|
|
3999
|
+
}
|
|
4000
|
+
return marker;
|
|
4001
|
+
}
|
|
4002
|
+
);
|
|
4003
|
+
|
|
4004
|
+
const update = storage.updateMemory(target.id, contentC, { sourceConnector: "provider-a" });
|
|
4005
|
+
try {
|
|
4006
|
+
await markerEntered.promise;
|
|
4007
|
+
assert.ok(
|
|
4008
|
+
capturedIdentities.some((identity) =>
|
|
4009
|
+
(Array.isArray(identity) ? identity : [identity]).includes(identityB)
|
|
4010
|
+
),
|
|
4011
|
+
"active rewrite must reacquire the current blocked target identity",
|
|
4012
|
+
);
|
|
4013
|
+
let recaptureCompleted = false;
|
|
4014
|
+
const recapture = peer
|
|
4015
|
+
.writeMemory("fact", contentB, {
|
|
4016
|
+
source: "test",
|
|
4017
|
+
sourceConnector: "provider-a",
|
|
4018
|
+
})
|
|
4019
|
+
.then((result) => {
|
|
4020
|
+
recaptureCompleted = true;
|
|
4021
|
+
return result;
|
|
4022
|
+
});
|
|
4023
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
4024
|
+
assert.equal(recaptureCompleted, false, "recapture must wait for the current target identity lock");
|
|
4025
|
+
releaseMarker.resolve();
|
|
4026
|
+
assert.equal(await update, true);
|
|
4027
|
+
assert.equal((await recapture).tombstoneBlocked, true);
|
|
4028
|
+
const after = await peer.readAllMemories();
|
|
4029
|
+
assert.equal(
|
|
4030
|
+
after.find((memory) => memory.path === targetMemory.path)?.content,
|
|
4031
|
+
contentC,
|
|
4032
|
+
"the stale update must write the requested content",
|
|
4033
|
+
);
|
|
4034
|
+
assert.equal(await peer.hasTombstoneBlockedExplicitCapture(contentB, "fact", "provider-a"), true);
|
|
4035
|
+
} finally {
|
|
4036
|
+
releaseMarker.resolve();
|
|
4037
|
+
prepareSpy.mock.restore();
|
|
4038
|
+
storageWithOverrides.readMemoryByPath = originalRead;
|
|
4039
|
+
storageWithOverrides.withTombstoneBlockedCaptureWriteLock = originalLock;
|
|
4040
|
+
await update.catch(() => false);
|
|
4041
|
+
}
|
|
4042
|
+
});
|
|
4043
|
+
});
|