@remnic/core 9.29.0 → 9.30.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.d.ts +11 -11
- package/dist/access-admin-ops-surface.js +19 -18
- package/dist/access-authorization-probe.d.ts +11 -11
- package/dist/access-authorization-probe.js +20 -19
- package/dist/access-boundary.d.ts +11 -11
- package/dist/access-boundary.js +19 -18
- package/dist/access-citation-namespace.js +2 -2
- package/dist/access-cli.js +45 -44
- package/dist/access-cli.js.map +1 -1
- package/dist/access-health-types.d.ts +1 -1
- package/dist/access-http-lcm-compaction.d.ts +11 -11
- package/dist/access-http.d.ts +11 -11
- package/dist/access-http.js +25 -24
- package/dist/access-identity-continuity-surface.d.ts +8 -8
- package/dist/access-identity-continuity-surface.js +19 -18
- package/dist/access-lcm-surface.d.ts +10 -10
- package/dist/access-lcm-surface.js +19 -18
- package/dist/access-mcp.d.ts +11 -11
- package/dist/access-mcp.js +23 -22
- package/dist/access-memory-search-fanout.d.ts +2 -2
- package/dist/{access-namespace-preflight-DCXBkDp0.d.ts → access-namespace-preflight-VfXssvoI.d.ts} +1 -1
- package/dist/access-namespace-preflight.d.ts +3 -3
- package/dist/access-namespace-preflight.js +19 -18
- package/dist/access-observe-write-surface.d.ts +10 -10
- package/dist/access-observe-write-surface.js +19 -18
- package/dist/access-operations-batch.js +20 -19
- package/dist/access-operations.d.ts +11 -11
- package/dist/access-operations.js +22 -21
- package/dist/access-recall-concurrency.d.ts +11 -11
- package/dist/access-recall-concurrency.js +19 -18
- package/dist/access-recall-response.d.ts +10 -10
- package/dist/access-recall-response.js +19 -18
- package/dist/access-recall-surface.d.ts +10 -10
- package/dist/access-recall-surface.js +19 -18
- package/dist/access-schema.d.ts +7 -7
- package/dist/access-schema.js +3 -3
- package/dist/{access-service-DvPW_M5j.d.ts → access-service-B5RwVsTV.d.ts} +7 -7
- package/dist/access-service.d.ts +11 -11
- package/dist/access-service.js +19 -18
- package/dist/access-surface-catalog.d.ts +11 -11
- package/dist/access-wearables-meetings-surface.d.ts +4 -3
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/active-recall.js +3 -3
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +8 -8
- package/dist/briefing.d.ts +3 -2
- package/dist/briefing.js +10 -9
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer.d.ts +3 -2
- package/dist/bulk-import/index.d.ts +3 -3
- package/dist/calibration.d.ts +1 -1
- package/dist/capabilities.d.ts +1 -1
- package/dist/{catalog-BIVgftWu.d.ts → catalog-V7inUQdp.d.ts} +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-consolidation.js +11 -10
- package/dist/causal-consolidation.js.map +1 -1
- package/dist/causal-trajectory-graph.d.ts +1 -1
- package/dist/{chunk-KUJQHXPH.js → chunk-2EUVEDF5.js} +2 -2
- package/dist/{chunk-2XC73SHY.js → chunk-4WKA6I65.js} +20 -20
- package/dist/{chunk-2XC73SHY.js.map → chunk-4WKA6I65.js.map} +1 -1
- package/dist/{chunk-AIK5W3IW.js → chunk-5TFP4CUL.js} +68 -2
- package/dist/chunk-5TFP4CUL.js.map +1 -0
- package/dist/{chunk-EVX44I7C.js → chunk-5W4ZLBEZ.js} +9 -9
- package/dist/{chunk-P3MH72CI.js → chunk-62U4J2GN.js} +2 -2
- package/dist/{chunk-YGHMSDLK.js → chunk-66772WBP.js} +2 -2
- package/dist/{chunk-EP5BVCRI.js → chunk-6BNE62PM.js} +2 -2
- package/dist/{chunk-AAP5T6QG.js → chunk-6DNMUYEE.js} +2 -2
- package/dist/{chunk-SG265CVF.js → chunk-6FUPCJUL.js} +7 -7
- package/dist/{chunk-WZ7UB7XQ.js → chunk-6U7F52DE.js} +19 -19
- package/dist/{chunk-E4OCKU67.js → chunk-7XC2HX75.js} +2 -2
- package/dist/{chunk-OVRQLAYR.js → chunk-AZ5BTEKX.js} +2 -2
- package/dist/{chunk-2HNUMF5V.js → chunk-CZRYLSSW.js} +3 -3
- package/dist/chunk-FBUE6PD3.js +1769 -0
- package/dist/chunk-FBUE6PD3.js.map +1 -0
- package/dist/{chunk-WONS4RZ2.js → chunk-FDAMS4PK.js} +2 -2
- package/dist/{chunk-WL5EBRDH.js → chunk-FFAZ2XFU.js} +9 -8
- package/dist/chunk-FFAZ2XFU.js.map +1 -0
- package/dist/{chunk-Q6VFVAKZ.js → chunk-GIBMDSJ2.js} +2 -2
- package/dist/{chunk-ZXAGJFHJ.js → chunk-H5CPS5JP.js} +3 -3
- package/dist/{chunk-NBDCZWP4.js → chunk-I5NLEENJ.js} +2 -2
- package/dist/{chunk-DRBXHRXE.js → chunk-IB26HVAJ.js} +2 -2
- package/dist/{chunk-XEQ7OT7U.js → chunk-IC6CBHCA.js} +3 -3
- package/dist/{chunk-CMRQ34J5.js → chunk-JCBWCYEJ.js} +2 -2
- package/dist/{chunk-3PDOVUHC.js → chunk-KXB4VXHY.js} +2 -2
- package/dist/{chunk-G6LBWZRF.js → chunk-LBOD4U5B.js} +7 -7
- package/dist/{chunk-ZGOQTOYA.js → chunk-LGHKJABJ.js} +3 -3
- package/dist/{chunk-T3GCXULH.js → chunk-M3C2V3PV.js} +10 -10
- package/dist/{chunk-OFHFH655.js → chunk-MJCGLM25.js} +8 -8
- package/dist/{chunk-JDZRYH2N.js → chunk-NHWB6HG2.js} +2 -2
- package/dist/{chunk-TC3VNKGX.js → chunk-NM2WP3B7.js} +4 -4
- package/dist/{chunk-IMA2EWLV.js → chunk-P5PXWTW7.js} +2 -2
- package/dist/chunk-PH3ZWL3W.js +786 -0
- package/dist/chunk-PH3ZWL3W.js.map +1 -0
- package/dist/{chunk-QMBNMPQM.js → chunk-PUS63Q3G.js} +24 -22
- package/dist/{chunk-QMBNMPQM.js.map → chunk-PUS63Q3G.js.map} +1 -1
- package/dist/{chunk-2QMKMVHH.js → chunk-QNWXCJ76.js} +2 -2
- package/dist/{chunk-FCDWWCLX.js → chunk-RGIWINDN.js} +427 -795
- package/dist/chunk-RGIWINDN.js.map +1 -0
- package/dist/{chunk-CSMCNOPH.js → chunk-SPJXKAVF.js} +3 -3
- package/dist/{chunk-4PBVXX7L.js → chunk-TH2LSRMG.js} +2 -2
- package/dist/{chunk-ESBQWC3U.js → chunk-VI2JQPNQ.js} +2 -2
- package/dist/{chunk-GGUBDXRR.js → chunk-W5EG6LE3.js} +2 -2
- package/dist/{chunk-DEBHXHJR.js → chunk-WIPEXG6R.js} +2 -2
- package/dist/{chunk-SH6YO3LT.js → chunk-WVO4G37W.js} +2 -2
- package/dist/{chunk-VVIW3PTA.js → chunk-YTQNQ4KB.js} +14 -14
- package/dist/{chunk-PKRIQL3V.js → chunk-YTSNAUXN.js} +2 -2
- package/dist/{chunk-QAOGGNE3.js → chunk-ZPSVBSOF.js} +4 -4
- package/dist/{cli-CxoSOiG0.d.ts → cli-C4GbETPC.d.ts} +5 -5
- package/dist/cli.d.ts +13 -13
- package/dist/cli.js +36 -35
- package/dist/compounding/engine.d.ts +3 -2
- package/dist/compounding/engine.js +10 -9
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.js +3 -3
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.js +10 -9
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/connectors/index.js +10 -9
- package/dist/consolidation-provenance-check.d.ts +3 -2
- package/dist/consolidation-undo.d.ts +3 -2
- package/dist/contradiction/index.d.ts +3 -2
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/corpus-watermark.d.ts +1 -1
- package/dist/corpus-watermark.js +12 -11
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer.d.ts +1 -1
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-retrieval.d.ts +3 -2
- package/dist/entity-retrieval.js +10 -9
- package/dist/entity-schema.d.ts +1 -1
- package/dist/explicit-capture.d.ts +52 -10
- package/dist/explicit-capture.js +13 -1
- package/dist/extraction-error-classification.d.ts +1 -1
- package/dist/extraction-faithfulness.d.ts +1 -1
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction-liveness.d.ts +1 -1
- package/dist/extraction-source-grounding-rules.d.ts +1 -1
- package/dist/extraction-source-grounding.d.ts +1 -1
- package/dist/extraction.d.ts +1 -1
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/{forget-HRPLVB3W.js → forget-4UY2EOKH.js} +3 -3
- package/dist/graph-dashboard-diff.d.ts +1 -1
- package/dist/graph-dashboard-key.d.ts +1 -1
- package/dist/graph-dashboard-parser.d.ts +1 -1
- package/dist/graph-edge-reinforcement.d.ts +1 -1
- package/dist/graph-snapshot.d.ts +1 -1
- package/dist/graph.d.ts +1 -1
- package/dist/importance.d.ts +1 -1
- package/dist/importers/index.d.ts +1 -1
- package/dist/in-flight-reads.d.ts +1 -1
- package/dist/index.d.ts +21 -21
- package/dist/index.js +47 -45
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +1 -1
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/local-model-endpoint.d.ts +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/maintenance/memory-governance.js +10 -9
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.d.ts +2 -2
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +10 -9
- package/dist/maintenance/rebuild-memory-projection.d.ts +3 -2
- package/dist/maintenance/rebuild-memory-projection.js +11 -10
- package/dist/{maintenance-Cb-PjxQd.d.ts → maintenance-CL30NWtc.d.ts} +3 -3
- package/dist/mcp-memory-inspector-app.d.ts +11 -11
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +3 -2
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +4 -3
- package/dist/namespaces/migrate.js +11 -10
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/storage.d.ts +4 -3
- package/dist/namespaces/storage.js +10 -9
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/offline-sync-impression-drain.d.ts +1 -1
- package/dist/offline-sync-impression-drain.js +5 -5
- package/dist/offline-sync.js +2 -2
- package/dist/operator-doctor-corpus.d.ts +1 -1
- package/dist/operator-doctor-corpus.js +13 -12
- package/dist/operator-doctor-replica.d.ts +1 -1
- package/dist/operator-toolkit.d.ts +4 -3
- package/dist/operator-toolkit.js +18 -17
- package/dist/orchestration/compression-guideline-coordinator.d.ts +3 -2
- package/dist/orchestration/maintenance.d.ts +5 -4
- package/dist/orchestration/maintenance.js +15 -14
- package/dist/{orchestrator-5GKqb_HL.d.ts → orchestrator-CKTCRwWV.d.ts} +7 -7
- package/dist/orchestrator.d.ts +8 -8
- package/dist/orchestrator.js +45 -44
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/{pipeline-3Qi1TpRo.d.ts → pipeline-B7wj4rRZ.d.ts} +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/proactive-contention.d.ts +1 -1
- package/dist/provenance.d.ts +1 -1
- package/dist/{purge-YSGX5BOG.js → purge-533FZLS4.js} +16 -9
- package/dist/purge-533FZLS4.js.map +1 -0
- package/dist/qmd-preflight.d.ts +2 -2
- package/dist/{qmd-D9JqzzRX.d.ts → qmd-q9tg4dHp.d.ts} +1 -1
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +2 -2
- package/dist/recall-concurrency-config.d.ts +1 -1
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-memory-map.d.ts +1 -1
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-state.d.ts +1 -1
- package/dist/recall-state.js +2 -2
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-timings.d.ts +1 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray.d.ts +1 -1
- package/dist/replay/normalizers/chatgpt.d.ts +1 -1
- package/dist/replay/normalizers/claude.d.ts +1 -1
- package/dist/replay/normalizers/openclaw.d.ts +1 -1
- package/dist/replay/normalizers/shared.d.ts +1 -1
- package/dist/replay/runner.d.ts +1 -1
- package/dist/replay/types.d.ts +1 -1
- package/dist/replica-divergence.d.ts +1 -1
- package/dist/replica-peers-config.d.ts +1 -1
- package/dist/resolve-auth-token.d.ts +1 -1
- package/dist/resume-bundles.js +4 -4
- package/dist/retrieval-agents.d.ts +2 -2
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/salvage-envelope.d.ts +1 -1
- package/dist/schemas.d.ts +22 -22
- package/dist/{scope-profiles-C4g6_UdP.d.ts → scope-profiles-CWDyglPM.d.ts} +1 -1
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/index.d.ts +1 -1
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/secure-store/index.js +2 -2
- package/dist/{semantic-consolidation-BrxqnzTW.d.ts → semantic-consolidation-BUE3deLn.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-consolidation.js +11 -10
- package/dist/semantic-rule-promotion.js +10 -9
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/semantic-rule-verifier.js +10 -9
- package/dist/{service-Bi1JgxdP.d.ts → service-CXgM0bQG.d.ts} +2 -2
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +1 -1
- package/dist/signal.d.ts +1 -1
- package/dist/{storage-CcH6yPk2.d.ts → storage-DysJq-XH.d.ts} +163 -27
- package/dist/storage.d.ts +3 -2
- package/dist/storage.js +12 -10
- package/dist/summarizer.d.ts +1 -1
- package/dist/summarizer.js +3 -3
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/summary-snapshot.js +2 -2
- package/dist/temporal-supersession.d.ts +3 -2
- package/dist/temporal-timeline-recall.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +3 -2
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/transfer/types.d.ts +12 -12
- package/dist/trust-score-stage.d.ts +1 -1
- package/dist/trust-score.d.ts +1 -1
- package/dist/{types-Dg0w9ADr.d.ts → types-aPPzhnjT.d.ts} +1 -1
- package/dist/types.d.ts +1 -1
- package/dist/utility-runtime.d.ts +1 -1
- package/dist/utils/serialize-mutations.d.ts +11 -7
- package/dist/utils/serialize-mutations.js +1 -1
- package/dist/verified-recall.js +10 -9
- package/dist/write-envelope.d.ts +1 -1
- package/package.json +2 -2
- package/src/access-observe-write-surface.ts +1 -1
- package/src/access-service-coding-write.test.ts +40 -0
- package/src/explicit-capture.test.ts +1110 -1
- package/src/explicit-capture.ts +534 -135
- package/src/maintenance/purge.test.ts +157 -1
- package/src/maintenance/purge.ts +15 -9
- package/src/secure-store/secure-fs.ts +78 -1
- package/src/storage/content-hash-index.ts +43 -17
- package/src/storage/entity-canonical-id-migration.ts +151 -23
- package/src/storage/tombstone-blocked-capture-index.ts +562 -0
- package/src/storage/tombstone-blocked-capture-mutation.ts +284 -0
- package/src/storage/tombstone-blocked-capture-sync.ts +872 -0
- package/src/storage-fact-hash-index-defer.test.ts +2900 -57
- package/src/storage.ts +266 -265
- package/src/testing/subjects/tombstone-blocked-capture.test.ts +181 -0
- package/src/utils/serialize-mutations.test.ts +4 -1
- package/src/utils/serialize-mutations.ts +26 -27
- package/dist/chunk-55MDBZLC.js +0 -490
- package/dist/chunk-55MDBZLC.js.map +0 -1
- package/dist/chunk-AIK5W3IW.js.map +0 -1
- package/dist/chunk-FCDWWCLX.js.map +0 -1
- package/dist/chunk-WL5EBRDH.js.map +0 -1
- package/dist/purge-YSGX5BOG.js.map +0 -1
- /package/dist/{chunk-KUJQHXPH.js.map → chunk-2EUVEDF5.js.map} +0 -0
- /package/dist/{chunk-EVX44I7C.js.map → chunk-5W4ZLBEZ.js.map} +0 -0
- /package/dist/{chunk-P3MH72CI.js.map → chunk-62U4J2GN.js.map} +0 -0
- /package/dist/{chunk-YGHMSDLK.js.map → chunk-66772WBP.js.map} +0 -0
- /package/dist/{chunk-EP5BVCRI.js.map → chunk-6BNE62PM.js.map} +0 -0
- /package/dist/{chunk-AAP5T6QG.js.map → chunk-6DNMUYEE.js.map} +0 -0
- /package/dist/{chunk-SG265CVF.js.map → chunk-6FUPCJUL.js.map} +0 -0
- /package/dist/{chunk-WZ7UB7XQ.js.map → chunk-6U7F52DE.js.map} +0 -0
- /package/dist/{chunk-E4OCKU67.js.map → chunk-7XC2HX75.js.map} +0 -0
- /package/dist/{chunk-OVRQLAYR.js.map → chunk-AZ5BTEKX.js.map} +0 -0
- /package/dist/{chunk-2HNUMF5V.js.map → chunk-CZRYLSSW.js.map} +0 -0
- /package/dist/{chunk-WONS4RZ2.js.map → chunk-FDAMS4PK.js.map} +0 -0
- /package/dist/{chunk-Q6VFVAKZ.js.map → chunk-GIBMDSJ2.js.map} +0 -0
- /package/dist/{chunk-ZXAGJFHJ.js.map → chunk-H5CPS5JP.js.map} +0 -0
- /package/dist/{chunk-NBDCZWP4.js.map → chunk-I5NLEENJ.js.map} +0 -0
- /package/dist/{chunk-DRBXHRXE.js.map → chunk-IB26HVAJ.js.map} +0 -0
- /package/dist/{chunk-XEQ7OT7U.js.map → chunk-IC6CBHCA.js.map} +0 -0
- /package/dist/{chunk-CMRQ34J5.js.map → chunk-JCBWCYEJ.js.map} +0 -0
- /package/dist/{chunk-3PDOVUHC.js.map → chunk-KXB4VXHY.js.map} +0 -0
- /package/dist/{chunk-G6LBWZRF.js.map → chunk-LBOD4U5B.js.map} +0 -0
- /package/dist/{chunk-ZGOQTOYA.js.map → chunk-LGHKJABJ.js.map} +0 -0
- /package/dist/{chunk-T3GCXULH.js.map → chunk-M3C2V3PV.js.map} +0 -0
- /package/dist/{chunk-OFHFH655.js.map → chunk-MJCGLM25.js.map} +0 -0
- /package/dist/{chunk-JDZRYH2N.js.map → chunk-NHWB6HG2.js.map} +0 -0
- /package/dist/{chunk-TC3VNKGX.js.map → chunk-NM2WP3B7.js.map} +0 -0
- /package/dist/{chunk-IMA2EWLV.js.map → chunk-P5PXWTW7.js.map} +0 -0
- /package/dist/{chunk-2QMKMVHH.js.map → chunk-QNWXCJ76.js.map} +0 -0
- /package/dist/{chunk-CSMCNOPH.js.map → chunk-SPJXKAVF.js.map} +0 -0
- /package/dist/{chunk-4PBVXX7L.js.map → chunk-TH2LSRMG.js.map} +0 -0
- /package/dist/{chunk-ESBQWC3U.js.map → chunk-VI2JQPNQ.js.map} +0 -0
- /package/dist/{chunk-GGUBDXRR.js.map → chunk-W5EG6LE3.js.map} +0 -0
- /package/dist/{chunk-DEBHXHJR.js.map → chunk-WIPEXG6R.js.map} +0 -0
- /package/dist/{chunk-SH6YO3LT.js.map → chunk-WVO4G37W.js.map} +0 -0
- /package/dist/{chunk-VVIW3PTA.js.map → chunk-YTQNQ4KB.js.map} +0 -0
- /package/dist/{chunk-PKRIQL3V.js.map → chunk-YTSNAUXN.js.map} +0 -0
- /package/dist/{chunk-QAOGGNE3.js.map → chunk-ZPSVBSOF.js.map} +0 -0
- /package/dist/{forget-HRPLVB3W.js.map → forget-4UY2EOKH.js.map} +0 -0
|
@@ -0,0 +1,1769 @@
|
|
|
1
|
+
import {
|
|
2
|
+
withHeldFileLock
|
|
3
|
+
} from "./chunk-FFAZ2XFU.js";
|
|
4
|
+
import {
|
|
5
|
+
markProjectedMemoryPathInvalid
|
|
6
|
+
} from "./chunk-3CSX3ETE.js";
|
|
7
|
+
import {
|
|
8
|
+
computeContentHash,
|
|
9
|
+
normalizeContent
|
|
10
|
+
} from "./chunk-W5H6ZRPT.js";
|
|
11
|
+
import {
|
|
12
|
+
SecureStoreLockedError,
|
|
13
|
+
readMaybeEncryptedFile,
|
|
14
|
+
readMaybeEncryptedFileFromChunks,
|
|
15
|
+
writeMaybeEncryptedFile,
|
|
16
|
+
writeMaybeEncryptedFileFromChunks
|
|
17
|
+
} from "./chunk-5TFP4CUL.js";
|
|
18
|
+
import {
|
|
19
|
+
isErrnoCode
|
|
20
|
+
} from "./chunk-5UZXUTVO.js";
|
|
21
|
+
import {
|
|
22
|
+
RECALL_FALLBACK_DIRS
|
|
23
|
+
} from "./chunk-VS2IYZRU.js";
|
|
24
|
+
import {
|
|
25
|
+
log
|
|
26
|
+
} from "./chunk-3WCASNQZ.js";
|
|
27
|
+
|
|
28
|
+
// src/storage/content-hash-index.ts
|
|
29
|
+
import { mkdir, stat } from "fs/promises";
|
|
30
|
+
import path from "path";
|
|
31
|
+
var CONTENT_HASH_INDEX_LOCK_STALE_MS = 3e4;
|
|
32
|
+
var CONTENT_HASH_INDEX_RETRY_MAX_ATTEMPTS = 5;
|
|
33
|
+
var CONTENT_HASH_INDEX_RETRY_BASE_MS = 500;
|
|
34
|
+
var CONTENT_HASH_INDEX_RETRY_MAX_DELAY_MS = 5e3;
|
|
35
|
+
var FactHashIndexNotAuthoritativeError = class extends Error {
|
|
36
|
+
constructor(stateDir) {
|
|
37
|
+
super(
|
|
38
|
+
`fact-hash index is not authoritative: the cross-process rebuild lock for ${stateDir} could not be acquired within the bounded retry budget`
|
|
39
|
+
);
|
|
40
|
+
this.name = "FactHashIndexNotAuthoritativeError";
|
|
41
|
+
}
|
|
42
|
+
};
|
|
43
|
+
var ContentHashIndex = class _ContentHashIndex {
|
|
44
|
+
hashes = /* @__PURE__ */ new Set();
|
|
45
|
+
dirty = false;
|
|
46
|
+
/**
|
|
47
|
+
* Hashes explicitly removed since the last successful save (issue #1909 review
|
|
48
|
+
* round 8 thread 3). Tracked separately so `saveMergingWithDisk` can be
|
|
49
|
+
* removal-AWARE: under the file lock it reads the latest on-disk set, DROPS
|
|
50
|
+
* these, then unions our additions — so a removal batch cannot resurrect a
|
|
51
|
+
* hash (union alone would) and cannot clobber a concurrent extraction's
|
|
52
|
+
* appended hash (blind overwrite would). Consumed + cleared by every save.
|
|
53
|
+
*/
|
|
54
|
+
removed = /* @__PURE__ */ new Set();
|
|
55
|
+
/**
|
|
56
|
+
* Hashes added by THIS instance since the last successful save (issue #1909
|
|
57
|
+
* review round 9 finding 3). Tracked separately from the loaded snapshot so a
|
|
58
|
+
* reconciling save publishes `(on-disk \ removed) ∪ added` — it re-applies only
|
|
59
|
+
* OUR additions, never the stale loaded entries. Without this, seeding the
|
|
60
|
+
* merge from `this.hashes` (loaded ∪ added) would resurrect a hash a peer
|
|
61
|
+
* removed on disk while this instance held a stale snapshot.
|
|
62
|
+
*/
|
|
63
|
+
added = /* @__PURE__ */ new Set();
|
|
64
|
+
filePath;
|
|
65
|
+
secureStoreKeyProvider;
|
|
66
|
+
secureStoreWriteKeyProvider;
|
|
67
|
+
memoryDir;
|
|
68
|
+
lockOptions;
|
|
69
|
+
/**
|
|
70
|
+
* Deferred reconcile-save durable-retry state (issue #1909 / PR #2016). At
|
|
71
|
+
* most ONE retry is ever armed (`reconcileRetryTimer` is the reentrancy guard)
|
|
72
|
+
* so successive timed-out saves cannot stack duplicate/parallel retries.
|
|
73
|
+
* `reconcileRetryBarrier` resolves when the chain settles (published or gave
|
|
74
|
+
* up) — awaited by tests and any caller wanting the eventual result.
|
|
75
|
+
*/
|
|
76
|
+
reconcileRetryTimer = null;
|
|
77
|
+
reconcileRetryAttempts = 0;
|
|
78
|
+
reconcileRetryBarrier = null;
|
|
79
|
+
reconcileRetryResolve = null;
|
|
80
|
+
/**
|
|
81
|
+
* On-disk fingerprint captured at the last point THIS instance's in-memory
|
|
82
|
+
* set matched disk — after `load()`, `save()`, or a reconcile publish. A peer
|
|
83
|
+
* process that advances the durable index changes its file generation even
|
|
84
|
+
* when byte size and mtime collide, so the freshness check remains safe
|
|
85
|
+
* without an O(file-size) read on the hot path.
|
|
86
|
+
*/
|
|
87
|
+
lastSyncedFingerprint = null;
|
|
88
|
+
constructor(stateDir, secureStoreKeyProvider = () => null, secureStoreWriteKeyProvider = secureStoreKeyProvider, memoryDir = path.dirname(stateDir), lockOptions = {}) {
|
|
89
|
+
this.filePath = path.join(stateDir, "fact-hashes.txt");
|
|
90
|
+
this.secureStoreKeyProvider = secureStoreKeyProvider;
|
|
91
|
+
this.secureStoreWriteKeyProvider = secureStoreWriteKeyProvider;
|
|
92
|
+
this.memoryDir = memoryDir;
|
|
93
|
+
this.lockOptions = lockOptions;
|
|
94
|
+
}
|
|
95
|
+
/** Load existing hashes from disk. Safe to call multiple times. */
|
|
96
|
+
async load() {
|
|
97
|
+
try {
|
|
98
|
+
const raw = await readMaybeEncryptedFile(this.filePath, this.secureStoreKeyProvider(), this.memoryDir);
|
|
99
|
+
for (const line of raw.split("\n")) {
|
|
100
|
+
const trimmed = line.trim();
|
|
101
|
+
if (trimmed.length > 0) {
|
|
102
|
+
this.hashes.add(trimmed);
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
log.debug(`content-hash index: loaded ${this.hashes.size} hashes`);
|
|
106
|
+
} catch (err) {
|
|
107
|
+
if (err instanceof SecureStoreLockedError) throw err;
|
|
108
|
+
if (!isErrnoCode(err, "ENOENT")) throw err;
|
|
109
|
+
log.debug("content-hash index: no existing index \u2014 starting fresh");
|
|
110
|
+
}
|
|
111
|
+
await this.captureSyncedFingerprint();
|
|
112
|
+
}
|
|
113
|
+
/** Cheap `stat` of the durable index file; null when it does not exist yet. */
|
|
114
|
+
async statIndexFile() {
|
|
115
|
+
try {
|
|
116
|
+
const s = await stat(this.filePath);
|
|
117
|
+
return {
|
|
118
|
+
mtimeMs: s.mtimeMs,
|
|
119
|
+
size: s.size,
|
|
120
|
+
dev: s.dev,
|
|
121
|
+
ino: s.ino,
|
|
122
|
+
ctimeMs: s.ctimeMs,
|
|
123
|
+
birthtimeMs: s.birthtimeMs
|
|
124
|
+
};
|
|
125
|
+
} catch (err) {
|
|
126
|
+
if (isErrnoCode(err, "ENOENT")) return null;
|
|
127
|
+
throw err;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/** Record the current on-disk fingerprint as this instance's synced baseline. */
|
|
131
|
+
async captureSyncedFingerprint() {
|
|
132
|
+
try {
|
|
133
|
+
this.lastSyncedFingerprint = await this.statIndexFile();
|
|
134
|
+
} catch (err) {
|
|
135
|
+
this.lastSyncedFingerprint = null;
|
|
136
|
+
log.debug(`content-hash index: could not fingerprint ${this.filePath}: ${err}`);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* True when the durable index file is unchanged since this instance last
|
|
141
|
+
* synced it (load / save / reconcile). False when a peer advanced it, when
|
|
142
|
+
* the file appeared or vanished, or when freshness cannot be established
|
|
143
|
+
* (stat error) — the caller then treats the in-memory snapshot as
|
|
144
|
+
* non-authoritative and confirms a dedup miss against the durable corpus.
|
|
145
|
+
* The stat includes file identity and generation metadata so same-size,
|
|
146
|
+
* same-mtime atomic replacements are detected without reading the file body.
|
|
147
|
+
*/
|
|
148
|
+
async isDiskFingerprintCurrent() {
|
|
149
|
+
let current;
|
|
150
|
+
try {
|
|
151
|
+
current = await this.statIndexFile();
|
|
152
|
+
} catch {
|
|
153
|
+
return false;
|
|
154
|
+
}
|
|
155
|
+
const last = this.lastSyncedFingerprint;
|
|
156
|
+
if (last === null || current === null) {
|
|
157
|
+
return last === null && current === null;
|
|
158
|
+
}
|
|
159
|
+
return current.mtimeMs === last.mtimeMs && current.size === last.size && current.dev === last.dev && current.ino === last.ino && current.ctimeMs === last.ctimeMs && current.birthtimeMs === last.birthtimeMs;
|
|
160
|
+
}
|
|
161
|
+
/** Check if content already exists in the index. */
|
|
162
|
+
has(content) {
|
|
163
|
+
return this.hashes.has(_ContentHashIndex.computeHash(content));
|
|
164
|
+
}
|
|
165
|
+
/** Add content hash to the index. */
|
|
166
|
+
add(content) {
|
|
167
|
+
const hash = _ContentHashIndex.computeHash(content);
|
|
168
|
+
this.removed.delete(hash);
|
|
169
|
+
this.hashes.add(hash);
|
|
170
|
+
this.added.add(hash);
|
|
171
|
+
this.dirty = true;
|
|
172
|
+
}
|
|
173
|
+
get size() {
|
|
174
|
+
return this.hashes.size;
|
|
175
|
+
}
|
|
176
|
+
/** Clear all loaded hashes so the next save rewrites the index from scratch. */
|
|
177
|
+
clear() {
|
|
178
|
+
if (this.hashes.size > 0) {
|
|
179
|
+
this.hashes.clear();
|
|
180
|
+
}
|
|
181
|
+
this.added.clear();
|
|
182
|
+
this.removed.clear();
|
|
183
|
+
this.cancelReconcileRetry();
|
|
184
|
+
this.dirty = true;
|
|
185
|
+
}
|
|
186
|
+
/** Persist index to disk if changed (plain whole-file overwrite). */
|
|
187
|
+
async save() {
|
|
188
|
+
if (!this.dirty) return;
|
|
189
|
+
await mkdir(path.dirname(this.filePath), { recursive: true });
|
|
190
|
+
const publishedAdded = new Set(this.added);
|
|
191
|
+
const publishedRemoved = new Set(this.removed);
|
|
192
|
+
const serialized = [...this.hashes].join("\n") + "\n";
|
|
193
|
+
await writeMaybeEncryptedFile(
|
|
194
|
+
this.filePath,
|
|
195
|
+
serialized,
|
|
196
|
+
this.secureStoreWriteKeyProvider(),
|
|
197
|
+
{},
|
|
198
|
+
this.memoryDir
|
|
199
|
+
);
|
|
200
|
+
for (const h of publishedAdded) this.added.delete(h);
|
|
201
|
+
for (const h of publishedRemoved) this.removed.delete(h);
|
|
202
|
+
this.dirty = this.added.size > 0 || this.removed.size > 0;
|
|
203
|
+
await this.captureSyncedFingerprint();
|
|
204
|
+
if (this.dirty) {
|
|
205
|
+
this.scheduleReconcileRetry();
|
|
206
|
+
} else {
|
|
207
|
+
this.cancelReconcileRetry();
|
|
208
|
+
}
|
|
209
|
+
log.debug(`content-hash index: saved ${this.hashes.size} hashes`);
|
|
210
|
+
}
|
|
211
|
+
/**
|
|
212
|
+
* Persist the index by RECONCILING with the latest on-disk state under a
|
|
213
|
+
* cross-process lock (issue #1909). Used by the extraction-persist batch save
|
|
214
|
+
* for BOTH append and removal batches, so appends and removals to the same
|
|
215
|
+
* index serialize against each other:
|
|
216
|
+
* final = (on-disk \ this.removed) ∪ this.added
|
|
217
|
+
* Note it re-applies only OUR local additions (`added`), NOT the loaded
|
|
218
|
+
* snapshot (review round 9 finding 3): seeding from `this.hashes` would
|
|
219
|
+
* resurrect a hash a peer removed on disk while we held a stale snapshot. This
|
|
220
|
+
* preserves a concurrent writer's appended hashes (a blind overwrite would
|
|
221
|
+
* drop them), never resurrects a removed hash, and never loses our additions.
|
|
222
|
+
*
|
|
223
|
+
* Hardening:
|
|
224
|
+
* - Dirty short-circuit (round 7 finding 3): when this instance neither added
|
|
225
|
+
* nor removed anything this run, skip the O(file) read+rewrite entirely.
|
|
226
|
+
* - Cross-process lock (round 7 finding 2): serialize the read→reconcile→write
|
|
227
|
+
* window on a per-file advisory lock. If the lock is NOT acquired (timeout),
|
|
228
|
+
* do NOT publish (round 9 finding 2): keep `dirty` set AND schedule a bounded
|
|
229
|
+
* durable background retry (PR #2016) that keeps re-attempting the locked
|
|
230
|
+
* publish until the deferred additions land — never an unlocked write that
|
|
231
|
+
* could clobber a peer, and never a silent "batch save complete" that leaves
|
|
232
|
+
* the addition in this process's memory only.
|
|
233
|
+
*/
|
|
234
|
+
async saveMergingWithDisk() {
|
|
235
|
+
if (!this.dirty) return;
|
|
236
|
+
await mkdir(path.dirname(this.filePath), { recursive: true });
|
|
237
|
+
const publish = async () => {
|
|
238
|
+
const publishedAdded = new Set(this.added);
|
|
239
|
+
const publishedRemoved = new Set(this.removed);
|
|
240
|
+
const merged = new Set(publishedAdded);
|
|
241
|
+
try {
|
|
242
|
+
const raw = await readMaybeEncryptedFile(
|
|
243
|
+
this.filePath,
|
|
244
|
+
this.secureStoreKeyProvider(),
|
|
245
|
+
this.memoryDir
|
|
246
|
+
);
|
|
247
|
+
for (const line of raw.split("\n")) {
|
|
248
|
+
const trimmed = line.trim();
|
|
249
|
+
if (trimmed.length > 0 && !publishedRemoved.has(trimmed)) merged.add(trimmed);
|
|
250
|
+
}
|
|
251
|
+
} catch (err) {
|
|
252
|
+
if (err instanceof SecureStoreLockedError) throw err;
|
|
253
|
+
if (!isErrnoCode(err, "ENOENT")) throw err;
|
|
254
|
+
}
|
|
255
|
+
await mkdir(path.dirname(this.filePath), { recursive: true });
|
|
256
|
+
await writeMaybeEncryptedFile(
|
|
257
|
+
this.filePath,
|
|
258
|
+
[...merged].join("\n") + "\n",
|
|
259
|
+
this.secureStoreWriteKeyProvider(),
|
|
260
|
+
{},
|
|
261
|
+
this.memoryDir
|
|
262
|
+
);
|
|
263
|
+
for (const h of publishedAdded) this.added.delete(h);
|
|
264
|
+
for (const h of publishedRemoved) this.removed.delete(h);
|
|
265
|
+
for (const h of this.added) merged.add(h);
|
|
266
|
+
for (const h of this.removed) merged.delete(h);
|
|
267
|
+
this.hashes = merged;
|
|
268
|
+
this.dirty = this.added.size > 0 || this.removed.size > 0;
|
|
269
|
+
await this.captureSyncedFingerprint();
|
|
270
|
+
};
|
|
271
|
+
await withHeldFileLock(
|
|
272
|
+
`${this.filePath}.lock`,
|
|
273
|
+
{
|
|
274
|
+
staleMs: CONTENT_HASH_INDEX_LOCK_STALE_MS,
|
|
275
|
+
...this.lockOptions.maxWaitMs !== void 0 ? { maxWaitMs: this.lockOptions.maxWaitMs } : {},
|
|
276
|
+
...this.lockOptions.pollMs !== void 0 ? { pollMs: this.lockOptions.pollMs } : {}
|
|
277
|
+
},
|
|
278
|
+
async (acquired) => {
|
|
279
|
+
if (!acquired) {
|
|
280
|
+
log.warn(
|
|
281
|
+
`content-hash index: lock not acquired for ${this.filePath}; deferring reconcile save (dirty retained, scheduling durable retry)`
|
|
282
|
+
);
|
|
283
|
+
this.scheduleReconcileRetry();
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
await publish();
|
|
287
|
+
if (this.dirty) {
|
|
288
|
+
this.scheduleReconcileRetry();
|
|
289
|
+
} else {
|
|
290
|
+
this.cancelReconcileRetry();
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
);
|
|
294
|
+
log.debug(`content-hash index: reconcile-saved ${this.hashes.size} hashes`);
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Rebuild-and-publish the index from an authoritative source (the durable
|
|
298
|
+
* corpus) under the SAME per-file cross-process lock the reconciling
|
|
299
|
+
* append/removal saves (`saveMergingWithDisk`) use (issue #1909 / PR #2016).
|
|
300
|
+
* The `populate` callback runs WHILE the lock is held and MUST repopulate this
|
|
301
|
+
* index from the corpus (`clear()` then `addByHash(...)` for every corpus
|
|
302
|
+
* hash); the rebuilt set is then published with a plain overwrite that is now
|
|
303
|
+
* serialized against every locked writer.
|
|
304
|
+
*
|
|
305
|
+
* This closes the finding where the previous unlocked clear→scan→save() could
|
|
306
|
+
* overwrite a peer's newer lock-merged or deferred additions: a locked writer
|
|
307
|
+
* that committed BEFORE the rebuild acquired the lock has its memory `.md` on
|
|
308
|
+
* disk and is re-scanned; one that commits AFTER the rebuild releases
|
|
309
|
+
* reconciles its additions on top of the freshly-published set. No union with
|
|
310
|
+
* the raw on-disk file is performed, so the rebuild keeps its
|
|
311
|
+
* garbage-collection property (orphaned hashes for deleted memories drop).
|
|
312
|
+
*
|
|
313
|
+
* Lock-acquisition failure is explicit and non-destructive: `populate` is NOT
|
|
314
|
+
* run and nothing is written (never an unlocked overwrite). Returns true when
|
|
315
|
+
* the rebuild was published under the lock, false when the lock timed out —
|
|
316
|
+
* the caller keeps the index non-authoritative and retries on next use. Uses
|
|
317
|
+
* the same non-reentrant file lock as the reconcile path, so it can never
|
|
318
|
+
* deadlock with the deferred reconcile-retry: a concurrent locked writer (or a
|
|
319
|
+
* fired retry) simply yields `acquired=false` after the bounded wait.
|
|
320
|
+
*/
|
|
321
|
+
async rebuildUnderLock(populate) {
|
|
322
|
+
await mkdir(path.dirname(this.filePath), { recursive: true });
|
|
323
|
+
let published = false;
|
|
324
|
+
await withHeldFileLock(
|
|
325
|
+
`${this.filePath}.lock`,
|
|
326
|
+
{
|
|
327
|
+
staleMs: CONTENT_HASH_INDEX_LOCK_STALE_MS,
|
|
328
|
+
...this.lockOptions.maxWaitMs !== void 0 ? { maxWaitMs: this.lockOptions.maxWaitMs } : {},
|
|
329
|
+
...this.lockOptions.pollMs !== void 0 ? { pollMs: this.lockOptions.pollMs } : {}
|
|
330
|
+
},
|
|
331
|
+
async (acquired) => {
|
|
332
|
+
if (!acquired) {
|
|
333
|
+
log.warn(
|
|
334
|
+
`content-hash index: lock not acquired for ${this.filePath}; deferring authoritative rebuild (index left non-authoritative, retried on next use)`
|
|
335
|
+
);
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
await populate();
|
|
339
|
+
await this.save();
|
|
340
|
+
published = true;
|
|
341
|
+
}
|
|
342
|
+
);
|
|
343
|
+
return published;
|
|
344
|
+
}
|
|
345
|
+
/**
|
|
346
|
+
* True while a background durable reconcile-save retry is armed (PR #2016).
|
|
347
|
+
* Test/introspection hook.
|
|
348
|
+
*/
|
|
349
|
+
get hasPendingReconcileRetry() {
|
|
350
|
+
return this.reconcileRetryTimer !== null;
|
|
351
|
+
}
|
|
352
|
+
/** True while this index has in-memory changes not yet published to disk. */
|
|
353
|
+
get hasPendingChanges() {
|
|
354
|
+
return this.dirty;
|
|
355
|
+
}
|
|
356
|
+
/**
|
|
357
|
+
* Resolve once any armed background reconcile-save retry chain has settled —
|
|
358
|
+
* either it published the deferred additions, or it exhausted its bounded
|
|
359
|
+
* attempts and fell back to the corpus-rebuild safety net. Resolves
|
|
360
|
+
* immediately when nothing is armed. Lets a caller (or a test) await the
|
|
361
|
+
* eventual durability outcome of a lock-timed-out save.
|
|
362
|
+
*/
|
|
363
|
+
async whenReconcileRetrySettled() {
|
|
364
|
+
await (this.reconcileRetryBarrier ?? Promise.resolve());
|
|
365
|
+
}
|
|
366
|
+
/**
|
|
367
|
+
* Drive any deferred lock-timeout reconcile save to completion INLINE at a
|
|
368
|
+
* lifecycle boundary (PR #2016 finding 3). The background retry timer is
|
|
369
|
+
* `unref`'d so it never keeps a long-lived daemon alive — but a short-lived
|
|
370
|
+
* writer (a one-shot CLI) can exit before it fires, leaving a durable fact
|
|
371
|
+
* `.md` whose hash never reached `fact-hashes.txt` for peers that already
|
|
372
|
+
* built their in-memory index. `orchestrator.destroy()` calls this so the
|
|
373
|
+
* addition publishes before the process exits; long-lived hosts only reach it
|
|
374
|
+
* at their own shutdown, so their in-flight retries keep running in the
|
|
375
|
+
* background until then. Bounded by the same attempt ceiling as the background
|
|
376
|
+
* retry; a permanently contended lock falls back to the corpus-rebuild-on-
|
|
377
|
+
* restart safety net (the fact `.md` is already durable). No deadlock: each
|
|
378
|
+
* attempt is the same non-reentrant, bounded-wait file lock.
|
|
379
|
+
*/
|
|
380
|
+
async flushReconcileRetry() {
|
|
381
|
+
if (!this.dirty) return;
|
|
382
|
+
const maxAttempts = this.lockOptions.retryMaxAttempts ?? CONTENT_HASH_INDEX_RETRY_MAX_ATTEMPTS;
|
|
383
|
+
const baseMs = this.lockOptions.retryBaseMs ?? CONTENT_HASH_INDEX_RETRY_BASE_MS;
|
|
384
|
+
for (let attempt = 0; this.dirty && attempt < maxAttempts; attempt += 1) {
|
|
385
|
+
if (this.reconcileRetryTimer) {
|
|
386
|
+
clearTimeout(this.reconcileRetryTimer);
|
|
387
|
+
this.reconcileRetryTimer = null;
|
|
388
|
+
}
|
|
389
|
+
await this.saveMergingWithDisk();
|
|
390
|
+
if (!this.dirty) break;
|
|
391
|
+
if (attempt < maxAttempts - 1) {
|
|
392
|
+
const wait = Math.min(baseMs * 2 ** attempt, CONTENT_HASH_INDEX_RETRY_MAX_DELAY_MS);
|
|
393
|
+
await new Promise((resolve) => setTimeout(resolve, wait));
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
if (this.reconcileRetryTimer) {
|
|
397
|
+
clearTimeout(this.reconcileRetryTimer);
|
|
398
|
+
this.reconcileRetryTimer = null;
|
|
399
|
+
}
|
|
400
|
+
this.reconcileRetryAttempts = 0;
|
|
401
|
+
this.settleReconcileRetry();
|
|
402
|
+
}
|
|
403
|
+
/**
|
|
404
|
+
* Schedule ONE bounded, exponentially-backed-off background retry of the
|
|
405
|
+
* deferred reconcile save (PR #2016). Reentrant-safe: an already-armed timer
|
|
406
|
+
* short-circuits, so successive timed-out saves never stack duplicate retries.
|
|
407
|
+
* When attempts are exhausted the chain settles without publishing — the
|
|
408
|
+
* durable fact `.md` is on disk and a process restart rebuilds the index from
|
|
409
|
+
* the corpus, so this degrades to that safety net, never a fatal failure.
|
|
410
|
+
*/
|
|
411
|
+
scheduleReconcileRetry() {
|
|
412
|
+
if (this.reconcileRetryTimer) return;
|
|
413
|
+
const maxAttempts = this.lockOptions.retryMaxAttempts ?? CONTENT_HASH_INDEX_RETRY_MAX_ATTEMPTS;
|
|
414
|
+
if (this.reconcileRetryAttempts >= maxAttempts) {
|
|
415
|
+
log.warn(
|
|
416
|
+
`content-hash index: exhausted ${maxAttempts} deferred reconcile-save retries for ${this.filePath}; ${this.added.size} addition(s) remain dirty in-memory (a process restart rebuilds the index from the durable corpus)`
|
|
417
|
+
);
|
|
418
|
+
this.reconcileRetryAttempts = 0;
|
|
419
|
+
this.settleReconcileRetry();
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
if (!this.reconcileRetryBarrier) {
|
|
423
|
+
this.reconcileRetryBarrier = new Promise((resolve) => {
|
|
424
|
+
this.reconcileRetryResolve = resolve;
|
|
425
|
+
});
|
|
426
|
+
}
|
|
427
|
+
const attempt = this.reconcileRetryAttempts + 1;
|
|
428
|
+
const baseMs = this.lockOptions.retryBaseMs ?? CONTENT_HASH_INDEX_RETRY_BASE_MS;
|
|
429
|
+
const delay = Math.min(baseMs * 2 ** (attempt - 1), CONTENT_HASH_INDEX_RETRY_MAX_DELAY_MS);
|
|
430
|
+
this.reconcileRetryTimer = setTimeout(() => {
|
|
431
|
+
this.reconcileRetryTimer = null;
|
|
432
|
+
this.reconcileRetryAttempts = attempt;
|
|
433
|
+
void this.saveMergingWithDisk().catch((err) => {
|
|
434
|
+
log.warn(`content-hash index: deferred reconcile-save retry failed for ${this.filePath}: ${err}`);
|
|
435
|
+
}).finally(() => {
|
|
436
|
+
if (this.dirty && !this.reconcileRetryTimer) this.settleReconcileRetry();
|
|
437
|
+
});
|
|
438
|
+
}, delay);
|
|
439
|
+
this.reconcileRetryTimer.unref?.();
|
|
440
|
+
}
|
|
441
|
+
/** Cancel any armed reconcile-save retry and settle the barrier (a publish landed). */
|
|
442
|
+
cancelReconcileRetry() {
|
|
443
|
+
if (this.reconcileRetryTimer) {
|
|
444
|
+
clearTimeout(this.reconcileRetryTimer);
|
|
445
|
+
this.reconcileRetryTimer = null;
|
|
446
|
+
}
|
|
447
|
+
this.reconcileRetryAttempts = 0;
|
|
448
|
+
this.settleReconcileRetry();
|
|
449
|
+
}
|
|
450
|
+
/** Resolve and clear the retry barrier if one is outstanding. */
|
|
451
|
+
settleReconcileRetry() {
|
|
452
|
+
const resolve = this.reconcileRetryResolve;
|
|
453
|
+
this.reconcileRetryBarrier = null;
|
|
454
|
+
this.reconcileRetryResolve = null;
|
|
455
|
+
resolve?.();
|
|
456
|
+
}
|
|
457
|
+
/** Remove a hash from the index (used when archiving/deleting). */
|
|
458
|
+
remove(content) {
|
|
459
|
+
const hash = _ContentHashIndex.computeHash(content);
|
|
460
|
+
this.added.delete(hash);
|
|
461
|
+
this.removed.add(hash);
|
|
462
|
+
this.hashes.delete(hash);
|
|
463
|
+
this.dirty = true;
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Remove a pre-computed SHA-256 hash directly from the index without
|
|
467
|
+
* re-hashing. Use this when the caller already holds the stored hash
|
|
468
|
+
* (e.g. `memory.frontmatter.contentHash`) to avoid the double-hash bug
|
|
469
|
+
* where `remove(hash)` would compute `hash(hash)` and never match the
|
|
470
|
+
* entry.
|
|
471
|
+
*/
|
|
472
|
+
removeByHash(hash) {
|
|
473
|
+
this.added.delete(hash);
|
|
474
|
+
this.removed.add(hash);
|
|
475
|
+
this.hashes.delete(hash);
|
|
476
|
+
this.dirty = true;
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* Add a pre-computed SHA-256 hash directly to the index without re-hashing.
|
|
480
|
+
* Use this when the caller already holds the stored hash
|
|
481
|
+
* (e.g. `memory.frontmatter.contentHash`) so that the index records the raw
|
|
482
|
+
* content hash rather than re-hashing the citation-annotated body.
|
|
483
|
+
*
|
|
484
|
+
* @internal Only called from `StorageManager.ensureFactHashIndexAuthoritative`.
|
|
485
|
+
* Not part of the public API — prefer `add(content)` for external callers.
|
|
486
|
+
*/
|
|
487
|
+
addByHash(hash) {
|
|
488
|
+
this.removed.delete(hash);
|
|
489
|
+
this.hashes.add(hash);
|
|
490
|
+
this.added.add(hash);
|
|
491
|
+
this.dirty = true;
|
|
492
|
+
}
|
|
493
|
+
/** Normalize content (delegates to content-hash.ts for a single source of truth). */
|
|
494
|
+
static normalizeContent(content) {
|
|
495
|
+
return normalizeContent(content);
|
|
496
|
+
}
|
|
497
|
+
/** Compute SHA-256 hash (delegates to content-hash.ts for a single source of truth). */
|
|
498
|
+
static computeHash(content) {
|
|
499
|
+
return computeContentHash(content);
|
|
500
|
+
}
|
|
501
|
+
};
|
|
502
|
+
|
|
503
|
+
// src/storage/tombstone-blocked-capture-mutation.ts
|
|
504
|
+
import { AsyncLocalStorage } from "async_hooks";
|
|
505
|
+
import { createHash } from "crypto";
|
|
506
|
+
import { mkdir as mkdir2 } from "fs/promises";
|
|
507
|
+
import path2 from "path";
|
|
508
|
+
var TOMBSTONE_CAPTURE_WRITE_LOCK_STALE_MS = 6e4;
|
|
509
|
+
var CAPTURE_WRITE_LOCK_MAX_ATTEMPTS = 3;
|
|
510
|
+
var CAPTURE_WRITE_LOCK_RETRY_BASE_MS = 25;
|
|
511
|
+
var CAPTURE_WRITE_LOCK_BUSY_MESSAGE = "tombstone-blocked capture write lock remained busy";
|
|
512
|
+
function isTombstoneBlockedCaptureWriteLockBusy(error) {
|
|
513
|
+
return error instanceof Error && error.message === CAPTURE_WRITE_LOCK_BUSY_MESSAGE;
|
|
514
|
+
}
|
|
515
|
+
function normalizeExplicitCaptureContent(value) {
|
|
516
|
+
return value.toLowerCase().replace(/\s+/g, " ").trim();
|
|
517
|
+
}
|
|
518
|
+
function buildExplicitCaptureDedupKey(content, category, sourceConnector) {
|
|
519
|
+
const encode = (label, value) => `${label} ${value.length} ${value}`;
|
|
520
|
+
return [
|
|
521
|
+
encode("category", category),
|
|
522
|
+
encode("connector", sourceConnector?.trim() ?? ""),
|
|
523
|
+
encode("content", normalizeExplicitCaptureContent(content))
|
|
524
|
+
].join(" ");
|
|
525
|
+
}
|
|
526
|
+
function captureIdentityHash(memory) {
|
|
527
|
+
return createHash("sha256").update(buildExplicitCaptureDedupKey(memory.content, memory.frontmatter.category, memory.frontmatter.sourceConnector)).digest("hex");
|
|
528
|
+
}
|
|
529
|
+
var TombstoneBlockedCaptureWriteLock = class {
|
|
530
|
+
constructor(options) {
|
|
531
|
+
this.options = options;
|
|
532
|
+
}
|
|
533
|
+
options;
|
|
534
|
+
context = new AsyncLocalStorage();
|
|
535
|
+
captureWriteLockPathForHash(identityHash) {
|
|
536
|
+
const filename = identityHash.length === 0 ? "explicit-capture-write.lock" : `explicit-capture-write-${identityHash}.lock`;
|
|
537
|
+
return path2.join(this.options.stateDir, "tombstone-blocked-capture", filename);
|
|
538
|
+
}
|
|
539
|
+
async withSingleCaptureWriteLock(lockPath, task) {
|
|
540
|
+
await mkdir2(path2.dirname(lockPath), { recursive: true });
|
|
541
|
+
const retry = /* @__PURE__ */ Symbol("retry");
|
|
542
|
+
const permanentFailure = /* @__PURE__ */ Symbol("permanent-failure");
|
|
543
|
+
const runWithHeldFileLock = this.options.withHeldFileLock ?? withHeldFileLock;
|
|
544
|
+
for (let attempt = 0; attempt < CAPTURE_WRITE_LOCK_MAX_ATTEMPTS; attempt += 1) {
|
|
545
|
+
const result = await runWithHeldFileLock(
|
|
546
|
+
lockPath,
|
|
547
|
+
{
|
|
548
|
+
staleMs: TOMBSTONE_CAPTURE_WRITE_LOCK_STALE_MS,
|
|
549
|
+
maxWaitMs: 1e3,
|
|
550
|
+
pollMs: 25
|
|
551
|
+
},
|
|
552
|
+
async (acquired, controller) => {
|
|
553
|
+
if (acquired) return await task();
|
|
554
|
+
return controller.failure === "error" ? permanentFailure : retry;
|
|
555
|
+
}
|
|
556
|
+
);
|
|
557
|
+
if (result === permanentFailure) {
|
|
558
|
+
throw new Error("tombstone-blocked capture write lock acquisition failed");
|
|
559
|
+
}
|
|
560
|
+
if (result !== retry) return result;
|
|
561
|
+
if (attempt === CAPTURE_WRITE_LOCK_MAX_ATTEMPTS - 1) break;
|
|
562
|
+
const delayState = Promise.withResolvers();
|
|
563
|
+
setTimeout(delayState.resolve, CAPTURE_WRITE_LOCK_RETRY_BASE_MS * 2 ** attempt);
|
|
564
|
+
await delayState.promise;
|
|
565
|
+
}
|
|
566
|
+
throw new Error(CAPTURE_WRITE_LOCK_BUSY_MESSAGE);
|
|
567
|
+
}
|
|
568
|
+
async withPhysicalCaptureWriteLocks(task, identityHashes) {
|
|
569
|
+
const lockPaths = [...new Set(identityHashes.map((hash) => this.captureWriteLockPathForHash(hash)))].sort();
|
|
570
|
+
const acquire = async (index) => {
|
|
571
|
+
if (index === lockPaths.length) return await task();
|
|
572
|
+
return await this.withSingleCaptureWriteLock(lockPaths[index], () => acquire(index + 1));
|
|
573
|
+
};
|
|
574
|
+
return await acquire(0);
|
|
575
|
+
}
|
|
576
|
+
async withCaptureWriteLock(task, identity) {
|
|
577
|
+
const identities = Array.isArray(identity) && identity.length > 0 ? identity : [typeof identity === "string" ? identity : ""];
|
|
578
|
+
return await this.withCaptureWriteLocks(task, identities, []);
|
|
579
|
+
}
|
|
580
|
+
async withCaptureWriteLockHashes(task, identityHashes) {
|
|
581
|
+
return await this.withCaptureWriteLocks(task, [], identityHashes);
|
|
582
|
+
}
|
|
583
|
+
async withCaptureWriteLockAndHashes(task, identity, identityHashes) {
|
|
584
|
+
const identities = Array.isArray(identity) && identity.length > 0 ? identity : [typeof identity === "string" ? identity : ""];
|
|
585
|
+
return await this.withCaptureWriteLocks(task, identities, identityHashes);
|
|
586
|
+
}
|
|
587
|
+
async withCaptureWriteLocks(task, identities, identityHashes) {
|
|
588
|
+
const current = this.context.getStore();
|
|
589
|
+
const hashToken = (key) => `\0${key.length === 0 ? "" : createHash("sha256").update(key).digest("hex")}`;
|
|
590
|
+
const identityHash = (key) => key.length === 0 ? "" : createHash("sha256").update(key).digest("hex");
|
|
591
|
+
const identityKeys = [...new Set(identities)];
|
|
592
|
+
const hashes = [...new Set(identityHashes)];
|
|
593
|
+
const missingIdentityHashes = identityKeys.filter((key) => !current || !current.has(key) && !current.has(hashToken(key))).map(identityHash);
|
|
594
|
+
const missingHashes = hashes.filter((hash) => !current || !current.has(`\0${hash}`));
|
|
595
|
+
const physicalMissing = [.../* @__PURE__ */ new Set([...missingIdentityHashes, ...missingHashes])];
|
|
596
|
+
if (current && physicalMissing.length > 0) {
|
|
597
|
+
const heldLockPaths = [...current].filter((token) => token.startsWith("\0")).map((token) => this.captureWriteLockPathForHash(token.slice(1)));
|
|
598
|
+
const requestedLockPaths = physicalMissing.map((hash) => this.captureWriteLockPathForHash(hash));
|
|
599
|
+
if (heldLockPaths.some((held) => requestedLockPaths.some((requested) => requested < held))) {
|
|
600
|
+
throw new Error(CAPTURE_WRITE_LOCK_BUSY_MESSAGE);
|
|
601
|
+
}
|
|
602
|
+
}
|
|
603
|
+
const next = new Set(current);
|
|
604
|
+
for (const key of identityKeys) {
|
|
605
|
+
next.add(key);
|
|
606
|
+
next.add(hashToken(key));
|
|
607
|
+
}
|
|
608
|
+
for (const hash of hashes) next.add(`\0${hash}`);
|
|
609
|
+
const run = () => this.context.run(next, task);
|
|
610
|
+
if (physicalMissing.length === 0) return await run();
|
|
611
|
+
return await this.withPhysicalCaptureWriteLocks(run, physicalMissing);
|
|
612
|
+
}
|
|
613
|
+
};
|
|
614
|
+
async function runTombstoneBlockedMutation(host, mutation) {
|
|
615
|
+
let lockIdentity = mutation.identity;
|
|
616
|
+
let mustLock = mutation.blocked || mutation.coordinate === true;
|
|
617
|
+
for (; ; ) {
|
|
618
|
+
let retryIdentity;
|
|
619
|
+
const mutate = async () => {
|
|
620
|
+
const current = await host.readCurrent();
|
|
621
|
+
const currentBlocked = host.isBlocked(current);
|
|
622
|
+
const currentQueuedReview = host.isQueuedReview(current);
|
|
623
|
+
if (current !== null && (currentBlocked || currentQueuedReview)) {
|
|
624
|
+
const currentIdentity = host.memoryIdentity(current);
|
|
625
|
+
const heldIdentities = Array.isArray(lockIdentity) ? lockIdentity : [lockIdentity];
|
|
626
|
+
if (!heldIdentities.includes(currentIdentity)) {
|
|
627
|
+
retryIdentity = currentIdentity;
|
|
628
|
+
return;
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
const rebuildMarker = mutation.blocked || currentBlocked ? await host.prepareWrite() : void 0;
|
|
632
|
+
try {
|
|
633
|
+
await host.writeStorageSecureFile(mutation.pathname, mutation.fileContent);
|
|
634
|
+
} catch (err) {
|
|
635
|
+
if (rebuildMarker) {
|
|
636
|
+
try {
|
|
637
|
+
await host.discardWrite(rebuildMarker);
|
|
638
|
+
} catch (cleanupError) {
|
|
639
|
+
host.markUntrusted();
|
|
640
|
+
host.logWarning(`storage.tombstoneBlocked failed to clear write marker: ${cleanupError}`);
|
|
641
|
+
}
|
|
642
|
+
}
|
|
643
|
+
throw err;
|
|
644
|
+
}
|
|
645
|
+
if (rebuildMarker) {
|
|
646
|
+
try {
|
|
647
|
+
await host.commitWrite(rebuildMarker);
|
|
648
|
+
} catch (err) {
|
|
649
|
+
host.markUntrusted();
|
|
650
|
+
host.logWarning(`storage.tombstoneBlocked committed write marker failed: ${err}`);
|
|
651
|
+
}
|
|
652
|
+
}
|
|
653
|
+
if (rebuildMarker) await mutation.beforeIndexUpdate?.();
|
|
654
|
+
await mutation.updateIndex(rebuildMarker, currentBlocked && current !== null ? current : void 0);
|
|
655
|
+
};
|
|
656
|
+
if (mustLock) {
|
|
657
|
+
await host.withCaptureWriteLock(mutate, lockIdentity);
|
|
658
|
+
} else {
|
|
659
|
+
await mutate();
|
|
660
|
+
}
|
|
661
|
+
if (retryIdentity === void 0) return;
|
|
662
|
+
const identities = Array.isArray(lockIdentity) ? [...lockIdentity, retryIdentity] : [lockIdentity, retryIdentity];
|
|
663
|
+
lockIdentity = [...new Set(identities)];
|
|
664
|
+
mustLock = true;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
function tombstoneBlocked(frontmatter) {
|
|
668
|
+
return frontmatter.status === "pending_review" && Boolean(frontmatter.blockedBy);
|
|
669
|
+
}
|
|
670
|
+
function isQueuedReviewFrontmatter(frontmatter) {
|
|
671
|
+
return frontmatter.status === "pending_review" && (frontmatter.tags ?? []).includes("queued-review");
|
|
672
|
+
}
|
|
673
|
+
function isQueuedReviewMemory(memory) {
|
|
674
|
+
return memory !== null && isQueuedReviewFrontmatter(memory.frontmatter);
|
|
675
|
+
}
|
|
676
|
+
async function runTombstoneBlockedChunkMutation(mutation) {
|
|
677
|
+
const persist = async () => {
|
|
678
|
+
if (mutation.blocked) {
|
|
679
|
+
const duplicate = await mutation.findDuplicate();
|
|
680
|
+
if (duplicate) return duplicate.frontmatter.id;
|
|
681
|
+
}
|
|
682
|
+
await mutation.persistMemory();
|
|
683
|
+
await mutation.afterWrite();
|
|
684
|
+
return mutation.id;
|
|
685
|
+
};
|
|
686
|
+
return mutation.blocked ? await mutation.withCaptureWriteLock(persist, mutation.identity) : await persist();
|
|
687
|
+
}
|
|
688
|
+
|
|
689
|
+
// src/storage/tombstone-blocked-capture-sync.ts
|
|
690
|
+
import { createHash as createHash2, randomUUID as randomUUID2 } from "crypto";
|
|
691
|
+
import { unlink as unlink2 } from "fs/promises";
|
|
692
|
+
import path4 from "path";
|
|
693
|
+
import { isDeepStrictEqual } from "util";
|
|
694
|
+
|
|
695
|
+
// src/storage/tombstone-blocked-capture-index.ts
|
|
696
|
+
import { execFile } from "child_process";
|
|
697
|
+
import { randomUUID } from "crypto";
|
|
698
|
+
import { mkdir as mkdir3, readFile, readdir, rename, stat as stat2, unlink, writeFile } from "fs/promises";
|
|
699
|
+
import path3 from "path";
|
|
700
|
+
import { promisify } from "util";
|
|
701
|
+
var REBUILD_MAX_ATTEMPTS = 3;
|
|
702
|
+
var execFileAsync = promisify(execFile);
|
|
703
|
+
var REBUILD_RETRY_BASE_MS = 50;
|
|
704
|
+
var ABANDONED_MARKER_MIN_AGE_MS = 6e4;
|
|
705
|
+
var MARKER_PROCESS_START_TOLERANCE_MS = 2e3;
|
|
706
|
+
var DOTNET_UNIX_EPOCH_TICKS = 621355968e9;
|
|
707
|
+
var MARKER_PROCESS_STARTED_AT_MS = Date.now() - process.uptime() * 1e3;
|
|
708
|
+
var TombstoneBlockedCaptureIndex = class {
|
|
709
|
+
options;
|
|
710
|
+
index = null;
|
|
711
|
+
loadPromise = null;
|
|
712
|
+
refreshPromise = null;
|
|
713
|
+
authoritative = false;
|
|
714
|
+
captureWriteLock;
|
|
715
|
+
constructor(options) {
|
|
716
|
+
this.options = options;
|
|
717
|
+
this.captureWriteLock = new TombstoneBlockedCaptureWriteLock(options);
|
|
718
|
+
}
|
|
719
|
+
createIndex() {
|
|
720
|
+
return new ContentHashIndex(
|
|
721
|
+
path3.join(this.options.stateDir, "tombstone-blocked-capture"),
|
|
722
|
+
this.options.secureStoreKeyProvider,
|
|
723
|
+
this.options.secureStoreWriteKeyProvider,
|
|
724
|
+
this.options.memoryDir,
|
|
725
|
+
this.options.lockOptions()
|
|
726
|
+
);
|
|
727
|
+
}
|
|
728
|
+
indexPath() {
|
|
729
|
+
return path3.join(this.options.stateDir, "tombstone-blocked-capture", "fact-hashes.txt");
|
|
730
|
+
}
|
|
731
|
+
rebuildMarkerPath() {
|
|
732
|
+
return path3.join(this.options.stateDir, "tombstone-blocked-capture", "rebuild-required");
|
|
733
|
+
}
|
|
734
|
+
async readRebuildMarkers() {
|
|
735
|
+
try {
|
|
736
|
+
const entries = (await readdir(this.rebuildMarkerPath())).filter((entry) => !entry.startsWith("."));
|
|
737
|
+
return (await Promise.all(
|
|
738
|
+
entries.map(async (entry) => {
|
|
739
|
+
const markerPath = path3.join(this.rebuildMarkerPath(), entry);
|
|
740
|
+
try {
|
|
741
|
+
const raw = (await readFile(markerPath, "utf8")).trim();
|
|
742
|
+
if (raw === "committed") return { path: markerPath, committed: true };
|
|
743
|
+
let value;
|
|
744
|
+
let malformed = false;
|
|
745
|
+
try {
|
|
746
|
+
const parsed = JSON.parse(raw);
|
|
747
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
748
|
+
malformed = true;
|
|
749
|
+
value = {};
|
|
750
|
+
} else {
|
|
751
|
+
value = parsed;
|
|
752
|
+
}
|
|
753
|
+
} catch {
|
|
754
|
+
malformed = true;
|
|
755
|
+
value = {};
|
|
756
|
+
}
|
|
757
|
+
const state = value.state;
|
|
758
|
+
if (state !== "pending" && state !== "committed") malformed = true;
|
|
759
|
+
const markerStat = await stat2(markerPath).catch(() => null);
|
|
760
|
+
const createdAt = typeof value.createdAt === "number" && Number.isFinite(value.createdAt) ? value.createdAt : markerStat?.mtimeMs;
|
|
761
|
+
if (typeof value.pid !== "number" || !Number.isInteger(value.pid)) malformed = true;
|
|
762
|
+
const ownerId = typeof value.ownerId === "string" && value.ownerId.length > 0 ? value.ownerId : path3.basename(markerPath);
|
|
763
|
+
if (typeof value.ownerId !== "string" || value.ownerId.length === 0) malformed = true;
|
|
764
|
+
const processStartedAtMs = typeof value.processStartedAtMs === "number" && Number.isFinite(value.processStartedAtMs) && value.processStartedAtMs > 0 ? value.processStartedAtMs : void 0;
|
|
765
|
+
return {
|
|
766
|
+
path: markerPath,
|
|
767
|
+
committed: state === "committed",
|
|
768
|
+
...typeof value.pid === "number" && Number.isInteger(value.pid) ? { pid: value.pid } : {},
|
|
769
|
+
...ownerId.length > 0 ? { ownerId } : {},
|
|
770
|
+
...typeof createdAt === "number" && Number.isFinite(createdAt) ? { createdAt } : {},
|
|
771
|
+
...processStartedAtMs === void 0 ? {} : { processStartedAtMs },
|
|
772
|
+
...malformed ? { malformed: true } : {}
|
|
773
|
+
};
|
|
774
|
+
} catch (err) {
|
|
775
|
+
if (isErrnoCode(err, "ENOENT")) return null;
|
|
776
|
+
throw err;
|
|
777
|
+
}
|
|
778
|
+
})
|
|
779
|
+
)).filter((marker) => marker !== null);
|
|
780
|
+
} catch (err) {
|
|
781
|
+
if (isErrnoCode(err, "ENOENT")) return [];
|
|
782
|
+
throw err;
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
async isAbandonedMarker(marker) {
|
|
786
|
+
if (marker.committed || marker.createdAt === void 0) return false;
|
|
787
|
+
if (Date.now() - marker.createdAt < ABANDONED_MARKER_MIN_AGE_MS) return false;
|
|
788
|
+
if (marker.malformed) return true;
|
|
789
|
+
return marker.pid !== void 0 && !await this.isProcessAlive(marker.pid, marker.processStartedAtMs);
|
|
790
|
+
}
|
|
791
|
+
async isProcessAlive(pid, processStartedAtMs) {
|
|
792
|
+
try {
|
|
793
|
+
process.kill(pid, 0);
|
|
794
|
+
} catch (err) {
|
|
795
|
+
return !isErrnoCode(err, "ESRCH");
|
|
796
|
+
}
|
|
797
|
+
if (processStartedAtMs === void 0) return true;
|
|
798
|
+
const runningStartedAtMs = await this.readProcessStartedAtMs(pid);
|
|
799
|
+
if (runningStartedAtMs === null) return true;
|
|
800
|
+
return runningStartedAtMs <= processStartedAtMs + MARKER_PROCESS_START_TOLERANCE_MS;
|
|
801
|
+
}
|
|
802
|
+
async readProcessStartedAtMs(pid) {
|
|
803
|
+
try {
|
|
804
|
+
if (process.platform === "win32") {
|
|
805
|
+
const { stdout: stdout2 } = await execFileAsync(
|
|
806
|
+
"powershell.exe",
|
|
807
|
+
[
|
|
808
|
+
"-NoLogo",
|
|
809
|
+
"-NoProfile",
|
|
810
|
+
"-NonInteractive",
|
|
811
|
+
"-Command",
|
|
812
|
+
`(Get-Process -Id ${pid} -ErrorAction Stop).StartTime.ToUniversalTime().Ticks`
|
|
813
|
+
],
|
|
814
|
+
{ encoding: "utf8", timeout: 1e3 }
|
|
815
|
+
);
|
|
816
|
+
const ticks = Number(stdout2.trim());
|
|
817
|
+
return Number.isFinite(ticks) ? (ticks - DOTNET_UNIX_EPOCH_TICKS) / 1e4 : null;
|
|
818
|
+
}
|
|
819
|
+
const { stdout } = await execFileAsync("ps", ["-p", String(pid), "-o", "lstart="], {
|
|
820
|
+
encoding: "utf8",
|
|
821
|
+
timeout: 1e3
|
|
822
|
+
});
|
|
823
|
+
const output = stdout.trim();
|
|
824
|
+
if (!output) return null;
|
|
825
|
+
const startedAtMs = Date.parse(output);
|
|
826
|
+
return Number.isFinite(startedAtMs) ? startedAtMs : null;
|
|
827
|
+
} catch {
|
|
828
|
+
return null;
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
async getClearableRebuildMarkerPaths(markers) {
|
|
832
|
+
const paths = await Promise.all(
|
|
833
|
+
markers.map(async (marker) => marker.committed || await this.isAbandonedMarker(marker) ? marker.path : null)
|
|
834
|
+
);
|
|
835
|
+
return paths.filter((markerPath) => markerPath !== null);
|
|
836
|
+
}
|
|
837
|
+
async getRebuildMarkers() {
|
|
838
|
+
return await this.readRebuildMarkers();
|
|
839
|
+
}
|
|
840
|
+
// A pending marker protects the pre-commit gap; only committed markers are safe for another writer to clear.
|
|
841
|
+
async writeMarkerAtomically(markerPath, payload) {
|
|
842
|
+
const temporaryPath = `${markerPath}.tmp-${process.pid}-${randomUUID()}`;
|
|
843
|
+
try {
|
|
844
|
+
await writeFile(temporaryPath, payload, "utf8");
|
|
845
|
+
await rename(temporaryPath, markerPath);
|
|
846
|
+
} finally {
|
|
847
|
+
await unlink(temporaryPath).catch((err) => {
|
|
848
|
+
if (!isErrnoCode(err, "ENOENT")) throw err;
|
|
849
|
+
});
|
|
850
|
+
}
|
|
851
|
+
}
|
|
852
|
+
async markRebuildRequired(committed = false) {
|
|
853
|
+
const markerDir = this.rebuildMarkerPath();
|
|
854
|
+
await mkdir3(markerDir, { recursive: true });
|
|
855
|
+
const ownerId = randomUUID();
|
|
856
|
+
const markerPath = path3.join(markerDir, ownerId);
|
|
857
|
+
await this.writeMarkerAtomically(
|
|
858
|
+
markerPath,
|
|
859
|
+
`${JSON.stringify({
|
|
860
|
+
state: committed ? "committed" : "pending",
|
|
861
|
+
pid: process.pid,
|
|
862
|
+
ownerId,
|
|
863
|
+
createdAt: Date.now(),
|
|
864
|
+
processStartedAtMs: MARKER_PROCESS_STARTED_AT_MS
|
|
865
|
+
})}
|
|
866
|
+
`
|
|
867
|
+
);
|
|
868
|
+
return markerPath;
|
|
869
|
+
}
|
|
870
|
+
async markRebuildCommitted(markerPath) {
|
|
871
|
+
await this.writeMarkerAtomically(
|
|
872
|
+
markerPath,
|
|
873
|
+
`${JSON.stringify({
|
|
874
|
+
state: "committed",
|
|
875
|
+
pid: process.pid,
|
|
876
|
+
ownerId: path3.basename(markerPath),
|
|
877
|
+
createdAt: Date.now(),
|
|
878
|
+
processStartedAtMs: MARKER_PROCESS_STARTED_AT_MS
|
|
879
|
+
})}
|
|
880
|
+
`
|
|
881
|
+
);
|
|
882
|
+
}
|
|
883
|
+
async clearRebuildRequired(markerPaths) {
|
|
884
|
+
await Promise.all(
|
|
885
|
+
markerPaths.map(async (markerPath) => {
|
|
886
|
+
try {
|
|
887
|
+
await unlink(markerPath);
|
|
888
|
+
} catch (err) {
|
|
889
|
+
if (!isErrnoCode(err, "ENOENT")) throw err;
|
|
890
|
+
}
|
|
891
|
+
})
|
|
892
|
+
);
|
|
893
|
+
}
|
|
894
|
+
async hasRebuildRequired(excludedMarkers = []) {
|
|
895
|
+
const excluded = new Set(excludedMarkers);
|
|
896
|
+
return (await this.getRebuildMarkers()).some((marker) => !excluded.has(marker.path));
|
|
897
|
+
}
|
|
898
|
+
async setAuthoritative(rebuilt) {
|
|
899
|
+
this.authoritative = rebuilt && !(await this.getRebuildMarkers()).some((marker) => !marker.committed);
|
|
900
|
+
}
|
|
901
|
+
isBlocked(memory) {
|
|
902
|
+
return memory.frontmatter.status === "pending_review" && Boolean(memory.frontmatter.blockedBy);
|
|
903
|
+
}
|
|
904
|
+
async rebuild(index) {
|
|
905
|
+
return await index.rebuildUnderLock(async () => {
|
|
906
|
+
index.clear();
|
|
907
|
+
const existing = [...await this.options.readAllMemories(), ...await this.options.readAllColdMemories()];
|
|
908
|
+
for (const memory of existing) {
|
|
909
|
+
if (!this.isBlocked(memory)) continue;
|
|
910
|
+
index.add(
|
|
911
|
+
buildExplicitCaptureDedupKey(memory.content, memory.frontmatter.category, memory.frontmatter.sourceConnector)
|
|
912
|
+
);
|
|
913
|
+
}
|
|
914
|
+
});
|
|
915
|
+
}
|
|
916
|
+
async hasPersistedIndexOrMarkers() {
|
|
917
|
+
try {
|
|
918
|
+
await stat2(this.indexPath());
|
|
919
|
+
return true;
|
|
920
|
+
} catch (err) {
|
|
921
|
+
if (!isErrnoCode(err, "ENOENT")) throw err;
|
|
922
|
+
return (await this.readRebuildMarkers()).length > 0;
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
async getIndex() {
|
|
926
|
+
if (this.index) return this.index;
|
|
927
|
+
if (!this.loadPromise) {
|
|
928
|
+
const index = this.createIndex();
|
|
929
|
+
this.loadPromise = (async () => {
|
|
930
|
+
await index.load();
|
|
931
|
+
const rebuildMarkers = await this.getRebuildMarkers();
|
|
932
|
+
const clearableRebuildMarkers = await this.getClearableRebuildMarkerPaths(rebuildMarkers);
|
|
933
|
+
let persisted = true;
|
|
934
|
+
try {
|
|
935
|
+
await stat2(this.indexPath());
|
|
936
|
+
if (rebuildMarkers.length > 0) persisted = false;
|
|
937
|
+
} catch (err) {
|
|
938
|
+
if (isErrnoCode(err, "ENOENT")) persisted = false;
|
|
939
|
+
else throw err;
|
|
940
|
+
}
|
|
941
|
+
if (!persisted) {
|
|
942
|
+
const lockOptions = this.options.lockOptions();
|
|
943
|
+
const maxAttempts = lockOptions.retryMaxAttempts ?? REBUILD_MAX_ATTEMPTS;
|
|
944
|
+
const baseMs = lockOptions.retryBaseMs ?? REBUILD_RETRY_BASE_MS;
|
|
945
|
+
let rebuilt = false;
|
|
946
|
+
for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
|
|
947
|
+
rebuilt = await this.rebuild(index);
|
|
948
|
+
if (rebuilt || attempt === maxAttempts - 1) break;
|
|
949
|
+
const wait = Math.min(baseMs * 2 ** attempt, CONTENT_HASH_INDEX_RETRY_MAX_DELAY_MS);
|
|
950
|
+
const delayState = Promise.withResolvers();
|
|
951
|
+
setTimeout(delayState.resolve, wait);
|
|
952
|
+
await delayState.promise;
|
|
953
|
+
}
|
|
954
|
+
if (!rebuilt) {
|
|
955
|
+
throw new Error("tombstone-blocked capture index rebuild lock unavailable");
|
|
956
|
+
}
|
|
957
|
+
await this.clearRebuildRequired(clearableRebuildMarkers);
|
|
958
|
+
}
|
|
959
|
+
await this.setAuthoritative(true);
|
|
960
|
+
this.index = index;
|
|
961
|
+
return index;
|
|
962
|
+
})().catch((err) => {
|
|
963
|
+
this.loadPromise = null;
|
|
964
|
+
throw err;
|
|
965
|
+
});
|
|
966
|
+
}
|
|
967
|
+
return this.loadPromise;
|
|
968
|
+
}
|
|
969
|
+
async withCaptureWriteLock(task, identity) {
|
|
970
|
+
return await this.captureWriteLock.withCaptureWriteLock(task, identity);
|
|
971
|
+
}
|
|
972
|
+
async withCaptureWriteLockHashes(task, identityHashes) {
|
|
973
|
+
return await this.captureWriteLock.withCaptureWriteLockHashes(task, identityHashes);
|
|
974
|
+
}
|
|
975
|
+
async withCaptureWriteLockAndHashes(task, identity, identityHashes) {
|
|
976
|
+
return await this.captureWriteLock.withCaptureWriteLockAndHashes(task, identity, identityHashes);
|
|
977
|
+
}
|
|
978
|
+
async reload() {
|
|
979
|
+
if (!this.refreshPromise) {
|
|
980
|
+
this.refreshPromise = (async () => {
|
|
981
|
+
this.index = null;
|
|
982
|
+
this.loadPromise = null;
|
|
983
|
+
return await this.getIndex();
|
|
984
|
+
})().finally(() => {
|
|
985
|
+
this.refreshPromise = null;
|
|
986
|
+
});
|
|
987
|
+
}
|
|
988
|
+
return this.refreshPromise;
|
|
989
|
+
}
|
|
990
|
+
/** Return blocked-row membership and authority from one index snapshot. */
|
|
991
|
+
async check(content, category, sourceConnector) {
|
|
992
|
+
const key = buildExplicitCaptureDedupKey(content, category, sourceConnector);
|
|
993
|
+
let index = await this.getIndex();
|
|
994
|
+
if (await this.hasRebuildRequired() || !await index.isDiskFingerprintCurrent()) {
|
|
995
|
+
index = await this.reload();
|
|
996
|
+
}
|
|
997
|
+
return { has: index.has(key), authoritative: this.authoritative };
|
|
998
|
+
}
|
|
999
|
+
/** Return whether a blocked row matches this explicit-capture identity. */
|
|
1000
|
+
async has(content, category, sourceConnector) {
|
|
1001
|
+
return (await this.check(content, category, sourceConnector)).has;
|
|
1002
|
+
}
|
|
1003
|
+
/** Return whether the in-memory index is safe to answer an authoritative miss. */
|
|
1004
|
+
async isAuthoritative() {
|
|
1005
|
+
const index = await this.getIndex();
|
|
1006
|
+
if (await this.hasRebuildRequired() || !await index.isDiskFingerprintCurrent()) {
|
|
1007
|
+
await this.reload();
|
|
1008
|
+
}
|
|
1009
|
+
return this.authoritative;
|
|
1010
|
+
}
|
|
1011
|
+
/** Reserve durable rebuild intent before a blocked memory file is committed. */
|
|
1012
|
+
async prepareWrite() {
|
|
1013
|
+
return this.markRebuildRequired();
|
|
1014
|
+
}
|
|
1015
|
+
/** Mark a prepared write committed once its memory file is durable. */
|
|
1016
|
+
async commitWrite(rebuildMarker) {
|
|
1017
|
+
await this.markRebuildCommitted(rebuildMarker);
|
|
1018
|
+
}
|
|
1019
|
+
/** Drop a pending marker when the protected memory write never commits. */
|
|
1020
|
+
async discardWrite(rebuildMarker) {
|
|
1021
|
+
await this.clearRebuildRequired([rebuildMarker]);
|
|
1022
|
+
}
|
|
1023
|
+
/** Add a newly persisted blocked row to the durable targeted index. */
|
|
1024
|
+
async add(memory, rebuildMarker) {
|
|
1025
|
+
if (!this.isBlocked(memory)) return;
|
|
1026
|
+
let index = await this.getIndex();
|
|
1027
|
+
if (await this.hasRebuildRequired(rebuildMarker ? [rebuildMarker] : []) || !await index.isDiskFingerprintCurrent()) {
|
|
1028
|
+
index = await this.reload();
|
|
1029
|
+
}
|
|
1030
|
+
index.add(
|
|
1031
|
+
buildExplicitCaptureDedupKey(memory.content, memory.frontmatter.category, memory.frontmatter.sourceConnector)
|
|
1032
|
+
);
|
|
1033
|
+
const marker = rebuildMarker ?? await this.markRebuildRequired(true);
|
|
1034
|
+
if (rebuildMarker) await this.markRebuildCommitted(rebuildMarker);
|
|
1035
|
+
await index.saveMergingWithDisk();
|
|
1036
|
+
await index.flushReconcileRetry();
|
|
1037
|
+
if (index.hasPendingChanges) {
|
|
1038
|
+
this.markUntrusted();
|
|
1039
|
+
return;
|
|
1040
|
+
}
|
|
1041
|
+
await this.clearRebuildRequired([marker]);
|
|
1042
|
+
await this.setAuthoritative(true);
|
|
1043
|
+
}
|
|
1044
|
+
/** Rebuild the index after a blocked row changes or is removed. */
|
|
1045
|
+
async rebuildIfLoaded(ownedMarker) {
|
|
1046
|
+
const index = this.index ?? (await this.hasPersistedIndexOrMarkers() ? this.getIndex() : null);
|
|
1047
|
+
if (!index) return;
|
|
1048
|
+
const rebuildMarkers = await this.getRebuildMarkers();
|
|
1049
|
+
const rebuildMarker = await this.markRebuildRequired(true);
|
|
1050
|
+
const clearableRebuildMarkers = await this.getClearableRebuildMarkerPaths(rebuildMarkers);
|
|
1051
|
+
const rebuilt = await this.rebuild(await index);
|
|
1052
|
+
if (rebuilt) {
|
|
1053
|
+
await this.clearRebuildRequired([
|
|
1054
|
+
...clearableRebuildMarkers,
|
|
1055
|
+
...ownedMarker === void 0 || clearableRebuildMarkers.includes(ownedMarker) ? [] : [ownedMarker],
|
|
1056
|
+
rebuildMarker
|
|
1057
|
+
]);
|
|
1058
|
+
}
|
|
1059
|
+
await this.setAuthoritative(rebuilt);
|
|
1060
|
+
}
|
|
1061
|
+
/** Rebuild when either side of a write is blocked and its identity changed. */
|
|
1062
|
+
async sync(before, after, rebuildMarker) {
|
|
1063
|
+
const beforeBlocked = this.isBlocked(before);
|
|
1064
|
+
const afterBlocked = this.isBlocked(after);
|
|
1065
|
+
if (!beforeBlocked && !afterBlocked) return;
|
|
1066
|
+
const beforeKey = buildExplicitCaptureDedupKey(
|
|
1067
|
+
before.content,
|
|
1068
|
+
before.frontmatter.category,
|
|
1069
|
+
before.frontmatter.sourceConnector
|
|
1070
|
+
);
|
|
1071
|
+
const afterKey = buildExplicitCaptureDedupKey(
|
|
1072
|
+
after.content,
|
|
1073
|
+
after.frontmatter.category,
|
|
1074
|
+
after.frontmatter.sourceConnector
|
|
1075
|
+
);
|
|
1076
|
+
if (beforeBlocked && afterBlocked && beforeKey === afterKey) {
|
|
1077
|
+
if (rebuildMarker) {
|
|
1078
|
+
await this.markRebuildCommitted(rebuildMarker);
|
|
1079
|
+
await this.clearRebuildRequired([rebuildMarker]);
|
|
1080
|
+
}
|
|
1081
|
+
return;
|
|
1082
|
+
}
|
|
1083
|
+
const existingMarkers = await this.getRebuildMarkers();
|
|
1084
|
+
const marker = rebuildMarker ?? await this.markRebuildRequired(true);
|
|
1085
|
+
const clearableExistingMarkers = await this.getClearableRebuildMarkerPaths(existingMarkers);
|
|
1086
|
+
if (rebuildMarker) await this.markRebuildCommitted(rebuildMarker);
|
|
1087
|
+
const rebuilt = await this.rebuild(await this.getIndex());
|
|
1088
|
+
if (rebuilt) {
|
|
1089
|
+
await this.clearRebuildRequired([
|
|
1090
|
+
...clearableExistingMarkers,
|
|
1091
|
+
marker
|
|
1092
|
+
]);
|
|
1093
|
+
}
|
|
1094
|
+
await this.setAuthoritative(rebuilt);
|
|
1095
|
+
}
|
|
1096
|
+
markUntrusted() {
|
|
1097
|
+
this.authoritative = false;
|
|
1098
|
+
}
|
|
1099
|
+
async failOpen(operation, action) {
|
|
1100
|
+
try {
|
|
1101
|
+
await action();
|
|
1102
|
+
} catch (err) {
|
|
1103
|
+
this.markUntrusted();
|
|
1104
|
+
log.warn(`storage.${operation} completed but failed to update tombstone-blocked capture index: ${err}`);
|
|
1105
|
+
}
|
|
1106
|
+
}
|
|
1107
|
+
async addWrittenMemory(pathname, frontmatter, content, rebuildMarker) {
|
|
1108
|
+
await this.failOpen("writeMemory", () => this.add({ path: pathname, frontmatter, content }, rebuildMarker));
|
|
1109
|
+
}
|
|
1110
|
+
async rebuildAfterInvalidation(ownedMarker) {
|
|
1111
|
+
await this.failOpen("invalidateMemory", () => this.rebuildIfLoaded(ownedMarker));
|
|
1112
|
+
}
|
|
1113
|
+
async syncUpdatedMemory(before, frontmatter, content, rebuildMarker) {
|
|
1114
|
+
await this.failOpen("updateMemory", () => this.sync(before, { ...before, frontmatter, content }, rebuildMarker));
|
|
1115
|
+
}
|
|
1116
|
+
async syncUpdatedFrontmatter(before, frontmatter, rebuildMarker) {
|
|
1117
|
+
await this.failOpen("writeMemoryFrontmatter", () => this.sync(before, { ...before, frontmatter }, rebuildMarker));
|
|
1118
|
+
}
|
|
1119
|
+
shouldRebuildAfterInvalidationForPath(filePath) {
|
|
1120
|
+
const relative = path3.relative(path3.resolve(this.options.memoryDir), path3.resolve(filePath));
|
|
1121
|
+
const category = relative.split(path3.sep)[0];
|
|
1122
|
+
return category === "cold" || RECALL_FALLBACK_DIRS.includes(category);
|
|
1123
|
+
}
|
|
1124
|
+
};
|
|
1125
|
+
|
|
1126
|
+
// src/storage/tombstone-blocked-capture-sync.ts
|
|
1127
|
+
var OFFLINE_SYNC_BUFFER_LIMIT_BYTES = 1048576;
|
|
1128
|
+
var OFFLINE_SYNC_STAGE_READ_BYTES = 64 * 1024;
|
|
1129
|
+
var OFFLINE_SYNC_FRONTMATTER_DELIMITER = Buffer.from("\n---\n", "utf8");
|
|
1130
|
+
function createExplicitContentNormalizer(onChunk) {
|
|
1131
|
+
let started = false;
|
|
1132
|
+
let pendingSpace = false;
|
|
1133
|
+
let output = "";
|
|
1134
|
+
const flush = () => {
|
|
1135
|
+
if (output.length > 0) {
|
|
1136
|
+
onChunk(output);
|
|
1137
|
+
output = "";
|
|
1138
|
+
}
|
|
1139
|
+
};
|
|
1140
|
+
return {
|
|
1141
|
+
push(text) {
|
|
1142
|
+
for (const character of text.toLowerCase()) {
|
|
1143
|
+
if (/\s/u.test(character)) {
|
|
1144
|
+
if (started) pendingSpace = true;
|
|
1145
|
+
continue;
|
|
1146
|
+
}
|
|
1147
|
+
if (pendingSpace) {
|
|
1148
|
+
output += " ";
|
|
1149
|
+
pendingSpace = false;
|
|
1150
|
+
}
|
|
1151
|
+
output += character;
|
|
1152
|
+
started = true;
|
|
1153
|
+
if (output.length >= OFFLINE_SYNC_STAGE_READ_BYTES) flush();
|
|
1154
|
+
}
|
|
1155
|
+
},
|
|
1156
|
+
finish: flush
|
|
1157
|
+
};
|
|
1158
|
+
}
|
|
1159
|
+
async function forEachStagedTextChunk(stagePath, offset, onText, key, memoryDir) {
|
|
1160
|
+
const decoder = new TextDecoder("utf-8");
|
|
1161
|
+
let remaining = offset;
|
|
1162
|
+
for await (const chunk of readMaybeEncryptedFileFromChunks(stagePath, key, memoryDir)) {
|
|
1163
|
+
const textChunk = remaining >= chunk.length ? null : chunk.subarray(remaining);
|
|
1164
|
+
remaining = Math.max(0, remaining - chunk.length);
|
|
1165
|
+
if (textChunk === null || textChunk.length === 0) continue;
|
|
1166
|
+
const text = decoder.decode(textChunk, { stream: true });
|
|
1167
|
+
if (text.length > 0) onText(text);
|
|
1168
|
+
}
|
|
1169
|
+
const finalText = decoder.decode();
|
|
1170
|
+
if (finalText.length > 0) onText(finalText);
|
|
1171
|
+
}
|
|
1172
|
+
async function findStagedBodyOffset(stagePath, key, memoryDir) {
|
|
1173
|
+
let scannedBytes = 0;
|
|
1174
|
+
let tail = Buffer.alloc(0);
|
|
1175
|
+
for await (const chunk of readMaybeEncryptedFileFromChunks(stagePath, key, memoryDir)) {
|
|
1176
|
+
const window = tail.length > 0 ? Buffer.concat([tail, chunk]) : chunk;
|
|
1177
|
+
const windowStart = scannedBytes - tail.length;
|
|
1178
|
+
for (let searchFrom = 0; ; ) {
|
|
1179
|
+
const delimiterStart = window.indexOf(OFFLINE_SYNC_FRONTMATTER_DELIMITER, searchFrom);
|
|
1180
|
+
if (delimiterStart < 0) break;
|
|
1181
|
+
if (windowStart + delimiterStart >= 4) {
|
|
1182
|
+
return windowStart + delimiterStart + OFFLINE_SYNC_FRONTMATTER_DELIMITER.length;
|
|
1183
|
+
}
|
|
1184
|
+
searchFrom = delimiterStart + 1;
|
|
1185
|
+
}
|
|
1186
|
+
scannedBytes += chunk.length;
|
|
1187
|
+
const tailLength = Math.min(OFFLINE_SYNC_FRONTMATTER_DELIMITER.length - 1, window.length);
|
|
1188
|
+
tail = Buffer.from(window.subarray(window.length - tailLength));
|
|
1189
|
+
}
|
|
1190
|
+
return null;
|
|
1191
|
+
}
|
|
1192
|
+
async function readStagedPrefix(stagePath, byteLimit, key, memoryDir) {
|
|
1193
|
+
const chunks = [];
|
|
1194
|
+
let bytesRead = 0;
|
|
1195
|
+
for await (const chunk of readMaybeEncryptedFileFromChunks(stagePath, key, memoryDir)) {
|
|
1196
|
+
const length = Math.min(chunk.length, byteLimit - bytesRead);
|
|
1197
|
+
if (length > 0) chunks.push(chunk.subarray(0, length));
|
|
1198
|
+
bytesRead += length;
|
|
1199
|
+
if (bytesRead >= byteLimit) break;
|
|
1200
|
+
}
|
|
1201
|
+
return Buffer.concat(chunks, bytesRead);
|
|
1202
|
+
}
|
|
1203
|
+
function parseTombstoneBlockedOfflineSyncMemory(filePath, content, memoryDir, parseFrontmatter, normalizeFrontmatterForPath, toMemoryPathRel) {
|
|
1204
|
+
const parsed = parseFrontmatter(content.toString("utf8"));
|
|
1205
|
+
if (!parsed) return null;
|
|
1206
|
+
return {
|
|
1207
|
+
path: filePath,
|
|
1208
|
+
frontmatter: normalizeFrontmatterForPath(parsed.frontmatter, toMemoryPathRel(memoryDir, filePath), parsed.content),
|
|
1209
|
+
content: parsed.content
|
|
1210
|
+
};
|
|
1211
|
+
}
|
|
1212
|
+
var TombstoneBlockedCaptureIndexHost = class {
|
|
1213
|
+
tombstoneBlockedCaptureIndex = null;
|
|
1214
|
+
async writeStorageSecureFileChunks(filePath, chunks) {
|
|
1215
|
+
const options = this.tombstoneBlockedCaptureIndexOptions();
|
|
1216
|
+
await writeMaybeEncryptedFileFromChunks(
|
|
1217
|
+
filePath,
|
|
1218
|
+
chunks,
|
|
1219
|
+
options.secureStoreWriteKeyProvider(),
|
|
1220
|
+
{},
|
|
1221
|
+
options.memoryDir
|
|
1222
|
+
);
|
|
1223
|
+
this.notifyCatalogWriteForPath(filePath);
|
|
1224
|
+
}
|
|
1225
|
+
getTombstoneBlockedCaptureIndex() {
|
|
1226
|
+
if (this.tombstoneBlockedCaptureIndex === null) {
|
|
1227
|
+
this.tombstoneBlockedCaptureIndex = new TombstoneBlockedCaptureIndex(this.tombstoneBlockedCaptureIndexOptions());
|
|
1228
|
+
}
|
|
1229
|
+
return this.tombstoneBlockedCaptureIndex;
|
|
1230
|
+
}
|
|
1231
|
+
async withTombstoneBlockedCaptureWriteLock(task, identity) {
|
|
1232
|
+
return await this.getTombstoneBlockedCaptureIndex().withCaptureWriteLock(task, identity);
|
|
1233
|
+
}
|
|
1234
|
+
async withTombstoneBlockedCaptureWriteLockAndHashes(task, identity, identityHashes) {
|
|
1235
|
+
return await this.getTombstoneBlockedCaptureIndex().withCaptureWriteLockAndHashes(task, identity, identityHashes);
|
|
1236
|
+
}
|
|
1237
|
+
async writeTombstoneBlockedMutation(blocked, pathname, fileContent, identity, updateIndex, beforeIndexUpdate, coordinate = false) {
|
|
1238
|
+
const host = {
|
|
1239
|
+
readCurrent: () => this.readMemoryByPath(pathname),
|
|
1240
|
+
isBlocked: (memory) => this.isTombstoneBlockedMemory(memory),
|
|
1241
|
+
isQueuedReview: (memory) => isQueuedReviewMemory(memory),
|
|
1242
|
+
memoryIdentity: (memory) => this.offlineSyncMemoryIdentity(memory),
|
|
1243
|
+
prepareWrite: () => this.getTombstoneBlockedCaptureIndex().prepareWrite(),
|
|
1244
|
+
commitWrite: (marker) => this.getTombstoneBlockedCaptureIndex().commitWrite(marker),
|
|
1245
|
+
discardWrite: (marker) => this.getTombstoneBlockedCaptureIndex().discardWrite(marker),
|
|
1246
|
+
markUntrusted: () => this.getTombstoneBlockedCaptureIndex().markUntrusted(),
|
|
1247
|
+
writeStorageSecureFile: (target, content) => this.writeStorageSecureFile(target, content),
|
|
1248
|
+
withCaptureWriteLock: (task, lockIdentity) => this.withTombstoneBlockedCaptureWriteLock(task, lockIdentity),
|
|
1249
|
+
logWarning: (message) => log.warn(message)
|
|
1250
|
+
};
|
|
1251
|
+
await runTombstoneBlockedMutation(host, {
|
|
1252
|
+
blocked,
|
|
1253
|
+
pathname,
|
|
1254
|
+
fileContent,
|
|
1255
|
+
identity,
|
|
1256
|
+
updateIndex,
|
|
1257
|
+
beforeIndexUpdate,
|
|
1258
|
+
coordinate
|
|
1259
|
+
});
|
|
1260
|
+
}
|
|
1261
|
+
async writeTombstoneBlockedMemory(pathname, fileContent, frontmatter, content, beforeIndexUpdate) {
|
|
1262
|
+
await this.writeTombstoneBlockedMutation(
|
|
1263
|
+
tombstoneBlocked(frontmatter),
|
|
1264
|
+
pathname,
|
|
1265
|
+
fileContent,
|
|
1266
|
+
buildExplicitCaptureDedupKey(content, frontmatter.category, frontmatter.sourceConnector),
|
|
1267
|
+
(rebuildMarker) => this.getTombstoneBlockedCaptureIndex().addWrittenMemory(pathname, frontmatter, content, rebuildMarker),
|
|
1268
|
+
beforeIndexUpdate
|
|
1269
|
+
);
|
|
1270
|
+
}
|
|
1271
|
+
async writeTombstoneBlockedChunk(pathname, fileContent, frontmatter, content, findDuplicate, afterWrite) {
|
|
1272
|
+
return await runTombstoneBlockedChunkMutation({
|
|
1273
|
+
blocked: tombstoneBlocked(frontmatter),
|
|
1274
|
+
id: frontmatter.id,
|
|
1275
|
+
identity: buildExplicitCaptureDedupKey(content, frontmatter.category, frontmatter.sourceConnector),
|
|
1276
|
+
findDuplicate,
|
|
1277
|
+
persistMemory: () => this.writeTombstoneBlockedMemory(pathname, fileContent, frontmatter, content),
|
|
1278
|
+
afterWrite,
|
|
1279
|
+
withCaptureWriteLock: (task, identity) => this.withTombstoneBlockedCaptureWriteLock(task, identity)
|
|
1280
|
+
});
|
|
1281
|
+
}
|
|
1282
|
+
async writeTombstoneBlockedUpdate(before, fileContent, frontmatter, content, beforeIndexUpdate) {
|
|
1283
|
+
await this.writeTombstoneBlockedMutation(
|
|
1284
|
+
tombstoneBlocked(before.frontmatter) || tombstoneBlocked(frontmatter),
|
|
1285
|
+
before.path,
|
|
1286
|
+
fileContent,
|
|
1287
|
+
[
|
|
1288
|
+
...tombstoneBlocked(before.frontmatter) ? [
|
|
1289
|
+
buildExplicitCaptureDedupKey(
|
|
1290
|
+
before.content,
|
|
1291
|
+
before.frontmatter.category,
|
|
1292
|
+
before.frontmatter.sourceConnector
|
|
1293
|
+
)
|
|
1294
|
+
] : [],
|
|
1295
|
+
...tombstoneBlocked(frontmatter) ? [buildExplicitCaptureDedupKey(content, frontmatter.category, frontmatter.sourceConnector)] : [],
|
|
1296
|
+
...isQueuedReviewMemory(before) ? [
|
|
1297
|
+
buildExplicitCaptureDedupKey(
|
|
1298
|
+
before.content,
|
|
1299
|
+
before.frontmatter.category,
|
|
1300
|
+
before.frontmatter.sourceConnector
|
|
1301
|
+
)
|
|
1302
|
+
] : [],
|
|
1303
|
+
...isQueuedReviewFrontmatter(frontmatter) ? [buildExplicitCaptureDedupKey(content, frontmatter.category, frontmatter.sourceConnector)] : []
|
|
1304
|
+
],
|
|
1305
|
+
(rebuildMarker, current) => this.getTombstoneBlockedCaptureIndex().syncUpdatedMemory(
|
|
1306
|
+
current ?? before,
|
|
1307
|
+
frontmatter,
|
|
1308
|
+
content,
|
|
1309
|
+
rebuildMarker
|
|
1310
|
+
),
|
|
1311
|
+
beforeIndexUpdate,
|
|
1312
|
+
isQueuedReviewMemory(before) || isQueuedReviewFrontmatter(frontmatter)
|
|
1313
|
+
);
|
|
1314
|
+
}
|
|
1315
|
+
async writeTombstoneBlockedFrontmatter(before, fileContent, frontmatter, beforeIndexUpdate) {
|
|
1316
|
+
await this.writeTombstoneBlockedMutation(
|
|
1317
|
+
tombstoneBlocked(before.frontmatter) || tombstoneBlocked(frontmatter),
|
|
1318
|
+
before.path,
|
|
1319
|
+
fileContent,
|
|
1320
|
+
[
|
|
1321
|
+
...tombstoneBlocked(before.frontmatter) ? [
|
|
1322
|
+
buildExplicitCaptureDedupKey(
|
|
1323
|
+
before.content,
|
|
1324
|
+
before.frontmatter.category,
|
|
1325
|
+
before.frontmatter.sourceConnector
|
|
1326
|
+
)
|
|
1327
|
+
] : [],
|
|
1328
|
+
...tombstoneBlocked(frontmatter) ? [buildExplicitCaptureDedupKey(before.content, frontmatter.category, frontmatter.sourceConnector)] : [],
|
|
1329
|
+
...isQueuedReviewMemory(before) ? [
|
|
1330
|
+
buildExplicitCaptureDedupKey(
|
|
1331
|
+
before.content,
|
|
1332
|
+
before.frontmatter.category,
|
|
1333
|
+
before.frontmatter.sourceConnector
|
|
1334
|
+
)
|
|
1335
|
+
] : [],
|
|
1336
|
+
...isQueuedReviewFrontmatter(frontmatter) ? [buildExplicitCaptureDedupKey(before.content, frontmatter.category, frontmatter.sourceConnector)] : []
|
|
1337
|
+
],
|
|
1338
|
+
(rebuildMarker, current) => this.getTombstoneBlockedCaptureIndex().syncUpdatedFrontmatter(
|
|
1339
|
+
current ?? before,
|
|
1340
|
+
frontmatter,
|
|
1341
|
+
rebuildMarker
|
|
1342
|
+
),
|
|
1343
|
+
beforeIndexUpdate,
|
|
1344
|
+
isQueuedReviewMemory(before) || isQueuedReviewFrontmatter(frontmatter)
|
|
1345
|
+
);
|
|
1346
|
+
}
|
|
1347
|
+
isTombstoneBlockedMemory(memory) {
|
|
1348
|
+
return memory !== null && memory.frontmatter.status === "pending_review" && Boolean(memory.frontmatter.blockedBy);
|
|
1349
|
+
}
|
|
1350
|
+
offlineSyncMemoryIdentity(memory) {
|
|
1351
|
+
return buildExplicitCaptureDedupKey(
|
|
1352
|
+
memory.content,
|
|
1353
|
+
memory.frontmatter.category,
|
|
1354
|
+
memory.frontmatter.sourceConnector
|
|
1355
|
+
);
|
|
1356
|
+
}
|
|
1357
|
+
async isDuplicateTombstoneBlockedOfflineSync(target, incoming, streamedIdentity) {
|
|
1358
|
+
const before = await this.readMemoryByPath(target);
|
|
1359
|
+
const duplicate = streamedIdentity ? true : await this.getTombstoneBlockedCaptureIndex().has(
|
|
1360
|
+
incoming.content,
|
|
1361
|
+
incoming.frontmatter.category,
|
|
1362
|
+
incoming.frontmatter.sourceConnector
|
|
1363
|
+
);
|
|
1364
|
+
if (!duplicate) return false;
|
|
1365
|
+
const incomingIdentity = streamedIdentity?.identityHash ?? this.offlineSyncMemoryIdentity(incoming);
|
|
1366
|
+
const options = this.tombstoneBlockedCaptureIndexOptions();
|
|
1367
|
+
const existing = [...await options.readAllMemories(), ...await options.readAllColdMemories()];
|
|
1368
|
+
const exactMatch = existing.some(
|
|
1369
|
+
(memory) => this.isTombstoneBlockedMemory(memory) && (streamedIdentity ? captureIdentityHash(memory) : this.offlineSyncMemoryIdentity(memory)) === incomingIdentity
|
|
1370
|
+
);
|
|
1371
|
+
if (!exactMatch) return false;
|
|
1372
|
+
if (before === null) return true;
|
|
1373
|
+
if (!this.isTombstoneBlockedMemory(before)) return false;
|
|
1374
|
+
const beforeIdentity = streamedIdentity ? captureIdentityHash(before) : this.offlineSyncMemoryIdentity(before);
|
|
1375
|
+
if (beforeIdentity !== incomingIdentity) return false;
|
|
1376
|
+
return streamedIdentity ? isDeepStrictEqual(before.frontmatter, incoming.frontmatter) : before.content === incoming.content && isDeepStrictEqual(before.frontmatter, incoming.frontmatter);
|
|
1377
|
+
}
|
|
1378
|
+
async invalidateAfterOfflineSyncMutation(filePath, ownedMarker) {
|
|
1379
|
+
this.invalidateAllMemoriesCache();
|
|
1380
|
+
this.invalidateKnowledgeIndexCache();
|
|
1381
|
+
this.markFactHashIndexNotAuthoritative();
|
|
1382
|
+
if (filePath.includes(`${path4.sep}cold${path4.sep}`)) {
|
|
1383
|
+
this.invalidateColdMemoriesCache();
|
|
1384
|
+
}
|
|
1385
|
+
await this.rebuildTombstoneBlockedCaptureAfterInvalidationForPath(filePath, ownedMarker);
|
|
1386
|
+
if (filePath.includes(`${path4.sep}artifacts${path4.sep}`)) {
|
|
1387
|
+
this.bumpArtifactWriteVersion();
|
|
1388
|
+
}
|
|
1389
|
+
this.bumpMemoryStatusVersion();
|
|
1390
|
+
}
|
|
1391
|
+
async runTombstoneBlockedArchive(memory, task) {
|
|
1392
|
+
if (!this.isTombstoneBlockedMemory(memory) && !isQueuedReviewMemory(memory)) {
|
|
1393
|
+
return await task(memory, () => {
|
|
1394
|
+
});
|
|
1395
|
+
}
|
|
1396
|
+
let archivedPath = null;
|
|
1397
|
+
const archived = await this.runTombstoneBlockedInvalidation(
|
|
1398
|
+
memory,
|
|
1399
|
+
async (current, rebuildMarker, markDurable) => {
|
|
1400
|
+
let archiveDurable = false;
|
|
1401
|
+
const markArchiveDurable = () => {
|
|
1402
|
+
archiveDurable = true;
|
|
1403
|
+
markDurable();
|
|
1404
|
+
};
|
|
1405
|
+
archivedPath = await task(current, markArchiveDurable);
|
|
1406
|
+
if (rebuildMarker !== void 0 && (archivedPath !== null || archiveDurable)) {
|
|
1407
|
+
await this.rebuildTombstoneBlockedCaptureAfterInvalidation(rebuildMarker);
|
|
1408
|
+
} else if (rebuildMarker !== void 0) {
|
|
1409
|
+
const index = this.getTombstoneBlockedCaptureIndex();
|
|
1410
|
+
try {
|
|
1411
|
+
await index.discardWrite(rebuildMarker);
|
|
1412
|
+
} catch {
|
|
1413
|
+
index.markUntrusted();
|
|
1414
|
+
}
|
|
1415
|
+
}
|
|
1416
|
+
return archivedPath !== null;
|
|
1417
|
+
}
|
|
1418
|
+
);
|
|
1419
|
+
return archived ? archivedPath : null;
|
|
1420
|
+
}
|
|
1421
|
+
async runTombstoneBlockedInvalidation(memory, task, propagateErrors = false, coordinate = false) {
|
|
1422
|
+
let lockIdentity = coordinate || this.isTombstoneBlockedMemory(memory) || isQueuedReviewMemory(memory) ? this.offlineSyncMemoryIdentity(memory) : void 0;
|
|
1423
|
+
for (; ; ) {
|
|
1424
|
+
const attempt = async () => {
|
|
1425
|
+
const options = this.tombstoneBlockedCaptureIndexOptions();
|
|
1426
|
+
const current = await this.readMemoryByPath(memory.path) ?? (await options.readAllMemories()).find((candidate) => candidate.frontmatter.id === memory.frontmatter.id) ?? (await options.readAllColdMemories()).find((candidate) => candidate.frontmatter.id === memory.frontmatter.id);
|
|
1427
|
+
if (!current) return { result: false };
|
|
1428
|
+
const blocked = this.isTombstoneBlockedMemory(current);
|
|
1429
|
+
const queuedReview = isQueuedReviewMemory(current);
|
|
1430
|
+
if (blocked || queuedReview) {
|
|
1431
|
+
const currentIdentity = this.offlineSyncMemoryIdentity(current);
|
|
1432
|
+
if (lockIdentity !== currentIdentity) return { retryIdentity: currentIdentity };
|
|
1433
|
+
}
|
|
1434
|
+
const blockedIndex = blocked ? this.getTombstoneBlockedCaptureIndex() : null;
|
|
1435
|
+
let rebuildMarker;
|
|
1436
|
+
let durable = false;
|
|
1437
|
+
try {
|
|
1438
|
+
if (blockedIndex) rebuildMarker = await blockedIndex.prepareWrite();
|
|
1439
|
+
const result2 = await task(current, rebuildMarker, () => {
|
|
1440
|
+
durable = true;
|
|
1441
|
+
});
|
|
1442
|
+
if (!result2 && rebuildMarker && !durable && blockedIndex) {
|
|
1443
|
+
try {
|
|
1444
|
+
await blockedIndex.discardWrite(rebuildMarker);
|
|
1445
|
+
} catch {
|
|
1446
|
+
blockedIndex.markUntrusted();
|
|
1447
|
+
}
|
|
1448
|
+
}
|
|
1449
|
+
return { result: result2 };
|
|
1450
|
+
} catch (error) {
|
|
1451
|
+
if (rebuildMarker && !durable && blockedIndex) {
|
|
1452
|
+
try {
|
|
1453
|
+
await blockedIndex.discardWrite(rebuildMarker);
|
|
1454
|
+
} catch {
|
|
1455
|
+
blockedIndex.markUntrusted();
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
if (propagateErrors) throw error;
|
|
1459
|
+
return { result: false };
|
|
1460
|
+
}
|
|
1461
|
+
};
|
|
1462
|
+
const result = lockIdentity === void 0 ? await attempt() : await this.withTombstoneBlockedCaptureWriteLock(attempt, lockIdentity);
|
|
1463
|
+
if (result.retryIdentity !== void 0) {
|
|
1464
|
+
lockIdentity = result.retryIdentity;
|
|
1465
|
+
continue;
|
|
1466
|
+
}
|
|
1467
|
+
return result.result;
|
|
1468
|
+
}
|
|
1469
|
+
}
|
|
1470
|
+
async deleteMemoryForMaintenance(memory, shouldDelete = () => true) {
|
|
1471
|
+
let deleted = null;
|
|
1472
|
+
const removed = await this.runTombstoneBlockedInvalidation(
|
|
1473
|
+
memory,
|
|
1474
|
+
async (current, rebuildMarker, markDurable) => {
|
|
1475
|
+
if (!shouldDelete(current)) return false;
|
|
1476
|
+
await unlink2(current.path);
|
|
1477
|
+
markDurable();
|
|
1478
|
+
deleted = current;
|
|
1479
|
+
const memoryDir = this.tombstoneBlockedCaptureIndexOptions().memoryDir;
|
|
1480
|
+
markProjectedMemoryPathInvalid(memoryDir, current.frontmatter.id);
|
|
1481
|
+
this.invalidateAllMemoriesCache();
|
|
1482
|
+
if (current.path.includes(`${path4.sep}cold${path4.sep}`)) {
|
|
1483
|
+
this.invalidateColdMemoriesCache();
|
|
1484
|
+
}
|
|
1485
|
+
await this.rebuildTombstoneBlockedCaptureAfterInvalidation(rebuildMarker);
|
|
1486
|
+
this.bumpMemoryCorpusVersion();
|
|
1487
|
+
this.bumpMemoryStatusVersion();
|
|
1488
|
+
return true;
|
|
1489
|
+
},
|
|
1490
|
+
true,
|
|
1491
|
+
true
|
|
1492
|
+
);
|
|
1493
|
+
return removed ? deleted : null;
|
|
1494
|
+
}
|
|
1495
|
+
async runTombstoneBlockedOfflineSyncMutation(target, after, write, coordinate = false, streamedIdentity) {
|
|
1496
|
+
for (; ; ) {
|
|
1497
|
+
const before = await this.readMemoryByPath(target);
|
|
1498
|
+
const targetIdentity = this.isTombstoneBlockedMemory(before) ? this.offlineSyncMemoryIdentity(before) : void 0;
|
|
1499
|
+
const blocked = coordinate || targetIdentity !== void 0 || this.isTombstoneBlockedMemory(after);
|
|
1500
|
+
const identities = [
|
|
1501
|
+
...targetIdentity === void 0 ? [] : [targetIdentity],
|
|
1502
|
+
...!streamedIdentity && this.isTombstoneBlockedMemory(after) ? [this.offlineSyncMemoryIdentity(after)] : [],
|
|
1503
|
+
...coordinate ? [target] : [],
|
|
1504
|
+
...coordinate && after === null && streamedIdentity === void 0 ? [""] : []
|
|
1505
|
+
];
|
|
1506
|
+
let retryIdentity;
|
|
1507
|
+
const mutate = async () => {
|
|
1508
|
+
const current = await this.readMemoryByPath(target);
|
|
1509
|
+
if (this.isTombstoneBlockedMemory(current)) {
|
|
1510
|
+
const currentIdentity = this.offlineSyncMemoryIdentity(current);
|
|
1511
|
+
if (currentIdentity !== targetIdentity) {
|
|
1512
|
+
retryIdentity = currentIdentity;
|
|
1513
|
+
return;
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
if (this.isTombstoneBlockedMemory(after) && await this.isDuplicateTombstoneBlockedOfflineSync(target, after, streamedIdentity)) {
|
|
1517
|
+
return;
|
|
1518
|
+
}
|
|
1519
|
+
const marker = blocked ? await this.getTombstoneBlockedCaptureIndex().prepareWrite() : void 0;
|
|
1520
|
+
let durable = false;
|
|
1521
|
+
try {
|
|
1522
|
+
await write();
|
|
1523
|
+
durable = true;
|
|
1524
|
+
if (marker) {
|
|
1525
|
+
try {
|
|
1526
|
+
await this.getTombstoneBlockedCaptureIndex().commitWrite(marker);
|
|
1527
|
+
} catch (err) {
|
|
1528
|
+
this.getTombstoneBlockedCaptureIndex().markUntrusted();
|
|
1529
|
+
log.warn(`storage.offlineSyncFile committed write marker failed: ${err}`);
|
|
1530
|
+
}
|
|
1531
|
+
}
|
|
1532
|
+
await this.invalidateAfterOfflineSyncMutation(target, marker);
|
|
1533
|
+
} catch (err) {
|
|
1534
|
+
if (marker && !durable) {
|
|
1535
|
+
try {
|
|
1536
|
+
await this.getTombstoneBlockedCaptureIndex().discardWrite(marker);
|
|
1537
|
+
} catch (cleanupError) {
|
|
1538
|
+
this.getTombstoneBlockedCaptureIndex().markUntrusted();
|
|
1539
|
+
log.warn(`storage.offlineSyncFile failed to clear write marker: ${cleanupError}`);
|
|
1540
|
+
}
|
|
1541
|
+
}
|
|
1542
|
+
throw err;
|
|
1543
|
+
}
|
|
1544
|
+
};
|
|
1545
|
+
if (!blocked) {
|
|
1546
|
+
await mutate();
|
|
1547
|
+
} else {
|
|
1548
|
+
if (streamedIdentity) {
|
|
1549
|
+
await this.withTombstoneBlockedCaptureWriteLockAndHashes(
|
|
1550
|
+
mutate,
|
|
1551
|
+
identities,
|
|
1552
|
+
[streamedIdentity.identityHash]
|
|
1553
|
+
);
|
|
1554
|
+
} else {
|
|
1555
|
+
await this.withTombstoneBlockedCaptureWriteLock(mutate, identities);
|
|
1556
|
+
}
|
|
1557
|
+
}
|
|
1558
|
+
if (retryIdentity !== void 0) continue;
|
|
1559
|
+
return;
|
|
1560
|
+
}
|
|
1561
|
+
}
|
|
1562
|
+
async writeTombstoneBlockedOfflineSyncFile(target, content) {
|
|
1563
|
+
await this.runTombstoneBlockedOfflineSyncMutation(
|
|
1564
|
+
target,
|
|
1565
|
+
this.tombstoneBlockedCaptureIndexOptions().parseMemory?.(target, content) ?? null,
|
|
1566
|
+
() => this.writeStorageSecureFile(target, content)
|
|
1567
|
+
);
|
|
1568
|
+
}
|
|
1569
|
+
async prepareTombstoneBlockedOfflineSyncChunks(target, chunks) {
|
|
1570
|
+
const parseMemory = this.tombstoneBlockedCaptureIndexOptions().parseMemory;
|
|
1571
|
+
if (!parseMemory) return { after: null, chunks };
|
|
1572
|
+
const iterator = chunks[Symbol.asyncIterator]();
|
|
1573
|
+
const buffered = [];
|
|
1574
|
+
let bufferedBytes = 0;
|
|
1575
|
+
let firstOverflow;
|
|
1576
|
+
for (; ; ) {
|
|
1577
|
+
const next = await iterator.next();
|
|
1578
|
+
if (next.done) break;
|
|
1579
|
+
if (bufferedBytes + next.value.length > OFFLINE_SYNC_BUFFER_LIMIT_BYTES) {
|
|
1580
|
+
firstOverflow = next.value;
|
|
1581
|
+
break;
|
|
1582
|
+
}
|
|
1583
|
+
bufferedBytes += next.value.length;
|
|
1584
|
+
buffered.push(next.value);
|
|
1585
|
+
}
|
|
1586
|
+
const overflow = firstOverflow;
|
|
1587
|
+
if (overflow === void 0) {
|
|
1588
|
+
let after = null;
|
|
1589
|
+
try {
|
|
1590
|
+
after = parseMemory(target, Buffer.concat(buffered));
|
|
1591
|
+
} catch {
|
|
1592
|
+
after = null;
|
|
1593
|
+
}
|
|
1594
|
+
const replay = (async function* () {
|
|
1595
|
+
for (const chunk of buffered) yield chunk;
|
|
1596
|
+
})();
|
|
1597
|
+
return { after, chunks: replay };
|
|
1598
|
+
}
|
|
1599
|
+
const options = this.tombstoneBlockedCaptureIndexOptions();
|
|
1600
|
+
const stagePath = path4.join(
|
|
1601
|
+
options.stateDir,
|
|
1602
|
+
"tombstone-blocked-capture",
|
|
1603
|
+
"offline-sync-staging",
|
|
1604
|
+
`${randomUUID2()}.stage`
|
|
1605
|
+
);
|
|
1606
|
+
const stagedChunks = (async function* () {
|
|
1607
|
+
for (const chunk of buffered) yield chunk;
|
|
1608
|
+
yield overflow;
|
|
1609
|
+
for (; ; ) {
|
|
1610
|
+
const next = await iterator.next();
|
|
1611
|
+
if (next.done) break;
|
|
1612
|
+
yield next.value;
|
|
1613
|
+
}
|
|
1614
|
+
})();
|
|
1615
|
+
try {
|
|
1616
|
+
await writeMaybeEncryptedFileFromChunks(
|
|
1617
|
+
stagePath,
|
|
1618
|
+
stagedChunks,
|
|
1619
|
+
options.secureStoreWriteKeyProvider(),
|
|
1620
|
+
{ atomic: false },
|
|
1621
|
+
options.memoryDir
|
|
1622
|
+
);
|
|
1623
|
+
return await this.describeStagedOfflineSyncChunks(target, stagePath, parseMemory);
|
|
1624
|
+
} catch (err) {
|
|
1625
|
+
await unlink2(stagePath).catch(() => {
|
|
1626
|
+
});
|
|
1627
|
+
throw err;
|
|
1628
|
+
}
|
|
1629
|
+
}
|
|
1630
|
+
async describeStagedOfflineSyncChunks(target, stagePath, parseMemory) {
|
|
1631
|
+
const options = this.tombstoneBlockedCaptureIndexOptions();
|
|
1632
|
+
const key = options.secureStoreKeyProvider();
|
|
1633
|
+
const bodyOffset = await findStagedBodyOffset(stagePath, key, options.memoryDir);
|
|
1634
|
+
if (bodyOffset === null) {
|
|
1635
|
+
return { after: null, chunks: this.readStagedOfflineSyncChunks(stagePath), stagePath };
|
|
1636
|
+
}
|
|
1637
|
+
const prefix = await readStagedPrefix(stagePath, bodyOffset, key, options.memoryDir);
|
|
1638
|
+
let after = null;
|
|
1639
|
+
try {
|
|
1640
|
+
after = parseMemory(target, prefix.subarray(0, bodyOffset));
|
|
1641
|
+
} catch {
|
|
1642
|
+
after = null;
|
|
1643
|
+
}
|
|
1644
|
+
if (after === null) {
|
|
1645
|
+
return { after: null, chunks: this.readStagedOfflineSyncChunks(stagePath), stagePath };
|
|
1646
|
+
}
|
|
1647
|
+
const streamedIdentity = await this.computeStagedOfflineSyncIdentity(stagePath, bodyOffset, after);
|
|
1648
|
+
return {
|
|
1649
|
+
after,
|
|
1650
|
+
chunks: this.readStagedOfflineSyncChunks(stagePath),
|
|
1651
|
+
stagePath,
|
|
1652
|
+
streamedIdentity
|
|
1653
|
+
};
|
|
1654
|
+
}
|
|
1655
|
+
async computeStagedOfflineSyncIdentity(stagePath, bodyOffset, after) {
|
|
1656
|
+
let normalizedLength = 0;
|
|
1657
|
+
const countNormalizer = createExplicitContentNormalizer((chunk) => {
|
|
1658
|
+
normalizedLength += chunk.length;
|
|
1659
|
+
});
|
|
1660
|
+
const options = this.tombstoneBlockedCaptureIndexOptions();
|
|
1661
|
+
const key = options.secureStoreKeyProvider();
|
|
1662
|
+
await forEachStagedTextChunk(
|
|
1663
|
+
stagePath,
|
|
1664
|
+
bodyOffset,
|
|
1665
|
+
(text) => countNormalizer.push(text),
|
|
1666
|
+
key,
|
|
1667
|
+
options.memoryDir
|
|
1668
|
+
);
|
|
1669
|
+
countNormalizer.finish();
|
|
1670
|
+
const category = after.frontmatter.category;
|
|
1671
|
+
const connector = after.frontmatter.sourceConnector?.trim() ?? "";
|
|
1672
|
+
const encode = (label, value) => `${label} ${value.length} ${value}`;
|
|
1673
|
+
const identityPrefix = [encode("category", category), encode("connector", connector), `content ${normalizedLength} `].join(" ");
|
|
1674
|
+
const identityHash = createHash2("sha256");
|
|
1675
|
+
identityHash.update(identityPrefix);
|
|
1676
|
+
const identityNormalizer = createExplicitContentNormalizer((chunk) => identityHash.update(chunk));
|
|
1677
|
+
await forEachStagedTextChunk(
|
|
1678
|
+
stagePath,
|
|
1679
|
+
bodyOffset,
|
|
1680
|
+
(text) => identityNormalizer.push(text),
|
|
1681
|
+
key,
|
|
1682
|
+
options.memoryDir
|
|
1683
|
+
);
|
|
1684
|
+
identityNormalizer.finish();
|
|
1685
|
+
return {
|
|
1686
|
+
identityHash: identityHash.digest("hex")
|
|
1687
|
+
};
|
|
1688
|
+
}
|
|
1689
|
+
async *readStagedOfflineSyncChunks(stagePath) {
|
|
1690
|
+
const options = this.tombstoneBlockedCaptureIndexOptions();
|
|
1691
|
+
for await (const chunk of readMaybeEncryptedFileFromChunks(
|
|
1692
|
+
stagePath,
|
|
1693
|
+
options.secureStoreKeyProvider(),
|
|
1694
|
+
options.memoryDir
|
|
1695
|
+
)) {
|
|
1696
|
+
yield chunk;
|
|
1697
|
+
}
|
|
1698
|
+
}
|
|
1699
|
+
async writeTombstoneBlockedOfflineSyncFileChunks(target, chunks) {
|
|
1700
|
+
const coordinate = this.getTombstoneBlockedCaptureIndex().shouldRebuildAfterInvalidationForPath(target);
|
|
1701
|
+
const prepared = coordinate ? await this.prepareTombstoneBlockedOfflineSyncChunks(target, chunks) : { after: null, chunks };
|
|
1702
|
+
const streamedIdentity = prepared.streamedIdentity;
|
|
1703
|
+
const mutate = async () => {
|
|
1704
|
+
try {
|
|
1705
|
+
await this.runTombstoneBlockedOfflineSyncMutation(
|
|
1706
|
+
target,
|
|
1707
|
+
prepared.after,
|
|
1708
|
+
() => this.writeStorageSecureFileChunks(target, prepared.chunks),
|
|
1709
|
+
coordinate,
|
|
1710
|
+
streamedIdentity
|
|
1711
|
+
);
|
|
1712
|
+
} finally {
|
|
1713
|
+
if (prepared.stagePath !== void 0) await unlink2(prepared.stagePath).catch(() => {
|
|
1714
|
+
});
|
|
1715
|
+
}
|
|
1716
|
+
};
|
|
1717
|
+
await mutate();
|
|
1718
|
+
}
|
|
1719
|
+
async writeOfflineSyncFile(filePath, content) {
|
|
1720
|
+
const target = this.assertManagedStoragePath(filePath, "storage.writeOfflineSyncFile");
|
|
1721
|
+
await this.writeTombstoneBlockedOfflineSyncFile(target, content);
|
|
1722
|
+
}
|
|
1723
|
+
async writeOfflineSyncFileChunks(filePath, chunks) {
|
|
1724
|
+
const target = this.assertManagedStoragePath(filePath, "storage.writeOfflineSyncFileChunks");
|
|
1725
|
+
await this.writeTombstoneBlockedOfflineSyncFileChunks(target, chunks);
|
|
1726
|
+
}
|
|
1727
|
+
async deleteOfflineSyncFile(filePath) {
|
|
1728
|
+
const target = this.assertManagedStoragePath(filePath, "storage.deleteOfflineSyncFile");
|
|
1729
|
+
await this.deleteTombstoneBlockedOfflineSyncFile(target);
|
|
1730
|
+
}
|
|
1731
|
+
async deleteTombstoneBlockedOfflineSyncFile(target) {
|
|
1732
|
+
await this.runTombstoneBlockedOfflineSyncMutation(target, null, async () => {
|
|
1733
|
+
await unlink2(target).catch((error) => {
|
|
1734
|
+
if (isErrnoCode(error, "ENOENT")) return;
|
|
1735
|
+
throw error;
|
|
1736
|
+
});
|
|
1737
|
+
});
|
|
1738
|
+
}
|
|
1739
|
+
async checkTombstoneBlockedExplicitCapture(...args) {
|
|
1740
|
+
return this.getTombstoneBlockedCaptureIndex().check(...args);
|
|
1741
|
+
}
|
|
1742
|
+
async hasTombstoneBlockedExplicitCapture(...args) {
|
|
1743
|
+
return this.getTombstoneBlockedCaptureIndex().has(...args);
|
|
1744
|
+
}
|
|
1745
|
+
async isTombstoneBlockedExplicitCaptureIndexAuthoritative() {
|
|
1746
|
+
return this.getTombstoneBlockedCaptureIndex().isAuthoritative();
|
|
1747
|
+
}
|
|
1748
|
+
async rebuildTombstoneBlockedCaptureAfterInvalidationForPath(filePath, ownedMarker) {
|
|
1749
|
+
const index = this.getTombstoneBlockedCaptureIndex();
|
|
1750
|
+
if (index.shouldRebuildAfterInvalidationForPath(filePath)) {
|
|
1751
|
+
await this.rebuildTombstoneBlockedCaptureAfterInvalidation(ownedMarker);
|
|
1752
|
+
}
|
|
1753
|
+
}
|
|
1754
|
+
async rebuildTombstoneBlockedCaptureAfterInvalidation(ownedMarker) {
|
|
1755
|
+
await this.getTombstoneBlockedCaptureIndex().rebuildAfterInvalidation(ownedMarker);
|
|
1756
|
+
}
|
|
1757
|
+
};
|
|
1758
|
+
|
|
1759
|
+
export {
|
|
1760
|
+
CONTENT_HASH_INDEX_RETRY_MAX_DELAY_MS,
|
|
1761
|
+
FactHashIndexNotAuthoritativeError,
|
|
1762
|
+
ContentHashIndex,
|
|
1763
|
+
isTombstoneBlockedCaptureWriteLockBusy,
|
|
1764
|
+
normalizeExplicitCaptureContent,
|
|
1765
|
+
buildExplicitCaptureDedupKey,
|
|
1766
|
+
parseTombstoneBlockedOfflineSyncMemory,
|
|
1767
|
+
TombstoneBlockedCaptureIndexHost
|
|
1768
|
+
};
|
|
1769
|
+
//# sourceMappingURL=chunk-FBUE6PD3.js.map
|