@remnic/core 10.0.48 → 10.0.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.js +63 -61
- package/dist/access-audit.js +2 -2
- package/dist/access-authorization-probe.js +64 -62
- package/dist/access-boundary.js +63 -61
- package/dist/access-cli.js +144 -142
- package/dist/access-cli.js.map +1 -1
- package/dist/access-extraction-force-flush.js +63 -61
- package/dist/access-http-briefing.js +64 -62
- package/dist/access-http-query.js +65 -63
- package/dist/access-http.js +85 -83
- package/dist/access-identity-continuity-surface.js +63 -61
- package/dist/access-lcm-surface.js +63 -61
- package/dist/access-mcp.js +74 -72
- package/dist/access-namespace-preflight.js +63 -61
- package/dist/access-observe-write-category.js +11 -10
- package/dist/access-observe-write-surface.js +63 -61
- package/dist/access-offline-manifest.js +8 -7
- package/dist/access-operations-batch.js +66 -64
- package/dist/access-operations.js +73 -71
- package/dist/access-recall-concurrency.js +63 -61
- package/dist/access-recall-response.js +63 -61
- package/dist/access-recall-surface.js +63 -61
- package/dist/access-schema.js +10 -9
- package/dist/access-service.d.ts.map +1 -1
- package/dist/access-service.js +63 -61
- package/dist/access-surface-catalog.js +65 -63
- package/dist/access-surface-catalog.js.map +1 -1
- package/dist/active-recall.js +23 -21
- package/dist/active-recall.js.map +1 -1
- package/dist/adapters/index.js +4 -4
- package/dist/adapters/registry.js +2 -2
- package/dist/briefing.js +13 -12
- package/dist/buffer.js +5 -5
- package/dist/calibration.js +3 -3
- package/dist/{capsule-crypto-GWVG7LGC.js → capsule-crypto-7FJQINUR.js} +2 -2
- package/dist/capture-write-lock-scope.d.ts +7 -0
- package/dist/capture-write-lock-scope.d.ts.map +1 -0
- package/dist/capture-write-lock-scope.js +10 -0
- package/dist/causal-behavior.js +2 -2
- package/dist/causal-chain.js +2 -2
- package/dist/causal-consolidation.js +16 -15
- package/dist/causal-consolidation.js.map +1 -1
- package/dist/causal-retrieval.js +2 -2
- package/dist/causal-trajectory-graph.js +3 -3
- package/dist/causal-trajectory.js +1 -1
- package/dist/{chunk-J6B6FK2X.js → chunk-2BY56HE6.js} +43 -43
- package/dist/{chunk-ME5LVISI.js → chunk-2J6BIOVN.js} +2 -2
- package/dist/{chunk-EIYJK7OD.js → chunk-2M36PR7R.js} +2 -2
- package/dist/{chunk-OWHERGF2.js → chunk-2NLLXCJG.js} +2 -2
- package/dist/{chunk-WOPZ2P5Z.js → chunk-3INLTKTW.js} +2 -2
- package/dist/{chunk-6P6MZDBL.js → chunk-3NL4ALUP.js} +2 -2
- package/dist/{chunk-YUE7J2HZ.js → chunk-3NYIXVCT.js} +2 -2
- package/dist/{chunk-B44NFHWI.js → chunk-3PVKV3XL.js} +2 -2
- package/dist/{chunk-H7BZBSP6.js → chunk-52II7PB6.js} +4 -4
- package/dist/{chunk-5KGYR5VN.js → chunk-5DVOBBTV.js} +2 -2
- package/dist/{chunk-WCGJK4NQ.js → chunk-5FNZ3G6X.js} +10 -10
- package/dist/{chunk-IJH6METJ.js → chunk-5IDQVE4U.js} +5 -5
- package/dist/{chunk-OXSTKCYK.js → chunk-6AEKSOWV.js} +33 -5
- package/dist/chunk-6AEKSOWV.js.map +1 -0
- package/dist/{chunk-2DPJFJFH.js → chunk-6TIVVOVU.js} +10 -10
- package/dist/{chunk-L46MG2GE.js → chunk-6W3QH3QW.js} +17 -14
- package/dist/chunk-6W3QH3QW.js.map +1 -0
- package/dist/{chunk-BUKK5SWA.js → chunk-7LCGFUMB.js} +4 -4
- package/dist/chunk-7VLBZKNB.js +26 -0
- package/dist/chunk-7VLBZKNB.js.map +1 -0
- package/dist/{chunk-CHSRA76M.js → chunk-7XFQSQYS.js} +2 -2
- package/dist/{chunk-A45QXZMU.js → chunk-A3WXV7YU.js} +2 -2
- package/dist/{chunk-LXN737FX.js → chunk-AAANTXCQ.js} +2 -2
- package/dist/{chunk-JNLYHJG7.js → chunk-AF4UIHON.js} +9 -9
- package/dist/{chunk-JGMJZBFV.js → chunk-AGCEJDG2.js} +2 -2
- package/dist/{chunk-I3IPXGFQ.js → chunk-AJJPXUE6.js} +4 -4
- package/dist/{chunk-KAUCVTAW.js → chunk-BJTFLP3J.js} +2 -2
- package/dist/{chunk-QR6W3FSW.js → chunk-BMG3SAXA.js} +3 -3
- package/dist/{chunk-QE2XQM3S.js → chunk-BWF3KNSQ.js} +2 -2
- package/dist/{chunk-BCDECPVO.js → chunk-C3HRYVGD.js} +2 -2
- package/dist/{chunk-RQLTCHUL.js → chunk-C7WDWD35.js} +10 -10
- package/dist/{chunk-YL564QS2.js → chunk-CMOYD5CA.js} +3 -3
- package/dist/{chunk-4J7BNL7B.js → chunk-CPBYKOTC.js} +2 -2
- package/dist/{chunk-PPDNG7WC.js → chunk-CRJX4OMN.js} +2 -2
- package/dist/{chunk-YFJHFEET.js → chunk-D76AHITY.js} +2 -2
- package/dist/{chunk-ZX5N5D6M.js → chunk-DAOMWGO2.js} +4 -4
- package/dist/{chunk-2RQR4TXV.js → chunk-DM6SOROF.js} +2 -2
- package/dist/{chunk-VWJ56T63.js → chunk-E3DL3CAL.js} +2 -2
- package/dist/{chunk-M7LA4QBY.js → chunk-E47UQBYW.js} +3 -3
- package/dist/{chunk-23ZMQONL.js → chunk-ERMT5WQM.js} +2 -2
- package/dist/{chunk-FAKRTS3Y.js → chunk-ESDOQEJM.js} +2 -2
- package/dist/{chunk-VTCNVGM6.js → chunk-FOKTW6GY.js} +2 -2
- package/dist/{chunk-TF5NZGPS.js → chunk-FYZXZUGU.js} +2 -2
- package/dist/{chunk-XTN54NML.js → chunk-G3T6YMBO.js} +4 -4
- package/dist/{chunk-2JNBPOZ2.js → chunk-GDYLZ6FR.js} +3 -3
- package/dist/{chunk-DZPTR2XQ.js → chunk-GFAEKSFR.js} +2 -2
- package/dist/{chunk-GLZWOIU3.js → chunk-HCPVL6SF.js} +3 -3
- package/dist/{chunk-NDMPRT3Z.js → chunk-HDT3DMUF.js} +2 -2
- package/dist/{chunk-BWRX322G.js → chunk-HI53EK3Y.js} +2 -2
- package/dist/{chunk-BFPF5N3H.js → chunk-HXFIILGF.js} +4 -4
- package/dist/chunk-JNA7LFMQ.js +41 -0
- package/dist/chunk-JNA7LFMQ.js.map +1 -0
- package/dist/{chunk-2V664H6O.js → chunk-JS44PBCM.js} +5 -5
- package/dist/{chunk-XKTHJHSQ.js → chunk-JZCBMKR5.js} +3 -3
- package/dist/{chunk-KRACAE73.js → chunk-KMTMMBEB.js} +6 -6
- package/dist/{chunk-FJ264QBP.js → chunk-KNCR7APE.js} +5 -5
- package/dist/{chunk-WUBGMV3G.js → chunk-KRAMCBIW.js} +9 -9
- package/dist/{chunk-DRXTMDAG.js → chunk-LCHFINL4.js} +2 -2
- package/dist/{chunk-PJBYVJKO.js → chunk-LXXIV7Y2.js} +2 -2
- package/dist/{chunk-6CC2YCKG.js → chunk-MCS4LIXP.js} +2 -2
- package/dist/{chunk-2NX7JTST.js → chunk-MPWOAIIH.js} +2 -2
- package/dist/{chunk-CMNL3QSL.js → chunk-N5PISU26.js} +2 -2
- package/dist/{chunk-PKPOO6X5.js → chunk-N6EM7ESC.js} +16 -16
- package/dist/{chunk-FWV2SXSL.js → chunk-NH37SP6V.js} +64 -64
- package/dist/{chunk-MZCTS3ZG.js → chunk-NMOEOQV4.js} +2 -2
- package/dist/{chunk-VECRSYQG.js → chunk-NRGQ5HJO.js} +1 -1
- package/dist/{chunk-3DKKZVXS.js → chunk-NUK5XU3P.js} +5 -5
- package/dist/{chunk-YBKQAVPZ.js → chunk-O4CA53UY.js} +77 -63
- package/dist/chunk-O4CA53UY.js.map +1 -0
- package/dist/{chunk-FPANDKMQ.js → chunk-O7GNSHFF.js} +2 -2
- package/dist/{chunk-5R2IF2KI.js → chunk-OD7ZXCUF.js} +4 -4
- package/dist/{chunk-RT2GN6KQ.js → chunk-PBWJVRKE.js} +2 -2
- package/dist/{chunk-EX27VKU6.js → chunk-PDTNQLPC.js} +2 -2
- package/dist/{chunk-LJCHDFLP.js → chunk-Q6PYCPHO.js} +2 -2
- package/dist/{chunk-XTVH2NAW.js → chunk-QFTNWDJ6.js} +3 -3
- package/dist/{chunk-6CWAY4R7.js → chunk-QZK6HG4J.js} +2 -2
- package/dist/{chunk-Q6UZREJK.js → chunk-RKRX4C4H.js} +2 -2
- package/dist/{chunk-UXAF4RQM.js → chunk-RZDDNH63.js} +2 -2
- package/dist/{chunk-CCTLRRV6.js → chunk-SA6P2KS3.js} +24 -19
- package/dist/chunk-SA6P2KS3.js.map +1 -0
- package/dist/{chunk-QFQ4UG3I.js → chunk-SVJ5JVZO.js} +4 -4
- package/dist/{chunk-CTHYR4QK.js → chunk-SW3WHVKN.js} +2 -2
- package/dist/{chunk-Y57GJDG5.js → chunk-SWRRZNHM.js} +14 -17
- package/dist/chunk-SWRRZNHM.js.map +1 -0
- package/dist/{chunk-B4XSXFKI.js → chunk-V4GPFCJ7.js} +2 -2
- package/dist/{chunk-7AIPWNM7.js → chunk-VCBS5GRQ.js} +2 -2
- package/dist/{chunk-LU57EEJW.js → chunk-VGRBKEBH.js} +3 -3
- package/dist/{chunk-M7L5A2AT.js → chunk-VKUMKVEU.js} +3 -3
- package/dist/{chunk-3EGL7XZO.js → chunk-WIBWONL4.js} +2 -2
- package/dist/{chunk-BGK3SFCC.js → chunk-WJCAAVZX.js} +2 -2
- package/dist/{chunk-BMOJFGVX.js → chunk-WQFHHJWK.js} +2 -2
- package/dist/{chunk-6HVLHIAB.js → chunk-X52SXKVC.js} +2 -2
- package/dist/{chunk-UVYI6VIX.js → chunk-X7Y7WX73.js} +1 -1
- package/dist/{chunk-XM747YJF.js → chunk-XFND63OB.js} +2 -2
- package/dist/{chunk-RUAIO5UA.js → chunk-XSJL62LE.js} +2 -2
- package/dist/{chunk-3SL52JKD.js → chunk-XVZA6VXC.js} +4 -4
- package/dist/{chunk-MXPVNDGO.js → chunk-YB3WN6XA.js} +8 -8
- package/dist/{chunk-JCYEDMOY.js → chunk-YY6PWFUC.js} +2 -2
- package/dist/{chunk-6Q56WDHG.js → chunk-ZLCBLDJO.js} +2 -2
- package/dist/cli.js +104 -102
- package/dist/coding/export-okf-codegraph.js +12 -11
- package/dist/coding/export-okf-codegraph.js.map +1 -1
- package/dist/coding/pre-action-gate.js +2 -2
- package/dist/compounding/engine.js +14 -13
- package/dist/config.js +23 -21
- package/dist/connectors/codex-materialize-runner.js +13 -12
- package/dist/connectors/index.js +13 -12
- package/dist/convergence-refresh.js +15 -14
- package/dist/corpus-watermark.js +15 -14
- package/dist/deep-recall-http-glue.js +11 -10
- package/dist/deep-recall.js +2 -2
- package/dist/embedding-fallback.js +3 -2
- package/dist/embedding-generation-marker.d.ts +16 -0
- package/dist/embedding-generation-marker.d.ts.map +1 -0
- package/dist/embedding-generation-marker.js +19 -0
- package/dist/embedding-generation-marker.js.map +1 -0
- package/dist/embedding-index-storage.d.ts +2 -4
- package/dist/embedding-index-storage.d.ts.map +1 -1
- package/dist/embedding-index-storage.js +2 -1
- package/dist/entity-retrieval.js +14 -13
- package/dist/event-order-recall.js +2 -2
- package/dist/explicit-capture.js +1 -1
- package/dist/external-wiki-access.js +64 -62
- package/dist/extraction-span-materialize.js +2 -2
- package/dist/extraction.js +23 -22
- package/dist/{first-start-migration-A6PHIBUQ.js → first-start-migration-QVD22WWS.js} +4 -4
- package/dist/{graph-edge-decay-3HVLMT6U.js → graph-edge-decay-IU75NV42.js} +6 -6
- package/dist/graph-edge-decay-IU75NV42.js.map +1 -0
- package/dist/graph-jsonl.js +4 -4
- package/dist/graph-snapshot.js +4 -4
- package/dist/graph.js +5 -5
- package/dist/harmonic-retrieval.js +13 -12
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +192 -188
- package/dist/lcm/engine.js +4 -4
- package/dist/lcm/index.js +9 -9
- package/dist/local-llm.js +3 -3
- package/dist/location/index.js +15 -14
- package/dist/maintenance/memory-governance.js +13 -12
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +13 -12
- package/dist/maintenance/rebuild-memory-projection-runner.js +15 -14
- package/dist/maintenance/rebuild-memory-projection.js +14 -13
- package/dist/memory-browse-operations.js +64 -62
- package/dist/memory-browse.js +5 -5
- package/dist/namespaces/migrate.js +14 -13
- package/dist/namespaces/search.js +6 -6
- package/dist/namespaces/storage.js +13 -12
- package/dist/offline-sync-changeset-generations.js +5 -4
- package/dist/offline-sync-embedding-generation.d.ts +12 -2
- package/dist/offline-sync-embedding-generation.d.ts.map +1 -1
- package/dist/offline-sync-embedding-generation.js +6 -3
- package/dist/offline-sync-snapshot-stream.js +8 -7
- package/dist/offline-sync.d.ts.map +1 -1
- package/dist/offline-sync.js +9 -8
- package/dist/operator-doctor-corpus.js +16 -15
- package/dist/operator-toolkit.js +35 -33
- package/dist/orchestration/maintenance.js +26 -25
- package/dist/orchestrator.js +146 -144
- package/dist/qmd.js +2 -2
- package/dist/recall-navigation-operations.js +64 -62
- package/dist/recall-navigation.js +63 -61
- package/dist/recall-state-view-admission.js +5 -5
- package/dist/recall-why-service.js +5 -5
- package/dist/resume-bundles.js +24 -22
- package/dist/search/factory.js +5 -5
- package/dist/search/index.js +9 -9
- package/dist/semantic-consolidation.js +14 -13
- package/dist/semantic-rule-promotion.js +13 -12
- package/dist/semantic-rule-verifier.js +13 -12
- package/dist/source-agent-qualifier.js +16 -15
- package/dist/standup/build.js +13 -12
- package/dist/storage.d.ts.map +1 -1
- package/dist/storage.js +12 -11
- package/dist/summarizer.js +4 -4
- package/dist/support-passport/index.js +66 -64
- package/dist/transfer/backup.js +2 -2
- package/dist/transfer/capsule-export.js +2 -2
- package/dist/transfer/capsule-import.js +2 -2
- package/dist/transfer/export-okf.js +13 -12
- package/dist/verified-recall.js +13 -12
- package/dist/who-knows-cli.js +14 -13
- package/dist/who-knows.js +13 -12
- package/package.json +2 -2
- package/src/access-service-peer-capture-backoff.test.ts +236 -0
- package/src/access-service.ts +6 -2
- package/src/capture-write-lock-scope.test.ts +32 -0
- package/src/capture-write-lock-scope.ts +30 -0
- package/src/embedding-generation-marker.ts +51 -0
- package/src/embedding-index-storage.ts +13 -29
- package/src/index.ts +1 -1
- package/src/offline-sync-embedding-delete-digest.test.ts +58 -0
- package/src/offline-sync-embedding-generation-marker.test.ts +245 -0
- package/src/offline-sync-embedding-generation.ts +47 -2
- package/src/offline-sync-embedding-lock-loss.test.ts +42 -0
- package/src/offline-sync.ts +4 -1
- package/src/storage.ts +2 -1
- package/dist/chunk-CCTLRRV6.js.map +0 -1
- package/dist/chunk-L46MG2GE.js.map +0 -1
- package/dist/chunk-OXSTKCYK.js.map +0 -1
- package/dist/chunk-Y57GJDG5.js.map +0 -1
- package/dist/chunk-YBKQAVPZ.js.map +0 -1
- /package/dist/{capsule-crypto-GWVG7LGC.js.map → capsule-crypto-7FJQINUR.js.map} +0 -0
- /package/dist/{graph-edge-decay-3HVLMT6U.js.map → capture-write-lock-scope.js.map} +0 -0
- /package/dist/{chunk-J6B6FK2X.js.map → chunk-2BY56HE6.js.map} +0 -0
- /package/dist/{chunk-ME5LVISI.js.map → chunk-2J6BIOVN.js.map} +0 -0
- /package/dist/{chunk-EIYJK7OD.js.map → chunk-2M36PR7R.js.map} +0 -0
- /package/dist/{chunk-OWHERGF2.js.map → chunk-2NLLXCJG.js.map} +0 -0
- /package/dist/{chunk-WOPZ2P5Z.js.map → chunk-3INLTKTW.js.map} +0 -0
- /package/dist/{chunk-6P6MZDBL.js.map → chunk-3NL4ALUP.js.map} +0 -0
- /package/dist/{chunk-YUE7J2HZ.js.map → chunk-3NYIXVCT.js.map} +0 -0
- /package/dist/{chunk-B44NFHWI.js.map → chunk-3PVKV3XL.js.map} +0 -0
- /package/dist/{chunk-H7BZBSP6.js.map → chunk-52II7PB6.js.map} +0 -0
- /package/dist/{chunk-5KGYR5VN.js.map → chunk-5DVOBBTV.js.map} +0 -0
- /package/dist/{chunk-WCGJK4NQ.js.map → chunk-5FNZ3G6X.js.map} +0 -0
- /package/dist/{chunk-IJH6METJ.js.map → chunk-5IDQVE4U.js.map} +0 -0
- /package/dist/{chunk-2DPJFJFH.js.map → chunk-6TIVVOVU.js.map} +0 -0
- /package/dist/{chunk-BUKK5SWA.js.map → chunk-7LCGFUMB.js.map} +0 -0
- /package/dist/{chunk-CHSRA76M.js.map → chunk-7XFQSQYS.js.map} +0 -0
- /package/dist/{chunk-A45QXZMU.js.map → chunk-A3WXV7YU.js.map} +0 -0
- /package/dist/{chunk-LXN737FX.js.map → chunk-AAANTXCQ.js.map} +0 -0
- /package/dist/{chunk-JNLYHJG7.js.map → chunk-AF4UIHON.js.map} +0 -0
- /package/dist/{chunk-JGMJZBFV.js.map → chunk-AGCEJDG2.js.map} +0 -0
- /package/dist/{chunk-I3IPXGFQ.js.map → chunk-AJJPXUE6.js.map} +0 -0
- /package/dist/{chunk-KAUCVTAW.js.map → chunk-BJTFLP3J.js.map} +0 -0
- /package/dist/{chunk-QR6W3FSW.js.map → chunk-BMG3SAXA.js.map} +0 -0
- /package/dist/{chunk-QE2XQM3S.js.map → chunk-BWF3KNSQ.js.map} +0 -0
- /package/dist/{chunk-BCDECPVO.js.map → chunk-C3HRYVGD.js.map} +0 -0
- /package/dist/{chunk-RQLTCHUL.js.map → chunk-C7WDWD35.js.map} +0 -0
- /package/dist/{chunk-YL564QS2.js.map → chunk-CMOYD5CA.js.map} +0 -0
- /package/dist/{chunk-4J7BNL7B.js.map → chunk-CPBYKOTC.js.map} +0 -0
- /package/dist/{chunk-PPDNG7WC.js.map → chunk-CRJX4OMN.js.map} +0 -0
- /package/dist/{chunk-YFJHFEET.js.map → chunk-D76AHITY.js.map} +0 -0
- /package/dist/{chunk-ZX5N5D6M.js.map → chunk-DAOMWGO2.js.map} +0 -0
- /package/dist/{chunk-2RQR4TXV.js.map → chunk-DM6SOROF.js.map} +0 -0
- /package/dist/{chunk-VWJ56T63.js.map → chunk-E3DL3CAL.js.map} +0 -0
- /package/dist/{chunk-M7LA4QBY.js.map → chunk-E47UQBYW.js.map} +0 -0
- /package/dist/{chunk-23ZMQONL.js.map → chunk-ERMT5WQM.js.map} +0 -0
- /package/dist/{chunk-FAKRTS3Y.js.map → chunk-ESDOQEJM.js.map} +0 -0
- /package/dist/{chunk-VTCNVGM6.js.map → chunk-FOKTW6GY.js.map} +0 -0
- /package/dist/{chunk-TF5NZGPS.js.map → chunk-FYZXZUGU.js.map} +0 -0
- /package/dist/{chunk-XTN54NML.js.map → chunk-G3T6YMBO.js.map} +0 -0
- /package/dist/{chunk-2JNBPOZ2.js.map → chunk-GDYLZ6FR.js.map} +0 -0
- /package/dist/{chunk-DZPTR2XQ.js.map → chunk-GFAEKSFR.js.map} +0 -0
- /package/dist/{chunk-GLZWOIU3.js.map → chunk-HCPVL6SF.js.map} +0 -0
- /package/dist/{chunk-NDMPRT3Z.js.map → chunk-HDT3DMUF.js.map} +0 -0
- /package/dist/{chunk-BWRX322G.js.map → chunk-HI53EK3Y.js.map} +0 -0
- /package/dist/{chunk-BFPF5N3H.js.map → chunk-HXFIILGF.js.map} +0 -0
- /package/dist/{chunk-2V664H6O.js.map → chunk-JS44PBCM.js.map} +0 -0
- /package/dist/{chunk-XKTHJHSQ.js.map → chunk-JZCBMKR5.js.map} +0 -0
- /package/dist/{chunk-KRACAE73.js.map → chunk-KMTMMBEB.js.map} +0 -0
- /package/dist/{chunk-FJ264QBP.js.map → chunk-KNCR7APE.js.map} +0 -0
- /package/dist/{chunk-WUBGMV3G.js.map → chunk-KRAMCBIW.js.map} +0 -0
- /package/dist/{chunk-DRXTMDAG.js.map → chunk-LCHFINL4.js.map} +0 -0
- /package/dist/{chunk-PJBYVJKO.js.map → chunk-LXXIV7Y2.js.map} +0 -0
- /package/dist/{chunk-6CC2YCKG.js.map → chunk-MCS4LIXP.js.map} +0 -0
- /package/dist/{chunk-2NX7JTST.js.map → chunk-MPWOAIIH.js.map} +0 -0
- /package/dist/{chunk-CMNL3QSL.js.map → chunk-N5PISU26.js.map} +0 -0
- /package/dist/{chunk-PKPOO6X5.js.map → chunk-N6EM7ESC.js.map} +0 -0
- /package/dist/{chunk-FWV2SXSL.js.map → chunk-NH37SP6V.js.map} +0 -0
- /package/dist/{chunk-MZCTS3ZG.js.map → chunk-NMOEOQV4.js.map} +0 -0
- /package/dist/{chunk-VECRSYQG.js.map → chunk-NRGQ5HJO.js.map} +0 -0
- /package/dist/{chunk-3DKKZVXS.js.map → chunk-NUK5XU3P.js.map} +0 -0
- /package/dist/{chunk-FPANDKMQ.js.map → chunk-O7GNSHFF.js.map} +0 -0
- /package/dist/{chunk-5R2IF2KI.js.map → chunk-OD7ZXCUF.js.map} +0 -0
- /package/dist/{chunk-RT2GN6KQ.js.map → chunk-PBWJVRKE.js.map} +0 -0
- /package/dist/{chunk-EX27VKU6.js.map → chunk-PDTNQLPC.js.map} +0 -0
- /package/dist/{chunk-LJCHDFLP.js.map → chunk-Q6PYCPHO.js.map} +0 -0
- /package/dist/{chunk-XTVH2NAW.js.map → chunk-QFTNWDJ6.js.map} +0 -0
- /package/dist/{chunk-6CWAY4R7.js.map → chunk-QZK6HG4J.js.map} +0 -0
- /package/dist/{chunk-Q6UZREJK.js.map → chunk-RKRX4C4H.js.map} +0 -0
- /package/dist/{chunk-UXAF4RQM.js.map → chunk-RZDDNH63.js.map} +0 -0
- /package/dist/{chunk-QFQ4UG3I.js.map → chunk-SVJ5JVZO.js.map} +0 -0
- /package/dist/{chunk-CTHYR4QK.js.map → chunk-SW3WHVKN.js.map} +0 -0
- /package/dist/{chunk-B4XSXFKI.js.map → chunk-V4GPFCJ7.js.map} +0 -0
- /package/dist/{chunk-7AIPWNM7.js.map → chunk-VCBS5GRQ.js.map} +0 -0
- /package/dist/{chunk-LU57EEJW.js.map → chunk-VGRBKEBH.js.map} +0 -0
- /package/dist/{chunk-M7L5A2AT.js.map → chunk-VKUMKVEU.js.map} +0 -0
- /package/dist/{chunk-3EGL7XZO.js.map → chunk-WIBWONL4.js.map} +0 -0
- /package/dist/{chunk-BGK3SFCC.js.map → chunk-WJCAAVZX.js.map} +0 -0
- /package/dist/{chunk-BMOJFGVX.js.map → chunk-WQFHHJWK.js.map} +0 -0
- /package/dist/{chunk-6HVLHIAB.js.map → chunk-X52SXKVC.js.map} +0 -0
- /package/dist/{chunk-UVYI6VIX.js.map → chunk-X7Y7WX73.js.map} +0 -0
- /package/dist/{chunk-XM747YJF.js.map → chunk-XFND63OB.js.map} +0 -0
- /package/dist/{chunk-RUAIO5UA.js.map → chunk-XSJL62LE.js.map} +0 -0
- /package/dist/{chunk-3SL52JKD.js.map → chunk-XVZA6VXC.js.map} +0 -0
- /package/dist/{chunk-MXPVNDGO.js.map → chunk-YB3WN6XA.js.map} +0 -0
- /package/dist/{chunk-JCYEDMOY.js.map → chunk-YY6PWFUC.js.map} +0 -0
- /package/dist/{chunk-6Q56WDHG.js.map → chunk-ZLCBLDJO.js.map} +0 -0
- /package/dist/{first-start-migration-A6PHIBUQ.js.map → first-start-migration-QVD22WWS.js.map} +0 -0
|
@@ -18,6 +18,13 @@ import { randomUUID } from "node:crypto";
|
|
|
18
18
|
import { constants as bufferConstants } from "node:buffer";
|
|
19
19
|
import { lstat, mkdir, mkdtemp, realpath, readFile, readdir, rename, rm, stat, writeFile } from "node:fs/promises";
|
|
20
20
|
import { log } from "./logger.js";
|
|
21
|
+
import {
|
|
22
|
+
GENERATION_MARKER_IN_FLIGHT,
|
|
23
|
+
beginGenerationMarker as writeGenerationMarkerInFlight,
|
|
24
|
+
completeGenerationMarker as writeGenerationMarkerComplete,
|
|
25
|
+
generationMarkerPath,
|
|
26
|
+
writeGenerationMarkerRaw,
|
|
27
|
+
} from "./embedding-generation-marker.js";
|
|
21
28
|
import { readEnvVar } from "./runtime/env.js";
|
|
22
29
|
import type { EmbeddingGenerationFence } from "./embedding-generation-lock.js";
|
|
23
30
|
|
|
@@ -166,21 +173,6 @@ function replacementBackupPath(shardDir: string): string {
|
|
|
166
173
|
return path.join(path.dirname(shardDir), "embeddings.pre-replace.tmp");
|
|
167
174
|
}
|
|
168
175
|
|
|
169
|
-
/**
|
|
170
|
-
* Warm-cache generation marker (`<stateDir>/embeddings.generation`).
|
|
171
|
-
* Two-phase publication barrier: in-flight before a publication's first
|
|
172
|
-
* destructive write, fresh unique value after its last write. A stamp is
|
|
173
|
-
* therefore stable only for a completed generation — readers can never
|
|
174
|
-
* cache a stamp that stays stable while generation bytes change, and an
|
|
175
|
-
* in-flight stamp never repeats, so probes cached during a failed
|
|
176
|
-
* completion keep revalidating (PR #3176 codex P2). Plain fs like the
|
|
177
|
-
* status file.
|
|
178
|
-
*/
|
|
179
|
-
const GENERATION_MARKER_BASENAME = "embeddings.generation";
|
|
180
|
-
|
|
181
|
-
/** Reserved in-flight marker value; never equals a completion marker. */
|
|
182
|
-
const GENERATION_MARKER_IN_FLIGHT = "in-flight";
|
|
183
|
-
|
|
184
176
|
/**
|
|
185
177
|
* Stable FNV-1a shard assignment: an entry always hashes to the same shard,
|
|
186
178
|
* so an update or removal never needs a manifest to find its file.
|
|
@@ -535,34 +527,26 @@ export class EmbeddingIndexFileStore {
|
|
|
535
527
|
|
|
536
528
|
/** Barrier first half: strict, so bytes can never change under a stable stamp. */
|
|
537
529
|
private async beginGenerationMarker(): Promise<void> {
|
|
538
|
-
await this.
|
|
530
|
+
await writeGenerationMarkerInFlight(generationMarkerPath(path.dirname(this.shardDir)));
|
|
539
531
|
}
|
|
540
532
|
|
|
541
533
|
/**
|
|
542
|
-
* Barrier second half: written after the last content write. Best-effort
|
|
543
|
-
*
|
|
544
|
-
* never repeats, so cached probes revalidate on every search until a
|
|
545
|
-
* publication completes.
|
|
534
|
+
* Barrier second half: written after the last content write. Best-effort;
|
|
535
|
+
* the in-flight fallback semantics live in the shared marker module.
|
|
546
536
|
*/
|
|
547
537
|
private async completeGenerationMarker(): Promise<void> {
|
|
548
|
-
await
|
|
549
|
-
log.warn(`embedding index: could not write generation completion marker: ${err instanceof Error ? err.message : String(err)}`);
|
|
550
|
-
});
|
|
538
|
+
await writeGenerationMarkerComplete(generationMarkerPath(path.dirname(this.shardDir)));
|
|
551
539
|
}
|
|
552
540
|
|
|
553
541
|
private async writeGenerationMarker(value: string): Promise<void> {
|
|
554
|
-
await this.
|
|
555
|
-
path.join(path.dirname(this.shardDir), GENERATION_MARKER_BASENAME),
|
|
556
|
-
value,
|
|
557
|
-
{ bypassSecureIo: true },
|
|
558
|
-
);
|
|
542
|
+
await writeGenerationMarkerRaw(generationMarkerPath(path.dirname(this.shardDir)), value);
|
|
559
543
|
}
|
|
560
544
|
|
|
561
545
|
/** Marker value, or null when absent/unreadable (pre-marker stores keep the stat stamp). */
|
|
562
546
|
private async readGenerationMarker(): Promise<string | null> {
|
|
563
547
|
try {
|
|
564
548
|
const raw = await readFile(
|
|
565
|
-
|
|
549
|
+
generationMarkerPath(path.dirname(this.shardDir)),
|
|
566
550
|
"utf-8",
|
|
567
551
|
);
|
|
568
552
|
const value = raw.trim();
|
package/src/index.ts
CHANGED
|
@@ -880,7 +880,7 @@ export {
|
|
|
880
880
|
type OfflineSyncSnapshot,
|
|
881
881
|
type OfflineSyncState,
|
|
882
882
|
} from "./offline-sync.js";
|
|
883
|
-
export { embeddingGenerationMembership, divergedEmbeddingGenerationDeferrals } from "./offline-sync-embedding-generation.js";
|
|
883
|
+
export { advanceEmbeddingGenerationMarker, embeddingGenerationMembership, divergedEmbeddingGenerationDeferrals } from "./offline-sync-embedding-generation.js";
|
|
884
884
|
export { EmbeddingIndexCapacityError, EmbeddingIndexFileStore, EmbeddingIndexStorageError, parseEmbeddingIndexDocument, serializeEmbeddingShard, shardEntriesForIndex, shardIndexOf, validateShardMembership } from "./embedding-index-storage.js";
|
|
885
885
|
|
|
886
886
|
// ---------------------------------------------------------------------------
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { createHash } from "node:crypto";
|
|
3
|
+
import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import test from "node:test";
|
|
7
|
+
import { withEmbeddingGenerationLock } from "./embedding-generation-lock.js";
|
|
8
|
+
import { advanceEmbeddingGenerationMarker, embeddingGenerationMembership } from "./offline-sync-embedding-generation.js";
|
|
9
|
+
|
|
10
|
+
// Issue #3182 review: a peer can publish a newer shard while converge waits
|
|
11
|
+
// for the generation lock; the stale pre-lock digest must not delete it.
|
|
12
|
+
test("a stale-digest deletion is rejected under the generation lock", async (t) => {
|
|
13
|
+
const root = await mkdtemp(path.join(os.tmpdir(), "remnic-3182-delete-digest-"));
|
|
14
|
+
t.after(() => rm(root, { recursive: true, force: true }));
|
|
15
|
+
const stateDir = path.join(root, "state");
|
|
16
|
+
const relative = "state/embeddings/shard-0000.json";
|
|
17
|
+
const target = path.join(root, relative);
|
|
18
|
+
const marker = path.join(stateDir, "embeddings.generation");
|
|
19
|
+
await mkdir(path.dirname(target), { recursive: true });
|
|
20
|
+
await writeFile(target, "older shard");
|
|
21
|
+
await writeFile(marker, "peer completed generation");
|
|
22
|
+
const staleSha256 = createHash("sha256").update("older shard").digest("hex");
|
|
23
|
+
|
|
24
|
+
let signalHeld: () => void;
|
|
25
|
+
const held = new Promise<void>((resolve) => { signalHeld = resolve; });
|
|
26
|
+
let releaseWriter: (() => void) | undefined;
|
|
27
|
+
const release = new Promise<void>((resolve) => { releaseWriter = resolve; });
|
|
28
|
+
assert.ok(releaseWriter);
|
|
29
|
+
const writer = withEmbeddingGenerationLock(stateDir, async () => {
|
|
30
|
+
signalHeld();
|
|
31
|
+
await release;
|
|
32
|
+
await writeFile(target, "newer peer shard");
|
|
33
|
+
});
|
|
34
|
+
await held;
|
|
35
|
+
|
|
36
|
+
let validated = false;
|
|
37
|
+
const deletion = advanceEmbeddingGenerationMarker(
|
|
38
|
+
root,
|
|
39
|
+
embeddingGenerationMembership(relative),
|
|
40
|
+
async () => {
|
|
41
|
+
assert.fail("deletion must never run when the digest is stale");
|
|
42
|
+
},
|
|
43
|
+
async () => {
|
|
44
|
+
validated = true;
|
|
45
|
+
if ((await readFile(target, "utf8")) !== "older shard") {
|
|
46
|
+
throw new Error("converge file changed before deletion");
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
);
|
|
50
|
+
const rejected = assert.rejects(deletion, /changed before deletion/);
|
|
51
|
+
releaseWriter();
|
|
52
|
+
await writer;
|
|
53
|
+
await rejected;
|
|
54
|
+
|
|
55
|
+
assert.equal(validated, true, "the digest precondition must run inside the lock");
|
|
56
|
+
assert.equal(await readFile(target, "utf8"), "newer peer shard");
|
|
57
|
+
assert.equal(await readFile(marker, "utf8"), "peer completed generation");
|
|
58
|
+
});
|
|
@@ -0,0 +1,245 @@
|
|
|
1
|
+
// Issue #3182: the chunk-level pull path must advance the node-local
|
|
2
|
+
// generation marker around direct shard writes; partial chunks and no-ops
|
|
3
|
+
// must not disturb a valid completed stamp. Stat pinning emulates a
|
|
4
|
+
// coarse-tick filesystem so only a marker advance can move the stamp.
|
|
5
|
+
import assert from "node:assert/strict";
|
|
6
|
+
import { createHash } from "node:crypto";
|
|
7
|
+
import { lstat, mkdir, mkdtemp, readFile, rm, utimes, writeFile } from "node:fs/promises";
|
|
8
|
+
import os from "node:os";
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import test from "node:test";
|
|
11
|
+
|
|
12
|
+
import {
|
|
13
|
+
EmbeddingIndexFileStore,
|
|
14
|
+
type EmbeddingIndexEntry,
|
|
15
|
+
type EmbeddingIndexFile,
|
|
16
|
+
shardIndexOf,
|
|
17
|
+
} from "./embedding-index-storage.js";
|
|
18
|
+
import { applyOfflineSyncFileContentChunk } from "./offline-sync.js";
|
|
19
|
+
|
|
20
|
+
// Pin the shard dir to one whole-millisecond tick, emulating a coarse-tick
|
|
21
|
+
// filesystem where the pre- and post-write directory stats alias (same
|
|
22
|
+
// technique as embedding-fallback-cross-process.test.ts).
|
|
23
|
+
const TICK_SEC = 1_700_000_000.5;
|
|
24
|
+
const MODEL = "text-embedding-3-small";
|
|
25
|
+
|
|
26
|
+
function shardRelFor(id: string): string {
|
|
27
|
+
return `state/embeddings/shard-${String(shardIndexOf(id, 64)).padStart(4, "0")}.json`;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function indexFile(entries: Record<string, EmbeddingIndexEntry>): EmbeddingIndexFile {
|
|
31
|
+
return { version: 1, provider: "openai", model: MODEL, entries };
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function shardDirOf(memoryDir: string): string {
|
|
35
|
+
return path.join(memoryDir, "state", "embeddings");
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function seedPublishedGeneration(memoryDir: string): Promise<EmbeddingIndexFileStore> {
|
|
39
|
+
const stateDir = path.join(memoryDir, "state");
|
|
40
|
+
const store = new EmbeddingIndexFileStore(
|
|
41
|
+
path.join(stateDir, "embeddings.json"),
|
|
42
|
+
path.join(stateDir, "embeddings"),
|
|
43
|
+
path.join(stateDir, "embedding-fallback-status.json"),
|
|
44
|
+
);
|
|
45
|
+
// Same fixture shape as embedding-index-storage.test.ts: a published
|
|
46
|
+
// sharded generation with one entry, completed through the store's
|
|
47
|
+
// two-phase marker.
|
|
48
|
+
await mkdir(path.join(stateDir, "embeddings"), { recursive: true });
|
|
49
|
+
await writeFile(
|
|
50
|
+
path.join(stateDir, "embeddings", shardRelFor("mem-old").split("/").pop()!),
|
|
51
|
+
JSON.stringify(indexFile({ "mem-old": { vector: [1, 0], path: "facts/old.md" } })),
|
|
52
|
+
);
|
|
53
|
+
await store.persist(indexFile({ "mem-old": { vector: [1, 0], path: "facts/old.md" } }), {
|
|
54
|
+
touchedIds: ["mem-old"],
|
|
55
|
+
memoryId: "mem-old",
|
|
56
|
+
});
|
|
57
|
+
return store;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
async function readMarker(memoryDir: string): Promise<string> {
|
|
61
|
+
return (await readFile(path.join(memoryDir, "state", "embeddings.generation"), "utf-8")).trim();
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
async function pinShardDirStat(memoryDir: string): Promise<number> {
|
|
65
|
+
const shardDir = shardDirOf(memoryDir);
|
|
66
|
+
await utimes(shardDir, TICK_SEC, TICK_SEC);
|
|
67
|
+
return (await lstat(shardDir)).ino;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
async function assertInoUnchanged(memoryDir: string, inoBefore: number): Promise<void> {
|
|
71
|
+
assert.equal(
|
|
72
|
+
(await lstat(shardDirOf(memoryDir))).ino,
|
|
73
|
+
inoBefore,
|
|
74
|
+
"precondition: the pull rewrote the shard in place (no directory swap)",
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
test("a completed offline-sync chunk write to a published shard moves the generation stamp", async () => {
|
|
79
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-3182-chunk-"));
|
|
80
|
+
try {
|
|
81
|
+
const store = await seedPublishedGeneration(memoryDir);
|
|
82
|
+
const shardRel = shardRelFor("mem-old");
|
|
83
|
+
const updated = Buffer.from(
|
|
84
|
+
JSON.stringify(indexFile({ "mem-old": { vector: [0, 1], path: "facts/old.md" } })),
|
|
85
|
+
"utf-8",
|
|
86
|
+
);
|
|
87
|
+
|
|
88
|
+
const inoBefore = await pinShardDirStat(memoryDir);
|
|
89
|
+
const markerBefore = await readMarker(memoryDir);
|
|
90
|
+
const stampBefore = await store.identityStamp();
|
|
91
|
+
assert.equal(await store.identityStamp(), stampBefore, "precondition: the completed stamp is stable");
|
|
92
|
+
|
|
93
|
+
// The chunk-level pull path used by converge pulls and CLI hydrate:
|
|
94
|
+
// no IO hooks, plain fs final write (writeSafeFileFromUpload).
|
|
95
|
+
const result = await applyOfflineSyncFileContentChunk({
|
|
96
|
+
root: memoryDir,
|
|
97
|
+
sourceId: "remote",
|
|
98
|
+
path: shardRel,
|
|
99
|
+
sha256: createHash("sha256").update(updated).digest("hex"),
|
|
100
|
+
bytes: updated.length,
|
|
101
|
+
mtimeMs: 1000,
|
|
102
|
+
offset: 0,
|
|
103
|
+
content: updated,
|
|
104
|
+
});
|
|
105
|
+
assert.equal(result.applied, true, "precondition: the shard write applied");
|
|
106
|
+
|
|
107
|
+
// Force the exact stat the stamp reads so only a marker advance can move it.
|
|
108
|
+
await utimes(shardDirOf(memoryDir), TICK_SEC, TICK_SEC);
|
|
109
|
+
await assertInoUnchanged(memoryDir, inoBefore);
|
|
110
|
+
|
|
111
|
+
assert.notEqual(
|
|
112
|
+
await readMarker(memoryDir),
|
|
113
|
+
markerBefore,
|
|
114
|
+
"a peer pull that published new generation bytes must advance the generation marker",
|
|
115
|
+
);
|
|
116
|
+
const stampAfter = await store.identityStamp();
|
|
117
|
+
assert.notEqual(
|
|
118
|
+
stampAfter,
|
|
119
|
+
stampBefore,
|
|
120
|
+
"a peer pull that changed generation bytes must move the warm-cache stamp even when the directory stat aliases",
|
|
121
|
+
);
|
|
122
|
+
} finally {
|
|
123
|
+
await rm(memoryDir, { recursive: true, force: true }).catch(() => undefined);
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("a partial chunk (staging only) must not invalidate a valid completed stamp", async () => {
|
|
128
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-3182-partial-"));
|
|
129
|
+
try {
|
|
130
|
+
const store = await seedPublishedGeneration(memoryDir);
|
|
131
|
+
const shardRel = shardRelFor("mem-old");
|
|
132
|
+
const onDiskBefore = await readFile(path.join(memoryDir, ...shardRel.split("/")));
|
|
133
|
+
|
|
134
|
+
const updated = Buffer.from(
|
|
135
|
+
JSON.stringify(indexFile({ "mem-old": { vector: [0, 1], path: "facts/old.md" } })),
|
|
136
|
+
"utf-8",
|
|
137
|
+
);
|
|
138
|
+
const inoBefore = await pinShardDirStat(memoryDir);
|
|
139
|
+
const markerBefore = await readMarker(memoryDir);
|
|
140
|
+
const stampBefore = await store.identityStamp();
|
|
141
|
+
|
|
142
|
+
const result = await applyOfflineSyncFileContentChunk({
|
|
143
|
+
root: memoryDir,
|
|
144
|
+
sourceId: "remote",
|
|
145
|
+
path: shardRel,
|
|
146
|
+
sha256: createHash("sha256").update(updated).digest("hex"),
|
|
147
|
+
bytes: updated.length,
|
|
148
|
+
mtimeMs: 1000,
|
|
149
|
+
offset: 0,
|
|
150
|
+
content: updated.subarray(0, Math.floor(updated.length / 2)),
|
|
151
|
+
});
|
|
152
|
+
assert.equal(result.done, false, "precondition: the chunk is partial");
|
|
153
|
+
assert.deepEqual(
|
|
154
|
+
await readFile(path.join(memoryDir, ...shardRel.split("/"))),
|
|
155
|
+
onDiskBefore,
|
|
156
|
+
"precondition: a partial chunk spools to staging, never to the generation",
|
|
157
|
+
);
|
|
158
|
+
|
|
159
|
+
await utimes(shardDirOf(memoryDir), TICK_SEC, TICK_SEC);
|
|
160
|
+
await assertInoUnchanged(memoryDir, inoBefore);
|
|
161
|
+
|
|
162
|
+
assert.equal(await readMarker(memoryDir), markerBefore, "a staging-only write must not touch the generation marker");
|
|
163
|
+
assert.equal(
|
|
164
|
+
await store.identityStamp(),
|
|
165
|
+
stampBefore,
|
|
166
|
+
"no generation bytes changed: the completed stamp must stay valid so warm caches keep working",
|
|
167
|
+
);
|
|
168
|
+
} finally {
|
|
169
|
+
await rm(memoryDir, { recursive: true, force: true }).catch(() => undefined);
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
|
|
173
|
+
test("a skipped no-op chunk (content already identical) must not invalidate the stamp", async () => {
|
|
174
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-3182-noop-"));
|
|
175
|
+
try {
|
|
176
|
+
const store = await seedPublishedGeneration(memoryDir);
|
|
177
|
+
const shardRel = shardRelFor("mem-old");
|
|
178
|
+
const onDisk = await readFile(path.join(memoryDir, ...shardRel.split("/")));
|
|
179
|
+
|
|
180
|
+
const inoBefore = await pinShardDirStat(memoryDir);
|
|
181
|
+
const markerBefore = await readMarker(memoryDir);
|
|
182
|
+
const stampBefore = await store.identityStamp();
|
|
183
|
+
|
|
184
|
+
const result = await applyOfflineSyncFileContentChunk({
|
|
185
|
+
root: memoryDir,
|
|
186
|
+
sourceId: "remote",
|
|
187
|
+
path: shardRel,
|
|
188
|
+
sha256: createHash("sha256").update(onDisk).digest("hex"),
|
|
189
|
+
bytes: onDisk.length,
|
|
190
|
+
mtimeMs: 1000,
|
|
191
|
+
offset: 0,
|
|
192
|
+
content: onDisk,
|
|
193
|
+
});
|
|
194
|
+
assert.equal(result.skipped, true, "precondition: identical content is a no-op");
|
|
195
|
+
|
|
196
|
+
await utimes(shardDirOf(memoryDir), TICK_SEC, TICK_SEC);
|
|
197
|
+
await assertInoUnchanged(memoryDir, inoBefore);
|
|
198
|
+
|
|
199
|
+
assert.equal(await readMarker(memoryDir), markerBefore, "a no-op must not touch the generation marker");
|
|
200
|
+
assert.equal(await store.identityStamp(), stampBefore, "no generation bytes changed: the stamp stays valid");
|
|
201
|
+
} finally {
|
|
202
|
+
await rm(memoryDir, { recursive: true, force: true }).catch(() => undefined);
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
test("a failed direct shard write leaves the marker in-flight: the stamp never repeats", async () => {
|
|
207
|
+
const memoryDir = await mkdtemp(path.join(os.tmpdir(), "remnic-3182-failed-"));
|
|
208
|
+
try {
|
|
209
|
+
const store = await seedPublishedGeneration(memoryDir);
|
|
210
|
+
const shardRel = shardRelFor("mem-old");
|
|
211
|
+
const updated = Buffer.from(
|
|
212
|
+
JSON.stringify(indexFile({ "mem-old": { vector: [0, 1], path: "facts/old.md" } })),
|
|
213
|
+
"utf-8",
|
|
214
|
+
);
|
|
215
|
+
|
|
216
|
+
const stampBefore = await store.identityStamp();
|
|
217
|
+
|
|
218
|
+
// Force the final destructive write to fail: the publish rename cannot
|
|
219
|
+
// replace the shard file with a directory in its place.
|
|
220
|
+
const shardAbs = path.join(memoryDir, ...shardRel.split("/"));
|
|
221
|
+
await rm(shardAbs);
|
|
222
|
+
await mkdir(shardAbs);
|
|
223
|
+
|
|
224
|
+
await assert.rejects(
|
|
225
|
+
applyOfflineSyncFileContentChunk({
|
|
226
|
+
root: memoryDir,
|
|
227
|
+
sourceId: "remote",
|
|
228
|
+
path: shardRel,
|
|
229
|
+
sha256: createHash("sha256").update(updated).digest("hex"),
|
|
230
|
+
bytes: updated.length,
|
|
231
|
+
mtimeMs: 1000,
|
|
232
|
+
offset: 0,
|
|
233
|
+
content: updated,
|
|
234
|
+
}),
|
|
235
|
+
);
|
|
236
|
+
|
|
237
|
+
assert.equal(await readMarker(memoryDir), "in-flight", "a failed publication must leave the in-flight barrier");
|
|
238
|
+
const probeA = await store.identityStamp();
|
|
239
|
+
const probeB = await store.identityStamp();
|
|
240
|
+
assert.notEqual(probeA, probeB, "an in-flight stamp never repeats, so cached probes keep revalidating");
|
|
241
|
+
assert.notEqual(probeA, stampBefore);
|
|
242
|
+
} finally {
|
|
243
|
+
await rm(memoryDir, { recursive: true, force: true }).catch(() => undefined);
|
|
244
|
+
}
|
|
245
|
+
});
|
|
@@ -45,7 +45,12 @@ import type {
|
|
|
45
45
|
OfflineSyncFileStagingWriteTarget,
|
|
46
46
|
OfflineSyncFileState,
|
|
47
47
|
} from "./offline-sync.js";
|
|
48
|
-
import { type SafeArchiveRoot, resolveSafeArchiveTarget } from "./transfer/fs-utils.js";
|
|
48
|
+
import { type SafeArchiveRoot, prepareSafeArchiveRoot, resolveSafeArchiveTarget } from "./transfer/fs-utils.js";
|
|
49
|
+
import {
|
|
50
|
+
GENERATION_MARKER_BASENAME,
|
|
51
|
+
beginGenerationMarker,
|
|
52
|
+
completeGenerationMarker,
|
|
53
|
+
} from "./embedding-generation-marker.js";
|
|
49
54
|
|
|
50
55
|
const EMBEDDING_MARKER_BASENAME = "embeddings.json";
|
|
51
56
|
export const EMBEDDING_SHARD_DIR_BASENAME = "embeddings";
|
|
@@ -95,7 +100,12 @@ export function embeddingMarkerDirOf(relPath: string): string | null {
|
|
|
95
100
|
* `.../embeddings` directory, the legacy marker belongs to the sibling
|
|
96
101
|
* `.../embeddings` directory of its state dir.
|
|
97
102
|
*/
|
|
98
|
-
export
|
|
103
|
+
export interface EmbeddingGenerationMembership {
|
|
104
|
+
kind: "shard" | "marker";
|
|
105
|
+
shardDir: string;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
export function embeddingGenerationMembership(relPath: string): EmbeddingGenerationMembership | null {
|
|
99
109
|
const shardDir = embeddingShardDirOf(relPath);
|
|
100
110
|
if (shardDir) return { kind: "shard", shardDir };
|
|
101
111
|
const markerDir = embeddingMarkerDirOf(relPath);
|
|
@@ -103,6 +113,41 @@ export function embeddingGenerationMembership(relPath: string): { kind: "shard"
|
|
|
103
113
|
return null;
|
|
104
114
|
}
|
|
105
115
|
|
|
116
|
+
/**
|
|
117
|
+
* Advance the node-local generation marker around one out-of-band destructive
|
|
118
|
+
* write or delete of a generation member (issue #3182); the fence reasserts
|
|
119
|
+
* lock ownership so a stale-broken lock never publishes a stable marker over
|
|
120
|
+
* a peer's write. Callers must not already hold the generation lock — the
|
|
121
|
+
* snapshot removal deletes members under it, which is why converge hooks its
|
|
122
|
+
* own call sites, not the shared IO adapter.
|
|
123
|
+
*/
|
|
124
|
+
export async function advanceEmbeddingGenerationMarker<T>(
|
|
125
|
+
rootAbs: string,
|
|
126
|
+
membership: EmbeddingGenerationMembership | null,
|
|
127
|
+
write: () => Promise<T>,
|
|
128
|
+
validate?: () => Promise<void>,
|
|
129
|
+
): Promise<T> {
|
|
130
|
+
if (!membership) {
|
|
131
|
+
await validate?.();
|
|
132
|
+
return write();
|
|
133
|
+
}
|
|
134
|
+
const root = await prepareSafeArchiveRoot(rootAbs, "advanceEmbeddingGenerationMarker", "root");
|
|
135
|
+
const stateDirRel = membership.shardDir.slice(0, -`/${EMBEDDING_SHARD_DIR_BASENAME}`.length);
|
|
136
|
+
const stateDirAbs = await resolveSafeArchiveTarget(root, stateDirRel);
|
|
137
|
+
const lockPath = embeddingGenerationLockPath(stateDirAbs);
|
|
138
|
+
return await withEmbeddingGenerationLock(stateDirAbs, async (section) => {
|
|
139
|
+
const markerAbs = await resolveSafeArchiveTarget(root, `${stateDirRel}/${GENERATION_MARKER_BASENAME}`);
|
|
140
|
+
await validate?.();
|
|
141
|
+
await assertEmbeddingGenerationLockHeld(lockPath, section);
|
|
142
|
+
await beginGenerationMarker(markerAbs);
|
|
143
|
+
await assertEmbeddingGenerationLockHeld(lockPath, section);
|
|
144
|
+
const result = await write();
|
|
145
|
+
await assertEmbeddingGenerationLockHeld(lockPath, section);
|
|
146
|
+
await completeGenerationMarker(markerAbs);
|
|
147
|
+
return result;
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
|
|
106
151
|
export interface IncomingEmbeddingGenerations {
|
|
107
152
|
/** Generation dirs with at least one incoming shard. */
|
|
108
153
|
shardDirs: Set<string>;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import fs from "node:fs/promises";
|
|
3
|
+
import { syncBuiltinESMExports } from "node:module";
|
|
4
|
+
import os from "node:os";
|
|
5
|
+
import path from "node:path";
|
|
6
|
+
import test from "node:test";
|
|
7
|
+
import { EmbeddingGenerationLockLostError, embeddingGenerationLockPath } from "./embedding-generation-lock.js";
|
|
8
|
+
import { advanceEmbeddingGenerationMarker, embeddingGenerationMembership } from "./offline-sync-embedding-generation.js";
|
|
9
|
+
|
|
10
|
+
// Lose ownership exactly after the in-flight rename, before any content write.
|
|
11
|
+
test("a direct embedding mutation rechecks ownership after its in-flight marker", async (t) => {
|
|
12
|
+
const root = await fs.mkdtemp(path.join(os.tmpdir(), "remnic-embedding-fence-"));
|
|
13
|
+
const stateDir = path.join(root, "state");
|
|
14
|
+
const relative = "state/embeddings/shard-0000.json";
|
|
15
|
+
const target = path.join(root, relative);
|
|
16
|
+
const marker = path.join(stateDir, "embeddings.generation");
|
|
17
|
+
await fs.mkdir(path.dirname(target), { recursive: true });
|
|
18
|
+
await fs.writeFile(target, "winning generation");
|
|
19
|
+
await fs.writeFile(marker, "completed generation");
|
|
20
|
+
const rename = fs.rename;
|
|
21
|
+
t.mock.method(fs, "rename", async (source: Parameters<typeof rename>[0], destination: Parameters<typeof rename>[1]) => {
|
|
22
|
+
await rename(source, destination);
|
|
23
|
+
if (String(destination) === marker && await fs.readFile(marker, "utf8") === "in-flight") {
|
|
24
|
+
await fs.unlink(embeddingGenerationLockPath(stateDir));
|
|
25
|
+
await fs.writeFile(marker, "peer completed generation");
|
|
26
|
+
}
|
|
27
|
+
});
|
|
28
|
+
syncBuiltinESMExports();
|
|
29
|
+
t.after(() => { t.mock.restoreAll(); syncBuiltinESMExports(); });
|
|
30
|
+
t.after(() => fs.rm(root, { recursive: true, force: true }));
|
|
31
|
+
let writes = 0;
|
|
32
|
+
await assert.rejects(
|
|
33
|
+
advanceEmbeddingGenerationMarker(root, embeddingGenerationMembership(relative), async () => {
|
|
34
|
+
writes++;
|
|
35
|
+
await fs.writeFile(target, "stale owner's generation");
|
|
36
|
+
}),
|
|
37
|
+
EmbeddingGenerationLockLostError,
|
|
38
|
+
);
|
|
39
|
+
assert.equal(writes, 0, "a lost owner must stop before touching generation bytes");
|
|
40
|
+
assert.equal(await fs.readFile(target, "utf8"), "winning generation");
|
|
41
|
+
assert.equal(await fs.readFile(marker, "utf8"), "peer completed generation");
|
|
42
|
+
});
|
package/src/offline-sync.ts
CHANGED
|
@@ -32,6 +32,7 @@ import { isMacOsReplicationJunkBasename } from "./utils/macos-replication-junk.j
|
|
|
32
32
|
import {
|
|
33
33
|
applyIncomingEmbeddingGenerations,
|
|
34
34
|
assertEmbeddingGenerationStillIncluded,
|
|
35
|
+
advanceEmbeddingGenerationMarker,
|
|
35
36
|
computeOmittedEmbeddingGenerationPaths,
|
|
36
37
|
divergedEmbeddingGenerationDeferrals,
|
|
37
38
|
embeddingGenerationMembership,
|
|
@@ -2466,7 +2467,9 @@ export async function applyOfflineSyncFileContentChunk(options: {
|
|
|
2466
2467
|
};
|
|
2467
2468
|
}
|
|
2468
2469
|
|
|
2469
|
-
await
|
|
2470
|
+
await advanceEmbeddingGenerationMarker(root.abs, embeddingGenerationMembership(relPath), () =>
|
|
2471
|
+
writeSafeFileFromUpload(root, relPath, upload, options.readFile, options.writeFileChunks, mtimeMs),
|
|
2472
|
+
);
|
|
2470
2473
|
return {
|
|
2471
2474
|
...baseResult,
|
|
2472
2475
|
applied: true,
|
package/src/storage.ts
CHANGED
|
@@ -17,6 +17,7 @@ import { computeLegacyContentHash, normalizeContent } from "./content-hash.js";
|
|
|
17
17
|
import { parseDriftProvenance, serializeDriftProvenance } from "./preferences/drift-types.js";
|
|
18
18
|
import { isAbortError, raceAbort } from "./abort-error.js";
|
|
19
19
|
import { checkCorpusReadAbort, type CorpusReadOptions } from "./corpus-read-cancellation.js";
|
|
20
|
+
import { scopedCaptureWriteLockLimits } from "./capture-write-lock-scope.js";
|
|
20
21
|
import { selectArtifactMatches, type ArtifactSearchOptions } from "./artifact-search.js";
|
|
21
22
|
import path from "node:path";
|
|
22
23
|
import { resolveSafeStoragePath } from "./storage-paths.js";
|
|
@@ -3241,7 +3242,7 @@ export class StorageManager extends TombstoneBlockedCaptureIndexHost {
|
|
|
3241
3242
|
* resolves to the legacy short bounds instead of the converge deadline.
|
|
3242
3243
|
*/
|
|
3243
3244
|
private captureLockLimitSource(): ContentHashIndexLockOptions | undefined {
|
|
3244
|
-
const options = this.factHashIndexLockOptions;
|
|
3245
|
+
const options = { ...this.factHashIndexLockOptions, ...scopedCaptureWriteLockLimits(this) };
|
|
3245
3246
|
const keys = ["captureWriteLockInitialDelayMs", "captureWriteLockMaxDelayMs", "captureWriteLockDeadlineMs", "captureWriteLockSignal"] as const;
|
|
3246
3247
|
return keys.some((key) => options[key] !== undefined) ? options : undefined;
|
|
3247
3248
|
}
|