@remnic/core 9.26.0 → 9.28.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/access-admin-ops-surface.d.ts +13 -10
- package/dist/access-admin-ops-surface.js +43 -38
- package/dist/access-audit.js +2 -2
- package/dist/access-authorization-probe.d.ts +13 -10
- package/dist/access-authorization-probe.js +44 -39
- package/dist/access-boundary.d.ts +13 -10
- package/dist/access-boundary.js +43 -38
- package/dist/access-cli.js +100 -94
- package/dist/access-cli.js.map +1 -1
- package/dist/access-health-types.d.ts +10 -1
- package/dist/access-http-lcm-compaction.d.ts +13 -10
- package/dist/access-http.d.ts +13 -10
- package/dist/access-http.js +53 -48
- package/dist/access-identity-continuity-surface.d.ts +9 -8
- package/dist/access-identity-continuity-surface.js +43 -38
- package/dist/access-lcm-surface.d.ts +13 -10
- package/dist/access-lcm-surface.js +43 -38
- package/dist/access-mcp.d.ts +13 -10
- package/dist/access-mcp.js +48 -43
- package/dist/access-memory-search-fanout.d.ts +2 -2
- package/dist/{access-namespace-preflight-wRSZEO5V.d.ts → access-namespace-preflight-BiQ_-iUJ.d.ts} +1 -1
- package/dist/access-namespace-preflight.d.ts +3 -3
- package/dist/access-namespace-preflight.js +43 -38
- package/dist/access-observe-write-surface.d.ts +13 -10
- package/dist/access-observe-write-surface.js +43 -38
- package/dist/access-operations-batch.js +45 -40
- package/dist/access-operations.d.ts +16 -13
- package/dist/access-operations.js +47 -42
- package/dist/access-recall-concurrency.d.ts +13 -10
- package/dist/access-recall-concurrency.js +43 -38
- package/dist/access-recall-response.d.ts +13 -10
- package/dist/access-recall-response.js +43 -38
- package/dist/access-recall-surface.d.ts +13 -10
- package/dist/access-recall-surface.js +43 -38
- package/dist/access-schema.d.ts +80 -80
- package/dist/access-schema.js +4 -4
- package/dist/{access-service-DL8FJnTj.d.ts → access-service-CXkIgcGh.d.ts} +12 -8
- package/dist/access-service.d.ts +13 -10
- package/dist/access-service.js +43 -38
- package/dist/access-surface-catalog.d.ts +13 -10
- package/dist/access-wearables-meetings-surface.d.ts +4 -3
- package/dist/action-confidence.d.ts +1 -1
- package/dist/active-memory-bridge.d.ts +1 -1
- package/dist/active-recall.d.ts +1 -1
- package/dist/active-recall.js +4 -2
- package/dist/active-recall.js.map +1 -1
- package/dist/adapters/index.js +7 -7
- package/dist/adapters/registry.js +3 -3
- package/dist/behavior-learner.d.ts +1 -1
- package/dist/behavior-signals.d.ts +1 -1
- package/dist/bootstrap.d.ts +9 -8
- package/dist/briefing.d.ts +3 -2
- package/dist/briefing.js +12 -10
- package/dist/buffer-surprise-report.d.ts +1 -1
- package/dist/buffer.d.ts +3 -2
- package/dist/bulk-import/index.d.ts +3 -3
- package/dist/calibration.d.ts +1 -1
- package/dist/capabilities.d.ts +1 -1
- package/dist/{capsule-crypto-YO5QJ6L3.js → capsule-crypto-7FJQINUR.js} +2 -2
- package/dist/{catalog-BLLxZ5L5.d.ts → catalog-Bouj8Gfg.d.ts} +1 -1
- package/dist/causal-behavior.d.ts +1 -1
- package/dist/causal-consolidation.d.ts +1 -1
- package/dist/causal-consolidation.js +13 -11
- package/dist/causal-consolidation.js.map +1 -1
- package/dist/causal-trajectory-graph.d.ts +1 -1
- package/dist/{chunk-7RZJWZ46.js → chunk-227UFSEQ.js} +5 -5
- package/dist/{chunk-VGEH3STZ.js → chunk-2BEJ4XCF.js} +2 -2
- package/dist/{chunk-LIIQPDFN.js → chunk-2FYNLDKJ.js} +24 -3
- package/dist/chunk-2FYNLDKJ.js.map +1 -0
- package/dist/{chunk-BXLOS5AJ.js → chunk-2NLLXCJG.js} +2 -2
- package/dist/{chunk-6NNNURLK.js → chunk-2PHX3I5C.js} +8 -8
- package/dist/{chunk-YN4AUGNF.js → chunk-2TQ2UFKH.js} +65 -60
- package/dist/chunk-2TQ2UFKH.js.map +1 -0
- package/dist/{chunk-XAAIUXJO.js → chunk-3BSJBIZ3.js} +2 -2
- package/dist/{chunk-JCVQV6QX.js → chunk-3UOSVTFR.js} +11 -11
- package/dist/{chunk-AJ7QCSIS.js → chunk-7CPOZH6Y.js} +2 -2
- package/dist/{chunk-EXM2Q555.js → chunk-AHFZXUZM.js} +7 -5
- package/dist/chunk-AHFZXUZM.js.map +1 -0
- package/dist/{chunk-M2KGXXTL.js → chunk-ALTP3MFS.js} +2 -2
- package/dist/{chunk-77CNV7DY.js → chunk-BLAKS3CP.js} +4 -4
- package/dist/{chunk-FE5FKTCI.js → chunk-BR4UFMLO.js} +2 -2
- package/dist/chunk-CVILPI77.js +432 -0
- package/dist/chunk-CVILPI77.js.map +1 -0
- package/dist/{chunk-H2PBM562.js → chunk-DKQM5Q6Q.js} +2 -2
- package/dist/{chunk-CSXHRAJJ.js → chunk-EG64LXKL.js} +3 -3
- package/dist/{chunk-PTN6UAUA.js → chunk-ES36HTSC.js} +4 -4
- package/dist/{chunk-WMDRLTNA.js → chunk-EZWWCN7I.js} +2 -2
- package/dist/{chunk-XKGPIWGI.js → chunk-FBZTWUK5.js} +5 -5
- package/dist/{chunk-GJJBPI6I.js → chunk-FOCEI3TN.js} +6 -3
- package/dist/chunk-FOCEI3TN.js.map +1 -0
- package/dist/{chunk-Y6E37WRG.js → chunk-GJPCP5CA.js} +2 -2
- package/dist/{chunk-IHMXPG4P.js → chunk-GQSMYXKL.js} +6 -4
- package/dist/{chunk-IHMXPG4P.js.map → chunk-GQSMYXKL.js.map} +1 -1
- package/dist/{chunk-TGQ2NTWH.js → chunk-IJ6WHMFB.js} +16 -2
- package/dist/{chunk-TGQ2NTWH.js.map → chunk-IJ6WHMFB.js.map} +1 -1
- package/dist/{chunk-VDX2J7OX.js → chunk-IPLYGWQF.js} +6 -6
- package/dist/{chunk-XNUBF6RF.js → chunk-IYT632D5.js} +3 -3
- package/dist/{chunk-2QSZNTDO.js → chunk-JBPKEARU.js} +7 -7
- package/dist/{chunk-67DSDRUB.js → chunk-JIF3VRKQ.js} +8 -8
- package/dist/{chunk-V25OWXAC.js → chunk-JQ7T5VDC.js} +1 -1
- package/dist/chunk-JQ7T5VDC.js.map +1 -0
- package/dist/{chunk-MIFF37NN.js → chunk-KKBTBUVT.js} +2 -2
- package/dist/{chunk-ZHP4SUPB.js → chunk-LIHC3T43.js} +2 -2
- package/dist/{chunk-PX7VDW7R.js → chunk-MJPV4LBH.js} +79 -79
- package/dist/{chunk-PX7VDW7R.js.map → chunk-MJPV4LBH.js.map} +1 -1
- package/dist/{chunk-UGH6VPPO.js → chunk-MWTMTW2T.js} +2 -2
- package/dist/{chunk-F6LSNPPB.js → chunk-MXJ3VP6S.js} +6 -6
- package/dist/{chunk-KPWLZBXZ.js → chunk-NBNDVNJ2.js} +186 -25
- package/dist/chunk-NBNDVNJ2.js.map +1 -0
- package/dist/{chunk-M5EPMNS4.js → chunk-P2CLYN6H.js} +2 -2
- package/dist/{chunk-7XXCIO6X.js → chunk-PXQM2HOG.js} +13 -6
- package/dist/{chunk-7XXCIO6X.js.map → chunk-PXQM2HOG.js.map} +1 -1
- package/dist/{chunk-UZK7VVPD.js → chunk-Q5VJUNXA.js} +1858 -272
- package/dist/chunk-Q5VJUNXA.js.map +1 -0
- package/dist/{chunk-KGTD2KZJ.js → chunk-Q6YJW6KE.js} +2 -2
- package/dist/{chunk-X3SQYAPP.js → chunk-R372P2EH.js} +2 -2
- package/dist/{chunk-HBVOJNV4.js → chunk-RKXZ7P4V.js} +3 -3
- package/dist/chunk-RMMVQQA6.js +70 -0
- package/dist/chunk-RMMVQQA6.js.map +1 -0
- package/dist/{chunk-E6EAB4J6.js → chunk-S24E4MBQ.js} +14 -14
- package/dist/chunk-S24E4MBQ.js.map +1 -0
- package/dist/{chunk-RYHWZ52A.js → chunk-S2BBISHU.js} +4 -4
- package/dist/{chunk-LLISELSY.js → chunk-TBMRQZDY.js} +2 -2
- package/dist/{chunk-FMLKOL2Z.js → chunk-TC3VNKGX.js} +4 -4
- package/dist/{chunk-7K5Q6COX.js → chunk-TVVEYCNW.js} +4 -4
- package/dist/{chunk-2XP3PWL3.js → chunk-UR5SIN7H.js} +53 -35
- package/dist/chunk-UR5SIN7H.js.map +1 -0
- package/dist/{chunk-NKDVXEFU.js → chunk-VFNI3RPT.js} +2 -2
- package/dist/chunk-VNA7W7QF.js +134 -0
- package/dist/chunk-VNA7W7QF.js.map +1 -0
- package/dist/{chunk-PQPSZICJ.js → chunk-VYYNBJXH.js} +5 -5
- package/dist/chunk-WNIUPHWV.js +152 -0
- package/dist/chunk-WNIUPHWV.js.map +1 -0
- package/dist/chunk-WRJVB3C2.js +118 -0
- package/dist/chunk-WRJVB3C2.js.map +1 -0
- package/dist/{chunk-SN3ZESWQ.js → chunk-WVVK2RFM.js} +6 -6
- package/dist/{chunk-DQEMWVMT.js → chunk-X7Y7WX73.js} +1 -1
- package/dist/{chunk-4KKV7W62.js → chunk-YOBNCD3Z.js} +6 -6
- package/dist/chunk-YXZG5TVN.js +40 -0
- package/dist/chunk-YXZG5TVN.js.map +1 -0
- package/dist/{chunk-4DJQYKMN.js → chunk-ZIKENA2C.js} +2 -2
- package/dist/chunk-ZIKENA2C.js.map +1 -0
- package/dist/{chunk-SK3EYPKC.js → chunk-ZN4UZWMT.js} +2 -2
- package/dist/{cli-BoouX6LX.d.ts → cli-DJbkaZBU.d.ts} +5 -5
- package/dist/cli.d.ts +14 -12
- package/dist/cli.js +79 -74
- package/dist/compounding/engine.d.ts +3 -2
- package/dist/compounding/engine.js +12 -10
- package/dist/compounding/preference-consolidator.d.ts +1 -1
- package/dist/compression-optimizer.d.ts +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.js +4 -2
- package/dist/connectors/codex-materialize-runner.d.ts +1 -1
- package/dist/connectors/codex-materialize-runner.js +12 -10
- package/dist/connectors/codex-materialize.d.ts +1 -1
- package/dist/connectors/index.d.ts +1 -1
- package/dist/connectors/index.js +12 -10
- package/dist/consolidation-provenance-check.d.ts +3 -2
- package/dist/consolidation-undo.d.ts +3 -2
- package/dist/contradiction/index.d.ts +3 -2
- package/dist/conversation-index/backend.d.ts +1 -1
- package/dist/conversation-index/backend.js +2 -2
- package/dist/conversation-index/chunker.d.ts +1 -1
- package/dist/conversation-index/faiss-adapter.d.ts +1 -1
- package/dist/conversation-index/indexer.d.ts +1 -1
- package/dist/conversation-index/search.d.ts +1 -1
- package/dist/corpus-watermark.d.ts +15 -2
- package/dist/corpus-watermark.js +25 -21
- package/dist/day-summary.d.ts +1 -1
- package/dist/delinearize.d.ts +1 -1
- package/dist/direct-answer-wiring.d.ts +1 -1
- package/dist/direct-answer.d.ts +1 -1
- package/dist/embedding-fallback.d.ts +1 -1
- package/dist/enrichment/index.d.ts +1 -1
- package/dist/entity-id-normalization.d.ts +4 -0
- package/dist/entity-id-normalization.js +10 -0
- package/dist/entity-id-normalization.js.map +1 -0
- package/dist/entity-retrieval-boundaries.d.ts +3 -0
- package/dist/entity-retrieval-boundaries.js +8 -0
- package/dist/entity-retrieval-boundaries.js.map +1 -0
- package/dist/entity-retrieval.d.ts +3 -2
- package/dist/entity-retrieval.js +13 -10
- package/dist/entity-schema.d.ts +1 -1
- package/dist/entity-schema.js +1 -1
- package/dist/explicit-capture.d.ts +9 -8
- package/dist/extraction-error-classification.d.ts +1 -1
- package/dist/extraction-faithfulness.d.ts +1 -1
- package/dist/extraction-judge-telemetry.d.ts +1 -1
- package/dist/extraction-judge-training.d.ts +1 -1
- package/dist/extraction-judge.d.ts +1 -1
- package/dist/extraction-liveness.d.ts +1 -1
- package/dist/extraction.d.ts +1 -1
- package/dist/fallback-llm.d.ts +1 -1
- package/dist/{first-start-migration-LEXL3XYL.js → first-start-migration-BKNGOFED.js} +4 -4
- package/dist/graph-dashboard-diff.d.ts +1 -1
- package/dist/graph-dashboard-key.d.ts +1 -1
- package/dist/graph-dashboard-parser.d.ts +1 -1
- package/dist/graph-edge-reinforcement.d.ts +1 -1
- package/dist/graph-snapshot.d.ts +1 -1
- package/dist/graph.d.ts +1 -1
- package/dist/importance.d.ts +1 -1
- package/dist/importers/index.d.ts +1 -1
- package/dist/in-flight-reads.d.ts +1 -1
- package/dist/index.d.ts +409 -407
- package/dist/index.js +147 -141
- package/dist/intent.d.ts +1 -1
- package/dist/lcm/engine.d.ts +1 -1
- package/dist/lcm/engine.js +3 -3
- package/dist/lcm/index.d.ts +1 -1
- package/dist/lcm/index.js +9 -9
- package/dist/lcm/tools.d.ts +1 -1
- package/dist/lifecycle.d.ts +1 -1
- package/dist/live-connectors-runner.d.ts +1 -1
- package/dist/local-llm.d.ts +1 -1
- package/dist/local-model-endpoint.d.ts +1 -1
- package/dist/maintenance/memory-governance.d.ts +1 -1
- package/dist/maintenance/memory-governance.js +12 -10
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.d.ts +3 -2
- package/dist/maintenance/rebuild-memory-lifecycle-ledger.js +12 -10
- package/dist/maintenance/rebuild-memory-projection.d.ts +3 -2
- package/dist/maintenance/rebuild-memory-projection.js +13 -11
- package/dist/{maintenance-DD22oIx9.d.ts → maintenance-DveW6azo.d.ts} +3 -3
- package/dist/mcp-memory-inspector-app.d.ts +13 -10
- package/dist/memory-action-policy.d.ts +1 -1
- package/dist/memory-cache.d.ts +1 -1
- package/dist/memory-lifecycle-ledger-utils.d.ts +1 -1
- package/dist/memory-projection-mutations.d.ts +4 -0
- package/dist/memory-projection-mutations.js +15 -0
- package/dist/memory-projection-mutations.js.map +1 -0
- package/dist/memory-projection-store.d.ts +1 -1
- package/dist/memory-provenance.d.ts +1 -1
- package/dist/memory-worth-outcomes.d.ts +3 -2
- package/dist/models-json.d.ts +1 -1
- package/dist/namespaces/migrate.d.ts +4 -3
- package/dist/namespaces/migrate.js +22 -20
- package/dist/namespaces/principal.d.ts +1 -1
- package/dist/namespaces/search.d.ts +1 -1
- package/dist/namespaces/search.js +9 -9
- package/dist/namespaces/storage.d.ts +4 -3
- package/dist/namespaces/storage.js +12 -10
- package/dist/native-knowledge.d.ts +1 -1
- package/dist/offline-sync-impression-drain.d.ts +1 -1
- package/dist/offline-sync-impression-drain.js +3 -3
- package/dist/operator-doctor-corpus.d.ts +22 -3
- package/dist/operator-doctor-corpus.js +32 -22
- package/dist/operator-doctor-replica.d.ts +52 -0
- package/dist/operator-doctor-replica.js +15 -0
- package/dist/operator-doctor-replica.js.map +1 -0
- package/dist/operator-toolkit.d.ts +7 -3
- package/dist/operator-toolkit.js +33 -27
- package/dist/orchestration/compression-guideline-coordinator.d.ts +3 -2
- package/dist/orchestration/maintenance.d.ts +5 -4
- package/dist/orchestration/maintenance.js +17 -15
- package/dist/{orchestrator-CFJ_TG6D.d.ts → orchestrator-Cvlgpe6I.d.ts} +7 -7
- package/dist/orchestrator.d.ts +9 -8
- package/dist/orchestrator.js +99 -93
- package/dist/patterns-cli.d.ts +1 -1
- package/dist/{pipeline-Bq5zU6Bp.d.ts → pipeline-BveYVaz7.d.ts} +1 -1
- package/dist/policy-runtime.d.ts +1 -1
- package/dist/proactive-contention.d.ts +1 -1
- package/dist/provenance.d.ts +1 -1
- package/dist/{qmd-3ILJkC70.d.ts → qmd-CLWDsbYR.d.ts} +1 -1
- package/dist/qmd-preflight.d.ts +2 -2
- package/dist/qmd-recall-cache.d.ts +1 -1
- package/dist/qmd.d.ts +2 -2
- package/dist/recall-concurrency-config.d.ts +1 -1
- package/dist/recall-disclosure-escalation.d.ts +1 -1
- package/dist/recall-explain-renderer.d.ts +1 -1
- package/dist/recall-explain-renderer.js +3 -3
- package/dist/recall-memory-map.d.ts +1 -1
- package/dist/recall-planner-llm.d.ts +1 -1
- package/dist/recall-state.d.ts +1 -1
- package/dist/recall-state.js +2 -2
- package/dist/recall-tag-filter.d.ts +1 -1
- package/dist/recall-timings.d.ts +1 -1
- package/dist/recall-xray-cli.d.ts +1 -1
- package/dist/recall-xray-cli.js +4 -4
- package/dist/recall-xray-renderer.d.ts +1 -1
- package/dist/recall-xray-renderer.js +3 -3
- package/dist/recall-xray.d.ts +1 -1
- package/dist/recall-xray.js +2 -2
- package/dist/replay/normalizers/chatgpt.d.ts +1 -1
- package/dist/replay/normalizers/claude.d.ts +1 -1
- package/dist/replay/normalizers/openclaw.d.ts +1 -1
- package/dist/replay/normalizers/shared.d.ts +1 -1
- package/dist/replay/runner.d.ts +1 -1
- package/dist/replay/types.d.ts +1 -1
- package/dist/replica-divergence.d.ts +279 -0
- package/dist/replica-divergence.js +28 -0
- package/dist/replica-divergence.js.map +1 -0
- package/dist/replica-peers-config.d.ts +5 -0
- package/dist/replica-peers-config.js +13 -0
- package/dist/replica-peers-config.js.map +1 -0
- package/dist/resolve-auth-token.d.ts +21 -2
- package/dist/resolve-auth-token.js +5 -1
- package/dist/resume-bundles.js +5 -3
- package/dist/retrieval-agents.d.ts +2 -2
- package/dist/retrieval-tiers.d.ts +1 -1
- package/dist/routing/engine.d.ts +1 -1
- package/dist/routing/store.d.ts +1 -1
- package/dist/salvage-envelope.d.ts +1 -1
- package/dist/schemas.d.ts +76 -76
- package/dist/{scope-profiles-D0-ff4xT.d.ts → scope-profiles-BVf3DZ7z.d.ts} +1 -1
- package/dist/search/embed-helper.d.ts +1 -1
- package/dist/search/factory.d.ts +1 -1
- package/dist/search/factory.js +8 -8
- package/dist/search/index.d.ts +1 -1
- package/dist/search/index.js +13 -13
- package/dist/search/lancedb-backend.d.ts +1 -1
- package/dist/search/lancedb-backend.js +2 -2
- package/dist/search/meilisearch-backend.d.ts +1 -1
- package/dist/search/meilisearch-backend.js +2 -2
- package/dist/search/noop-backend.d.ts +1 -1
- package/dist/search/orama-backend.d.ts +1 -1
- package/dist/search/orama-backend.js +2 -2
- package/dist/search/port.d.ts +1 -1
- package/dist/search/remote-backend.d.ts +1 -1
- package/dist/{semantic-consolidation-4Tfh7c1g.d.ts → semantic-consolidation-DXLV47y2.d.ts} +1 -1
- package/dist/semantic-consolidation.d.ts +2 -2
- package/dist/semantic-consolidation.js +13 -11
- package/dist/semantic-rule-promotion.js +12 -10
- package/dist/semantic-rule-verifier.d.ts +1 -1
- package/dist/semantic-rule-verifier.js +12 -10
- package/dist/{service-Z8uzoAST.d.ts → service-DbgPTeMJ.d.ts} +2 -2
- package/dist/session-observer-bands.d.ts +1 -1
- package/dist/session-observer-state.d.ts +1 -1
- package/dist/shared-context/manager.d.ts +9 -9
- package/dist/signal.d.ts +1 -1
- package/dist/{storage-BZVMn8yJ.d.ts → storage-DOzTm8wG.d.ts} +13 -22
- package/dist/storage.d.ts +3 -2
- package/dist/storage.js +13 -10
- package/dist/summarizer.d.ts +1 -1
- package/dist/summary-snapshot.d.ts +1 -1
- package/dist/temporal-supersession.d.ts +3 -2
- package/dist/temporal-timeline-recall.d.ts +1 -1
- package/dist/temporal-validity.d.ts +1 -1
- package/dist/threading.d.ts +1 -1
- package/dist/tier-migration.d.ts +3 -2
- package/dist/tier-routing.d.ts +1 -1
- package/dist/topics.d.ts +1 -1
- package/dist/transcript.d.ts +1 -1
- package/dist/transfer/autodetect.js +1 -1
- package/dist/transfer/backup.js +2 -2
- package/dist/transfer/capsule-export.js +3 -3
- package/dist/transfer/capsule-import.js +2 -2
- package/dist/transfer/types.d.ts +66 -66
- package/dist/trust-score-stage.d.ts +1 -1
- package/dist/trust-score.d.ts +1 -1
- package/dist/{types-DHBxIF0s.d.ts → types-BA7W-_xy.d.ts} +53 -4
- package/dist/types.d.ts +1 -1
- package/dist/types.js +1 -1
- package/dist/utility-runtime.d.ts +1 -1
- package/dist/verified-recall.js +12 -10
- package/dist/write-envelope.d.ts +1 -1
- package/package.json +2 -2
- package/src/access-health-types.ts +8 -0
- package/src/access-http.test.ts +4 -0
- package/src/access-service-health.test.ts +23 -0
- package/src/access-service.ts +27 -4
- package/src/cli.ts +9 -7
- package/src/config.ts +2 -1
- package/src/corpus-watermark.test.ts +21 -0
- package/src/corpus-watermark.ts +25 -3
- package/src/entity-id-normalization.ts +55 -0
- package/src/entity-retrieval-boundaries.ts +134 -0
- package/src/entity-retrieval.ts +253 -27
- package/src/entity-schema.ts +2 -1
- package/src/memory-projection-mutations.ts +132 -0
- package/src/namespaces/storage.ts +5 -3
- package/src/operator-doctor-corpus.ts +41 -0
- package/src/operator-doctor-replica.test.ts +261 -0
- package/src/operator-doctor-replica.ts +129 -0
- package/src/operator-toolkit.ts +5 -4
- package/src/orchestration/orchestrator-init.ts +2 -2
- package/src/replica-divergence.test.ts +1475 -0
- package/src/replica-divergence.ts +910 -0
- package/src/replica-peers-config.ts +250 -0
- package/src/resolve-auth-token.ts +38 -0
- package/src/storage/entity-canonical-id-migration-adapter.ts +72 -0
- package/src/storage/entity-canonical-id-migration-runner.ts +68 -0
- package/src/storage/entity-canonical-id-migration.ts +904 -0
- package/src/storage/entity-store.ts +44 -0
- package/src/storage/memory-frontmatter-metadata.ts +17 -0
- package/src/storage/memory-migration-serialization.ts +37 -0
- package/src/storage/memory-read-store.ts +3 -2
- package/src/storage/profile-header.ts +898 -0
- package/src/storage-profile-timestamp.test.ts +3358 -0
- package/src/storage.ts +73 -77
- package/src/testing/subjects/profile-header-write.test.ts +71 -0
- package/src/types.ts +3 -3
- package/dist/chunk-2XP3PWL3.js.map +0 -1
- package/dist/chunk-4DJQYKMN.js.map +0 -1
- package/dist/chunk-E6EAB4J6.js.map +0 -1
- package/dist/chunk-EXM2Q555.js.map +0 -1
- package/dist/chunk-GJJBPI6I.js.map +0 -1
- package/dist/chunk-KPWLZBXZ.js.map +0 -1
- package/dist/chunk-LIIQPDFN.js.map +0 -1
- package/dist/chunk-UZK7VVPD.js.map +0 -1
- package/dist/chunk-V25OWXAC.js.map +0 -1
- package/dist/chunk-YN4AUGNF.js.map +0 -1
- /package/dist/{capsule-crypto-YO5QJ6L3.js.map → capsule-crypto-7FJQINUR.js.map} +0 -0
- /package/dist/{chunk-7RZJWZ46.js.map → chunk-227UFSEQ.js.map} +0 -0
- /package/dist/{chunk-VGEH3STZ.js.map → chunk-2BEJ4XCF.js.map} +0 -0
- /package/dist/{chunk-BXLOS5AJ.js.map → chunk-2NLLXCJG.js.map} +0 -0
- /package/dist/{chunk-6NNNURLK.js.map → chunk-2PHX3I5C.js.map} +0 -0
- /package/dist/{chunk-XAAIUXJO.js.map → chunk-3BSJBIZ3.js.map} +0 -0
- /package/dist/{chunk-JCVQV6QX.js.map → chunk-3UOSVTFR.js.map} +0 -0
- /package/dist/{chunk-AJ7QCSIS.js.map → chunk-7CPOZH6Y.js.map} +0 -0
- /package/dist/{chunk-M2KGXXTL.js.map → chunk-ALTP3MFS.js.map} +0 -0
- /package/dist/{chunk-77CNV7DY.js.map → chunk-BLAKS3CP.js.map} +0 -0
- /package/dist/{chunk-FE5FKTCI.js.map → chunk-BR4UFMLO.js.map} +0 -0
- /package/dist/{chunk-H2PBM562.js.map → chunk-DKQM5Q6Q.js.map} +0 -0
- /package/dist/{chunk-CSXHRAJJ.js.map → chunk-EG64LXKL.js.map} +0 -0
- /package/dist/{chunk-PTN6UAUA.js.map → chunk-ES36HTSC.js.map} +0 -0
- /package/dist/{chunk-WMDRLTNA.js.map → chunk-EZWWCN7I.js.map} +0 -0
- /package/dist/{chunk-XKGPIWGI.js.map → chunk-FBZTWUK5.js.map} +0 -0
- /package/dist/{chunk-Y6E37WRG.js.map → chunk-GJPCP5CA.js.map} +0 -0
- /package/dist/{chunk-VDX2J7OX.js.map → chunk-IPLYGWQF.js.map} +0 -0
- /package/dist/{chunk-XNUBF6RF.js.map → chunk-IYT632D5.js.map} +0 -0
- /package/dist/{chunk-2QSZNTDO.js.map → chunk-JBPKEARU.js.map} +0 -0
- /package/dist/{chunk-67DSDRUB.js.map → chunk-JIF3VRKQ.js.map} +0 -0
- /package/dist/{chunk-MIFF37NN.js.map → chunk-KKBTBUVT.js.map} +0 -0
- /package/dist/{chunk-ZHP4SUPB.js.map → chunk-LIHC3T43.js.map} +0 -0
- /package/dist/{chunk-UGH6VPPO.js.map → chunk-MWTMTW2T.js.map} +0 -0
- /package/dist/{chunk-F6LSNPPB.js.map → chunk-MXJ3VP6S.js.map} +0 -0
- /package/dist/{chunk-M5EPMNS4.js.map → chunk-P2CLYN6H.js.map} +0 -0
- /package/dist/{chunk-KGTD2KZJ.js.map → chunk-Q6YJW6KE.js.map} +0 -0
- /package/dist/{chunk-X3SQYAPP.js.map → chunk-R372P2EH.js.map} +0 -0
- /package/dist/{chunk-HBVOJNV4.js.map → chunk-RKXZ7P4V.js.map} +0 -0
- /package/dist/{chunk-RYHWZ52A.js.map → chunk-S2BBISHU.js.map} +0 -0
- /package/dist/{chunk-LLISELSY.js.map → chunk-TBMRQZDY.js.map} +0 -0
- /package/dist/{chunk-FMLKOL2Z.js.map → chunk-TC3VNKGX.js.map} +0 -0
- /package/dist/{chunk-7K5Q6COX.js.map → chunk-TVVEYCNW.js.map} +0 -0
- /package/dist/{chunk-NKDVXEFU.js.map → chunk-VFNI3RPT.js.map} +0 -0
- /package/dist/{chunk-PQPSZICJ.js.map → chunk-VYYNBJXH.js.map} +0 -0
- /package/dist/{chunk-SN3ZESWQ.js.map → chunk-WVVK2RFM.js.map} +0 -0
- /package/dist/{chunk-DQEMWVMT.js.map → chunk-X7Y7WX73.js.map} +0 -0
- /package/dist/{chunk-4KKV7W62.js.map → chunk-YOBNCD3Z.js.map} +0 -0
- /package/dist/{chunk-SK3EYPKC.js.map → chunk-ZN4UZWMT.js.map} +0 -0
- /package/dist/{first-start-migration-LEXL3XYL.js.map → first-start-migration-BKNGOFED.js.map} +0 -0
|
@@ -0,0 +1,1475 @@
|
|
|
1
|
+
import assert from "node:assert/strict";
|
|
2
|
+
import { createServer, type RequestListener } from "node:http";
|
|
3
|
+
import type { AddressInfo, Socket } from "node:net";
|
|
4
|
+
import test from "node:test";
|
|
5
|
+
|
|
6
|
+
import { computeCorpusWatermark, type CorpusWatermark } from "./corpus-watermark.js";
|
|
7
|
+
import {
|
|
8
|
+
MAX_PEER_RESPONSE_BYTES,
|
|
9
|
+
ReplicaDivergenceMonitor,
|
|
10
|
+
type FetchLike,
|
|
11
|
+
type ReplicaDivergenceStatus,
|
|
12
|
+
compareReplicaWatermarks,
|
|
13
|
+
filterReplicaReportByCaps,
|
|
14
|
+
gateReportByCensus,
|
|
15
|
+
pollReplicaPeers,
|
|
16
|
+
redactPeerUrl,
|
|
17
|
+
} from "./replica-divergence.js";
|
|
18
|
+
import { parseReplicaPeersConfig, resolveReplicaPeersConfig } from "./replica-peers-config.js";
|
|
19
|
+
|
|
20
|
+
// ---------------------------------------------------------------------------
|
|
21
|
+
// Helpers
|
|
22
|
+
// ---------------------------------------------------------------------------
|
|
23
|
+
|
|
24
|
+
const THRESHOLDS = { maxFileCountDelta: 100, maxWatermarkAgeDeltaMs: 900_000 } as const;
|
|
25
|
+
|
|
26
|
+
function watermark(namespace: string, overrides: Partial<CorpusWatermark> = {}): CorpusWatermark {
|
|
27
|
+
return {
|
|
28
|
+
namespace,
|
|
29
|
+
memoryFileCount: 10,
|
|
30
|
+
newestPartition: "2026-03-08",
|
|
31
|
+
newestWriteAt: "2026-03-08T00:00:00.000Z",
|
|
32
|
+
digest: `digest-${namespace}`,
|
|
33
|
+
computedAt: new Date().toISOString(),
|
|
34
|
+
...overrides,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
interface Peer {
|
|
39
|
+
url: string;
|
|
40
|
+
port: number;
|
|
41
|
+
requests: { pathname: string; authorization: string | undefined }[];
|
|
42
|
+
close: () => Promise<void>;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
async function startPeer(handler: RequestListener): Promise<Peer> {
|
|
46
|
+
const requests: Peer["requests"] = [];
|
|
47
|
+
const sockets = new Set<Socket>();
|
|
48
|
+
const server = createServer((req, res) => {
|
|
49
|
+
requests.push({ pathname: new URL(req.url ?? "/", "http://127.0.0.1").pathname, authorization: req.headers.authorization });
|
|
50
|
+
handler(req, res);
|
|
51
|
+
});
|
|
52
|
+
server.on("connection", (socket) => {
|
|
53
|
+
sockets.add(socket);
|
|
54
|
+
socket.on("close", () => sockets.delete(socket));
|
|
55
|
+
});
|
|
56
|
+
const listening = Promise.withResolvers<void>();
|
|
57
|
+
server.once("error", listening.reject);
|
|
58
|
+
server.listen(0, "127.0.0.1", listening.resolve);
|
|
59
|
+
await listening.promise;
|
|
60
|
+
const address = server.address() as AddressInfo;
|
|
61
|
+
return {
|
|
62
|
+
url: `http://127.0.0.1:${address.port}`,
|
|
63
|
+
port: address.port,
|
|
64
|
+
requests,
|
|
65
|
+
close: async () => {
|
|
66
|
+
for (const socket of sockets) socket.destroy();
|
|
67
|
+
await new Promise<void>((resolve) => server.close(() => resolve()));
|
|
68
|
+
},
|
|
69
|
+
};
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** A `/health`-style handler that serves `corpus` under the given path (default engram prefix). */
|
|
73
|
+
function corpusHandler(corpus: CorpusWatermark[], servedPath = "/engram/v1/health"): RequestListener {
|
|
74
|
+
return (req, res) => {
|
|
75
|
+
const pathname = new URL(req.url ?? "/", "http://127.0.0.1").pathname;
|
|
76
|
+
if (pathname === servedPath) {
|
|
77
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
78
|
+
res.end(JSON.stringify({ ok: true, corpus }));
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
res.writeHead(404, { "content-type": "application/json" });
|
|
82
|
+
res.end(JSON.stringify({ error: "not_found" }));
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ---------------------------------------------------------------------------
|
|
87
|
+
// Config parsing (§1/§17/§24/§39)
|
|
88
|
+
// ---------------------------------------------------------------------------
|
|
89
|
+
|
|
90
|
+
test("parseReplicaPeersConfig: defaults are opt-out and match the documented values", () => {
|
|
91
|
+
const config = parseReplicaPeersConfig({});
|
|
92
|
+
assert.equal(config.enabled, false);
|
|
93
|
+
assert.equal(config.peers.length, 0);
|
|
94
|
+
assert.equal(config.pollIntervalMs, 300_000);
|
|
95
|
+
assert.equal(config.requestTimeoutMs, 10_000);
|
|
96
|
+
assert.equal(config.maxFileCountDelta, 100);
|
|
97
|
+
assert.equal(config.maxWatermarkAgeDeltaMs, 900_000);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test("parseReplicaPeersConfig: string \"false\"/\"0\" are falsy (§24 CLI/JSON string coercion)", () => {
|
|
101
|
+
assert.equal(parseReplicaPeersConfig({ replicaPeers: { enabled: "false" } }).enabled, false);
|
|
102
|
+
assert.equal(parseReplicaPeersConfig({ replicaPeers: { enabled: "0" } }).enabled, false);
|
|
103
|
+
assert.equal(parseReplicaPeersConfig({ replicaPeers: { enabled: "true" } }).enabled, true);
|
|
104
|
+
assert.equal(parseReplicaPeersConfig({ replicaPeers: { enabled: "1" } }).enabled, true);
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("parseReplicaPeersConfig: string numbers coerce; interval floors are honored", () => {
|
|
108
|
+
const config = parseReplicaPeersConfig({
|
|
109
|
+
replicaPeers: { pollIntervalMs: "60000", requestTimeoutMs: "5000", maxFileCountDelta: "0" },
|
|
110
|
+
});
|
|
111
|
+
assert.equal(config.pollIntervalMs, 60_000);
|
|
112
|
+
assert.equal(config.requestTimeoutMs, 5_000);
|
|
113
|
+
assert.equal(config.maxFileCountDelta, 0, "0 is a valid strictest threshold (schema minimum 0, code accepts 0)");
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
test("parseReplicaPeersConfig: valid peers parse with url + token", () => {
|
|
117
|
+
const config = parseReplicaPeersConfig({
|
|
118
|
+
replicaPeers: { enabled: true, peers: [{ url: "https://backup.example:4318", token: "tok" }] },
|
|
119
|
+
});
|
|
120
|
+
assert.equal(config.peers.length, 1);
|
|
121
|
+
assert.equal(config.peers[0]?.url, "https://backup.example:4318");
|
|
122
|
+
assert.equal(config.peers[0]?.token, "tok");
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
test("parseReplicaPeersConfig: rejects invalid inputs instead of silently defaulting (§39)", () => {
|
|
126
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: [{ url: "not a url" }] } }), /valid URL/);
|
|
127
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: [{ url: "ftp://host" }] } }), /http\(s\)/);
|
|
128
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: "nope" } }), /must be an array/);
|
|
129
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: [42] } }), /must be an object/);
|
|
130
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: [{}] } }), /url must be a non-empty string/);
|
|
131
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: [{ url: "http://h", token: 5 }] } }), /token must be/);
|
|
132
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: [{ url: "http://h", token: "" }] } }), /non-empty string/, "an explicit empty token is a config error, not unauthenticated");
|
|
133
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: [{ url: "http://h", token: " " }] } }), /non-empty string/, "a whitespace-only token is a config error too");
|
|
134
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: [] }), /must be a plain object/);
|
|
135
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: [{ url: "https://peer.example?tenant=x" }] } }), /clean base URL/, "a query component corrupts path concatenation and is rejected");
|
|
136
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: [{ url: "https://peer.example/#frag" }] } }), /clean base URL/, "a fragment component is rejected");
|
|
137
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { peers: [{ url: "https://user:pass@peer.example" }] } }), /clean base URL/, "embedded credentials are rejected (Node fetch refuses them)");
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
test("parseReplicaPeersConfig: rejects fractional / non-positive intervals (§1/§17)", () => {
|
|
141
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { pollIntervalMs: 1.5 } }), /integer/);
|
|
142
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { pollIntervalMs: 0 } }), /greater than or equal to 1/);
|
|
143
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { requestTimeoutMs: -5 } }), /greater than or equal to 1/);
|
|
144
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { maxFileCountDelta: -1 } }), /greater than or equal to 0/);
|
|
145
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { maxWatermarkAgeDeltaMs: 2.5 } }), /integer/);
|
|
146
|
+
assert.throws(() => parseReplicaPeersConfig({ replicaPeers: { requestTimeoutMs: 2_147_483_648 } }), /no greater than/, "a requestTimeoutMs above the Node timer ceiling is rejected, not silently clamped to 1ms");
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("resolveReplicaPeersConfig: the read boundary caps requestTimeoutMs at the timer ceiling (falls back, never throws)", () => {
|
|
150
|
+
// The lenient read boundary (/health, doctor) must apply the same timer ceiling
|
|
151
|
+
// as the strict parser: a legacy config with an over-ceiling timeout would clamp
|
|
152
|
+
// to ~1ms and mark every peer unreachable. It falls back to the default, never throws.
|
|
153
|
+
const resolved = resolveReplicaPeersConfig({ requestTimeoutMs: 2_147_483_648, enabled: true });
|
|
154
|
+
assert.equal(resolved.requestTimeoutMs, 10_000, "an over-ceiling timeout falls back to the default at the read boundary");
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
test("parseReplicaPeersConfig: ${ENV} expansion in url and token, throws on unset", () => {
|
|
158
|
+
process.env.REPLICA_TEST_HOST = "backup.example:4318";
|
|
159
|
+
process.env.REPLICA_TEST_TOKEN = "env-token";
|
|
160
|
+
try {
|
|
161
|
+
const config = parseReplicaPeersConfig({
|
|
162
|
+
replicaPeers: { peers: [{ url: "https://${REPLICA_TEST_HOST}", token: "${REPLICA_TEST_TOKEN}" }] },
|
|
163
|
+
});
|
|
164
|
+
assert.equal(config.peers[0]?.url, "https://backup.example:4318");
|
|
165
|
+
assert.equal(config.peers[0]?.token, "env-token");
|
|
166
|
+
assert.throws(
|
|
167
|
+
() => parseReplicaPeersConfig({ replicaPeers: { peers: [{ url: "https://${REPLICA_TEST_UNSET_VAR}" }] } }),
|
|
168
|
+
/is not set/,
|
|
169
|
+
);
|
|
170
|
+
} finally {
|
|
171
|
+
delete process.env.REPLICA_TEST_HOST;
|
|
172
|
+
delete process.env.REPLICA_TEST_TOKEN;
|
|
173
|
+
}
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
// ---------------------------------------------------------------------------
|
|
177
|
+
// compareReplicaWatermarks (pure)
|
|
178
|
+
// ---------------------------------------------------------------------------
|
|
179
|
+
|
|
180
|
+
test("compareReplicaWatermarks: a one-sided null newest-write is divergence, not agreement (round 6, codex)", () => {
|
|
181
|
+
const local = [watermark("default", { newestWriteAt: "2026-03-08T00:00:00.000Z" })];
|
|
182
|
+
const peer = [watermark("default", { newestWriteAt: null })]; // same count+digest, but no dated write
|
|
183
|
+
const result = compareReplicaWatermarks(local, peer, THRESHOLDS);
|
|
184
|
+
assert.equal(result.state, "diverged", "a missing measurement on one side cannot certify convergence");
|
|
185
|
+
assert.ok(result.namespaces[0]?.reasons.includes("newest_write_presence_mismatch"));
|
|
186
|
+
// Two genuinely undated but otherwise-matching corpora still converge.
|
|
187
|
+
const bothNull = compareReplicaWatermarks(
|
|
188
|
+
[watermark("default", { newestWriteAt: null })],
|
|
189
|
+
[watermark("default", { newestWriteAt: null })],
|
|
190
|
+
THRESHOLDS,
|
|
191
|
+
);
|
|
192
|
+
assert.equal(bothNull.state, "converged", "two undated but matching corpora agree");
|
|
193
|
+
});
|
|
194
|
+
|
|
195
|
+
test("compareReplicaWatermarks: identical corpora converge", () => {
|
|
196
|
+
const local = [watermark("default")];
|
|
197
|
+
const peer = [watermark("default")];
|
|
198
|
+
const result = compareReplicaWatermarks(local, peer, THRESHOLDS);
|
|
199
|
+
assert.equal(result.state, "converged");
|
|
200
|
+
assert.equal(result.divergedNamespaceCount, 0);
|
|
201
|
+
assert.equal(result.namespaces[0]?.digestMatch, true);
|
|
202
|
+
});
|
|
203
|
+
|
|
204
|
+
test("compareReplicaWatermarks: file-count delta beyond threshold diverges and reports the delta", () => {
|
|
205
|
+
const local = [watermark("default", { memoryFileCount: 190_000, digest: "d" })];
|
|
206
|
+
const peer = [watermark("default", { memoryFileCount: 340_000, digest: "d" })];
|
|
207
|
+
const result = compareReplicaWatermarks(local, peer, THRESHOLDS);
|
|
208
|
+
assert.equal(result.state, "diverged");
|
|
209
|
+
const delta = result.namespaces[0];
|
|
210
|
+
assert.equal(delta?.fileCountDelta, 150_000);
|
|
211
|
+
assert.ok(delta?.reasons.includes("file_count_delta=150000"));
|
|
212
|
+
});
|
|
213
|
+
|
|
214
|
+
test("compareReplicaWatermarks: file-count delta within threshold stays converged", () => {
|
|
215
|
+
const local = [watermark("default", { memoryFileCount: 100, digest: "d" })];
|
|
216
|
+
const peer = [watermark("default", { memoryFileCount: 150, digest: "d" })];
|
|
217
|
+
const result = compareReplicaWatermarks(local, peer, THRESHOLDS);
|
|
218
|
+
assert.equal(result.state, "converged", "delta 50 is within maxFileCountDelta 100");
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test("compareReplicaWatermarks: equal counts but DIFFERENT digest diverges (split-brain)", () => {
|
|
222
|
+
const local = [watermark("default", { memoryFileCount: 500, digest: "aaa" })];
|
|
223
|
+
const peer = [watermark("default", { memoryFileCount: 500, digest: "bbb" })];
|
|
224
|
+
const result = compareReplicaWatermarks(local, peer, THRESHOLDS);
|
|
225
|
+
assert.equal(result.state, "diverged", "same size, different content must be caught by the digest");
|
|
226
|
+
assert.equal(result.namespaces[0]?.digestMatch, false);
|
|
227
|
+
assert.ok(result.namespaces[0]?.reasons.includes("digest_mismatch"));
|
|
228
|
+
});
|
|
229
|
+
|
|
230
|
+
test("compareReplicaWatermarks: stale peer watermark (age delta beyond threshold) diverges", () => {
|
|
231
|
+
const local = [watermark("default", { newestWriteAt: "2026-03-08T12:00:00.000Z", digest: "d", memoryFileCount: 10 })];
|
|
232
|
+
const peer = [watermark("default", { newestWriteAt: "2026-03-08T00:00:00.000Z", digest: "d", memoryFileCount: 10 })];
|
|
233
|
+
const result = compareReplicaWatermarks(local, peer, THRESHOLDS);
|
|
234
|
+
assert.equal(result.state, "diverged");
|
|
235
|
+
assert.equal(result.namespaces[0]?.writeAgeDeltaMs, 12 * 3_600_000);
|
|
236
|
+
assert.ok(result.namespaces[0]?.reasons.some((reason) => reason.startsWith("write_age_delta_ms=")));
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
test("compareReplicaWatermarks: a one-sided namespace is reported; local_only advisory, peer_only diverges", () => {
|
|
240
|
+
const local = [watermark("default"), watermark("team-a")];
|
|
241
|
+
const peer = [watermark("default"), watermark("team-b")];
|
|
242
|
+
const result = compareReplicaWatermarks(local, peer, THRESHOLDS);
|
|
243
|
+
const byNamespace = new Map(result.namespaces.map((delta) => [delta.namespace, delta]));
|
|
244
|
+
// local_only is advisory (a namespace-restricted peer token may hide it): reported, not diverged.
|
|
245
|
+
assert.equal(byNamespace.get("team-a")?.presence, "local_only");
|
|
246
|
+
assert.equal(byNamespace.get("team-a")?.diverged, false);
|
|
247
|
+
// peer_only is a real divergence — the peer holds a namespace we lack.
|
|
248
|
+
assert.equal(byNamespace.get("team-b")?.presence, "peer_only");
|
|
249
|
+
assert.equal(byNamespace.get("team-b")?.diverged, true);
|
|
250
|
+
assert.equal(byNamespace.get("default")?.diverged, false);
|
|
251
|
+
assert.equal(result.state, "diverged", "peer_only still diverges the peer");
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
test("compareReplicaWatermarks: a local-only namespace is ambiguous, not converged (round 2, codex P1)", () => {
|
|
255
|
+
// A namespace-scoped peer token returns only `default`, and an unrestricted
|
|
256
|
+
// peer that genuinely LOST `team-a` returns exactly the same shape. Round 1
|
|
257
|
+
// called this advisory and certified `converged`, which would hide a peer
|
|
258
|
+
// that dropped an entire namespace. It is neither divergence nor health:
|
|
259
|
+
// the comparison resolves the peer to `unknown` so nothing is certified on
|
|
260
|
+
// evidence that cannot distinguish the two cases.
|
|
261
|
+
const local = [watermark("default", { digest: "d" }), watermark("team-a", { digest: "d" })];
|
|
262
|
+
const peer = [watermark("default", { digest: "d" })];
|
|
263
|
+
const result = compareReplicaWatermarks(local, peer, THRESHOLDS);
|
|
264
|
+
assert.equal(result.state, "unknown", "an ambiguous local-only namespace must not read as converged");
|
|
265
|
+
assert.notEqual(result.state, "converged", "the round-1 contract would have hidden a peer that lost a namespace");
|
|
266
|
+
assert.equal(result.divergedNamespaceCount, 0, "ambiguous is not divergence either");
|
|
267
|
+
const teamA = result.namespaces.find((delta) => delta.namespace === "team-a");
|
|
268
|
+
assert.equal(teamA?.presence, "local_only");
|
|
269
|
+
assert.equal(teamA?.diverged, false);
|
|
270
|
+
});
|
|
271
|
+
|
|
272
|
+
test("compareReplicaWatermarks: no shared namespace is unknown, never converged (round 6, codex P1)", () => {
|
|
273
|
+
// A local empty corpus and a peer that scoped out every namespace both present
|
|
274
|
+
// as empty maps. Zero overlap is zero evidence of agreement — it must not fall
|
|
275
|
+
// through to converged.
|
|
276
|
+
const empty = compareReplicaWatermarks([], [], THRESHOLDS);
|
|
277
|
+
assert.equal(empty.state, "unknown", "no shared namespace cannot be certified converged");
|
|
278
|
+
assert.equal(empty.reason, "no_shared_namespaces");
|
|
279
|
+
assert.notEqual(empty.state, "converged");
|
|
280
|
+
|
|
281
|
+
// But a genuinely empty single-namespace deployment still shares that one
|
|
282
|
+
// namespace on both sides (0 files == 0 files) and converges normally.
|
|
283
|
+
const emptyDefault = watermark("default", {
|
|
284
|
+
memoryFileCount: 0,
|
|
285
|
+
newestWriteAt: null,
|
|
286
|
+
newestPartition: null,
|
|
287
|
+
digest: "empty",
|
|
288
|
+
});
|
|
289
|
+
const single = compareReplicaWatermarks([emptyDefault], [emptyDefault], THRESHOLDS);
|
|
290
|
+
assert.equal(single.state, "converged", "an empty but SHARED namespace is still evidence of agreement");
|
|
291
|
+
});
|
|
292
|
+
|
|
293
|
+
test("compareReplicaWatermarks: real watermarks with equal count but different day distribution diverge", async () => {
|
|
294
|
+
const baseDir = "/tmp/replica-fixture";
|
|
295
|
+
const localPaths = ["facts/2026-03-01/a.md", "facts/2026-03-01/b.md"].map((rel) => `${baseDir}/${rel}`);
|
|
296
|
+
const peerPaths = ["facts/2026-03-01/a.md", "facts/2026-03-02/b.md"].map((rel) => `${baseDir}/${rel}`);
|
|
297
|
+
const now = new Date("2026-03-08T00:00:00.000Z");
|
|
298
|
+
const local = await computeCorpusWatermark({ namespace: "default", paths: localPaths, baseDir, now });
|
|
299
|
+
const peer = await computeCorpusWatermark({ namespace: "default", paths: peerPaths, baseDir, now });
|
|
300
|
+
assert.equal(local.memoryFileCount, peer.memoryFileCount, "both have 2 files");
|
|
301
|
+
assert.notEqual(local.digest, peer.digest, "different day distribution changes the census digest");
|
|
302
|
+
const result = compareReplicaWatermarks([local], [peer], THRESHOLDS);
|
|
303
|
+
assert.equal(result.state, "diverged");
|
|
304
|
+
assert.equal(result.namespaces[0]?.digestMatch, false);
|
|
305
|
+
});
|
|
306
|
+
|
|
307
|
+
// ---------------------------------------------------------------------------
|
|
308
|
+
// fetchPeerWatermarks / pollReplicaPeers (real HTTP peer on 127.0.0.1)
|
|
309
|
+
// ---------------------------------------------------------------------------
|
|
310
|
+
|
|
311
|
+
test("pollReplicaPeers: converged peer over the engram prefix this server serves (no fallback needed)", async () => {
|
|
312
|
+
const peer = await startPeer(corpusHandler([watermark("default")], "/engram/v1/health"));
|
|
313
|
+
try {
|
|
314
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: peer.url }] } });
|
|
315
|
+
const report = await pollReplicaPeers({ config, localWatermarks: [watermark("default")] });
|
|
316
|
+
assert.equal(report.peers.length, 1);
|
|
317
|
+
assert.equal(report.peers[0]?.state, "converged");
|
|
318
|
+
const paths = peer.requests.map((request) => request.pathname);
|
|
319
|
+
assert.deepEqual(paths, ["/engram/v1/health"], "the served path is tried first, so remnic is never requested");
|
|
320
|
+
} finally {
|
|
321
|
+
await peer.close();
|
|
322
|
+
}
|
|
323
|
+
});
|
|
324
|
+
|
|
325
|
+
test("pollReplicaPeers: falls back to the remnic prefix when engram 404s (round 6, codex P2)", async () => {
|
|
326
|
+
const peer = await startPeer(corpusHandler([watermark("default")], "/remnic/v1/health"));
|
|
327
|
+
try {
|
|
328
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: peer.url }] } });
|
|
329
|
+
const report = await pollReplicaPeers({ config, localWatermarks: [watermark("default")] });
|
|
330
|
+
assert.equal(report.peers[0]?.state, "converged");
|
|
331
|
+
const paths = peer.requests.map((request) => request.pathname);
|
|
332
|
+
assert.deepEqual(paths, ["/engram/v1/health", "/remnic/v1/health"], "engram is tried first, remnic is the fallback");
|
|
333
|
+
} finally {
|
|
334
|
+
await peer.close();
|
|
335
|
+
}
|
|
336
|
+
});
|
|
337
|
+
|
|
338
|
+
test("pollReplicaPeers: file-count divergence end to end reports concrete deltas", async () => {
|
|
339
|
+
const peer = await startPeer(corpusHandler([watermark("default", { memoryFileCount: 340_000, digest: "x" })]));
|
|
340
|
+
try {
|
|
341
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: peer.url }] } });
|
|
342
|
+
const report = await pollReplicaPeers({
|
|
343
|
+
config,
|
|
344
|
+
localWatermarks: [watermark("default", { memoryFileCount: 190_000, digest: "x" })],
|
|
345
|
+
});
|
|
346
|
+
assert.equal(report.peers[0]?.state, "diverged");
|
|
347
|
+
assert.equal(report.peers[0]?.namespaces[0]?.fileCountDelta, 150_000);
|
|
348
|
+
} finally {
|
|
349
|
+
await peer.close();
|
|
350
|
+
}
|
|
351
|
+
});
|
|
352
|
+
|
|
353
|
+
test("pollReplicaPeers: a 500 is reported unreachable and is NOT conflated with converged (§22)", async () => {
|
|
354
|
+
const peer = await startPeer((_req, res) => {
|
|
355
|
+
res.writeHead(500);
|
|
356
|
+
res.end("boom");
|
|
357
|
+
});
|
|
358
|
+
try {
|
|
359
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: peer.url }] } });
|
|
360
|
+
const report = await pollReplicaPeers({ config, localWatermarks: [watermark("default")] });
|
|
361
|
+
assert.equal(report.peers[0]?.state, "unreachable");
|
|
362
|
+
assert.notEqual(report.peers[0]?.state, "converged");
|
|
363
|
+
assert.equal(report.peers[0]?.reason, "http_500");
|
|
364
|
+
} finally {
|
|
365
|
+
await peer.close();
|
|
366
|
+
}
|
|
367
|
+
});
|
|
368
|
+
|
|
369
|
+
test("pollReplicaPeers: a connection refused is unreachable, never converged (§22)", async () => {
|
|
370
|
+
// Bind then close to get a port that is deterministically refused.
|
|
371
|
+
const dead = await startPeer(corpusHandler([watermark("default")]));
|
|
372
|
+
const url = dead.url;
|
|
373
|
+
await dead.close();
|
|
374
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url }] } });
|
|
375
|
+
const report = await pollReplicaPeers({ config, localWatermarks: [watermark("default")] });
|
|
376
|
+
assert.equal(report.peers[0]?.state, "unreachable");
|
|
377
|
+
assert.notEqual(report.peers[0]?.state, "converged");
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
test("pollReplicaPeers: a stalled peer times out to unreachable, never converged (§22)", async () => {
|
|
381
|
+
const peer = await startPeer(() => {
|
|
382
|
+
// Never respond — the client's bounded timeout must abort.
|
|
383
|
+
});
|
|
384
|
+
try {
|
|
385
|
+
const config = parseReplicaPeersConfig({
|
|
386
|
+
replicaPeers: { enabled: true, requestTimeoutMs: 100, peers: [{ url: peer.url }] },
|
|
387
|
+
});
|
|
388
|
+
const report = await pollReplicaPeers({ config, localWatermarks: [watermark("default")] });
|
|
389
|
+
assert.equal(report.peers[0]?.state, "unreachable");
|
|
390
|
+
assert.notEqual(report.peers[0]?.state, "converged");
|
|
391
|
+
assert.equal(report.peers[0]?.reason, "timeout");
|
|
392
|
+
} finally {
|
|
393
|
+
await peer.close();
|
|
394
|
+
}
|
|
395
|
+
});
|
|
396
|
+
|
|
397
|
+
test("pollReplicaPeers: a 2xx payload without corpus is unknown, not converged (§22)", async () => {
|
|
398
|
+
const peer = await startPeer((_req, res) => {
|
|
399
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
400
|
+
res.end(JSON.stringify({ ok: true, extraction: {} })); // no `corpus`
|
|
401
|
+
});
|
|
402
|
+
try {
|
|
403
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: peer.url }] } });
|
|
404
|
+
const report = await pollReplicaPeers({ config, localWatermarks: [watermark("default")] });
|
|
405
|
+
assert.equal(report.peers[0]?.state, "unknown");
|
|
406
|
+
assert.notEqual(report.peers[0]?.state, "converged");
|
|
407
|
+
assert.equal(report.peers[0]?.reason, "missing_corpus");
|
|
408
|
+
} finally {
|
|
409
|
+
await peer.close();
|
|
410
|
+
}
|
|
411
|
+
});
|
|
412
|
+
|
|
413
|
+
test("pollReplicaPeers: disabled config performs NO network I/O (peer receives zero requests)", async () => {
|
|
414
|
+
const peer = await startPeer(corpusHandler([watermark("default")]));
|
|
415
|
+
try {
|
|
416
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: false, peers: [{ url: peer.url }] } });
|
|
417
|
+
const report = await pollReplicaPeers({ config, localWatermarks: [watermark("default")] });
|
|
418
|
+
assert.equal(report.enabled, false);
|
|
419
|
+
assert.equal(report.polledAt, null);
|
|
420
|
+
assert.equal(peer.requests.length, 0, "a disabled monitor must never touch the network");
|
|
421
|
+
} finally {
|
|
422
|
+
await peer.close();
|
|
423
|
+
}
|
|
424
|
+
});
|
|
425
|
+
|
|
426
|
+
test("pollReplicaPeers: the peer token is sent as a Bearer header but never leaks to the report or logs", async () => {
|
|
427
|
+
const secret = "SUPER-SECRET-PEER-TOKEN";
|
|
428
|
+
const peer = await startPeer(corpusHandler([watermark("default", { digest: "different", memoryFileCount: 999_999 })]));
|
|
429
|
+
const logs: string[] = [];
|
|
430
|
+
try {
|
|
431
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: peer.url, token: secret }] } });
|
|
432
|
+
const report = await pollReplicaPeers({ config, localWatermarks: [watermark("default")], log: (line) => logs.push(line) });
|
|
433
|
+
const authHeaders = peer.requests.map((request) => request.authorization);
|
|
434
|
+
assert.ok(authHeaders.includes(`Bearer ${secret}`), "the token IS sent (so the leak check is not vacuous)");
|
|
435
|
+
assert.ok(!JSON.stringify(report).includes(secret), "the token must not appear in the report payload");
|
|
436
|
+
assert.equal(report.peers[0]?.state, "diverged", "the peer diverged, so a warn line is emitted");
|
|
437
|
+
assert.ok(logs.length > 0, "a divergence is logged");
|
|
438
|
+
assert.ok(!logs.some((line) => line.includes(secret)), "the token must not appear in any log line");
|
|
439
|
+
} finally {
|
|
440
|
+
await peer.close();
|
|
441
|
+
}
|
|
442
|
+
});
|
|
443
|
+
|
|
444
|
+
test("pollReplicaPeers: a corpus with a malformed entry is unknown, not converged (review round 1)", async () => {
|
|
445
|
+
const peer = await startPeer((req, res) => {
|
|
446
|
+
if (new URL(req.url ?? "/", "http://127.0.0.1").pathname === "/engram/v1/health") {
|
|
447
|
+
res.writeHead(200, { "content-type": "application/json" });
|
|
448
|
+
res.end(JSON.stringify({ ok: true, corpus: [watermark("default"), {}] }));
|
|
449
|
+
return;
|
|
450
|
+
}
|
|
451
|
+
res.writeHead(404);
|
|
452
|
+
res.end();
|
|
453
|
+
});
|
|
454
|
+
try {
|
|
455
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: peer.url }] } });
|
|
456
|
+
const report = await pollReplicaPeers({ config, localWatermarks: [watermark("default")] });
|
|
457
|
+
assert.equal(report.peers[0]?.state, "unknown");
|
|
458
|
+
assert.equal(report.peers[0]?.reason, "malformed_corpus");
|
|
459
|
+
assert.notEqual(report.peers[0]?.state, "converged");
|
|
460
|
+
} finally {
|
|
461
|
+
await peer.close();
|
|
462
|
+
}
|
|
463
|
+
});
|
|
464
|
+
|
|
465
|
+
test("pollReplicaPeers: a SecretRef token with no resolver degrades to unreachable, never throws (review round 1)", async () => {
|
|
466
|
+
const peer = await startPeer(corpusHandler([watermark("default")]));
|
|
467
|
+
try {
|
|
468
|
+
const config = parseReplicaPeersConfig({
|
|
469
|
+
replicaPeers: { enabled: true, peers: [{ url: peer.url, token: { source: "exec", provider: "kc_peer" } }] },
|
|
470
|
+
});
|
|
471
|
+
// Resolving the SecretRef throws (no resolver); the whole poll must NOT reject.
|
|
472
|
+
const report = await pollReplicaPeers({ config, localWatermarks: [watermark("default")] });
|
|
473
|
+
assert.equal(report.peers[0]?.state, "unreachable");
|
|
474
|
+
assert.equal(report.peers[0]?.reason, "token_error");
|
|
475
|
+
assert.notEqual(report.peers[0]?.state, "converged");
|
|
476
|
+
assert.equal(peer.requests.length, 0, "no request is sent when the token cannot be resolved");
|
|
477
|
+
} finally {
|
|
478
|
+
await peer.close();
|
|
479
|
+
}
|
|
480
|
+
});
|
|
481
|
+
|
|
482
|
+
test("round 6 (codex P2): a stalling peer-token resolver times out instead of hanging the fetch", { timeout: 5000 }, async () => {
|
|
483
|
+
// A SecretRef whose host resolver never settles must not hang fetchPeerWatermarks;
|
|
484
|
+
// that would wedge the monitor's single-flight refresh and block doctor's batch.
|
|
485
|
+
// The per-peer deadline now bounds token resolution too. The 5s test timeout is a
|
|
486
|
+
// guard: on the fix, the poll returns in ~one requestTimeoutMs; a revert hangs.
|
|
487
|
+
const stall = () => Promise.withResolvers<string>().promise; // never settles
|
|
488
|
+
const config = parseReplicaPeersConfig({
|
|
489
|
+
replicaPeers: {
|
|
490
|
+
enabled: true,
|
|
491
|
+
requestTimeoutMs: 100,
|
|
492
|
+
peers: [{ url: "http://127.0.0.1:4318", token: { source: "exec", provider: "kc_peer" } }],
|
|
493
|
+
},
|
|
494
|
+
});
|
|
495
|
+
const report = await pollReplicaPeers({
|
|
496
|
+
config,
|
|
497
|
+
localWatermarks: [watermark("default")],
|
|
498
|
+
resolveSecretRef: stall,
|
|
499
|
+
fetchImpl: async () => ({ ok: true, status: 200, json: async () => ({ corpus: [watermark("default")] }) }),
|
|
500
|
+
});
|
|
501
|
+
assert.equal(report.peers[0]?.state, "unreachable", "a stalled token resolver degrades to unreachable, not a hang");
|
|
502
|
+
assert.equal(report.peers[0]?.reason, "timeout");
|
|
503
|
+
assert.notEqual(report.peers[0]?.state, "converged");
|
|
504
|
+
});
|
|
505
|
+
|
|
506
|
+
test("pollReplicaPeers: a provided resolveSecretRef resolves a SecretRef token and sends it as Bearer (review round 1)", async () => {
|
|
507
|
+
const peer = await startPeer(corpusHandler([watermark("default")]));
|
|
508
|
+
try {
|
|
509
|
+
const config = parseReplicaPeersConfig({
|
|
510
|
+
replicaPeers: { enabled: true, peers: [{ url: peer.url, token: { source: "exec", provider: "kc_peer" } }] },
|
|
511
|
+
});
|
|
512
|
+
const report = await pollReplicaPeers({
|
|
513
|
+
config,
|
|
514
|
+
localWatermarks: [watermark("default")],
|
|
515
|
+
resolveSecretRef: async () => "RESOLVED-PEER-BEARER",
|
|
516
|
+
});
|
|
517
|
+
assert.equal(report.peers[0]?.state, "converged");
|
|
518
|
+
assert.ok(
|
|
519
|
+
peer.requests.some((request) => request.authorization === "Bearer RESOLVED-PEER-BEARER"),
|
|
520
|
+
"the host-resolved SecretRef token is sent to the peer",
|
|
521
|
+
);
|
|
522
|
+
} finally {
|
|
523
|
+
await peer.close();
|
|
524
|
+
}
|
|
525
|
+
});
|
|
526
|
+
|
|
527
|
+
// ---------------------------------------------------------------------------
|
|
528
|
+
// Capability filtering (issue #2156 finding B parity)
|
|
529
|
+
// ---------------------------------------------------------------------------
|
|
530
|
+
|
|
531
|
+
function reportWithNamespaces(): ReplicaDivergenceStatus {
|
|
532
|
+
const comparison = compareReplicaWatermarks(
|
|
533
|
+
[watermark("default", { digest: "a" }), watermark("team-secret", { digest: "a" })],
|
|
534
|
+
[watermark("default", { digest: "a" }), watermark("team-secret", { digest: "b" })], // team-secret diverges
|
|
535
|
+
THRESHOLDS,
|
|
536
|
+
);
|
|
537
|
+
return {
|
|
538
|
+
enabled: true,
|
|
539
|
+
pending: false,
|
|
540
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
541
|
+
peers: [{ peer: "127.0.0.1:4318", state: comparison.state, polledAt: "2026-03-08T00:00:00.000Z", namespaces: comparison.namespaces, divergedNamespaceCount: comparison.divergedNamespaceCount }],
|
|
542
|
+
};
|
|
543
|
+
}
|
|
544
|
+
|
|
545
|
+
test("filterReplicaReportByCaps: a namespace-restricted token cannot see other namespaces' divergence", () => {
|
|
546
|
+
const report = reportWithNamespaces();
|
|
547
|
+
const restricted = filterReplicaReportByCaps(report, { version: 1, namespaces: ["default"] });
|
|
548
|
+
const visibleNamespaces = restricted.peers[0]?.namespaces.map((delta) => delta.namespace) ?? [];
|
|
549
|
+
assert.deepEqual(visibleNamespaces, ["default"], "only the permitted namespace is visible");
|
|
550
|
+
assert.equal(restricted.peers[0]?.state, "converged", "divergence in a hidden namespace must not leak as a diverged verdict");
|
|
551
|
+
assert.equal(restricted.peers[0]?.divergedNamespaceCount, 0);
|
|
552
|
+
});
|
|
553
|
+
|
|
554
|
+
test("filterReplicaReportByCaps: an unrestricted token sees the full report", () => {
|
|
555
|
+
const report = reportWithNamespaces();
|
|
556
|
+
const full = filterReplicaReportByCaps(report, { version: 1 });
|
|
557
|
+
assert.equal(full.peers[0]?.namespaces.length, 2);
|
|
558
|
+
assert.equal(full.peers[0]?.state, "diverged");
|
|
559
|
+
});
|
|
560
|
+
|
|
561
|
+
test("filterReplicaReportByCaps: an unreachable peer's state is preserved for a restricted token", () => {
|
|
562
|
+
const report: ReplicaDivergenceStatus = {
|
|
563
|
+
enabled: true,
|
|
564
|
+
pending: false,
|
|
565
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
566
|
+
peers: [{ peer: "127.0.0.1:4318", state: "unreachable", polledAt: "2026-03-08T00:00:00.000Z", namespaces: [], divergedNamespaceCount: 0, reason: "timeout" }],
|
|
567
|
+
};
|
|
568
|
+
const restricted = filterReplicaReportByCaps(report, { version: 1, namespaces: ["default"] });
|
|
569
|
+
assert.equal(restricted.peers[0]?.state, "unreachable", "reachability is not namespace data and must survive filtering");
|
|
570
|
+
assert.equal(restricted.peers[0]?.reason, "timeout");
|
|
571
|
+
});
|
|
572
|
+
|
|
573
|
+
// ---------------------------------------------------------------------------
|
|
574
|
+
// redaction
|
|
575
|
+
// ---------------------------------------------------------------------------
|
|
576
|
+
|
|
577
|
+
test("redactPeerUrl: strips userinfo, path, and query so an embedded credential cannot leak", () => {
|
|
578
|
+
assert.equal(redactPeerUrl("https://user:s3cr3t@host.example:4318/health?token=abc"), "host.example:4318");
|
|
579
|
+
assert.equal(redactPeerUrl("http://127.0.0.1:9999"), "127.0.0.1:9999");
|
|
580
|
+
assert.ok(!redactPeerUrl("https://user:s3cr3t@host.example:4318/x").includes("s3cr3t"));
|
|
581
|
+
});
|
|
582
|
+
|
|
583
|
+
// ---------------------------------------------------------------------------
|
|
584
|
+
// ReplicaDivergenceMonitor (SWR / single-flight)
|
|
585
|
+
// ---------------------------------------------------------------------------
|
|
586
|
+
|
|
587
|
+
test("ReplicaDivergenceMonitor: disabled config never polls and reports disabled", () => {
|
|
588
|
+
let computed = 0;
|
|
589
|
+
const monitor = new ReplicaDivergenceMonitor();
|
|
590
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: false, peers: [{ url: "http://127.0.0.1:1" }] } });
|
|
591
|
+
const report = monitor.getReport({ config, computeLocalWatermarks: async () => { computed += 1; return { watermarks: [], complete: true }; } });
|
|
592
|
+
assert.equal(report.enabled, false);
|
|
593
|
+
assert.equal(computed, 0, "a disabled monitor never computes local watermarks or polls");
|
|
594
|
+
});
|
|
595
|
+
|
|
596
|
+
test("ReplicaDivergenceMonitor: stale-while-revalidate serves cached state and single-flights the poll", async () => {
|
|
597
|
+
let clock = 1_000;
|
|
598
|
+
const fetchImpl: FetchLike = async () => ({ ok: true, status: 200, json: async () => ({ corpus: [watermark("default", { memoryFileCount: 999, digest: "z" })] }) });
|
|
599
|
+
let computeCalls = 0;
|
|
600
|
+
const computeLocalWatermarks = async () => {
|
|
601
|
+
computeCalls += 1;
|
|
602
|
+
return { watermarks: [watermark("default", { memoryFileCount: 1, digest: "a" })], complete: true };
|
|
603
|
+
};
|
|
604
|
+
const monitor = new ReplicaDivergenceMonitor({ clock: () => clock, fetchImpl });
|
|
605
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: true, pollIntervalMs: 60_000, peers: [{ url: "http://127.0.0.1:4318" }] } });
|
|
606
|
+
|
|
607
|
+
// First probe is cold: returns a never-polled placeholder and triggers a background poll.
|
|
608
|
+
const cold = monitor.getReport({ config, computeLocalWatermarks });
|
|
609
|
+
assert.equal(cold.polledAt, null);
|
|
610
|
+
await monitor.whenIdle();
|
|
611
|
+
|
|
612
|
+
// Second probe (within TTL) serves the completed report without re-polling.
|
|
613
|
+
const warm = monitor.getReport({ config, computeLocalWatermarks });
|
|
614
|
+
assert.equal(warm.peers[0]?.state, "diverged");
|
|
615
|
+
assert.equal(computeCalls, 1, "single-flight: exactly one poll within the TTL");
|
|
616
|
+
monitor.getReport({ config, computeLocalWatermarks });
|
|
617
|
+
await monitor.whenIdle();
|
|
618
|
+
assert.equal(computeCalls, 1, "still one — the cached report is fresh");
|
|
619
|
+
|
|
620
|
+
// After the TTL elapses, a probe triggers a refresh.
|
|
621
|
+
clock += 60_001;
|
|
622
|
+
monitor.getReport({ config, computeLocalWatermarks });
|
|
623
|
+
await monitor.whenIdle();
|
|
624
|
+
assert.equal(computeCalls, 2, "a probe past the poll interval re-polls");
|
|
625
|
+
});
|
|
626
|
+
|
|
627
|
+
test("ReplicaDivergenceMonitor: applies capability filtering at read time", async () => {
|
|
628
|
+
const fetchImpl: FetchLike = async () => ({
|
|
629
|
+
ok: true,
|
|
630
|
+
status: 200,
|
|
631
|
+
json: async () => ({ corpus: [watermark("default", { digest: "a" }), watermark("team-secret", { digest: "b" })] }),
|
|
632
|
+
});
|
|
633
|
+
const computeLocalWatermarks = async () => ({
|
|
634
|
+
watermarks: [watermark("default", { digest: "a" }), watermark("team-secret", { digest: "a" })],
|
|
635
|
+
complete: true,
|
|
636
|
+
});
|
|
637
|
+
const monitor = new ReplicaDivergenceMonitor({ fetchImpl });
|
|
638
|
+
const config = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } });
|
|
639
|
+
monitor.getReport({ config, computeLocalWatermarks });
|
|
640
|
+
await monitor.whenIdle();
|
|
641
|
+
const restricted = monitor.getReport({ config, computeLocalWatermarks, caps: { version: 1, namespaces: ["default"] } });
|
|
642
|
+
assert.deepEqual(restricted.peers[0]?.namespaces.map((delta) => delta.namespace), ["default"]);
|
|
643
|
+
assert.equal(restricted.peers[0]?.state, "converged", "the hidden team-secret divergence must not leak");
|
|
644
|
+
});
|
|
645
|
+
|
|
646
|
+
test("ReplicaDivergenceMonitor: warming state is distinguishable from no-peers configured (review round 1)", async () => {
|
|
647
|
+
const fetchImpl: FetchLike = async () => ({ ok: true, status: 200, json: async () => ({ corpus: [watermark("default")] }) });
|
|
648
|
+
const computeLocalWatermarks = async () => ({ watermarks: [watermark("default")], complete: true });
|
|
649
|
+
const monitor = new ReplicaDivergenceMonitor({ fetchImpl });
|
|
650
|
+
const withPeers = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } });
|
|
651
|
+
const noPeers = parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [] } });
|
|
652
|
+
|
|
653
|
+
// Enabled + peers but no completed poll yet -> pending:true (in-progress/failed).
|
|
654
|
+
const warming = monitor.getReport({ config: withPeers, computeLocalWatermarks });
|
|
655
|
+
assert.equal(warming.pending, true);
|
|
656
|
+
assert.equal(warming.peers.length, 0);
|
|
657
|
+
|
|
658
|
+
// Enabled but no peers configured -> pending:false, distinct from warming.
|
|
659
|
+
assert.equal(monitor.getReport({ config: noPeers, computeLocalWatermarks }).pending, false);
|
|
660
|
+
|
|
661
|
+
// After the poll completes -> pending:false with the real report.
|
|
662
|
+
await monitor.whenIdle();
|
|
663
|
+
const polled = monitor.getReport({ config: withPeers, computeLocalWatermarks });
|
|
664
|
+
assert.equal(polled.pending, false);
|
|
665
|
+
assert.equal(polled.peers[0]?.state, "converged");
|
|
666
|
+
});
|
|
667
|
+
|
|
668
|
+
test("ReplicaDivergenceMonitor: a partial config with no replicaPeers block reads as disabled, never crashes (review round 1)", () => {
|
|
669
|
+
const monitor = new ReplicaDivergenceMonitor();
|
|
670
|
+
// A duck-typed/legacy orchestrator whose config has no `replicaPeers` block:
|
|
671
|
+
// health() -> getReport must degrade to disabled, not throw on `config.enabled`.
|
|
672
|
+
const report = monitor.getReport({ config: undefined, computeLocalWatermarks: async () => ({ watermarks: [], complete: true }) });
|
|
673
|
+
assert.equal(report.enabled, false);
|
|
674
|
+
assert.equal(report.pending, false);
|
|
675
|
+
assert.equal(report.peers.length, 0);
|
|
676
|
+
});
|
|
677
|
+
|
|
678
|
+
// ---------------------------------------------------------------------------
|
|
679
|
+
// Review round 2 regressions
|
|
680
|
+
// ---------------------------------------------------------------------------
|
|
681
|
+
|
|
682
|
+
test("round 2 (cursor): poll TTL runs from completion, so a slow poll cannot self-expire", async () => {
|
|
683
|
+
// expiresAt was computed from the poll's START. A poll slower than
|
|
684
|
+
// pollIntervalMs was therefore already stale when stored, so every probe
|
|
685
|
+
// scheduled another peer poll — unbounded re-polling under slow peers,
|
|
686
|
+
// exactly when peers can least afford it.
|
|
687
|
+
let now = 1_000_000;
|
|
688
|
+
let polls = 0;
|
|
689
|
+
const fetchImpl: FetchLike = async () => {
|
|
690
|
+
polls += 1;
|
|
691
|
+
now += 60_000; // the poll itself outlasts the 30s interval below
|
|
692
|
+
return { ok: true, status: 200, json: async () => ({ corpus: [watermark("default")] }) };
|
|
693
|
+
};
|
|
694
|
+
const monitor = new ReplicaDivergenceMonitor({ fetchImpl, clock: () => now });
|
|
695
|
+
const config = parseReplicaPeersConfig({
|
|
696
|
+
replicaPeers: { enabled: true, pollIntervalMs: 30_000, peers: [{ url: "http://127.0.0.1:4318" }] },
|
|
697
|
+
});
|
|
698
|
+
const computeLocalWatermarks = async () => ({ watermarks: [watermark("default")], complete: true });
|
|
699
|
+
|
|
700
|
+
monitor.getReport({ config, computeLocalWatermarks });
|
|
701
|
+
await monitor.whenIdle();
|
|
702
|
+
assert.equal(polls, 1, "first probe triggers exactly one poll");
|
|
703
|
+
|
|
704
|
+
// Immediately after completion the entry must still be fresh: the TTL starts
|
|
705
|
+
// when the poll FINISHED, so no second poll is scheduled.
|
|
706
|
+
monitor.getReport({ config, computeLocalWatermarks });
|
|
707
|
+
await monitor.whenIdle();
|
|
708
|
+
assert.equal(polls, 1, "a poll slower than the interval must not be born expired");
|
|
709
|
+
|
|
710
|
+
now += 31_000; // now past the interval measured from completion
|
|
711
|
+
monitor.getReport({ config, computeLocalWatermarks });
|
|
712
|
+
await monitor.whenIdle();
|
|
713
|
+
assert.equal(polls, 2, "the cache still expires once the interval genuinely elapses");
|
|
714
|
+
});
|
|
715
|
+
|
|
716
|
+
test("round 6 (coderabbit): a persistently failing poll backs off one interval, not one scan per probe", async () => {
|
|
717
|
+
// this.cached is set only on success; without backoff every /health probe found
|
|
718
|
+
// no fresh cache and rescheduled a full local corpus scan + peer fan-out.
|
|
719
|
+
let clock = 1_000_000;
|
|
720
|
+
let computeCalls = 0;
|
|
721
|
+
const computeLocalWatermarks = async () => {
|
|
722
|
+
computeCalls += 1;
|
|
723
|
+
throw new Error("local corpus unreadable");
|
|
724
|
+
};
|
|
725
|
+
const fetchImpl: FetchLike = async () => ({ ok: true, status: 200, json: async () => ({ corpus: [watermark("default")] }) });
|
|
726
|
+
const monitor = new ReplicaDivergenceMonitor({ clock: () => clock, fetchImpl });
|
|
727
|
+
const config = parseReplicaPeersConfig({
|
|
728
|
+
replicaPeers: { enabled: true, pollIntervalMs: 60_000, peers: [{ url: "http://127.0.0.1:4318" }] },
|
|
729
|
+
});
|
|
730
|
+
|
|
731
|
+
const first = monitor.getReport({ config, computeLocalWatermarks });
|
|
732
|
+
await monitor.whenIdle();
|
|
733
|
+
assert.equal(computeCalls, 1, "the first probe triggers exactly one scan");
|
|
734
|
+
assert.equal(first.pending, true, "a never-completed poll reports pending, never converged");
|
|
735
|
+
|
|
736
|
+
for (let i = 0; i < 5; i += 1) {
|
|
737
|
+
const during = monitor.getReport({ config, computeLocalWatermarks });
|
|
738
|
+
await monitor.whenIdle();
|
|
739
|
+
assert.equal(during.pending, true, "probes during backoff stay pending, never converged");
|
|
740
|
+
}
|
|
741
|
+
assert.equal(computeCalls, 1, "sequential probes within one interval must NOT reschedule the scan");
|
|
742
|
+
|
|
743
|
+
clock += 61_000;
|
|
744
|
+
monitor.getReport({ config, computeLocalWatermarks });
|
|
745
|
+
await monitor.whenIdle();
|
|
746
|
+
assert.equal(computeCalls, 2, "the backoff releases after pollIntervalMs elapses");
|
|
747
|
+
});
|
|
748
|
+
|
|
749
|
+
test("round 2 (codex P2): the file-count threshold tolerates a real sub-threshold delta", async () => {
|
|
750
|
+
// Any nonzero count delta also perturbs the census digest, so flagging
|
|
751
|
+
// digest mismatch unconditionally made maxFileCountDelta unreachable for
|
|
752
|
+
// REAL watermarks — a delta of 1 always reported diverged. Digest mismatch
|
|
753
|
+
// is now the equal-count (split-brain) signal only.
|
|
754
|
+
const baseDir = "/tmp/replica-threshold-fixture";
|
|
755
|
+
const localPaths = ["facts/2026-03-01/a.md", "facts/2026-03-01/b.md"].map((rel) => `${baseDir}/${rel}`);
|
|
756
|
+
const peerPaths = [...localPaths, `${baseDir}/facts/2026-03-01/c.md`];
|
|
757
|
+
const now = new Date("2026-03-08T00:00:00.000Z");
|
|
758
|
+
const local = await computeCorpusWatermark({ namespace: "default", paths: localPaths, baseDir, now });
|
|
759
|
+
const peer = await computeCorpusWatermark({ namespace: "default", paths: peerPaths, baseDir, now });
|
|
760
|
+
assert.notEqual(local.digest, peer.digest, "a real count delta necessarily changes the digest");
|
|
761
|
+
|
|
762
|
+
const tolerant = compareReplicaWatermarks([local], [peer], THRESHOLDS);
|
|
763
|
+
assert.equal(tolerant.state, "converged", "a delta of 1 is within maxFileCountDelta=100 and must be tolerated");
|
|
764
|
+
|
|
765
|
+
const strict = compareReplicaWatermarks([local], [peer], { maxFileCountDelta: 0, maxWatermarkAgeDeltaMs: 900_000 });
|
|
766
|
+
assert.equal(strict.state, "diverged", "the same pair still diverges once the threshold is tightened");
|
|
767
|
+
});
|
|
768
|
+
|
|
769
|
+
test("round 2 (codex P2): semantically malformed peer watermarks read as unknown, never converged", async () => {
|
|
770
|
+
// Structurally present but invalid values used to pass: a peer could return
|
|
771
|
+
// memoryFileCount: -1 with an unparseable timestamp and be certified
|
|
772
|
+
// converged. Corrupt telemetry must be unknown, not health.
|
|
773
|
+
const cases: Array<[string, unknown]> = [
|
|
774
|
+
["negative count", { ...watermark("default"), memoryFileCount: -1 }],
|
|
775
|
+
["fractional count", { ...watermark("default"), memoryFileCount: 1.5 }],
|
|
776
|
+
["unparseable timestamp", { ...watermark("default"), newestWriteAt: "not-a-date" }],
|
|
777
|
+
["non-string digest", { ...watermark("default"), digest: 42 }],
|
|
778
|
+
];
|
|
779
|
+
for (const [label, corpus] of cases) {
|
|
780
|
+
const fetchImpl: FetchLike = async () => ({ ok: true, status: 200, json: async () => ({ corpus: [corpus] }) });
|
|
781
|
+
const status = await pollReplicaPeers({
|
|
782
|
+
config: parseReplicaPeersConfig({
|
|
783
|
+
replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] },
|
|
784
|
+
}),
|
|
785
|
+
localWatermarks: [watermark("default")],
|
|
786
|
+
fetchImpl,
|
|
787
|
+
});
|
|
788
|
+
assert.equal(status.peers[0]?.state, "unknown", `${label} must resolve to unknown`);
|
|
789
|
+
assert.notEqual(status.peers[0]?.state, "converged", `${label} must never be certified converged`);
|
|
790
|
+
}
|
|
791
|
+
});
|
|
792
|
+
|
|
793
|
+
test("round 6 (codex P1): a stale peer census reads as unknown, never converged", async () => {
|
|
794
|
+
// CorpusWatermarkCache serves a prior watermark when a refresh fails, so a peer
|
|
795
|
+
// that changed after its last successful scan presents an old computedAt. A
|
|
796
|
+
// census older than maxWatermarkAgeDeltaMs cannot certify convergence.
|
|
797
|
+
const now = Date.parse("2026-03-08T12:00:00.000Z");
|
|
798
|
+
const staleAt = new Date(now - 20 * 60_000).toISOString(); // 20 min > default 15 min bound
|
|
799
|
+
const staleImpl: FetchLike = async () => ({
|
|
800
|
+
ok: true,
|
|
801
|
+
status: 200,
|
|
802
|
+
json: async () => ({ corpus: [watermark("default", { computedAt: staleAt })] }),
|
|
803
|
+
});
|
|
804
|
+
const stale = await pollReplicaPeers({
|
|
805
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
806
|
+
localWatermarks: [watermark("default")],
|
|
807
|
+
now: new Date(now),
|
|
808
|
+
fetchImpl: staleImpl,
|
|
809
|
+
});
|
|
810
|
+
assert.equal(stale.peers[0]?.state, "unknown", "a stale peer census must not be certified converged");
|
|
811
|
+
assert.equal(stale.peers[0]?.reason, "peer_census_stale");
|
|
812
|
+
assert.notEqual(stale.peers[0]?.state, "converged");
|
|
813
|
+
|
|
814
|
+
// A fresh census (default helper computedAt = now) still converges.
|
|
815
|
+
const freshImpl: FetchLike = async () => ({ ok: true, status: 200, json: async () => ({ corpus: [watermark("default")] }) });
|
|
816
|
+
const fresh = await pollReplicaPeers({
|
|
817
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
818
|
+
localWatermarks: [watermark("default")],
|
|
819
|
+
fetchImpl: freshImpl,
|
|
820
|
+
});
|
|
821
|
+
assert.equal(fresh.peers[0]?.state, "converged", "a fresh peer census still converges");
|
|
822
|
+
});
|
|
823
|
+
|
|
824
|
+
test("round 6 (codex P2): a peer census computed during a slow poll is fresh, not future-stale", { timeout: 5000 }, async () => {
|
|
825
|
+
// polledAt is captured before the fan-out; a slow/queued peer computes its census
|
|
826
|
+
// AFTER polledAt. Freshness is measured at RESPONSE time, so that fresh census is
|
|
827
|
+
// not falsely flagged peer_census_stale under a small age bound. Real timers
|
|
828
|
+
// (ts-no-test-timers exception): the delay past polledAt IS the behavior under test.
|
|
829
|
+
const fetchImpl: FetchLike = async () => {
|
|
830
|
+
const gate = Promise.withResolvers<void>();
|
|
831
|
+
setTimeout(() => gate.resolve(), 120); // push the response well past polledAt
|
|
832
|
+
await gate.promise;
|
|
833
|
+
return { ok: true, status: 200, json: async () => ({ corpus: [watermark("default", { computedAt: new Date().toISOString() })] }) };
|
|
834
|
+
};
|
|
835
|
+
const status = await pollReplicaPeers({
|
|
836
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, maxWatermarkAgeDeltaMs: 50, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
837
|
+
localWatermarks: [watermark("default")],
|
|
838
|
+
fetchImpl,
|
|
839
|
+
});
|
|
840
|
+
assert.equal(status.peers[0]?.state, "converged", "a census computed during the request is fresh at response time, not future-stale");
|
|
841
|
+
assert.notEqual(status.peers[0]?.reason, "peer_census_stale");
|
|
842
|
+
});
|
|
843
|
+
|
|
844
|
+
test("round 6 (cursor): maxWatermarkAgeDeltaMs=0 (any-gap divergence mode) does NOT gate census staleness", async () => {
|
|
845
|
+
// 0 is the documented strictest DIVERGENCE mode ("flag any write-age gap"); it
|
|
846
|
+
// must NOT be read as a 0ms census-freshness bound that marks every peer stale
|
|
847
|
+
// and blocks convergence entirely.
|
|
848
|
+
const now = Date.parse("2026-03-08T12:00:00.000Z");
|
|
849
|
+
const oldButMatching = watermark("default", { computedAt: new Date(now - 60 * 60_000).toISOString() });
|
|
850
|
+
const fetchImpl: FetchLike = async () => ({ ok: true, status: 200, json: async () => ({ corpus: [oldButMatching] }) });
|
|
851
|
+
const status = await pollReplicaPeers({
|
|
852
|
+
config: parseReplicaPeersConfig({
|
|
853
|
+
replicaPeers: { enabled: true, maxWatermarkAgeDeltaMs: 0, peers: [{ url: "http://127.0.0.1:4318" }] },
|
|
854
|
+
}),
|
|
855
|
+
localWatermarks: [watermark("default", { computedAt: new Date(now).toISOString() })],
|
|
856
|
+
now: new Date(now),
|
|
857
|
+
fetchImpl,
|
|
858
|
+
});
|
|
859
|
+
assert.equal(status.peers[0]?.state, "converged", "a 0 age bound must not certify every peer as peer_census_stale");
|
|
860
|
+
assert.notEqual(status.peers[0]?.reason, "peer_census_stale");
|
|
861
|
+
});
|
|
862
|
+
|
|
863
|
+
test("round 6 (codex P2): a future-dated peer census is rejected, never indefinitely fresh", async () => {
|
|
864
|
+
// A `9999` computedAt makes `now - computedAt` negative, which a one-sided
|
|
865
|
+
// "> maxAge" check reads as fresh forever. A materially future-dated census is
|
|
866
|
+
// corrupt/clock-skewed telemetry and must not certify convergence.
|
|
867
|
+
const now = Date.parse("2026-03-08T12:00:00.000Z");
|
|
868
|
+
const futureImpl: FetchLike = async () => ({
|
|
869
|
+
ok: true,
|
|
870
|
+
status: 200,
|
|
871
|
+
json: async () => ({ corpus: [watermark("default", { computedAt: "9999-01-01T00:00:00.000Z" })] }),
|
|
872
|
+
});
|
|
873
|
+
const status = await pollReplicaPeers({
|
|
874
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
875
|
+
localWatermarks: [watermark("default")],
|
|
876
|
+
now: new Date(now),
|
|
877
|
+
fetchImpl: futureImpl,
|
|
878
|
+
});
|
|
879
|
+
assert.equal(status.peers[0]?.state, "unknown", "a future-dated census must not be certified converged");
|
|
880
|
+
assert.equal(status.peers[0]?.reason, "peer_census_stale");
|
|
881
|
+
assert.notEqual(status.peers[0]?.state, "converged");
|
|
882
|
+
});
|
|
883
|
+
|
|
884
|
+
test("round 3 (codex P2): a peer repeating a namespace is malformed, not converged", async () => {
|
|
885
|
+
// compareReplicaWatermarks builds a Map, so a later duplicate silently wins:
|
|
886
|
+
// a mismatching watermark followed by a matching one would certify converged.
|
|
887
|
+
const fetchImpl: FetchLike = async () => ({
|
|
888
|
+
ok: true,
|
|
889
|
+
status: 200,
|
|
890
|
+
json: async () => ({
|
|
891
|
+
corpus: [
|
|
892
|
+
{ ...watermark("default"), digest: "does-not-match", memoryFileCount: 9_999 },
|
|
893
|
+
watermark("default"),
|
|
894
|
+
],
|
|
895
|
+
}),
|
|
896
|
+
});
|
|
897
|
+
const status = await pollReplicaPeers({
|
|
898
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
899
|
+
localWatermarks: [watermark("default")],
|
|
900
|
+
fetchImpl,
|
|
901
|
+
});
|
|
902
|
+
assert.equal(status.peers[0]?.state, "unknown", "a duplicated namespace must read as malformed");
|
|
903
|
+
assert.notEqual(status.peers[0]?.state, "converged", "the shadowed mismatch must not be certified healthy");
|
|
904
|
+
assert.equal(status.peers[0]?.reason, "malformed_corpus");
|
|
905
|
+
});
|
|
906
|
+
|
|
907
|
+
test("round 3 (codex P2): a present-but-invalid replicaPeers.enabled is rejected, not read as false", () => {
|
|
908
|
+
// `enabled: 1` used to coerce to undefined and fall back to false — silently
|
|
909
|
+
// leaving monitoring OFF after an operator tried to turn it on.
|
|
910
|
+
for (const bad of [1, 0, "yes-please", {}]) {
|
|
911
|
+
assert.throws(
|
|
912
|
+
() => parseReplicaPeersConfig({ replicaPeers: { enabled: bad, peers: [] } }),
|
|
913
|
+
/replicaPeers\.enabled must be a boolean/,
|
|
914
|
+
`enabled: ${JSON.stringify(bad)} must be rejected`,
|
|
915
|
+
);
|
|
916
|
+
}
|
|
917
|
+
// Recognized tokens and absence still work.
|
|
918
|
+
assert.equal(parseReplicaPeersConfig({ replicaPeers: { enabled: "false", peers: [] } }).enabled, false);
|
|
919
|
+
assert.equal(parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [] } }).enabled, true);
|
|
920
|
+
assert.equal(parseReplicaPeersConfig({ replicaPeers: { peers: [] } }).enabled, false, "absent -> default");
|
|
921
|
+
});
|
|
922
|
+
|
|
923
|
+
test("round 4 (cursor): an incomplete local census cannot certify a peer converged", async () => {
|
|
924
|
+
// computeServiceCorpusWatermarks DROPS a namespace whose scan failed, so an
|
|
925
|
+
// unscanned tenant never enters the comparison and its divergence would be
|
|
926
|
+
// invisible. Doctor already gated on this; /health did not.
|
|
927
|
+
const fetchImpl: FetchLike = async () => ({
|
|
928
|
+
ok: true,
|
|
929
|
+
status: 200,
|
|
930
|
+
json: async () => ({ corpus: [watermark("default")] }),
|
|
931
|
+
});
|
|
932
|
+
const config = parseReplicaPeersConfig({
|
|
933
|
+
replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] },
|
|
934
|
+
});
|
|
935
|
+
|
|
936
|
+
const complete = new ReplicaDivergenceMonitor({ fetchImpl });
|
|
937
|
+
complete.getReport({ config, computeLocalWatermarks: async () => ({ watermarks: [watermark("default")], complete: true }) });
|
|
938
|
+
await complete.whenIdle();
|
|
939
|
+
const ok = complete.getReport({ config, computeLocalWatermarks: async () => ({ watermarks: [watermark("default")], complete: true }) });
|
|
940
|
+
assert.equal(ok.peers[0]?.state, "converged", "a complete census still certifies convergence");
|
|
941
|
+
assert.equal(ok.censusComplete, true);
|
|
942
|
+
|
|
943
|
+
const partial = new ReplicaDivergenceMonitor({ fetchImpl });
|
|
944
|
+
const incomplete = async () => ({ watermarks: [watermark("default")], complete: false });
|
|
945
|
+
partial.getReport({ config, computeLocalWatermarks: incomplete });
|
|
946
|
+
await partial.whenIdle();
|
|
947
|
+
const gated = partial.getReport({ config, computeLocalWatermarks: incomplete });
|
|
948
|
+
assert.equal(gated.censusComplete, false);
|
|
949
|
+
assert.equal(gated.peers[0]?.state, "unknown", "an incomplete census downgrades converged to unknown");
|
|
950
|
+
assert.equal(gated.peers[0]?.reason, "local_census_incomplete");
|
|
951
|
+
});
|
|
952
|
+
|
|
953
|
+
test("round 4 (codex P2): a present null peer token is rejected, not silently unauthenticated", () => {
|
|
954
|
+
// Dropping `token: null` polls without the credential and surfaces a healthy
|
|
955
|
+
// peer as http_401, hiding the real configuration error.
|
|
956
|
+
assert.throws(
|
|
957
|
+
() =>
|
|
958
|
+
parseReplicaPeersConfig({
|
|
959
|
+
replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318", token: null }] },
|
|
960
|
+
}),
|
|
961
|
+
/token must be a string or a SecretRef object/,
|
|
962
|
+
);
|
|
963
|
+
// An OMITTED token still selects unauthenticated polling.
|
|
964
|
+
const omitted = parseReplicaPeersConfig({
|
|
965
|
+
replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] },
|
|
966
|
+
});
|
|
967
|
+
assert.equal(omitted.peers[0]?.token, undefined);
|
|
968
|
+
});
|
|
969
|
+
|
|
970
|
+
/** A both-sides, in-agreement namespace delta for report-shaped fixtures. */
|
|
971
|
+
function delta(namespace: string) {
|
|
972
|
+
return {
|
|
973
|
+
namespace,
|
|
974
|
+
presence: "both" as const,
|
|
975
|
+
fileCountDelta: 0,
|
|
976
|
+
writeAgeDeltaMs: 0,
|
|
977
|
+
digestMatch: true,
|
|
978
|
+
diverged: false,
|
|
979
|
+
reasons: [] as string[],
|
|
980
|
+
localFileCount: 10,
|
|
981
|
+
peerFileCount: 10,
|
|
982
|
+
localNewestWriteAt: "2026-03-08T00:00:00.000Z",
|
|
983
|
+
peerNewestWriteAt: "2026-03-08T00:00:00.000Z",
|
|
984
|
+
};
|
|
985
|
+
}
|
|
986
|
+
|
|
987
|
+
test("round 5 (cursor/codex): a capability filter cannot clear a census-level unknown", () => {
|
|
988
|
+
// The cap filter recomputes a peer's state from its VISIBLE deltas. A
|
|
989
|
+
// census-level unknown says the LOCAL set was partial, which no amount of
|
|
990
|
+
// namespace filtering makes safe — it must survive the recompute.
|
|
991
|
+
const report: ReplicaDivergenceStatus = {
|
|
992
|
+
enabled: true,
|
|
993
|
+
pending: false,
|
|
994
|
+
censusComplete: false,
|
|
995
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
996
|
+
peers: [
|
|
997
|
+
{
|
|
998
|
+
peer: "127.0.0.1:4318",
|
|
999
|
+
state: "unknown",
|
|
1000
|
+
reason: "local_census_incomplete",
|
|
1001
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1002
|
+
divergedNamespaceCount: 0,
|
|
1003
|
+
namespaces: [
|
|
1004
|
+
delta("default"),
|
|
1005
|
+
delta("team-secret"),
|
|
1006
|
+
],
|
|
1007
|
+
},
|
|
1008
|
+
],
|
|
1009
|
+
};
|
|
1010
|
+
const filtered = filterReplicaReportByCaps(report, { namespaces: ["default"] } as never);
|
|
1011
|
+
assert.equal(filtered.peers[0]?.state, "unknown", "an incomplete census must not be filtered into converged");
|
|
1012
|
+
assert.equal(filtered.peers[0]?.reason, "local_census_incomplete", "and its reason must survive");
|
|
1013
|
+
assert.equal(filtered.peers[0]?.namespaces.length, 1, "hidden namespaces are still filtered out");
|
|
1014
|
+
});
|
|
1015
|
+
|
|
1016
|
+
test("round 6 (coderabbit): a cap filter that hides every namespace yields unknown, not converged", async () => {
|
|
1017
|
+
// A restricted token whose capabilities hide EVERY namespace of a converged
|
|
1018
|
+
// peer leaves zero visible deltas — no evidence of agreement. The recompute
|
|
1019
|
+
// must resolve to unknown, not a convergence claim derived from nothing.
|
|
1020
|
+
const report: ReplicaDivergenceStatus = {
|
|
1021
|
+
enabled: true,
|
|
1022
|
+
pending: false,
|
|
1023
|
+
censusComplete: true,
|
|
1024
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1025
|
+
peers: [
|
|
1026
|
+
{
|
|
1027
|
+
peer: "127.0.0.1:4318",
|
|
1028
|
+
state: "converged",
|
|
1029
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1030
|
+
divergedNamespaceCount: 0,
|
|
1031
|
+
namespaces: [delta("team-secret")],
|
|
1032
|
+
},
|
|
1033
|
+
],
|
|
1034
|
+
};
|
|
1035
|
+
const filtered = filterReplicaReportByCaps(report, { namespaces: ["default"] } as never);
|
|
1036
|
+
assert.equal(filtered.peers[0]?.state, "unknown", "hiding every namespace cannot leave a converged verdict");
|
|
1037
|
+
assert.equal(filtered.peers[0]?.reason, "no_shared_namespaces");
|
|
1038
|
+
assert.equal(filtered.peers[0]?.namespaces.length, 0, "the hidden namespace is filtered out");
|
|
1039
|
+
});
|
|
1040
|
+
|
|
1041
|
+
test("round 6 (codex P2): capability filtering re-applies the census gate (no false split-brain for a restricted caller)", () => {
|
|
1042
|
+
// Incomplete census + a peer with a REAL shared divergence in a HIDDEN namespace
|
|
1043
|
+
// AND a visible peer_only namespace. The full-report gate keeps the peer diverged
|
|
1044
|
+
// (the shared divergence is real). Filtering hides that namespace, leaving only
|
|
1045
|
+
// the peer_only delta — which, against an incomplete census, is a false
|
|
1046
|
+
// split-brain. The re-applied gate must downgrade the FILTERED view to unknown.
|
|
1047
|
+
const report: ReplicaDivergenceStatus = {
|
|
1048
|
+
enabled: true,
|
|
1049
|
+
pending: false,
|
|
1050
|
+
censusComplete: false,
|
|
1051
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1052
|
+
peers: [
|
|
1053
|
+
{
|
|
1054
|
+
peer: "127.0.0.1:4318",
|
|
1055
|
+
state: "diverged",
|
|
1056
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1057
|
+
divergedNamespaceCount: 2,
|
|
1058
|
+
namespaces: [
|
|
1059
|
+
{ ...delta("team-secret"), diverged: true, digestMatch: false, reasons: ["digest_mismatch"] },
|
|
1060
|
+
{
|
|
1061
|
+
namespace: "team-a",
|
|
1062
|
+
presence: "peer_only",
|
|
1063
|
+
localFileCount: null,
|
|
1064
|
+
peerFileCount: 5,
|
|
1065
|
+
fileCountDelta: null,
|
|
1066
|
+
localNewestWriteAt: null,
|
|
1067
|
+
peerNewestWriteAt: null,
|
|
1068
|
+
writeAgeDeltaMs: null,
|
|
1069
|
+
digestMatch: null,
|
|
1070
|
+
diverged: true,
|
|
1071
|
+
reasons: ["namespace_absent_locally"],
|
|
1072
|
+
},
|
|
1073
|
+
],
|
|
1074
|
+
},
|
|
1075
|
+
],
|
|
1076
|
+
};
|
|
1077
|
+
// The presenting token can see team-a but NOT the diverged team-secret.
|
|
1078
|
+
const filtered = filterReplicaReportByCaps(report, { namespaces: ["team-a"] } as never);
|
|
1079
|
+
assert.equal(filtered.peers[0]?.state, "unknown", "a hidden shared divergence must not leave the visible peer_only as a false split-brain");
|
|
1080
|
+
assert.equal(filtered.peers[0]?.reason, "local_census_incomplete");
|
|
1081
|
+
assert.notEqual(filtered.peers[0]?.state, "diverged");
|
|
1082
|
+
assert.equal(filtered.peers[0]?.namespaces.length, 1, "only the visible namespace remains");
|
|
1083
|
+
});
|
|
1084
|
+
|
|
1085
|
+
test("round 6 (codex P2): an incomplete census downgrades peer-only divergence but keeps real shared divergence", () => {
|
|
1086
|
+
// A namespace the local scan DROPPED but the peer reports looks like peer_only
|
|
1087
|
+
// (=diverged). Against an incomplete local census that is a false split-brain --
|
|
1088
|
+
// we may hold that namespace and simply failed to read it -- so it must read
|
|
1089
|
+
// `unknown`. A genuine shared-namespace divergence still stands.
|
|
1090
|
+
const peerOnly: ReplicaDivergenceStatus = {
|
|
1091
|
+
enabled: true,
|
|
1092
|
+
pending: false,
|
|
1093
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1094
|
+
peers: [
|
|
1095
|
+
{
|
|
1096
|
+
peer: "127.0.0.1:4318",
|
|
1097
|
+
state: "diverged",
|
|
1098
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1099
|
+
divergedNamespaceCount: 1,
|
|
1100
|
+
namespaces: [
|
|
1101
|
+
{
|
|
1102
|
+
namespace: "team-a",
|
|
1103
|
+
presence: "peer_only",
|
|
1104
|
+
localFileCount: null,
|
|
1105
|
+
peerFileCount: 5,
|
|
1106
|
+
fileCountDelta: null,
|
|
1107
|
+
localNewestWriteAt: null,
|
|
1108
|
+
peerNewestWriteAt: null,
|
|
1109
|
+
writeAgeDeltaMs: null,
|
|
1110
|
+
digestMatch: null,
|
|
1111
|
+
diverged: true,
|
|
1112
|
+
reasons: ["namespace_absent_locally"],
|
|
1113
|
+
},
|
|
1114
|
+
],
|
|
1115
|
+
},
|
|
1116
|
+
],
|
|
1117
|
+
};
|
|
1118
|
+
const gatedPeerOnly = gateReportByCensus(peerOnly, false);
|
|
1119
|
+
assert.equal(gatedPeerOnly.peers[0]?.state, "unknown", "peer-only divergence is unsafe against a partial local census");
|
|
1120
|
+
assert.equal(gatedPeerOnly.peers[0]?.reason, "local_census_incomplete");
|
|
1121
|
+
assert.notEqual(gatedPeerOnly.peers[0]?.state, "diverged");
|
|
1122
|
+
|
|
1123
|
+
// A real shared-namespace divergence survives an incomplete census.
|
|
1124
|
+
const shared: ReplicaDivergenceStatus = {
|
|
1125
|
+
enabled: true,
|
|
1126
|
+
pending: false,
|
|
1127
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1128
|
+
peers: [
|
|
1129
|
+
{
|
|
1130
|
+
peer: "127.0.0.1:4318",
|
|
1131
|
+
state: "diverged",
|
|
1132
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1133
|
+
divergedNamespaceCount: 1,
|
|
1134
|
+
namespaces: [{ ...delta("default"), diverged: true, digestMatch: false, reasons: ["digest_mismatch"] }],
|
|
1135
|
+
},
|
|
1136
|
+
],
|
|
1137
|
+
};
|
|
1138
|
+
const gatedShared = gateReportByCensus(shared, false);
|
|
1139
|
+
assert.equal(gatedShared.peers[0]?.state, "diverged", "a real shared-namespace divergence stands regardless of census completeness");
|
|
1140
|
+
assert.equal(gatedShared.censusComplete, false);
|
|
1141
|
+
});
|
|
1142
|
+
|
|
1143
|
+
test("round 6 (codex P2): a mixed incomplete census keeps shared divergence but neutralizes peer-only deltas", () => {
|
|
1144
|
+
// Namespace `default` has a REAL shared divergence; the failed local scan omitted
|
|
1145
|
+
// team-b, so it shows peer_only. The peer must stay diverged (default is real) but
|
|
1146
|
+
// team-b must NOT count as divergence — its absence is unverifiable under an
|
|
1147
|
+
// incomplete census (round 6, codex).
|
|
1148
|
+
const report: ReplicaDivergenceStatus = {
|
|
1149
|
+
enabled: true,
|
|
1150
|
+
pending: false,
|
|
1151
|
+
censusComplete: false,
|
|
1152
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1153
|
+
peers: [
|
|
1154
|
+
{
|
|
1155
|
+
peer: "127.0.0.1:4318",
|
|
1156
|
+
state: "diverged",
|
|
1157
|
+
polledAt: "2026-03-08T00:00:00.000Z",
|
|
1158
|
+
divergedNamespaceCount: 2,
|
|
1159
|
+
namespaces: [
|
|
1160
|
+
{ ...delta("default"), diverged: true, digestMatch: false, reasons: ["digest_mismatch"] },
|
|
1161
|
+
{
|
|
1162
|
+
namespace: "team-b",
|
|
1163
|
+
presence: "peer_only",
|
|
1164
|
+
localFileCount: null,
|
|
1165
|
+
peerFileCount: 7,
|
|
1166
|
+
fileCountDelta: null,
|
|
1167
|
+
localNewestWriteAt: null,
|
|
1168
|
+
peerNewestWriteAt: null,
|
|
1169
|
+
writeAgeDeltaMs: null,
|
|
1170
|
+
digestMatch: null,
|
|
1171
|
+
diverged: true,
|
|
1172
|
+
reasons: ["namespace_absent_locally"],
|
|
1173
|
+
},
|
|
1174
|
+
],
|
|
1175
|
+
},
|
|
1176
|
+
],
|
|
1177
|
+
};
|
|
1178
|
+
const peer = gateReportByCensus(report, false).peers[0];
|
|
1179
|
+
assert.equal(peer?.state, "diverged", "the real shared divergence keeps the peer diverged");
|
|
1180
|
+
assert.equal(peer?.divergedNamespaceCount, 1, "only the shared divergence counts; the peer_only delta is neutralized");
|
|
1181
|
+
assert.equal(peer?.namespaces.find((d) => d.namespace === "team-b")?.diverged, false, "the untrusted peer_only delta must not read as divergence");
|
|
1182
|
+
assert.equal(peer?.namespaces.find((d) => d.namespace === "default")?.diverged, true, "the shared divergence delta is preserved");
|
|
1183
|
+
});
|
|
1184
|
+
|
|
1185
|
+
test("round 6 (coderabbit): the dual-prefix fallback inherits the REMAINING shared deadline", async () => {
|
|
1186
|
+
// Two independent 300ms budgets would pass the old assertions; this asserts the
|
|
1187
|
+
// SECOND attempt's signal is bounded by what the first left, not a fresh full
|
|
1188
|
+
// budget. The preferred (engram) path burns most of the 300ms budget then 404s.
|
|
1189
|
+
//
|
|
1190
|
+
// Real timers are unavoidable here (ts-no-test-timers exception): the assertion
|
|
1191
|
+
// is about `AbortSignal.timeout` firing against a real `Date.now()` deadline
|
|
1192
|
+
// shared across two fetch attempts, which node:test cannot drive with a fake
|
|
1193
|
+
// clock — the elapsed delay IS the behavior under test.
|
|
1194
|
+
let fallbackSignalAborted: boolean | undefined;
|
|
1195
|
+
const fetchImpl: FetchLike = async (url, init) => {
|
|
1196
|
+
if (String(url).includes("/engram/v1/")) {
|
|
1197
|
+
const burned = Promise.withResolvers<void>();
|
|
1198
|
+
setTimeout(() => burned.resolve(), 220);
|
|
1199
|
+
await burned.promise;
|
|
1200
|
+
return { ok: false, status: 404, json: async () => ({}) };
|
|
1201
|
+
}
|
|
1202
|
+
// The fallback inherits the ~80ms remainder. Wait PAST that remainder but well
|
|
1203
|
+
// under a fresh 300ms budget: with the shared deadline the signal has aborted
|
|
1204
|
+
// by now; with a per-attempt fresh budget it would not have.
|
|
1205
|
+
const waited = Promise.withResolvers<void>();
|
|
1206
|
+
setTimeout(() => waited.resolve(), 150);
|
|
1207
|
+
await waited.promise;
|
|
1208
|
+
fallbackSignalAborted = init?.signal?.aborted;
|
|
1209
|
+
return { ok: true, status: 200, json: async () => ({ corpus: [watermark("default")] }) };
|
|
1210
|
+
};
|
|
1211
|
+
await pollReplicaPeers({
|
|
1212
|
+
config: parseReplicaPeersConfig({
|
|
1213
|
+
replicaPeers: { enabled: true, requestTimeoutMs: 300, peers: [{ url: "http://127.0.0.1:4318" }] },
|
|
1214
|
+
}),
|
|
1215
|
+
localWatermarks: [watermark("default")],
|
|
1216
|
+
fetchImpl,
|
|
1217
|
+
});
|
|
1218
|
+
assert.equal(
|
|
1219
|
+
fallbackSignalAborted,
|
|
1220
|
+
true,
|
|
1221
|
+
"the fallback's signal is bounded by the REMAINING shared budget (aborted <150ms in), not a fresh 300ms one",
|
|
1222
|
+
);
|
|
1223
|
+
});
|
|
1224
|
+
|
|
1225
|
+
test("round 7 (codex P2): an oversized peer body is refused, not buffered", async () => {
|
|
1226
|
+
// A configured peer is a trust boundary: response.json() would buffer an
|
|
1227
|
+
// unbounded corpus whole, and several peers are polled concurrently.
|
|
1228
|
+
let delivered = 0;
|
|
1229
|
+
const chunk = new TextEncoder().encode("x".repeat(1024 * 1024));
|
|
1230
|
+
const fetchImpl: FetchLike = async () => ({
|
|
1231
|
+
ok: true,
|
|
1232
|
+
status: 200,
|
|
1233
|
+
json: async () => ({ corpus: [] }),
|
|
1234
|
+
body: {
|
|
1235
|
+
getReader: () => ({
|
|
1236
|
+
read: async () => {
|
|
1237
|
+
delivered += chunk.byteLength;
|
|
1238
|
+
// Far exceeds the cap; the reader must stop early, not run forever.
|
|
1239
|
+
return delivered > MAX_PEER_RESPONSE_BYTES * 4
|
|
1240
|
+
? { done: true, value: undefined }
|
|
1241
|
+
: { done: false, value: chunk };
|
|
1242
|
+
},
|
|
1243
|
+
cancel: async () => undefined,
|
|
1244
|
+
}),
|
|
1245
|
+
},
|
|
1246
|
+
});
|
|
1247
|
+
const status = await pollReplicaPeers({
|
|
1248
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
1249
|
+
localWatermarks: [watermark("default")],
|
|
1250
|
+
fetchImpl,
|
|
1251
|
+
});
|
|
1252
|
+
assert.equal(status.peers[0]?.state, "unknown");
|
|
1253
|
+
assert.equal(status.peers[0]?.reason, "response_too_large");
|
|
1254
|
+
assert.ok(
|
|
1255
|
+
delivered <= MAX_PEER_RESPONSE_BYTES + chunk.byteLength,
|
|
1256
|
+
`must stop at the cap, buffered ${delivered} bytes`,
|
|
1257
|
+
);
|
|
1258
|
+
});
|
|
1259
|
+
|
|
1260
|
+
test("round 7 (codex P2): a noncanonical peer namespace key is malformed", async () => {
|
|
1261
|
+
// "default " would compare as a DISTINCT namespace and manufacture phantom
|
|
1262
|
+
// local_only/peer_only deltas for the same logical tenant.
|
|
1263
|
+
for (const bad of ["default ", "../escape", "team\u0000a"]) {
|
|
1264
|
+
const fetchImpl: FetchLike = async () => ({
|
|
1265
|
+
ok: true,
|
|
1266
|
+
status: 200,
|
|
1267
|
+
json: async () => ({ corpus: [{ ...watermark("default"), namespace: bad }] }),
|
|
1268
|
+
});
|
|
1269
|
+
const status = await pollReplicaPeers({
|
|
1270
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
1271
|
+
localWatermarks: [watermark("default")],
|
|
1272
|
+
fetchImpl,
|
|
1273
|
+
});
|
|
1274
|
+
assert.equal(status.peers[0]?.state, "unknown", `${JSON.stringify(bad)} must not be accepted`);
|
|
1275
|
+
assert.equal(status.peers[0]?.reason, "malformed_corpus");
|
|
1276
|
+
}
|
|
1277
|
+
});
|
|
1278
|
+
|
|
1279
|
+
test("round 7 (codex P1): a peer advertising an incomplete census cannot converge", async () => {
|
|
1280
|
+
// The peer omits corpus entries whose scan failed; a namespace present only
|
|
1281
|
+
// on the peer could be missing, leaving the comparison seeing false agreement.
|
|
1282
|
+
const fetchImpl: FetchLike = async () => ({
|
|
1283
|
+
ok: true,
|
|
1284
|
+
status: 200,
|
|
1285
|
+
json: async () => ({ corpus: [watermark("default")], replica: { censusComplete: false } }),
|
|
1286
|
+
});
|
|
1287
|
+
const status = await pollReplicaPeers({
|
|
1288
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
1289
|
+
localWatermarks: [watermark("default")],
|
|
1290
|
+
fetchImpl,
|
|
1291
|
+
});
|
|
1292
|
+
assert.equal(status.peers[0]?.state, "unknown");
|
|
1293
|
+
assert.equal(status.peers[0]?.reason, "peer_census_incomplete");
|
|
1294
|
+
|
|
1295
|
+
// A peer that advertises a COMPLETE census still converges.
|
|
1296
|
+
const healthy: FetchLike = async () => ({
|
|
1297
|
+
ok: true,
|
|
1298
|
+
status: 200,
|
|
1299
|
+
json: async () => ({ corpus: [watermark("default")], replica: { censusComplete: true } }),
|
|
1300
|
+
});
|
|
1301
|
+
const ok = await pollReplicaPeers({
|
|
1302
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
1303
|
+
localWatermarks: [watermark("default")],
|
|
1304
|
+
fetchImpl: healthy,
|
|
1305
|
+
});
|
|
1306
|
+
assert.equal(ok.peers[0]?.state, "converged");
|
|
1307
|
+
});
|
|
1308
|
+
|
|
1309
|
+
test("round 8 (codex P1): corpusComplete describing THIS response wins over the replica block", async () => {
|
|
1310
|
+
// The peer's `corpus` array and `replica.censusComplete` came from two
|
|
1311
|
+
// independently-cached scans and could disagree; the flag that describes the
|
|
1312
|
+
// shipped array is authoritative.
|
|
1313
|
+
const mixed: FetchLike = async () => ({
|
|
1314
|
+
ok: true,
|
|
1315
|
+
status: 200,
|
|
1316
|
+
json: async () => ({
|
|
1317
|
+
corpus: [watermark("default")],
|
|
1318
|
+
corpusComplete: false, // the array we were actually given is partial
|
|
1319
|
+
replica: { censusComplete: true }, // the peer's separate monitor scan was fine
|
|
1320
|
+
}),
|
|
1321
|
+
});
|
|
1322
|
+
const status = await pollReplicaPeers({
|
|
1323
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
1324
|
+
localWatermarks: [watermark("default")],
|
|
1325
|
+
fetchImpl: mixed,
|
|
1326
|
+
});
|
|
1327
|
+
assert.equal(status.peers[0]?.state, "unknown", "a partial corpus must not converge on a stale complete flag");
|
|
1328
|
+
assert.equal(status.peers[0]?.reason, "peer_census_incomplete");
|
|
1329
|
+
});
|
|
1330
|
+
|
|
1331
|
+
test("round 8 (codex P2): an unterminated env placeholder in a peer token is rejected", () => {
|
|
1332
|
+
assert.throws(
|
|
1333
|
+
() =>
|
|
1334
|
+
parseReplicaPeersConfig({
|
|
1335
|
+
replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318", token: "${PEER_TOKEN" }] },
|
|
1336
|
+
}),
|
|
1337
|
+
/unterminated environment variable placeholder/,
|
|
1338
|
+
"a typo'd placeholder must fail at parse, not be sent literally as a bearer token",
|
|
1339
|
+
);
|
|
1340
|
+
});
|
|
1341
|
+
|
|
1342
|
+
test("round 9 (cursor): a cached poll is re-gated by the census shipped with THIS response", async () => {
|
|
1343
|
+
// Poll once while the census is complete, then serve the cache while the
|
|
1344
|
+
// local scan has degraded. The response must not pair `converged` peers with
|
|
1345
|
+
// an incomplete corpus.
|
|
1346
|
+
let now = 0;
|
|
1347
|
+
const monitor = new ReplicaDivergenceMonitor({ clock: () => now, fetchImpl: async () => ({
|
|
1348
|
+
ok: true, status: 200,
|
|
1349
|
+
json: async () => ({ corpus: [watermark("default")], corpusComplete: true }),
|
|
1350
|
+
}) });
|
|
1351
|
+
const config = parseReplicaPeersConfig({
|
|
1352
|
+
replicaPeers: { enabled: true, pollIntervalMs: 60_000, peers: [{ url: "http://127.0.0.1:4318" }] },
|
|
1353
|
+
});
|
|
1354
|
+
const complete = async () => ({ watermarks: [watermark("default")], complete: true });
|
|
1355
|
+
monitor.getReport({ config, computeLocalWatermarks: complete });
|
|
1356
|
+
await monitor.whenIdle();
|
|
1357
|
+
const fresh = monitor.getReport({ config, computeLocalWatermarks: complete });
|
|
1358
|
+
assert.equal(fresh.peers[0]?.state, "converged", "baseline: a complete census can certify convergence");
|
|
1359
|
+
|
|
1360
|
+
now += 1_000; // still inside the poll TTL, so the cache is served
|
|
1361
|
+
const degraded = monitor.getReport({ config, computeLocalWatermarks: complete, localCensusComplete: false });
|
|
1362
|
+
assert.notEqual(degraded.peers[0]?.state, "converged", "an incomplete census cannot certify a cached verdict");
|
|
1363
|
+
assert.equal(degraded.censusComplete, false);
|
|
1364
|
+
});
|
|
1365
|
+
|
|
1366
|
+
test("round 9 (codex P2): a present but non-boolean corpusComplete is malformed telemetry", async () => {
|
|
1367
|
+
const bad: FetchLike = async () => ({
|
|
1368
|
+
ok: true, status: 200,
|
|
1369
|
+
json: async () => ({ corpus: [watermark("default")], corpusComplete: "false" }),
|
|
1370
|
+
});
|
|
1371
|
+
const status = await pollReplicaPeers({
|
|
1372
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
1373
|
+
localWatermarks: [watermark("default")],
|
|
1374
|
+
fetchImpl: bad,
|
|
1375
|
+
});
|
|
1376
|
+
assert.equal(status.peers[0]?.state, "unknown");
|
|
1377
|
+
assert.equal(status.peers[0]?.reason, "malformed_corpus", "a corrupt flag must not fall through to a weaker signal");
|
|
1378
|
+
});
|
|
1379
|
+
|
|
1380
|
+
test("round 9 (codex P2): a token expanding to whitespace is rejected, not silently unauthenticated", () => {
|
|
1381
|
+
process.env.REMNIC_TEST_BLANK_PEER_TOKEN = " ";
|
|
1382
|
+
try {
|
|
1383
|
+
assert.throws(
|
|
1384
|
+
() =>
|
|
1385
|
+
parseReplicaPeersConfig({
|
|
1386
|
+
replicaPeers: {
|
|
1387
|
+
enabled: true,
|
|
1388
|
+
peers: [{ url: "http://127.0.0.1:4318", token: "${REMNIC_TEST_BLANK_PEER_TOKEN}" }],
|
|
1389
|
+
},
|
|
1390
|
+
}),
|
|
1391
|
+
/expanded to an empty value/,
|
|
1392
|
+
);
|
|
1393
|
+
} finally {
|
|
1394
|
+
delete process.env.REMNIC_TEST_BLANK_PEER_TOKEN;
|
|
1395
|
+
}
|
|
1396
|
+
});
|
|
1397
|
+
|
|
1398
|
+
test("round 10 (codex P2): a malformed LEGACY completeness flag is rejected too", async () => {
|
|
1399
|
+
// The round-9 fix gated `corpusComplete`; the legacy fallback kept accepting
|
|
1400
|
+
// a string, which slips past the `=== false` branch and certifies a partial.
|
|
1401
|
+
const bad: FetchLike = async () => ({
|
|
1402
|
+
ok: true, status: 200,
|
|
1403
|
+
json: async () => ({ corpus: [watermark("default")], replica: { censusComplete: "false" } }),
|
|
1404
|
+
});
|
|
1405
|
+
const status = await pollReplicaPeers({
|
|
1406
|
+
config: parseReplicaPeersConfig({ replicaPeers: { enabled: true, peers: [{ url: "http://127.0.0.1:4318" }] } }),
|
|
1407
|
+
localWatermarks: [watermark("default")],
|
|
1408
|
+
fetchImpl: bad,
|
|
1409
|
+
});
|
|
1410
|
+
assert.equal(status.peers[0]?.state, "unknown");
|
|
1411
|
+
assert.equal(status.peers[0]?.reason, "malformed_corpus");
|
|
1412
|
+
});
|
|
1413
|
+
|
|
1414
|
+
test("round 10 (codex P2): an expanded peer URL is stored canonically", () => {
|
|
1415
|
+
process.env.REMNIC_TEST_PEER_URL = " http://127.0.0.1:4318 ";
|
|
1416
|
+
try {
|
|
1417
|
+
const config = parseReplicaPeersConfig({
|
|
1418
|
+
replicaPeers: { enabled: true, peers: [{ url: "${REMNIC_TEST_PEER_URL}" }] },
|
|
1419
|
+
});
|
|
1420
|
+
// The health path is appended by concatenation, so retained whitespace would
|
|
1421
|
+
// build " http://... /engram/v1/health" and read as an unreachable peer.
|
|
1422
|
+
assert.equal(config.peers[0]?.url, "http://127.0.0.1:4318");
|
|
1423
|
+
} finally {
|
|
1424
|
+
delete process.env.REMNIC_TEST_PEER_URL;
|
|
1425
|
+
}
|
|
1426
|
+
});
|
|
1427
|
+
|
|
1428
|
+
test("round 10 (codex P2): a cached verdict cannot outlive the census freshness bound", async () => {
|
|
1429
|
+
let now = 0;
|
|
1430
|
+
const monitor = new ReplicaDivergenceMonitor({ clock: () => now, fetchImpl: async () => ({
|
|
1431
|
+
ok: true, status: 200,
|
|
1432
|
+
json: async () => ({ corpus: [watermark("default")], corpusComplete: true }),
|
|
1433
|
+
}) });
|
|
1434
|
+
// A poll interval far longer than the freshness bound: without the cap, a
|
|
1435
|
+
// converged verdict would be served for a full day past its own staleness.
|
|
1436
|
+
const config = parseReplicaPeersConfig({
|
|
1437
|
+
replicaPeers: {
|
|
1438
|
+
enabled: true, pollIntervalMs: 86_400_000, maxWatermarkAgeDeltaMs: 900_000,
|
|
1439
|
+
peers: [{ url: "http://127.0.0.1:4318" }],
|
|
1440
|
+
},
|
|
1441
|
+
});
|
|
1442
|
+
const census = async () => ({ watermarks: [watermark("default")], complete: true });
|
|
1443
|
+
monitor.getReport({ config, computeLocalWatermarks: census });
|
|
1444
|
+
await monitor.whenIdle();
|
|
1445
|
+
let polls = 0;
|
|
1446
|
+
const counting = async () => { polls += 1; return { watermarks: [watermark("default")], complete: true }; };
|
|
1447
|
+
now += 900_001; // past the freshness bound, far short of the poll interval
|
|
1448
|
+
monitor.getReport({ config, computeLocalWatermarks: counting });
|
|
1449
|
+
await monitor.whenIdle();
|
|
1450
|
+
assert.equal(polls, 1, "the cache must expire at the freshness bound, not the poll interval");
|
|
1451
|
+
});
|
|
1452
|
+
|
|
1453
|
+
test("round 11: maxWatermarkAgeDeltaMs=0 disables the freshness cap, it does not zero the cache", async () => {
|
|
1454
|
+
// 0 is the documented strict write-age mode. Reading it as a zero-length TTL
|
|
1455
|
+
// re-scans the whole corpus on every probe (§17 zero footgun).
|
|
1456
|
+
let now = 0;
|
|
1457
|
+
const monitor = new ReplicaDivergenceMonitor({ clock: () => now, fetchImpl: async () => ({
|
|
1458
|
+
ok: true, status: 200,
|
|
1459
|
+
json: async () => ({ corpus: [watermark("default")], corpusComplete: true }),
|
|
1460
|
+
}) });
|
|
1461
|
+
const config = parseReplicaPeersConfig({
|
|
1462
|
+
replicaPeers: {
|
|
1463
|
+
enabled: true, pollIntervalMs: 300_000, maxWatermarkAgeDeltaMs: 0,
|
|
1464
|
+
peers: [{ url: "http://127.0.0.1:4318" }],
|
|
1465
|
+
},
|
|
1466
|
+
});
|
|
1467
|
+
let polls = 0;
|
|
1468
|
+
const census = async () => { polls += 1; return { watermarks: [watermark("default")], complete: true }; };
|
|
1469
|
+
monitor.getReport({ config, computeLocalWatermarks: census });
|
|
1470
|
+
await monitor.whenIdle();
|
|
1471
|
+
now += 1_000; // well inside pollIntervalMs
|
|
1472
|
+
monitor.getReport({ config, computeLocalWatermarks: census });
|
|
1473
|
+
await monitor.whenIdle();
|
|
1474
|
+
assert.equal(polls, 1, "the cache must still honor pollIntervalMs when the freshness gate is disabled");
|
|
1475
|
+
});
|