@torrent-tv/proxy 2.87.8 → 2.87.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/opsx/apply.md +155 -155
- package/.claude/commands/opsx/archive.md +160 -160
- package/.claude/commands/opsx/explore.md +174 -174
- package/.claude/commands/opsx/propose.md +109 -109
- package/.claude/commands/opsx/sync.md +143 -143
- package/.claude/skills/openspec-apply-change/SKILL.md +159 -159
- package/.claude/skills/openspec-archive-change/SKILL.md +117 -117
- package/.claude/skills/openspec-explore/SKILL.md +289 -289
- package/.claude/skills/openspec-propose/SKILL.md +113 -113
- package/.claude/skills/openspec-sync-specs/SKILL.md +147 -147
- package/.dockerignore +5 -5
- package/AGENTS.md +322 -322
- package/CHANGELOG.md +4 -0
- package/Dockerfile +30 -30
- package/LICENSE +16 -16
- package/README.md +369 -369
- package/assets/calibration/NOTICE.md +100 -100
- package/assets/diagnostics/sctpstate.gdb +31 -31
- package/bin/cli.js +684 -684
- package/bin/thread-pool.js +30 -30
- package/biome.json +411 -411
- package/docs/container-architecture.md +218 -218
- package/docs/download-architecture.md +261 -261
- package/docs/encode-architecture.md +729 -729
- package/docs/encode-run-state.md +96 -96
- package/docs/logs.md +97 -97
- package/docs/socks5-egress-research.md +114 -114
- package/docs/storage-architecture.md +267 -267
- package/knip.json +14 -14
- package/openspec/changes/adaptive-bitrate/design.md +92 -92
- package/openspec/changes/adaptive-bitrate/proposal.md +56 -56
- package/openspec/changes/adaptive-bitrate/specs/adaptive-bitrate/spec.md +51 -51
- package/openspec/changes/adaptive-bitrate/tasks.md +40 -40
- package/openspec/changes/binary-distribution/.openspec.yaml +2 -2
- package/openspec/changes/binary-distribution/proposal.md +53 -53
- package/openspec/changes/chunked-request-bodies/design.md +81 -81
- package/openspec/changes/chunked-request-bodies/proposal.md +53 -53
- package/openspec/changes/chunked-request-bodies/specs/chunked-request-bodies/spec.md +30 -30
- package/openspec/changes/chunked-request-bodies/tasks.md +40 -40
- package/openspec/changes/cold-start/design.md +98 -98
- package/openspec/changes/cold-start/proposal.md +62 -62
- package/openspec/changes/cold-start/specs/cold-start/spec.md +47 -47
- package/openspec/changes/cold-start/tasks.md +48 -48
- package/openspec/changes/disk-cap/.openspec.yaml +2 -2
- package/openspec/changes/disk-cap/proposal.md +37 -37
- package/openspec/changes/disk-cap/specs/disk-cap/spec.md +25 -25
- package/openspec/changes/disk-cap/tasks.md +19 -19
- package/openspec/changes/proxy-observability/.openspec.yaml +2 -2
- package/openspec/changes/proxy-observability/design.md +38 -38
- package/openspec/changes/proxy-observability/proposal.md +49 -49
- package/openspec/changes/proxy-observability/specs/observability/spec.md +33 -33
- package/openspec/changes/proxy-observability/tasks.md +21 -21
- package/openspec/changes/seek-debounce/design.md +118 -118
- package/openspec/changes/seek-debounce/proposal.md +59 -59
- package/openspec/changes/seek-debounce/specs/seek-debounce/spec.md +41 -41
- package/openspec/changes/seek-debounce/tasks.md +33 -33
- package/openspec/changes/subtitle-language/.openspec.yaml +2 -2
- package/openspec/changes/subtitle-language/proposal.md +50 -50
- package/openspec/changes/subtitle-language/specs/subtitle-language/spec.md +34 -34
- package/openspec/changes/subtitle-language/tasks.md +26 -26
- package/openspec/changes/track-selection/.openspec.yaml +2 -2
- package/openspec/changes/track-selection/design.md +47 -47
- package/openspec/changes/track-selection/proposal.md +45 -45
- package/openspec/changes/track-selection/specs/track-selection/spec.md +41 -41
- package/openspec/changes/track-selection/tasks.md +20 -20
- package/openspec/changes/transcode-quality/.openspec.yaml +2 -2
- package/openspec/changes/transcode-quality/proposal.md +51 -51
- package/openspec/changes/transcode-quality/specs/transcode-quality/spec.md +139 -139
- package/openspec/changes/transcode-quality/tasks.md +63 -63
- package/openspec/config.yaml +38 -38
- package/package.json +1 -1
- package/public/.well-known/appspecific/com.chrome.devtools.json +1 -1
- package/research/delivery-wedge-stand-ladder-2026-09-13.md +329 -329
- package/research/double-spawn-2026-09-10.md +171 -171
- package/research/handover-2026-09-13.md +209 -209
- package/research/handover-reader-claims-removal-2026-09-12.md +166 -166
- package/research/piece-withdrawn-but-still-claimed-2026-09-12.md +175 -175
- package/research/preset-benchmark-size-scaling-2026-08-22.md +55 -55
- package/research/priority-map-is-the-truth-2026-09-12.md +225 -225
- package/routes/api/client-logs/post.js +86 -86
- package/routes/api/delivery-sink/get.js +72 -72
- package/routes/api/link-probe/get.js +51 -51
- package/routes/api/playback-plan/audio-tracks/post.js +29 -29
- package/routes/api/playback-plan/post.js +53 -53
- package/routes/api/sources/files/get.js +100 -100
- package/routes/api/sources/post.js +36 -36
- package/routes/api/sources/stats/get.js +132 -132
- package/routes/api/subtitles/get.js +230 -230
- package/routes/api/transcode-sessions/fragment-far/post.js +60 -60
- package/routes/api/transcode-sessions/net-report/post.js +129 -129
- package/routes/api/transcode-sessions/post.js +160 -160
- package/routes/api/transcode-sessions/progress/get.js +27 -27
- package/routes/api/transcode-sessions/release/post.js +41 -41
- package/routes/api/transcode-sessions/seek/post.js +63 -63
- package/routes/health/get.js +13 -13
- package/routes/healthz/get.js +13 -13
- package/routes/stream/get.js +361 -361
- package/routes/transcode/audio-file/get.js +64 -64
- package/routes/transcode/audio-warm/get.js +84 -84
- package/routes/transcode/session-file/get.js +52 -52
- package/routes/transcode/variant-file/get.js +95 -95
- package/routes/transcode/variant-warm/get.js +84 -84
- package/scripts/render-run-graph.js +173 -173
- package/services/encode/CoverageMap.js +428 -428
- package/services/encode/CushionReport.js +345 -345
- package/services/encode/EncodeAdmission.js +332 -332
- package/services/encode/EncodeOrchestrator.js +1217 -1217
- package/services/encode/EncodePlan.js +1366 -1366
- package/services/encode/EncodeRun.js +776 -776
- package/services/encode/EncodeRuns.js +1229 -1229
- package/services/encode/Encoder.js +142 -142
- package/services/encode/EncoderSelection.js +38 -38
- package/services/encode/LocalObservations.js +359 -359
- package/services/encode/NvencEncoder.js +50 -50
- package/services/encode/OutputOpening.js +755 -755
- package/services/encode/OutputTimes.js +579 -579
- package/services/encode/QsvEncoder.js +58 -58
- package/services/encode/Renditions.js +2162 -2162
- package/services/encode/RunProgress.js +84 -84
- package/services/encode/SegmentDemand.js +95 -95
- package/services/encode/SoftwareEncoder.js +121 -121
- package/services/encode/V4l2m2mEncoder.js +53 -53
- package/services/encode/VaapiEncoder.js +66 -66
- package/services/encode/args.js +416 -416
- package/services/encode/available-share.js +89 -89
- package/services/encode/calibration.js +361 -361
- package/services/encode/contention.js +128 -128
- package/services/encode/decode-cost-fit.js +363 -363
- package/services/encode/encode-exit.js +114 -114
- package/services/encode/encode-run-state.js +551 -551
- package/services/encode/encoder-readings.js +45 -45
- package/services/encode/fingerprint.js +322 -322
- package/services/encode/host-load.js +174 -174
- package/services/encode/hwaccel.js +1894 -1894
- package/services/encode/index.js +9 -9
- package/services/encode/learned-median.js +90 -90
- package/services/encode/output/EncodedOutput.js +57 -57
- package/services/encode/output/Output.js +131 -131
- package/services/encode/output/OutputCatalog.js +414 -414
- package/services/encode/output/OutputSpec.js +390 -390
- package/services/encode/output/Timeline.js +308 -308
- package/services/encode/output/cut-grid.js +107 -107
- package/services/encode/output/index.js +20 -20
- package/services/encode/output/ladder.js +32 -32
- package/services/encode/output/playlists.js +284 -284
- package/services/encode/output/rates.js +125 -125
- package/services/encode/output-key-format.js +21 -21
- package/services/encode/quality/EncodeCost.js +1083 -1083
- package/services/encode/quality/HostLoad.js +525 -525
- package/services/encode/quality/HostTimings.js +249 -249
- package/services/encode/quality/OutputQualityState.js +28 -28
- package/services/encode/quality/QualityController.js +935 -935
- package/services/encode/quality/QualityOffer.js +416 -416
- package/services/encode/quality/drain-threat.js +47 -47
- package/services/encode/quality/link-budget.js +254 -254
- package/services/encode/quality/output-format.js +266 -266
- package/services/encode/quality/serving-output.js +216 -216
- package/services/encode/quality/visible-rung.js +57 -57
- package/services/encode/run-budget.js +97 -97
- package/services/encode/run-command.js +747 -747
- package/services/encode/run-costs.js +155 -155
- package/services/encode/run-inputs.js +42 -42
- package/services/encode/segment-formats/fmp4.js +437 -437
- package/services/encode/segment-formats/index.js +77 -77
- package/services/encode/segment-formats/init-compat.js +324 -324
- package/services/encode/segment-formats/mp4-boxes.js +311 -311
- package/services/encode/segment-formats/mpegts.js +134 -134
- package/services/encode/start-stop-cost.js +178 -178
- package/services/encode/throughput.js +97 -97
- package/services/encode/torrent-cost.js +88 -88
- package/services/identity/installation-id.js +38 -38
- package/services/media/ContainerOrchestrator.js +149 -149
- package/services/media/KeyframeTables.js +316 -316
- package/services/media/SourceFile.js +298 -298
- package/services/media/SubtitleCues.js +520 -520
- package/services/media/SubtitleOrchestrator.js +167 -167
- package/services/media/container/AviContainer.js +266 -266
- package/services/media/container/Container.js +603 -603
- package/services/media/container/ContainerFactory.js +56 -56
- package/services/media/container/KeyframeTable.js +272 -272
- package/services/media/container/MatroskaContainer.js +1494 -1494
- package/services/media/container/Mp4Container.js +1244 -1244
- package/services/media/container/SubtitleFileContainer.js +322 -322
- package/services/media/container/ebml-reader.js +159 -159
- package/services/media/ffmpeg-banner.js +260 -260
- package/services/media/keyframe-probe.js +157 -157
- package/services/media/media-info-probe.js +93 -93
- package/services/media/playback-planner.js +1 -1
- package/services/media/tracks/AudioTrack.js +131 -131
- package/services/media/tracks/ContainerTrack.js +75 -75
- package/services/media/tracks/ImageSubtitleTrack.js +19 -19
- package/services/media/tracks/SubtitleTrack.js +38 -38
- package/services/media/tracks/TextSubtitleTrack.js +287 -287
- package/services/media/tracks/VideoTrack.js +33 -33
- package/services/media/tracks/index.js +15 -15
- package/services/media/tracks/language-detect.js +174 -174
- package/services/server/OutputLifecycle.js +418 -418
- package/services/server/SegmentServing.js +1146 -1146
- package/services/server/ViewerRequests.js +522 -522
- package/services/server/WaitLedger.js +142 -142
- package/services/server/controllers/SubtitleController.js +109 -109
- package/services/server/http-responses.js +60 -60
- package/services/server/transcode-session-files.js +325 -325
- package/services/server/wire-outputs.js +983 -983
- package/services/storage/Diagnostics.js +202 -202
- package/services/storage/MachineBudget.js +235 -235
- package/services/storage/allowance.js +107 -107
- package/services/storage/core-dumps.js +106 -106
- package/services/storage/files/CompletedFiles.js +391 -391
- package/services/storage/files/piece-from-whole-file.js +118 -118
- package/services/storage/free.js +71 -71
- package/services/storage/keep.js +48 -48
- package/services/storage/machine-memory.js +56 -56
- package/services/storage/memory-report.js +575 -575
- package/services/storage/piece-store/piece-disk-store.js +516 -516
- package/services/storage/piece-store/piece-lru.js +495 -495
- package/services/storage/piece-store/shared-piece-store.js +1880 -1880
- package/services/storage/returns.js +103 -103
- package/services/storage/segment-store/SegmentStore.js +1196 -1196
- package/services/storage/wire.js +175 -175
- package/services/torrent/Contents.js +324 -324
- package/services/torrent/demand/DemandRegister.js +202 -202
- package/services/torrent/demand/Urgency.js +137 -137
- package/services/torrent/demand/Window.js +118 -118
- package/services/torrent/demand/index.js +8 -8
- package/services/torrent/demand/pieces.js +140 -140
- package/services/torrent/download/SwarmSelection.js +386 -386
- package/services/torrent/download/registry.js +102 -102
- package/services/torrent/download/withdraw-claim.js +80 -80
- package/services/torrent/files.js +455 -455
- package/services/torrent/naming.js +619 -619
- package/services/torrent/piece-store-of.js +29 -29
- package/services/torrent/supply-margin.js +262 -262
- package/services/torrent/worker/background-fill.js +213 -213
- package/services/torrent/worker/channel.js +271 -271
- package/services/torrent/worker/fastest-wires.js +342 -342
- package/services/torrent/worker/handle-state.js +23 -23
- package/services/torrent/worker/held-bytes.js +155 -155
- package/services/torrent/worker/install-webrtc-shim.js +32 -32
- package/services/torrent/worker/piece-reader.js +1078 -1078
- package/services/torrent/worker/resume-warm.js +107 -107
- package/services/torrent/worker/webrtc-shim.js +219 -219
- package/services/transport/delivery-probe.js +842 -842
- package/services/transport/health-collector.js +53 -53
- package/services/transport/nat-classifier.js +231 -231
- package/services/transport/packet-witness.js +808 -808
- package/services/transport/port-mapper.js +235 -235
- package/services/transport/registry-api.js +81 -81
- package/services/transport/tunnel-client.js +447 -447
- package/services/transport/usrsctp-state.js +175 -175
- package/services/transport/webrtc-manager.js +484 -484
- package/services/viewer/Assignments.js +340 -340
- package/services/viewer/PriorityMap.js +390 -390
- package/services/viewer/PriorityOrchestrator.js +325 -325
- package/services/viewer/Viewer.js +657 -657
- package/services/viewer/Viewers.js +368 -368
- package/services/viewer/active-output.js +37 -37
- package/services/viewer/audio-start.js +46 -46
- package/services/viewer/buffer-trend.js +81 -81
- package/services/viewer/choices.js +647 -647
- package/services/viewer/positions.js +126 -126
- package/services/viewer/report-intake.js +46 -46
- package/store/source-registry.js +71 -71
- package/test/asset-paths.test.js +28 -28
- package/test/audio-track-bitrate.test.js +40 -40
- package/test/auto-quality-step.test.js +707 -707
- package/test/available-share.test.js +76 -76
- package/test/background-fill.test.js +190 -190
- package/test/client-log-file.test.js +110 -110
- package/test/completed-files.test.js +115 -115
- package/test/concurrent-cost.test.js +148 -148
- package/test/container-media-info.test.js +228 -228
- package/test/contention.test.js +76 -76
- package/test/core-dumps.test.js +57 -57
- package/test/coverage-follows-the-disk.test.js +228 -228
- package/test/coverage-map.test.js +195 -195
- package/test/cue-push-carries-what-the-walk-found.test.js +58 -58
- package/test/cut-times-timeline.test.js +65 -65
- package/test/cuts-follow-published-grid.test.js +109 -109
- package/test/data-channel-frame.test.js +63 -63
- package/test/dead-torrent-handle.test.js +52 -52
- package/test/declared-rates.test.js +116 -116
- package/test/declared-tracks.test.js +35 -35
- package/test/decode-cost-fit.test.js +202 -202
- package/test/decode-cost.test.js +462 -462
- package/test/delivery-probe.test.js +393 -393
- package/test/demand-register.test.js +206 -206
- package/test/dht-bootstrap.test.js +72 -72
- package/test/drain-threat.test.js +65 -65
- package/test/duplicate-add.test.js +48 -48
- package/test/encode-admission.test.js +146 -146
- package/test/encode-cost.test.js +283 -283
- package/test/encode-exit.test.js +111 -111
- package/test/encode-orchestrator.test.js +419 -419
- package/test/encode-plan-viewers.test.js +718 -718
- package/test/encode-plan.test.js +545 -545
- package/test/encode-run-state.test.js +314 -314
- package/test/encode-run.test.js +161 -161
- package/test/encode-slope.test.js +46 -46
- package/test/encoded-output.test.js +39 -39
- package/test/encoder-kinds.test.js +122 -122
- package/test/encoder-readings.test.js +46 -46
- package/test/every-claimant-is-told-its-share.test.js +152 -152
- package/test/failed-encoder-still-serves-what-exists.test.js +354 -354
- package/test/failed-start-breaker.test.js +202 -202
- package/test/fastest-wires.test.js +143 -143
- package/test/ffmpeg-stream-counts.test.js +78 -78
- package/test/file-edges.test.js +191 -191
- package/test/fmp4-self-contained.test.js +80 -80
- package/test/fmp4-timestamps.test.js +140 -140
- package/test/fmp4-track-completeness.test.js +41 -41
- package/test/format-keyed-outputs.test.js +78 -78
- package/test/health-metrics.test.js +38 -38
- package/test/held-request-width.test.js +157 -157
- package/test/helpers/encode-run.js +136 -136
- package/test/helpers/held-file.js +29 -29
- package/test/helpers/manager.js +57 -57
- package/test/helpers/output-spec.js +63 -63
- package/test/host-load.test.js +68 -68
- package/test/index-accuracy-is-about-the-file.test.js +128 -128
- package/test/init-compat.test.js +219 -219
- package/test/input-lost-quiets-the-plan.test.js +261 -261
- package/test/input-unavailable.test.js +48 -48
- package/test/installation-id.test.js +39 -39
- package/test/keeping-period.test.js +83 -83
- package/test/keyframe-index-accuracy.test.js +188 -188
- package/test/keyframe-table-budget.test.js +102 -102
- package/test/keyframes-belong-to-the-file.test.js +249 -249
- package/test/learned-median.test.js +37 -37
- package/test/letting-go-asks-who-is-watching.test.js +191 -191
- package/test/limit-row-by-area.test.js +60 -60
- package/test/link-budget.test.js +111 -111
- package/test/link-probe.test.js +56 -56
- package/test/local-encoding-data.test.js +273 -273
- package/test/logger-repeats.test.js +134 -134
- package/test/look-ahead-coverage.test.js +56 -56
- package/test/manager-surface.test.js +292 -292
- package/test/matroska-cues-track.test.js +192 -192
- package/test/memory-budget.test.js +390 -390
- package/test/move-cost.test.js +216 -216
- package/test/mp4-subtitles.test.js +173 -173
- package/test/one-authority.test.js +285 -285
- package/test/one-budget.test.js +149 -149
- package/test/one-output-one-session.test.js +204 -204
- package/test/one-resource-divided.test.js +185 -185
- package/test/one-serving-output-per-height.test.js +458 -458
- package/test/orchestrator-wired.test.js +206 -206
- package/test/output-catalog.test.js +39 -39
- package/test/output-shape.test.js +21 -21
- package/test/output-spec.test.js +251 -251
- package/test/output-speed.test.js +88 -88
- package/test/outputs-being-prepared.test.js +40 -40
- package/test/packet-witness-ring.test.js +236 -236
- package/test/packet-witness.test.js +148 -148
- package/test/piece-disk-store.test.js +381 -381
- package/test/piece-from-whole-file.test.js +129 -129
- package/test/piece-lru.test.js +319 -319
- package/test/piece-reader.test.js +162 -162
- package/test/piece-selection-offset.test.js +85 -85
- package/test/piece-store-eviction.test.js +560 -560
- package/test/piece-store-never-refuses.test.js +153 -153
- package/test/piece-store-reservations.test.js +318 -318
- package/test/piece-store-slow-disk.test.js +159 -159
- package/test/piece-tail.test.js +97 -97
- package/test/plan-host-timings.test.js +68 -68
- package/test/plan-is-steady.test.js +225 -225
- package/test/priority-map-download.test.js +174 -174
- package/test/priority-map-emptied.test.js +204 -204
- package/test/priority-map-per-output.test.js +215 -215
- package/test/priority-map.test.js +163 -163
- package/test/probe-wedge-certainty.test.js +60 -60
- package/test/produced-copy-choice.test.js +213 -213
- package/test/published-timeline.test.js +83 -83
- package/test/quality-offer.test.js +162 -162
- package/test/quality-variants.test.js +1229 -1229
- package/test/rate-control-in-the-output.test.js +180 -180
- package/test/read-survives-withdrawal.test.js +164 -164
- package/test/read-window.test.js +365 -365
- package/test/resume-warm.test.js +39 -39
- package/test/run-graph-drift.test.js +37 -37
- package/test/run-inputs.test.js +80 -80
- package/test/run-intervals.test.js +100 -100
- package/test/run-position-follows-published-grid.test.js +89 -89
- package/test/run-progress.test.js +65 -65
- package/test/same-height-switch.test.js +664 -664
- package/test/seek-frees-the-viewers-requests.test.js +217 -217
- package/test/seek-landing.test.js +109 -109
- package/test/seek-target-not-superseded.test.js +195 -195
- package/test/segment-closed-on-both-branches.test.js +96 -96
- package/test/segment-cut-times.test.js +63 -63
- package/test/segment-demand.test.js +80 -80
- package/test/segment-hold-supersede.test.js +91 -91
- package/test/segment-judged-by-its-own-header.test.js +221 -221
- package/test/segment-serve-wiring.test.js +475 -475
- package/test/segment-store-eviction.test.js +247 -247
- package/test/segment-store.test.js +323 -323
- package/test/segment-true-start.test.js +86 -86
- package/test/segments-are-shared.test.js +176 -176
- package/test/send-chunk-bytes.test.js +86 -86
- package/test/serving-existing-output.test.js +102 -102
- package/test/serving-output.test.js +102 -102
- package/test/shared-piece-store.test.js +310 -310
- package/test/sidecar-naming.test.js +142 -142
- package/test/source-file.test.js +133 -133
- package/test/source-files-route.test.js +103 -103
- package/test/source-registry.test.js +18 -18
- package/test/split-a-stretch.test.js +139 -139
- package/test/stale-request-after-seek.test.js +158 -158
- package/test/startup-readings.test.js +124 -124
- package/test/stream-route.test.js +182 -182
- package/test/subtitle-cue-framing.test.js +200 -200
- package/test/subtitle-cue-source.test.js +102 -102
- package/test/subtitle-cue-walk.test.js +370 -370
- package/test/subtitle-cursor.test.js +63 -63
- package/test/subtitle-defaults.test.js +118 -118
- package/test/subtitle-language.test.js +252 -252
- package/test/subtitle-track-numbering.test.js +386 -386
- package/test/supply-margin.test.js +177 -177
- package/test/swarm-follows-readers.test.js +174 -174
- package/test/swarm-reach.test.js +130 -130
- package/test/swarm-selection.test.js +220 -220
- package/test/tail-duplication.test.js +167 -167
- package/test/the-channel-has-both-ends.test.js +91 -91
- package/test/the-torrent-thread-serves-bytes.test.js +107 -107
- package/test/thread-pool.test.js +53 -53
- package/test/timeline.test.js +84 -84
- package/test/torrent-contents.test.js +235 -235
- package/test/torrent-cost.test.js +71 -71
- package/test/torrent-files.test.js +221 -221
- package/test/torrent-naming.test.js +255 -255
- package/test/tracks-begin-together.test.js +209 -209
- package/test/tunnel-renewal.test.js +167 -167
- package/test/two-viewers-one-picture.test.js +407 -407
- package/test/upload-hurry.test.js +186 -186
- package/test/usrsctp-state.test.js +131 -131
- package/test/video-facts.test.js +102 -102
- package/test/viewer-assignments.test.js +143 -143
- package/test/viewer-outputs.test.js +285 -285
- package/test/viewer-presence.test.js +116 -116
- package/test/viewer-requests-admission.test.js +89 -89
- package/test/viewer-states.test.js +189 -189
- package/test/viewer-subtitle-subscription.test.js +96 -96
- package/test/viewer.test.js +139 -139
- package/test/viewing-generations.test.js +390 -390
- package/test/visible-rung.test.js +37 -37
- package/test/wait-ledger.test.js +100 -100
- package/test/webrtc-shim.test.js +149 -149
- package/test/wedge-certainty.test.js +131 -131
- package/test/withdraw-piece-claim.test.js +212 -212
- package/test/worker-channel.test.js +158 -158
- package/utils/client-log-file.js +151 -151
- package/utils/logger.js +312 -312
- package/utils/parse-range.js +25 -25
- package/utils/perf.js +59 -59
- package/utils/torrent-source-key.js +38 -38
|
@@ -1,842 +1,842 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file Numbered delivery probes, and the verdict they make possible.
|
|
3
|
-
*
|
|
4
|
-
* The delivery freeze (roadmap item 11) looks identical from the proxy in two
|
|
5
|
-
* cases that need opposite fixes: usrsctp stopped transmitting, or the browser
|
|
6
|
-
* closed its receive window because the page stopped draining the channel. The
|
|
7
|
-
* proxy's own counters cannot separate them — libdatachannel's `bytesSent`
|
|
8
|
-
* counts bytes ACCEPTED into usrsctp, not bytes put on the wire — so the
|
|
9
|
-
* reading has to come from the far end.
|
|
10
|
-
*
|
|
11
|
-
* Two facts make that cheap. The reverse direction keeps working throughout the
|
|
12
|
-
* freeze (browser to proxy requests arrive and are answered for the whole
|
|
13
|
-
* episode, 88 min in the 2026-08-24 case), so the browser can always report.
|
|
14
|
-
* And SCTP orders per STREAM, so a probe on a channel opened UNORDERED and
|
|
15
|
-
* WITHOUT retransmission passes head-of-line blocking in another stream but
|
|
16
|
-
* neither a closed receive window nor a transmitter that stopped.
|
|
17
|
-
*
|
|
18
|
-
* So: number a probe every {@link PROBE_INTERVAL_MS} on every channel of the
|
|
19
|
-
* connection, have the browser echo back the highest number it has seen on
|
|
20
|
-
* each, and read the gaps:
|
|
21
|
-
*
|
|
22
|
-
* every channel current to flowing
|
|
23
|
-
* ordered behind, unreliable current to a retransmission stuck in a stream
|
|
24
|
-
* both behind, echoes still arriving to the association stopped transmitting
|
|
25
|
-
* no echo at all to the reverse direction went too
|
|
26
|
-
*
|
|
27
|
-
* The verdict is computed from the gaps, not chosen, and every line prints the
|
|
28
|
-
* numbers that produced it.
|
|
29
|
-
*/
|
|
30
|
-
|
|
31
|
-
/**
|
|
32
|
-
* How often a probe is numbered and sent on every channel.
|
|
33
|
-
*
|
|
34
|
-
* Half a second, because the transport heartbeat is five and that was the whole
|
|
35
|
-
* resolution the 2026-08-24 episode had: the onset could be placed no closer
|
|
36
|
-
* than the five seconds between two lines. Sending is cheap - a probe is a few
|
|
37
|
-
* dozen bytes - and the interval is NOT the verdict: a healthy burst can hold a
|
|
38
|
-
* probe up behind queued data, so how many probes may be outstanding is worked
|
|
39
|
-
* out per channel by {@link allowedGap} from the bytes queued ahead of the
|
|
40
|
-
* probe and the rate they are leaving at.
|
|
41
|
-
*/
|
|
42
|
-
export const PROBE_INTERVAL_MS = 500;
|
|
43
|
-
|
|
44
|
-
/**
|
|
45
|
-
* How many probes may be outstanding on a channel before it counts as behind.
|
|
46
|
-
*
|
|
47
|
-
* DERIVED per channel, not chosen. A probe is handed to the same association as
|
|
48
|
-
* the data, and SCTP orders per stream but SCHEDULES per association: one
|
|
49
|
-
* congestion window, one send buffer. So a probe waits for whatever is queued
|
|
50
|
-
* ahead of it whichever channel it rides on - the unordered one included - and
|
|
51
|
-
* a fixed threshold cannot tell that wait from a stopped association. Measured
|
|
52
|
-
* 2026-08-26: `association-stopped` printed with all three channels at gap 4-7
|
|
53
|
-
* while 110-150 Mbps crossed that same association and 7.34 GB went through it
|
|
54
|
-
* without a single failure.
|
|
55
|
-
*
|
|
56
|
-
* What the wait costs is arithmetic on two measured quantities: the bytes
|
|
57
|
-
* queued ahead of the probe, and the rate at which this connection is getting
|
|
58
|
-
* bytes away. Add one round trip for the echo to come back. A probe is behind
|
|
59
|
-
* only when it is later than that.
|
|
60
|
-
*
|
|
61
|
-
* There is a third term, and leaving it out made this worse than the constant
|
|
62
|
-
* it replaced. The browser answers on ITS own schedule, not ours: it batches
|
|
63
|
-
* what it has seen and echoes on a timer, and that timer is throttled to about
|
|
64
|
-
* once a second whenever the tab is hidden. So with an empty queue the
|
|
65
|
-
* allowance collapsed to one probe and the bound to half a second, while echoes
|
|
66
|
-
* legitimately arrived every second - measured 2026-08-27, 67 `association-
|
|
67
|
-
* stopped` and 66 `reverse-direction-gone` against 84 `flowing` on a connection
|
|
68
|
-
* carrying 3.4 MB/s with every queue at zero. The peer's own cadence is
|
|
69
|
-
* measurable on the same connection, so it is measured and added rather than
|
|
70
|
-
* assumed.
|
|
71
|
-
*
|
|
72
|
-
* There is a fourth term, and without it the third one arrives too late to
|
|
73
|
-
* help. `echoIntervalMs` is the widest gap between two echoes this connection
|
|
74
|
-
* has SHOWN, so it can only grow after a late echo has landed — and a browser
|
|
75
|
-
* whose tab has just been hidden goes quiet before it has taught us anything.
|
|
76
|
-
* The browser measures its own event-loop delay and puts it in every echo, so
|
|
77
|
-
* the delay is known BEFORE the silence rather than after it: an echo timer
|
|
78
|
-
* set for half a second cannot fire until the loop runs, and a loop reported
|
|
79
|
-
* as 4297 ms behind cannot answer sooner than that. Field 2026-09-03: a viewer
|
|
80
|
-
* paused at 15:24:21, the tab went hidden at 15:24:35, `loopLag` climbed
|
|
81
|
-
* 681 → 1881 → 4297 → 5957 ms, and at 15:26:05 the probes read `gap 12 of 11`
|
|
82
|
-
* and printed `association-stopped` on a connection that was `flowing` again
|
|
83
|
-
* seven seconds later. Adding the peer's own reported lag makes that allowance
|
|
84
|
-
* 15 instead of 11, and the gap of 12 is then what it was — a browser whose
|
|
85
|
-
* timers are frozen, not an association that stopped.
|
|
86
|
-
*
|
|
87
|
-
* @param {{ queuedBytes: number, bytesPerSecond: number, rttMs: number, echoIntervalMs?: number, peerLoopLagMs?: number, intervalMs?: number }} state
|
|
88
|
-
* @returns {number | null} Probes that may legitimately be outstanding, or null
|
|
89
|
-
* when no rate has been measured yet and nothing can be said.
|
|
90
|
-
*/
|
|
91
|
-
export function allowedGap({
|
|
92
|
-
queuedBytes,
|
|
93
|
-
bytesPerSecond,
|
|
94
|
-
rttMs,
|
|
95
|
-
echoIntervalMs = 0,
|
|
96
|
-
peerLoopLagMs = 0,
|
|
97
|
-
intervalMs = PROBE_INTERVAL_MS
|
|
98
|
-
}) {
|
|
99
|
-
if (!(bytesPerSecond > 0) || !(intervalMs > 0)) {
|
|
100
|
-
return null;
|
|
101
|
-
}
|
|
102
|
-
// At least one: a probe sent and not yet echoed is the ordinary state.
|
|
103
|
-
return Math.max(1, Math.ceil(allowedWaitMs({ queuedBytes, bytesPerSecond, rttMs, echoIntervalMs, peerLoopLagMs }) / intervalMs));
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
/**
|
|
107
|
-
* How long a probe may legitimately take to be reported back, in milliseconds.
|
|
108
|
-
*
|
|
109
|
-
* THE QUANTITY {@link allowedGap} COMPUTES AND THEN THROWS AWAY by dividing it
|
|
110
|
-
* into probes. Probes are the wrong unit and always were: the same probe goes
|
|
111
|
-
* down every channel INCLUDING the one carrying the film, and SCTP schedules
|
|
112
|
-
* per association, so a probe waits behind queued video exactly as a segment
|
|
113
|
-
* does. What is behind is then not the peer's answer but the probe itself —
|
|
114
|
-
* and the honest measure of that is the age of the newest probe the peer has
|
|
115
|
-
* seen, which this is compared against.
|
|
116
|
-
*
|
|
117
|
-
* Four terms, every one measured: the queue's own drain time at the rate this
|
|
118
|
-
* connection is achieving, the crossing, the peer's reporting cadence, and how
|
|
119
|
-
* late the peer's event loop is running.
|
|
120
|
-
*
|
|
121
|
-
* @param {{ queuedBytes: number, bytesPerSecond: number, rttMs: number,
|
|
122
|
-
* echoIntervalMs?: number, peerLoopLagMs?: number }} state
|
|
123
|
-
* @returns {number}
|
|
124
|
-
*/
|
|
125
|
-
export function allowedWaitMs({ queuedBytes, bytesPerSecond, rttMs, echoIntervalMs = 0, peerLoopLagMs = 0 }) {
|
|
126
|
-
if (!(bytesPerSecond > 0)) {
|
|
127
|
-
return 0;
|
|
128
|
-
}
|
|
129
|
-
const drainMs = (Math.max(queuedBytes, 0) / bytesPerSecond) * 1000;
|
|
130
|
-
return drainMs + Math.max(rttMs, 0) + Math.max(echoIntervalMs, 0) + Math.max(peerLoopLagMs, 0);
|
|
131
|
-
}
|
|
132
|
-
|
|
133
|
-
/**
|
|
134
|
-
* How far back the send times of probes are kept.
|
|
135
|
-
*
|
|
136
|
-
* A probe older than the point at which the reverse direction is called gone
|
|
137
|
-
* can say nothing further, and that bound is itself derived per connection —
|
|
138
|
-
* this is the outer edge of it, kept so the map cannot grow with the session.
|
|
139
|
-
*/
|
|
140
|
-
const PROBE_HISTORY_MS = 10 * 60 * 1000;
|
|
141
|
-
|
|
142
|
-
/** The label the browser gives the unordered, non-retransmitting channel. */
|
|
143
|
-
export const UNRELIABLE_LABEL = "proxy-fast";
|
|
144
|
-
|
|
145
|
-
/**
|
|
146
|
-
* How old an echo may be before the reverse direction counts as gone, when
|
|
147
|
-
* nothing better can be derived.
|
|
148
|
-
*
|
|
149
|
-
* Used only while no rate has been measured. Otherwise the caller passes
|
|
150
|
-
* `echoStaleMs`, worked out the same way as {@link allowedGap}: the browser
|
|
151
|
-
* only echoes a probe it has RECEIVED, so an echo waits behind our own queue
|
|
152
|
-
* exactly as the probe did.
|
|
153
|
-
*/
|
|
154
|
-
const ECHO_STALE_FALLBACK_MS = 5_000;
|
|
155
|
-
|
|
156
|
-
/** How often the probe state is written to the log while nothing changes. */
|
|
157
|
-
const REPORT_INTERVAL_MS = 5_000;
|
|
158
|
-
|
|
159
|
-
/**
|
|
160
|
-
* Whether the `seen` counter has stopped advancing for longer than this
|
|
161
|
-
* connection's own history says a healthy gap between two advances ever
|
|
162
|
-
* takes.
|
|
163
|
-
*
|
|
164
|
-
* The `association-stopped` verdict alone is not enough to act on: a
|
|
165
|
-
* connection can sit BEHIND by a bounded, roughly constant amount for
|
|
166
|
-
* minutes (measured 2026-08-28, session on a backgrounded tab — gap held at
|
|
167
|
-
* 6-7 probes for 95+ seconds while `seen` kept climbing right along with
|
|
168
|
-
* `sent`) without anything being wrong. What a true wedge shows instead,
|
|
169
|
-
* measured the same day against a session already known to be one
|
|
170
|
-
* (`d85ae4f5`): `seen` FROZEN at one value for over a minute while `sent`
|
|
171
|
-
* climbs unbounded. So the question is not "is there a gap" but "has the
|
|
172
|
-
* highest-seen number stopped moving at all, for longer than it has ever
|
|
173
|
-
* legitimately taken this connection to report an advance" — the same shape
|
|
174
|
-
* as {@link wedgeIsCertain} in `transport/data-channel-handler.js`, applied to the
|
|
175
|
-
* probe's own counter instead of the transport's byte counter.
|
|
176
|
-
*
|
|
177
|
-
* THE FLOOR IS NOT ONE PROBE INTERVAL. "The longest gap this connection has
|
|
178
|
-
* ever shown" is nothing at all in its first minutes, and the floor under it
|
|
179
|
-
* was a single interval — 500 ms — so any two-second quiet stretch while the
|
|
180
|
-
* browser filled its cushion read as longer than anything healthy ever seen.
|
|
181
|
-
* Field 2026-09-11: two captures of 180 s each, triggered at `wedged 1s` and
|
|
182
|
-
* `wedged 2s`, on a connection with `rtt=5ms`, every queue at 0 B and the
|
|
183
|
-
* viewer watching; 65 MB of them were left on the addon's disk.
|
|
184
|
-
*
|
|
185
|
-
* What replaces it is the connection's own answer to how long a legitimate
|
|
186
|
-
* report may take: the peer reports twice a second, the report has to cross,
|
|
187
|
-
* and the peer's own event loop may be late — all three measured, all three
|
|
188
|
-
* already summed for `echoStaleMs`. A stretch shorter than that proves nothing
|
|
189
|
-
* whatever about the association.
|
|
190
|
-
*
|
|
191
|
-
* @param {{ stuckForMs: number, longestHealthySeenGapMs: number, intervalMs?: number,
|
|
192
|
-
* legitimateReportMs?: number }} state
|
|
193
|
-
* @returns {{ isCertain: boolean, needMs: number }}
|
|
194
|
-
*/
|
|
195
|
-
export function probeWedgeIsCertain({
|
|
196
|
-
stuckForMs,
|
|
197
|
-
longestHealthySeenGapMs,
|
|
198
|
-
intervalMs = PROBE_INTERVAL_MS,
|
|
199
|
-
legitimateReportMs = 0
|
|
200
|
-
}) {
|
|
201
|
-
const floorMs = Math.max(intervalMs, Number.isFinite(legitimateReportMs) ? legitimateReportMs : 0);
|
|
202
|
-
const needMs = Math.max(longestHealthySeenGapMs, floorMs);
|
|
203
|
-
return { isCertain: stuckForMs >= needMs, needMs };
|
|
204
|
-
}
|
|
205
|
-
|
|
206
|
-
/**
|
|
207
|
-
* One connection's probe state.
|
|
208
|
-
*
|
|
209
|
-
* @typedef {Object} ProbeConnection
|
|
210
|
-
* @property {string} tag
|
|
211
|
-
* @property {Map<import('node-datachannel').DataChannel, string>} channels
|
|
212
|
-
* @property {number} seq - Highest probe number sent.
|
|
213
|
-
* @property {number} sentAt - When that probe was sent.
|
|
214
|
-
* @property {Map<string, number>} seen - Label to the highest number the browser reported.
|
|
215
|
-
* @property {number} echoAt - When the last echo arrived (0 = never).
|
|
216
|
-
* @property {number} echoes - How many echoes have arrived.
|
|
217
|
-
* @property {string} verdict - Last verdict reported, so a change is logged at once.
|
|
218
|
-
* @property {number} reportedAt - When the state was last written to the log.
|
|
219
|
-
* @property {number} lastSeenAdvanceAt - When any label's `seen` value last increased (0 = never yet).
|
|
220
|
-
* @property {number} longestHealthySeenGapMs - The longest gap between two advances this connection has shown while not flagged as a wedge.
|
|
221
|
-
* @property {number | null} peerBytes - The far end's transport-level received total, as last reported.
|
|
222
|
-
* @property {number | null} peerBytesAtTick - The same, as it stood at the previous tick.
|
|
223
|
-
* @property {number | null} peerLoopLagMs - The far end's own event-loop delay, as last reported.
|
|
224
|
-
* @property {string | null} peerVisibility - Whether the far end's tab is visible or hidden, as last reported.
|
|
225
|
-
* @property {boolean} probeCaptureStarted - One evidence-gathering attempt per wedge; reset once `seen` advances again.
|
|
226
|
-
* @property {ReturnType<typeof setInterval> | null} timer
|
|
227
|
-
*/
|
|
228
|
-
|
|
229
|
-
/**
|
|
230
|
-
* Read the gaps and say what they mean.
|
|
231
|
-
*
|
|
232
|
-
* Exported so the rule is testable without a connection: the same numbers
|
|
233
|
-
* always produce the same word.
|
|
234
|
-
*
|
|
235
|
-
* `allowed` carries, per channel label, how many probes may legitimately be
|
|
236
|
-
* outstanding right now — {@link allowedGap} computes it from that channel's
|
|
237
|
-
* own queue and the connection's measured rate. A label with no entry, or an
|
|
238
|
-
* entry of null, cannot be judged: with no rate measured there is nothing to
|
|
239
|
-
* divide the queue by, and the verdict says that rather than inventing one.
|
|
240
|
-
*
|
|
241
|
-
* `peerBytesAdvancing` is the fact that outranks every gap here. The gaps are
|
|
242
|
-
* counted against an allowance whose only load-dependent term is OUR OWN
|
|
243
|
-
* queue — and that queue is empty by construction, because it drains the moment
|
|
244
|
-
* libdatachannel accepts the bytes, whether or not usrsctp then puts them on
|
|
245
|
-
* the wire. So a browser filling its cushion as fast as the link allows shows
|
|
246
|
-
* an empty queue, a small allowance and a large gap, which reads exactly like a
|
|
247
|
-
* stopped association. Measured 2026-08-28: four `association-stopped` in the
|
|
248
|
-
* first two minutes of a healthy session, on a connection whose every queue was
|
|
249
|
-
* at 0 B and whose viewer never saw the picture stop. Deepening the browser's
|
|
250
|
-
* cushion from 30 s to 120 s (roadmap item 4) made the burst four times longer
|
|
251
|
-
* and the false verdict correspondingly likelier.
|
|
252
|
-
*
|
|
253
|
-
* What separates the two is not the size of the backlog — it is large in both —
|
|
254
|
-
* but whether bytes are still arriving. The browser reports its own
|
|
255
|
-
* transport-level received total with every echo, so the question is answered
|
|
256
|
-
* by a counter rather than by a threshold: while that total is advancing, bytes
|
|
257
|
-
* ARE crossing and no verdict of a stopped association can stand, however far
|
|
258
|
-
* behind the probes are. `null` where the far end does not report it, and then
|
|
259
|
-
* the rule falls back to what it says without the term.
|
|
260
|
-
*
|
|
261
|
-
* `peerLoopLagMs` and `peerVisibility` are printed but never compared against
|
|
262
|
-
* anything here: they are what made the allowance as wide as it is, and a
|
|
263
|
-
* reader of the log cannot check that arithmetic unless the terms are on the
|
|
264
|
-
* same line as the result.
|
|
265
|
-
*
|
|
266
|
-
* @param {{ seq: number, seen: Map<string, number> | Record<string, number>, labels: string[], echoes: number, echoAgeMs: number | null, allowed?: Map<string, number | null> | Record<string, number | null>, echoStaleMs?: number, peerBytesAdvancing?: boolean | null, peerLoopLagMs?: number | null, peerVisibility?: string | null, peerPending?: number | null }} state
|
|
267
|
-
* @returns {{ verdict: string, detail: string }}
|
|
268
|
-
*/
|
|
269
|
-
export function readProbeState(state) {
|
|
270
|
-
const seenOf = (label) =>
|
|
271
|
-
state.seen instanceof Map ? state.seen.get(label) : state.seen?.[label];
|
|
272
|
-
/**
|
|
273
|
-
* @param {unknown} source
|
|
274
|
-
* @param {string} label
|
|
275
|
-
* @returns {number | null}
|
|
276
|
-
*/
|
|
277
|
-
const timeOf = (source, label) => {
|
|
278
|
-
const value = source instanceof Map ? source.get(label) : /** @type {any} */ (source)?.[label];
|
|
279
|
-
return Number.isFinite(value) ? Number(value) : null;
|
|
280
|
-
};
|
|
281
|
-
const allowedOf = (label) => {
|
|
282
|
-
const source = state.allowed;
|
|
283
|
-
const value = source instanceof Map ? source.get(label) : source?.[label];
|
|
284
|
-
return Number.isInteger(value) ? Number(value) : null;
|
|
285
|
-
};
|
|
286
|
-
const parts = [];
|
|
287
|
-
let isOrderedBehind = false;
|
|
288
|
-
let isUnreliableBehind = false;
|
|
289
|
-
let isUnreliableKnown = false;
|
|
290
|
-
let isJudgeable = false;
|
|
291
|
-
for (const label of state.labels) {
|
|
292
|
-
const seen = seenOf(label);
|
|
293
|
-
const gap = Number.isInteger(seen) ? state.seq - Number(seen) : null;
|
|
294
|
-
const allowance = allowedOf(label);
|
|
295
|
-
// BY TIME WHERE IT IS KNOWN. The count is what the line prints, because it
|
|
296
|
-
// is what a reader recognises; what decides is a time.
|
|
297
|
-
//
|
|
298
|
-
// The measured one-way delay first: it is the forward direction itself,
|
|
299
|
-
// with the two clocks reconciled from the exchange. The age of the newest
|
|
300
|
-
// reported probe is the fallback, and it is larger than the thing itself by
|
|
301
|
-
// the peer's reporting cadence and the way back — which is why its own
|
|
302
|
-
// allowance carries both of those and the one-way allowance carries
|
|
303
|
-
// neither.
|
|
304
|
-
const oneWay = timeOf(state.oneWayMs, label);
|
|
305
|
-
const oneWayAllowed = timeOf(state.allowedOneWayMs, label);
|
|
306
|
-
const lagMs = oneWay !== null && oneWayAllowed !== null ? oneWay : timeOf(state.behindMs, label);
|
|
307
|
-
const mayWaitMs = oneWay !== null && oneWayAllowed !== null
|
|
308
|
-
? oneWayAllowed
|
|
309
|
-
: timeOf(state.allowedWaitMs, label);
|
|
310
|
-
const lagText = lagMs === null || mayWaitMs === null
|
|
311
|
-
? ""
|
|
312
|
-
: ` ${Math.round(lagMs)}ms of ${Math.round(mayWaitMs)}ms${oneWay !== null ? " one way" : ""}`;
|
|
313
|
-
parts.push(`${label}=${seen ?? "?"}(gap ${gap ?? "?"} of ${allowance ?? "?"}${lagText})`);
|
|
314
|
-
if (allowance === null) {
|
|
315
|
-
continue;
|
|
316
|
-
}
|
|
317
|
-
isJudgeable = true;
|
|
318
|
-
const isBehind = lagMs !== null && mayWaitMs !== null
|
|
319
|
-
? lagMs > mayWaitMs
|
|
320
|
-
: gap === null || gap > allowance;
|
|
321
|
-
if (label === UNRELIABLE_LABEL) {
|
|
322
|
-
isUnreliableKnown = true;
|
|
323
|
-
isUnreliableBehind = isBehind;
|
|
324
|
-
} else if (isBehind) {
|
|
325
|
-
isOrderedBehind = true;
|
|
326
|
-
}
|
|
327
|
-
}
|
|
328
|
-
const isPeerAdvancing = state.peerBytesAdvancing === true;
|
|
329
|
-
const detail =
|
|
330
|
-
`sent=${state.seq} ${parts.join(" ")} ` +
|
|
331
|
-
`echoAge=${state.echoAgeMs === null ? "never" : `${state.echoAgeMs}ms`}` +
|
|
332
|
-
(state.peerBytesAdvancing === null || state.peerBytesAdvancing === undefined
|
|
333
|
-
? ""
|
|
334
|
-
: ` peerBytes=${isPeerAdvancing ? "advancing" : "still"}`) +
|
|
335
|
-
(Number.isFinite(state.peerLoopLagMs) ? ` peerLoopLag=${Math.round(Number(state.peerLoopLagMs))}ms` : "") +
|
|
336
|
-
(state.peerVisibility ? ` peerTab=${state.peerVisibility}` : "") +
|
|
337
|
-
// WHAT THE PEER SAYS IT IS STILL WAITING FOR. The far end has sent this
|
|
338
|
-
// twice a second since the probe echo was built and nothing read it, while
|
|
339
|
-
// every verdict here was reasoned out from this side alone. It is evidence
|
|
340
|
-
// and not a verdict: the receiver declares its own transport lost, because
|
|
341
|
-
// it knows first — on 2026-09-06 the browser said so sixteen minutes before
|
|
342
|
-
// this end did.
|
|
343
|
-
(Number.isFinite(state.peerPending) ? ` peerWaiting=${Number(state.peerPending)}` : "");
|
|
344
|
-
|
|
345
|
-
if (state.echoes === 0) {
|
|
346
|
-
return { verdict: "no-echo-yet", detail };
|
|
347
|
-
}
|
|
348
|
-
if (!isJudgeable) {
|
|
349
|
-
return { verdict: "no-rate-yet", detail };
|
|
350
|
-
}
|
|
351
|
-
const staleAfterMs = Number.isFinite(state.echoStaleMs) && state.echoStaleMs > 0
|
|
352
|
-
? state.echoStaleMs
|
|
353
|
-
: ECHO_STALE_FALLBACK_MS;
|
|
354
|
-
if (state.echoAgeMs !== null && state.echoAgeMs > staleAfterMs) {
|
|
355
|
-
return { verdict: "reverse-direction-gone", detail };
|
|
356
|
-
}
|
|
357
|
-
if (!isOrderedBehind && !(isUnreliableKnown && isUnreliableBehind)) {
|
|
358
|
-
return { verdict: "flowing", detail };
|
|
359
|
-
}
|
|
360
|
-
// Bytes are still arriving at the far end. Whatever the probe gaps say, this
|
|
361
|
-
// association has not stopped — the probes are behind a backlog, which is
|
|
362
|
-
// what filling a cushion looks like from here.
|
|
363
|
-
if (isPeerAdvancing) {
|
|
364
|
-
return { verdict: "flowing", detail };
|
|
365
|
-
}
|
|
366
|
-
if (isOrderedBehind && isUnreliableKnown && !isUnreliableBehind) {
|
|
367
|
-
return { verdict: "stream-stuck", detail };
|
|
368
|
-
}
|
|
369
|
-
if (isOrderedBehind) {
|
|
370
|
-
return {
|
|
371
|
-
verdict: isUnreliableKnown ? "association-stopped" : "ordered-behind-no-comparison",
|
|
372
|
-
detail
|
|
373
|
-
};
|
|
374
|
-
}
|
|
375
|
-
return { verdict: "unreliable-behind-only", detail };
|
|
376
|
-
}
|
|
377
|
-
|
|
378
|
-
/**
|
|
379
|
-
* Create the probe service. One instance serves every session.
|
|
380
|
-
*
|
|
381
|
-
* @param {Object} options
|
|
382
|
-
* @param {(message: string) => void} options.log
|
|
383
|
-
* @param {number} [options.intervalMs]
|
|
384
|
-
* @param {(sessionId: string) => object | null} [options.getTransportSnapshot]
|
|
385
|
-
* Needed only to hand the witness a remote endpoint when this probe is the
|
|
386
|
-
* one declaring a wedge.
|
|
387
|
-
* @param {{ maybeCapture: (trigger: object) => boolean }} [options.witness]
|
|
388
|
-
* @param {{ maybeRead: (reasonText: string) => boolean }} [options.usrsctpState]
|
|
389
|
-
* @returns {{
|
|
390
|
-
* attach: (sessionId: string, tag: string, label: string, channel: import('node-datachannel').DataChannel) => void,
|
|
391
|
-
* detach: (sessionId: string, channel: import('node-datachannel').DataChannel) => void,
|
|
392
|
-
* noteEcho: (sessionId: string, echo: object) => void,
|
|
393
|
-
* dispose: () => void
|
|
394
|
-
* }}
|
|
395
|
-
*/
|
|
396
|
-
export function createDeliveryProbe({
|
|
397
|
-
log,
|
|
398
|
-
intervalMs = PROBE_INTERVAL_MS,
|
|
399
|
-
readDelivery,
|
|
400
|
-
getTransportSnapshot,
|
|
401
|
-
witness,
|
|
402
|
-
usrsctpState
|
|
403
|
-
}) {
|
|
404
|
-
/** @type {Map<string, ProbeConnection>} */
|
|
405
|
-
const connections = new Map();
|
|
406
|
-
|
|
407
|
-
/**
|
|
408
|
-
* Send this tick's probe on every channel of one connection, then report.
|
|
409
|
-
*
|
|
410
|
-
* @param {ProbeConnection} connection
|
|
411
|
-
* @returns {void}
|
|
412
|
-
*/
|
|
413
|
-
function tick(connection) {
|
|
414
|
-
const now = Date.now();
|
|
415
|
-
connection.seq += 1;
|
|
416
|
-
connection.sentAt = now;
|
|
417
|
-
// WHEN each probe went out, so that what the peer reports can be read as a
|
|
418
|
-
// time rather than as a count of probes. Pruned to the oldest probe any
|
|
419
|
-
// judgement could still be about: once a probe is older than the point at
|
|
420
|
-
// which the reverse direction is called gone, its age says nothing further.
|
|
421
|
-
connection.sentAtBySeq.set(connection.seq, now);
|
|
422
|
-
for (const [probeNumber, sentAt] of connection.sentAtBySeq) {
|
|
423
|
-
if (now - sentAt > PROBE_HISTORY_MS) {
|
|
424
|
-
connection.sentAtBySeq.delete(probeNumber);
|
|
425
|
-
} else {
|
|
426
|
-
break;
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
const message = JSON.stringify({ type: "probe", seq: connection.seq, sentAt: now });
|
|
430
|
-
for (const channel of connection.channels.keys()) {
|
|
431
|
-
try {
|
|
432
|
-
channel.sendMessage(message);
|
|
433
|
-
} catch {
|
|
434
|
-
// A channel closing between the check and the send is ordinary.
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
|
-
|
|
438
|
-
// What this connection is getting away, and how far behind it therefore
|
|
439
|
-
// sits. Both come from the send-queue watcher, which measures them anyway.
|
|
440
|
-
const delivery = typeof readDelivery === "function" ? readDelivery(connection.id) : null;
|
|
441
|
-
const bytesPerSecond = Number(delivery?.bytesPerSecond) || 0;
|
|
442
|
-
const rttMs = Number(delivery?.rttMs) || 0;
|
|
443
|
-
/** @type {Map<string, number | null>} */
|
|
444
|
-
const allowed = new Map();
|
|
445
|
-
/** How long each channel's newest unreported probe may legitimately be. */
|
|
446
|
-
const allowedWait = new Map();
|
|
447
|
-
/** The same, for the measured one-way time where the clocks are reconciled. */
|
|
448
|
-
const allowedOneWay = new Map();
|
|
449
|
-
for (const [channel, label] of connection.channels) {
|
|
450
|
-
let queuedBytes = 0;
|
|
451
|
-
try {
|
|
452
|
-
queuedBytes = typeof channel.bufferedAmount === "function" ? channel.bufferedAmount() : 0;
|
|
453
|
-
} catch {
|
|
454
|
-
queuedBytes = 0;
|
|
455
|
-
}
|
|
456
|
-
const allowance = allowedGap({
|
|
457
|
-
queuedBytes,
|
|
458
|
-
bytesPerSecond,
|
|
459
|
-
rttMs,
|
|
460
|
-
echoIntervalMs: connection.echoIntervalMs,
|
|
461
|
-
peerLoopLagMs: connection.peerLoopLagMs ?? 0,
|
|
462
|
-
intervalMs
|
|
463
|
-
});
|
|
464
|
-
const waitMs = allowedWaitMs({
|
|
465
|
-
queuedBytes,
|
|
466
|
-
bytesPerSecond,
|
|
467
|
-
rttMs,
|
|
468
|
-
echoIntervalMs: connection.echoIntervalMs,
|
|
469
|
-
peerLoopLagMs: connection.peerLoopLagMs ?? 0
|
|
470
|
-
});
|
|
471
|
-
const heldWait = allowedWait.get(label);
|
|
472
|
-
if (!Number.isFinite(heldWait) || waitMs > heldWait) {
|
|
473
|
-
allowedWait.set(label, waitMs);
|
|
474
|
-
}
|
|
475
|
-
// What a probe may take ONE WAY: the queue's own drain time and half the
|
|
476
|
-
// crossing. Neither the peer's reporting cadence nor its loop delay
|
|
477
|
-
// belongs here — with the clocks reconciled they are no longer in the
|
|
478
|
-
// measurement they would have to be allowed for.
|
|
479
|
-
const oneWayAllowed = allowedWaitMs({ queuedBytes, bytesPerSecond, rttMs: rttMs / 2 });
|
|
480
|
-
const heldOneWay = allowedOneWay.get(label);
|
|
481
|
-
if (!Number.isFinite(heldOneWay) || oneWayAllowed > heldOneWay) {
|
|
482
|
-
allowedOneWay.set(label, oneWayAllowed);
|
|
483
|
-
}
|
|
484
|
-
// Several channels can carry one label only in malformed cases; the
|
|
485
|
-
// larger allowance is the safer of the two.
|
|
486
|
-
const held = allowed.get(label);
|
|
487
|
-
if (allowance !== null && (!Number.isInteger(held) || allowance > held)) {
|
|
488
|
-
allowed.set(label, allowance);
|
|
489
|
-
} else if (!allowed.has(label)) {
|
|
490
|
-
allowed.set(label, allowance);
|
|
491
|
-
}
|
|
492
|
-
}
|
|
493
|
-
const widest = [...allowed.values()].reduce(
|
|
494
|
-
(most, value) => (Number.isInteger(value) && value > most ? value : most),
|
|
495
|
-
0
|
|
496
|
-
);
|
|
497
|
-
// Advancing SINCE THE PREVIOUS TICK, not since the connection began: the
|
|
498
|
-
// question is whether bytes are crossing now. Prefer channel bytes —
|
|
499
|
-
// transport's bytesReceived advances on SACKs (140 B/s on a wedge) and
|
|
500
|
-
// gives a false "advancing" every other tick, while channel bytes are flat.
|
|
501
|
-
let peerBytesAdvancing = null;
|
|
502
|
-
if (connection.peerChannelBytes !== null) {
|
|
503
|
-
peerBytesAdvancing =
|
|
504
|
-
connection.peerChannelBytesAtTick === null ||
|
|
505
|
-
connection.peerChannelBytes > connection.peerChannelBytesAtTick;
|
|
506
|
-
connection.peerChannelBytesAtTick = connection.peerChannelBytes;
|
|
507
|
-
// keep transport's tick in sync for fallback logging
|
|
508
|
-
connection.peerBytesAtTick = connection.peerBytes;
|
|
509
|
-
} else if (connection.peerBytes !== null) {
|
|
510
|
-
peerBytesAdvancing =
|
|
511
|
-
connection.peerBytesAtTick === null || connection.peerBytes > connection.peerBytesAtTick;
|
|
512
|
-
connection.peerBytesAtTick = connection.peerBytes;
|
|
513
|
-
}
|
|
514
|
-
const { verdict, detail } = readProbeState({
|
|
515
|
-
seq: connection.seq,
|
|
516
|
-
seen: connection.seen,
|
|
517
|
-
peerBytesAdvancing,
|
|
518
|
-
peerPending: connection.peerPending,
|
|
519
|
-
labels: [...new Set(connection.channels.values())],
|
|
520
|
-
echoes: connection.echoes,
|
|
521
|
-
echoAgeMs: connection.echoAt === 0 ? null : now - connection.echoAt,
|
|
522
|
-
peerLoopLagMs: connection.peerLoopLagMs,
|
|
523
|
-
peerVisibility: connection.peerVisibility,
|
|
524
|
-
allowed,
|
|
525
|
-
// WHAT IS BEHIND, IN TIME. For each channel: how old the newest probe the
|
|
526
|
-
// peer has reported seeing is. The same probe goes down every channel
|
|
527
|
-
// including the one carrying the film, and SCTP schedules per
|
|
528
|
-
// association, so a probe waits behind queued video exactly as a segment
|
|
529
|
-
// does — which means a count of outstanding probes measures the queue,
|
|
530
|
-
// not the association. This is the queue's own time, against the time the
|
|
531
|
-
// queue is allowed to take.
|
|
532
|
-
behindMs: Object.fromEntries(
|
|
533
|
-
[...connection.seen.entries()].map(([label, newestSeenNumber]) => {
|
|
534
|
-
const sentAt = connection.sentAtBySeq.get(newestSeenNumber);
|
|
535
|
-
return [label, Number.isFinite(sentAt) ? now - sentAt : null];
|
|
536
|
-
})
|
|
537
|
-
),
|
|
538
|
-
allowedWaitMs: Object.fromEntries(allowedWait.entries()),
|
|
539
|
-
// The measured forward delay, where the two clocks have been reconciled.
|
|
540
|
-
// Preferred over the age above because it is the thing itself: the age
|
|
541
|
-
// also carries the peer's reporting cadence and the way back.
|
|
542
|
-
oneWayMs: Object.fromEntries(connection.oneWayMs.entries()),
|
|
543
|
-
allowedOneWayMs: Object.fromEntries(allowedOneWay.entries()),
|
|
544
|
-
// Same arithmetic for the echo's own age: the peer cannot answer sooner
|
|
545
|
-
// than its own cadence allows, nor sooner than its own event loop runs.
|
|
546
|
-
echoStaleMs:
|
|
547
|
-
widest > 0
|
|
548
|
-
? widest * intervalMs +
|
|
549
|
-
rttMs +
|
|
550
|
-
connection.echoIntervalMs +
|
|
551
|
-
Math.max(connection.peerLoopLagMs ?? 0, 0)
|
|
552
|
-
: 0
|
|
553
|
-
});
|
|
554
|
-
if (verdict !== connection.verdict || now - connection.reportedAt >= REPORT_INTERVAL_MS) {
|
|
555
|
-
connection.verdict = verdict;
|
|
556
|
-
connection.reportedAt = now;
|
|
557
|
-
log(`[dc-probe] ${connection.tag} ${verdict} — ${detail} at=${new Date(now).toISOString()}`);
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
// `association-stopped` alone is not certainty — see probeWedgeIsCertain.
|
|
561
|
-
// A connection that is merely lagging by a bounded amount reaches this
|
|
562
|
-
// verdict too (a hidden tab's own echo cadence, measured 2026-08-28), and
|
|
563
|
-
// `seen` keeps advancing right along with it. Only a `seen` value that has
|
|
564
|
-
// stopped moving ENTIRELY, for longer than this connection has ever shown
|
|
565
|
-
// as a legitimate gap, is the wedge this exists to catch.
|
|
566
|
-
const stuckForMs = connection.lastSeenAdvanceAt === 0 ? 0 : now - connection.lastSeenAdvanceAt;
|
|
567
|
-
if (verdict === "association-stopped") {
|
|
568
|
-
const { isCertain, needMs } = probeWedgeIsCertain({
|
|
569
|
-
stuckForMs,
|
|
570
|
-
longestHealthySeenGapMs: connection.longestHealthySeenGapMs,
|
|
571
|
-
// The same sum the echo's own staleness is judged by: the peer reports
|
|
572
|
-
// twice a second, the report has to cross, and the peer's loop may be
|
|
573
|
-
// late. Nothing shorter than that says anything.
|
|
574
|
-
// BOTH cadences, because they beat against each other: a probe sent
|
|
575
|
-
// just after the peer composed a report shows up only in the NEXT one.
|
|
576
|
-
// So the longest legitimate gap between two advances is one probe
|
|
577
|
-
// interval plus one report interval plus the crossing plus whatever
|
|
578
|
-
// the peer's own loop is behind by.
|
|
579
|
-
legitimateReportMs:
|
|
580
|
-
intervalMs +
|
|
581
|
-
connection.echoIntervalMs +
|
|
582
|
-
rttMs +
|
|
583
|
-
Math.max(connection.peerLoopLagMs ?? 0, 0)
|
|
584
|
-
});
|
|
585
|
-
if (isCertain && !connection.probeCaptureStarted) {
|
|
586
|
-
connection.probeCaptureStarted = true;
|
|
587
|
-
const reasonText =
|
|
588
|
-
`probe seen-counter unmoved ${Math.round(stuckForMs / 1000)}s against the ` +
|
|
589
|
-
`${(needMs / 1000).toFixed(1)}s this connection's own history says is legitimate`;
|
|
590
|
-
if (witness) {
|
|
591
|
-
const snapshot = getTransportSnapshot?.(connection.id) ?? null;
|
|
592
|
-
const started = witness.maybeCapture({
|
|
593
|
-
sessionId: connection.id,
|
|
594
|
-
tag: connection.tag,
|
|
595
|
-
label: "probe",
|
|
596
|
-
remote: snapshot?.remote ?? null,
|
|
597
|
-
queuedBytes: 0,
|
|
598
|
-
stuckForMs
|
|
599
|
-
});
|
|
600
|
-
if (!started) {
|
|
601
|
-
connection.probeCaptureStarted = false;
|
|
602
|
-
}
|
|
603
|
-
}
|
|
604
|
-
if (usrsctpState) {
|
|
605
|
-
usrsctpState.maybeRead(reasonText);
|
|
606
|
-
}
|
|
607
|
-
}
|
|
608
|
-
} else {
|
|
609
|
-
// Not association-stopped any more: whatever was flagged has cleared,
|
|
610
|
-
// and a later wedge on the same connection deserves its own attempt.
|
|
611
|
-
connection.probeCaptureStarted = false;
|
|
612
|
-
}
|
|
613
|
-
}
|
|
614
|
-
|
|
615
|
-
return {
|
|
616
|
-
attach(sessionId, tag, label, channel) {
|
|
617
|
-
let connection = connections.get(sessionId);
|
|
618
|
-
if (!connection) {
|
|
619
|
-
connection = {
|
|
620
|
-
id: sessionId,
|
|
621
|
-
tag,
|
|
622
|
-
// The longest this peer has ever taken between two echoes. Its own
|
|
623
|
-
// schedule, measured rather than assumed - a hidden tab answers
|
|
624
|
-
// about once a second because the browser throttles the timer.
|
|
625
|
-
echoIntervalMs: 0,
|
|
626
|
-
channels: new Map(),
|
|
627
|
-
seq: 0,
|
|
628
|
-
sentAt: 0,
|
|
629
|
-
seen: new Map(),
|
|
630
|
-
echoAt: 0,
|
|
631
|
-
echoes: 0,
|
|
632
|
-
// The far end's own transport-level received total, and its value at
|
|
633
|
-
// the previous tick. Null until a browser that reports it has echoed.
|
|
634
|
-
/** @type {number | null} */
|
|
635
|
-
peerBytes: null,
|
|
636
|
-
// Requests the far end says it is still waiting for, as last reported.
|
|
637
|
-
peerPending: null,
|
|
638
|
-
/** @type {number | null} */
|
|
639
|
-
peerBytesAtTick: null,
|
|
640
|
-
// Sum of per-channel bytes (report.channels[].bytes). Transport's
|
|
641
|
-
// bytesReceived advances on SACKs (140 B/s on a wedge) and gives a
|
|
642
|
-
// false "advancing" every other tick — channel bytes are flat on a wedge.
|
|
643
|
-
/** @type {number | null} */
|
|
644
|
-
peerChannelBytes: null,
|
|
645
|
-
/** @type {number | null} */
|
|
646
|
-
peerChannelBytesAtTick: null,
|
|
647
|
-
// The far end's own event-loop delay and tab state, as last reported.
|
|
648
|
-
// The lag is a term in every allowance below; the tab state is only
|
|
649
|
-
// printed, so that a wide allowance can be read back to its cause.
|
|
650
|
-
/** @type {number | null} */
|
|
651
|
-
peerLoopLagMs: null,
|
|
652
|
-
/** @type {string | null} */
|
|
653
|
-
peerVisibility: null,
|
|
654
|
-
verdict: "",
|
|
655
|
-
reportedAt: 0,
|
|
656
|
-
lastSeenAdvanceAt: 0,
|
|
657
|
-
longestHealthySeenGapMs: 0,
|
|
658
|
-
sentAtBySeq: new Map(),
|
|
659
|
-
// THE TWO CLOCKS, separated by the exchange rather than assumed to
|
|
660
|
-
// agree. Every report carries when each channel's newest probe
|
|
661
|
-
// arrived at the peer and when the report itself left; with the time
|
|
662
|
-
// this proxy stamped into that probe, and the time the report
|
|
663
|
-
// arrives, that is the four timestamps an offset is computed from.
|
|
664
|
-
//
|
|
665
|
-
// The estimate is kept from the report with the SMALLEST round trip
|
|
666
|
-
// seen, because that is the one that queued the least — and the two
|
|
667
|
-
// directions here are as unequal as they get, film one way and almost
|
|
668
|
-
// nothing the other, which is exactly where an offset taken from a
|
|
669
|
-
// busy moment is wrong by half the difference.
|
|
670
|
-
/** @type {number | null} */
|
|
671
|
-
clockOffsetMs: null,
|
|
672
|
-
/** @type {number} */
|
|
673
|
-
offsetFromRoundTripMs: Number.POSITIVE_INFINITY,
|
|
674
|
-
/** How long the newest probe took to reach the peer, by channel. @type {Map<string, number>} */
|
|
675
|
-
oneWayMs: new Map(),
|
|
676
|
-
probeCaptureStarted: false,
|
|
677
|
-
timer: null
|
|
678
|
-
};
|
|
679
|
-
connections.set(sessionId, connection);
|
|
680
|
-
}
|
|
681
|
-
connection.channels.set(channel, label);
|
|
682
|
-
if (connection.timer === null) {
|
|
683
|
-
const held = connection;
|
|
684
|
-
connection.timer = setInterval(() => tick(held), intervalMs);
|
|
685
|
-
// The probe must never be the reason a process stays alive.
|
|
686
|
-
if (typeof connection.timer.unref === "function") {
|
|
687
|
-
connection.timer.unref();
|
|
688
|
-
}
|
|
689
|
-
}
|
|
690
|
-
},
|
|
691
|
-
|
|
692
|
-
detach(sessionId, channel) {
|
|
693
|
-
const connection = connections.get(sessionId);
|
|
694
|
-
if (!connection) {
|
|
695
|
-
return;
|
|
696
|
-
}
|
|
697
|
-
connection.channels.delete(channel);
|
|
698
|
-
if (connection.channels.size === 0) {
|
|
699
|
-
if (connection.timer !== null) {
|
|
700
|
-
clearInterval(connection.timer);
|
|
701
|
-
connection.timer = null;
|
|
702
|
-
}
|
|
703
|
-
if (connections.get(sessionId) === connection) {
|
|
704
|
-
connections.delete(sessionId);
|
|
705
|
-
}
|
|
706
|
-
}
|
|
707
|
-
},
|
|
708
|
-
|
|
709
|
-
noteEcho(sessionId, echo) {
|
|
710
|
-
const connection = connections.get(sessionId);
|
|
711
|
-
if (!connection || !echo || typeof echo !== "object") {
|
|
712
|
-
return;
|
|
713
|
-
}
|
|
714
|
-
const now = Date.now();
|
|
715
|
-
const seen = echo.seen;
|
|
716
|
-
if (seen && typeof seen === "object") {
|
|
717
|
-
let advanced = false;
|
|
718
|
-
for (const [label, value] of Object.entries(seen)) {
|
|
719
|
-
if (!Number.isInteger(value)) {
|
|
720
|
-
continue;
|
|
721
|
-
}
|
|
722
|
-
const previous = connection.seen.get(label);
|
|
723
|
-
if (!Number.isInteger(previous) || value > previous) {
|
|
724
|
-
advanced = true;
|
|
725
|
-
}
|
|
726
|
-
connection.seen.set(label, value);
|
|
727
|
-
}
|
|
728
|
-
// What a wedge shows is this counter frozen, not merely behind — see
|
|
729
|
-
// probeWedgeIsCertain. The gap since the last time ANY label moved is
|
|
730
|
-
// this connection's own answer to "how long may a healthy report take
|
|
731
|
-
// to arrive", recorded only while nothing is currently flagged (the
|
|
732
|
-
// same guard `longestHealthyFlatMs` uses): a stretch already under
|
|
733
|
-
// suspicion must not teach the detector to tolerate it.
|
|
734
|
-
if (advanced) {
|
|
735
|
-
if (connection.lastSeenAdvanceAt !== 0 && !connection.probeCaptureStarted) {
|
|
736
|
-
const gap = now - connection.lastSeenAdvanceAt;
|
|
737
|
-
if (gap > connection.longestHealthySeenGapMs) {
|
|
738
|
-
connection.longestHealthySeenGapMs = gap;
|
|
739
|
-
}
|
|
740
|
-
}
|
|
741
|
-
connection.lastSeenAdvanceAt = now;
|
|
742
|
-
}
|
|
743
|
-
}
|
|
744
|
-
// THE FOUR TIMESTAMPS. `t1` is when this proxy sent the probe the peer
|
|
745
|
-
// is reporting, `t2` when the peer received it, `t3` when the peer sent
|
|
746
|
-
// this report, `t4` is now. From them the round trip is
|
|
747
|
-
// `(t4 - t1) - (t3 - t2)` — the peer's own thinking time removed — and
|
|
748
|
-
// the difference between the two clocks is `((t2 - t1) + (t3 - t4)) / 2`.
|
|
749
|
-
//
|
|
750
|
-
// The offset is kept from the report whose round trip was SMALLEST: that
|
|
751
|
-
// arithmetic assumes the two directions are equally quick, which here
|
|
752
|
-
// they are not — film one way, almost nothing the other — and the least
|
|
753
|
-
// queued sample is the one where the assumption costs least. A fresh
|
|
754
|
-
// minimum replaces the estimate, so a clock that jumps is followed within
|
|
755
|
-
// a few reports rather than believed for ever.
|
|
756
|
-
const peerSentAt = Number(echo?.sentAt);
|
|
757
|
-
const seenAt = echo?.seenAt;
|
|
758
|
-
if (Number.isFinite(peerSentAt) && seenAt && typeof seenAt === "object") {
|
|
759
|
-
for (const [label, peerSawAtRaw] of Object.entries(seenAt)) {
|
|
760
|
-
const peerSawAt = Number(peerSawAtRaw);
|
|
761
|
-
const newestSeenNumber = connection.seen.get(label);
|
|
762
|
-
const weSentAt = Number.isInteger(newestSeenNumber)
|
|
763
|
-
? connection.sentAtBySeq.get(newestSeenNumber)
|
|
764
|
-
: undefined;
|
|
765
|
-
if (!Number.isFinite(peerSawAt) || !Number.isFinite(weSentAt)) {
|
|
766
|
-
continue;
|
|
767
|
-
}
|
|
768
|
-
const roundTrip = (now - weSentAt) - (peerSentAt - peerSawAt);
|
|
769
|
-
if (roundTrip >= 0 && roundTrip < connection.offsetFromRoundTripMs) {
|
|
770
|
-
connection.offsetFromRoundTripMs = roundTrip;
|
|
771
|
-
connection.clockOffsetMs = ((peerSawAt - weSentAt) + (peerSentAt - now)) / 2;
|
|
772
|
-
}
|
|
773
|
-
if (connection.clockOffsetMs !== null) {
|
|
774
|
-
// What the probe itself took, one way, with the clocks reconciled.
|
|
775
|
-
// Negative only if the offset is stale, and then it says so rather
|
|
776
|
-
// than being hidden.
|
|
777
|
-
connection.oneWayMs.set(label, peerSawAt - weSentAt - connection.clockOffsetMs);
|
|
778
|
-
}
|
|
779
|
-
}
|
|
780
|
-
}
|
|
781
|
-
// What the far end says it has received at the transport level. It is the
|
|
782
|
-
// one figure that separates a backlog from a stopped association, and it
|
|
783
|
-
// arrives on the direction that goes on working through a freeze.
|
|
784
|
-
// How many requests the far end says it is still waiting for. Theirs to
|
|
785
|
-
// count: this end knows what it has answered, not what reached them.
|
|
786
|
-
const peerPending = Number(echo?.report?.pending);
|
|
787
|
-
connection.peerPending = Number.isFinite(peerPending) && peerPending >= 0 ? peerPending : null;
|
|
788
|
-
const peerBytes = Number(echo?.report?.transportBytesReceived);
|
|
789
|
-
if (Number.isFinite(peerBytes) && peerBytes >= 0) {
|
|
790
|
-
connection.peerBytes = peerBytes;
|
|
791
|
-
}
|
|
792
|
-
// Sum of per-channel bytes. Transport's bytesReceived advances on SACKs
|
|
793
|
-
// (140 B/s on a wedge) and gives a false "advancing" every other tick —
|
|
794
|
-
// channel bytes are flat on a wedge, so they are the correct signal.
|
|
795
|
-
const peerChannels = echo?.report?.channels;
|
|
796
|
-
if (peerChannels && typeof peerChannels === "object") {
|
|
797
|
-
let bytesAcrossChannels = 0;
|
|
798
|
-
let hasChannelReport = false;
|
|
799
|
-
for (const channelReport of Object.values(peerChannels)) {
|
|
800
|
-
if (channelReport && Number.isFinite(channelReport.bytes)) {
|
|
801
|
-
bytesAcrossChannels += channelReport.bytes;
|
|
802
|
-
hasChannelReport = true;
|
|
803
|
-
}
|
|
804
|
-
}
|
|
805
|
-
if (hasChannelReport) {
|
|
806
|
-
connection.peerChannelBytes = bytesAcrossChannels;
|
|
807
|
-
}
|
|
808
|
-
}
|
|
809
|
-
// How far behind the far end's own event loop is running. A browser that
|
|
810
|
-
// cannot run its timers cannot answer a probe, and every allowance here
|
|
811
|
-
// is a wait the answer has to fit inside — so this is a term in the
|
|
812
|
-
// arithmetic, not a note. It is the peer's own measurement, taken on the
|
|
813
|
-
// peer, and it arrives on the direction that survives a freeze.
|
|
814
|
-
const peerLoopLag = Number(echo?.report?.loopLagMs);
|
|
815
|
-
if (Number.isFinite(peerLoopLag) && peerLoopLag >= 0) {
|
|
816
|
-
connection.peerLoopLagMs = peerLoopLag;
|
|
817
|
-
}
|
|
818
|
-
const peerVisibility = echo?.report?.visibility;
|
|
819
|
-
if (typeof peerVisibility === "string" && peerVisibility.length > 0) {
|
|
820
|
-
connection.peerVisibility = peerVisibility;
|
|
821
|
-
}
|
|
822
|
-
if (connection.echoAt !== 0) {
|
|
823
|
-
const sinceLast = now - connection.echoAt;
|
|
824
|
-
if (sinceLast > connection.echoIntervalMs) {
|
|
825
|
-
connection.echoIntervalMs = sinceLast;
|
|
826
|
-
}
|
|
827
|
-
}
|
|
828
|
-
connection.echoAt = now;
|
|
829
|
-
connection.echoes += 1;
|
|
830
|
-
},
|
|
831
|
-
|
|
832
|
-
dispose() {
|
|
833
|
-
for (const connection of connections.values()) {
|
|
834
|
-
if (connection.timer !== null) {
|
|
835
|
-
clearInterval(connection.timer);
|
|
836
|
-
connection.timer = null;
|
|
837
|
-
}
|
|
838
|
-
}
|
|
839
|
-
connections.clear();
|
|
840
|
-
}
|
|
841
|
-
};
|
|
842
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* @file Numbered delivery probes, and the verdict they make possible.
|
|
3
|
+
*
|
|
4
|
+
* The delivery freeze (roadmap item 11) looks identical from the proxy in two
|
|
5
|
+
* cases that need opposite fixes: usrsctp stopped transmitting, or the browser
|
|
6
|
+
* closed its receive window because the page stopped draining the channel. The
|
|
7
|
+
* proxy's own counters cannot separate them — libdatachannel's `bytesSent`
|
|
8
|
+
* counts bytes ACCEPTED into usrsctp, not bytes put on the wire — so the
|
|
9
|
+
* reading has to come from the far end.
|
|
10
|
+
*
|
|
11
|
+
* Two facts make that cheap. The reverse direction keeps working throughout the
|
|
12
|
+
* freeze (browser to proxy requests arrive and are answered for the whole
|
|
13
|
+
* episode, 88 min in the 2026-08-24 case), so the browser can always report.
|
|
14
|
+
* And SCTP orders per STREAM, so a probe on a channel opened UNORDERED and
|
|
15
|
+
* WITHOUT retransmission passes head-of-line blocking in another stream but
|
|
16
|
+
* neither a closed receive window nor a transmitter that stopped.
|
|
17
|
+
*
|
|
18
|
+
* So: number a probe every {@link PROBE_INTERVAL_MS} on every channel of the
|
|
19
|
+
* connection, have the browser echo back the highest number it has seen on
|
|
20
|
+
* each, and read the gaps:
|
|
21
|
+
*
|
|
22
|
+
* every channel current to flowing
|
|
23
|
+
* ordered behind, unreliable current to a retransmission stuck in a stream
|
|
24
|
+
* both behind, echoes still arriving to the association stopped transmitting
|
|
25
|
+
* no echo at all to the reverse direction went too
|
|
26
|
+
*
|
|
27
|
+
* The verdict is computed from the gaps, not chosen, and every line prints the
|
|
28
|
+
* numbers that produced it.
|
|
29
|
+
*/
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* How often a probe is numbered and sent on every channel.
|
|
33
|
+
*
|
|
34
|
+
* Half a second, because the transport heartbeat is five and that was the whole
|
|
35
|
+
* resolution the 2026-08-24 episode had: the onset could be placed no closer
|
|
36
|
+
* than the five seconds between two lines. Sending is cheap - a probe is a few
|
|
37
|
+
* dozen bytes - and the interval is NOT the verdict: a healthy burst can hold a
|
|
38
|
+
* probe up behind queued data, so how many probes may be outstanding is worked
|
|
39
|
+
* out per channel by {@link allowedGap} from the bytes queued ahead of the
|
|
40
|
+
* probe and the rate they are leaving at.
|
|
41
|
+
*/
|
|
42
|
+
export const PROBE_INTERVAL_MS = 500;
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* How many probes may be outstanding on a channel before it counts as behind.
|
|
46
|
+
*
|
|
47
|
+
* DERIVED per channel, not chosen. A probe is handed to the same association as
|
|
48
|
+
* the data, and SCTP orders per stream but SCHEDULES per association: one
|
|
49
|
+
* congestion window, one send buffer. So a probe waits for whatever is queued
|
|
50
|
+
* ahead of it whichever channel it rides on - the unordered one included - and
|
|
51
|
+
* a fixed threshold cannot tell that wait from a stopped association. Measured
|
|
52
|
+
* 2026-08-26: `association-stopped` printed with all three channels at gap 4-7
|
|
53
|
+
* while 110-150 Mbps crossed that same association and 7.34 GB went through it
|
|
54
|
+
* without a single failure.
|
|
55
|
+
*
|
|
56
|
+
* What the wait costs is arithmetic on two measured quantities: the bytes
|
|
57
|
+
* queued ahead of the probe, and the rate at which this connection is getting
|
|
58
|
+
* bytes away. Add one round trip for the echo to come back. A probe is behind
|
|
59
|
+
* only when it is later than that.
|
|
60
|
+
*
|
|
61
|
+
* There is a third term, and leaving it out made this worse than the constant
|
|
62
|
+
* it replaced. The browser answers on ITS own schedule, not ours: it batches
|
|
63
|
+
* what it has seen and echoes on a timer, and that timer is throttled to about
|
|
64
|
+
* once a second whenever the tab is hidden. So with an empty queue the
|
|
65
|
+
* allowance collapsed to one probe and the bound to half a second, while echoes
|
|
66
|
+
* legitimately arrived every second - measured 2026-08-27, 67 `association-
|
|
67
|
+
* stopped` and 66 `reverse-direction-gone` against 84 `flowing` on a connection
|
|
68
|
+
* carrying 3.4 MB/s with every queue at zero. The peer's own cadence is
|
|
69
|
+
* measurable on the same connection, so it is measured and added rather than
|
|
70
|
+
* assumed.
|
|
71
|
+
*
|
|
72
|
+
* There is a fourth term, and without it the third one arrives too late to
|
|
73
|
+
* help. `echoIntervalMs` is the widest gap between two echoes this connection
|
|
74
|
+
* has SHOWN, so it can only grow after a late echo has landed — and a browser
|
|
75
|
+
* whose tab has just been hidden goes quiet before it has taught us anything.
|
|
76
|
+
* The browser measures its own event-loop delay and puts it in every echo, so
|
|
77
|
+
* the delay is known BEFORE the silence rather than after it: an echo timer
|
|
78
|
+
* set for half a second cannot fire until the loop runs, and a loop reported
|
|
79
|
+
* as 4297 ms behind cannot answer sooner than that. Field 2026-09-03: a viewer
|
|
80
|
+
* paused at 15:24:21, the tab went hidden at 15:24:35, `loopLag` climbed
|
|
81
|
+
* 681 → 1881 → 4297 → 5957 ms, and at 15:26:05 the probes read `gap 12 of 11`
|
|
82
|
+
* and printed `association-stopped` on a connection that was `flowing` again
|
|
83
|
+
* seven seconds later. Adding the peer's own reported lag makes that allowance
|
|
84
|
+
* 15 instead of 11, and the gap of 12 is then what it was — a browser whose
|
|
85
|
+
* timers are frozen, not an association that stopped.
|
|
86
|
+
*
|
|
87
|
+
* @param {{ queuedBytes: number, bytesPerSecond: number, rttMs: number, echoIntervalMs?: number, peerLoopLagMs?: number, intervalMs?: number }} state
|
|
88
|
+
* @returns {number | null} Probes that may legitimately be outstanding, or null
|
|
89
|
+
* when no rate has been measured yet and nothing can be said.
|
|
90
|
+
*/
|
|
91
|
+
export function allowedGap({
|
|
92
|
+
queuedBytes,
|
|
93
|
+
bytesPerSecond,
|
|
94
|
+
rttMs,
|
|
95
|
+
echoIntervalMs = 0,
|
|
96
|
+
peerLoopLagMs = 0,
|
|
97
|
+
intervalMs = PROBE_INTERVAL_MS
|
|
98
|
+
}) {
|
|
99
|
+
if (!(bytesPerSecond > 0) || !(intervalMs > 0)) {
|
|
100
|
+
return null;
|
|
101
|
+
}
|
|
102
|
+
// At least one: a probe sent and not yet echoed is the ordinary state.
|
|
103
|
+
return Math.max(1, Math.ceil(allowedWaitMs({ queuedBytes, bytesPerSecond, rttMs, echoIntervalMs, peerLoopLagMs }) / intervalMs));
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* How long a probe may legitimately take to be reported back, in milliseconds.
|
|
108
|
+
*
|
|
109
|
+
* THE QUANTITY {@link allowedGap} COMPUTES AND THEN THROWS AWAY by dividing it
|
|
110
|
+
* into probes. Probes are the wrong unit and always were: the same probe goes
|
|
111
|
+
* down every channel INCLUDING the one carrying the film, and SCTP schedules
|
|
112
|
+
* per association, so a probe waits behind queued video exactly as a segment
|
|
113
|
+
* does. What is behind is then not the peer's answer but the probe itself —
|
|
114
|
+
* and the honest measure of that is the age of the newest probe the peer has
|
|
115
|
+
* seen, which this is compared against.
|
|
116
|
+
*
|
|
117
|
+
* Four terms, every one measured: the queue's own drain time at the rate this
|
|
118
|
+
* connection is achieving, the crossing, the peer's reporting cadence, and how
|
|
119
|
+
* late the peer's event loop is running.
|
|
120
|
+
*
|
|
121
|
+
* @param {{ queuedBytes: number, bytesPerSecond: number, rttMs: number,
|
|
122
|
+
* echoIntervalMs?: number, peerLoopLagMs?: number }} state
|
|
123
|
+
* @returns {number}
|
|
124
|
+
*/
|
|
125
|
+
export function allowedWaitMs({ queuedBytes, bytesPerSecond, rttMs, echoIntervalMs = 0, peerLoopLagMs = 0 }) {
|
|
126
|
+
if (!(bytesPerSecond > 0)) {
|
|
127
|
+
return 0;
|
|
128
|
+
}
|
|
129
|
+
const drainMs = (Math.max(queuedBytes, 0) / bytesPerSecond) * 1000;
|
|
130
|
+
return drainMs + Math.max(rttMs, 0) + Math.max(echoIntervalMs, 0) + Math.max(peerLoopLagMs, 0);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
/**
|
|
134
|
+
* How far back the send times of probes are kept.
|
|
135
|
+
*
|
|
136
|
+
* A probe older than the point at which the reverse direction is called gone
|
|
137
|
+
* can say nothing further, and that bound is itself derived per connection —
|
|
138
|
+
* this is the outer edge of it, kept so the map cannot grow with the session.
|
|
139
|
+
*/
|
|
140
|
+
const PROBE_HISTORY_MS = 10 * 60 * 1000;
|
|
141
|
+
|
|
142
|
+
/** The label the browser gives the unordered, non-retransmitting channel. */
|
|
143
|
+
export const UNRELIABLE_LABEL = "proxy-fast";
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* How old an echo may be before the reverse direction counts as gone, when
|
|
147
|
+
* nothing better can be derived.
|
|
148
|
+
*
|
|
149
|
+
* Used only while no rate has been measured. Otherwise the caller passes
|
|
150
|
+
* `echoStaleMs`, worked out the same way as {@link allowedGap}: the browser
|
|
151
|
+
* only echoes a probe it has RECEIVED, so an echo waits behind our own queue
|
|
152
|
+
* exactly as the probe did.
|
|
153
|
+
*/
|
|
154
|
+
const ECHO_STALE_FALLBACK_MS = 5_000;
|
|
155
|
+
|
|
156
|
+
/** How often the probe state is written to the log while nothing changes. */
|
|
157
|
+
const REPORT_INTERVAL_MS = 5_000;
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Whether the `seen` counter has stopped advancing for longer than this
|
|
161
|
+
* connection's own history says a healthy gap between two advances ever
|
|
162
|
+
* takes.
|
|
163
|
+
*
|
|
164
|
+
* The `association-stopped` verdict alone is not enough to act on: a
|
|
165
|
+
* connection can sit BEHIND by a bounded, roughly constant amount for
|
|
166
|
+
* minutes (measured 2026-08-28, session on a backgrounded tab — gap held at
|
|
167
|
+
* 6-7 probes for 95+ seconds while `seen` kept climbing right along with
|
|
168
|
+
* `sent`) without anything being wrong. What a true wedge shows instead,
|
|
169
|
+
* measured the same day against a session already known to be one
|
|
170
|
+
* (`d85ae4f5`): `seen` FROZEN at one value for over a minute while `sent`
|
|
171
|
+
* climbs unbounded. So the question is not "is there a gap" but "has the
|
|
172
|
+
* highest-seen number stopped moving at all, for longer than it has ever
|
|
173
|
+
* legitimately taken this connection to report an advance" — the same shape
|
|
174
|
+
* as {@link wedgeIsCertain} in `transport/data-channel-handler.js`, applied to the
|
|
175
|
+
* probe's own counter instead of the transport's byte counter.
|
|
176
|
+
*
|
|
177
|
+
* THE FLOOR IS NOT ONE PROBE INTERVAL. "The longest gap this connection has
|
|
178
|
+
* ever shown" is nothing at all in its first minutes, and the floor under it
|
|
179
|
+
* was a single interval — 500 ms — so any two-second quiet stretch while the
|
|
180
|
+
* browser filled its cushion read as longer than anything healthy ever seen.
|
|
181
|
+
* Field 2026-09-11: two captures of 180 s each, triggered at `wedged 1s` and
|
|
182
|
+
* `wedged 2s`, on a connection with `rtt=5ms`, every queue at 0 B and the
|
|
183
|
+
* viewer watching; 65 MB of them were left on the addon's disk.
|
|
184
|
+
*
|
|
185
|
+
* What replaces it is the connection's own answer to how long a legitimate
|
|
186
|
+
* report may take: the peer reports twice a second, the report has to cross,
|
|
187
|
+
* and the peer's own event loop may be late — all three measured, all three
|
|
188
|
+
* already summed for `echoStaleMs`. A stretch shorter than that proves nothing
|
|
189
|
+
* whatever about the association.
|
|
190
|
+
*
|
|
191
|
+
* @param {{ stuckForMs: number, longestHealthySeenGapMs: number, intervalMs?: number,
|
|
192
|
+
* legitimateReportMs?: number }} state
|
|
193
|
+
* @returns {{ isCertain: boolean, needMs: number }}
|
|
194
|
+
*/
|
|
195
|
+
export function probeWedgeIsCertain({
|
|
196
|
+
stuckForMs,
|
|
197
|
+
longestHealthySeenGapMs,
|
|
198
|
+
intervalMs = PROBE_INTERVAL_MS,
|
|
199
|
+
legitimateReportMs = 0
|
|
200
|
+
}) {
|
|
201
|
+
const floorMs = Math.max(intervalMs, Number.isFinite(legitimateReportMs) ? legitimateReportMs : 0);
|
|
202
|
+
const needMs = Math.max(longestHealthySeenGapMs, floorMs);
|
|
203
|
+
return { isCertain: stuckForMs >= needMs, needMs };
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* One connection's probe state.
|
|
208
|
+
*
|
|
209
|
+
* @typedef {Object} ProbeConnection
|
|
210
|
+
* @property {string} tag
|
|
211
|
+
* @property {Map<import('node-datachannel').DataChannel, string>} channels
|
|
212
|
+
* @property {number} seq - Highest probe number sent.
|
|
213
|
+
* @property {number} sentAt - When that probe was sent.
|
|
214
|
+
* @property {Map<string, number>} seen - Label to the highest number the browser reported.
|
|
215
|
+
* @property {number} echoAt - When the last echo arrived (0 = never).
|
|
216
|
+
* @property {number} echoes - How many echoes have arrived.
|
|
217
|
+
* @property {string} verdict - Last verdict reported, so a change is logged at once.
|
|
218
|
+
* @property {number} reportedAt - When the state was last written to the log.
|
|
219
|
+
* @property {number} lastSeenAdvanceAt - When any label's `seen` value last increased (0 = never yet).
|
|
220
|
+
* @property {number} longestHealthySeenGapMs - The longest gap between two advances this connection has shown while not flagged as a wedge.
|
|
221
|
+
* @property {number | null} peerBytes - The far end's transport-level received total, as last reported.
|
|
222
|
+
* @property {number | null} peerBytesAtTick - The same, as it stood at the previous tick.
|
|
223
|
+
* @property {number | null} peerLoopLagMs - The far end's own event-loop delay, as last reported.
|
|
224
|
+
* @property {string | null} peerVisibility - Whether the far end's tab is visible or hidden, as last reported.
|
|
225
|
+
* @property {boolean} probeCaptureStarted - One evidence-gathering attempt per wedge; reset once `seen` advances again.
|
|
226
|
+
* @property {ReturnType<typeof setInterval> | null} timer
|
|
227
|
+
*/
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Read the gaps and say what they mean.
|
|
231
|
+
*
|
|
232
|
+
* Exported so the rule is testable without a connection: the same numbers
|
|
233
|
+
* always produce the same word.
|
|
234
|
+
*
|
|
235
|
+
* `allowed` carries, per channel label, how many probes may legitimately be
|
|
236
|
+
* outstanding right now — {@link allowedGap} computes it from that channel's
|
|
237
|
+
* own queue and the connection's measured rate. A label with no entry, or an
|
|
238
|
+
* entry of null, cannot be judged: with no rate measured there is nothing to
|
|
239
|
+
* divide the queue by, and the verdict says that rather than inventing one.
|
|
240
|
+
*
|
|
241
|
+
* `peerBytesAdvancing` is the fact that outranks every gap here. The gaps are
|
|
242
|
+
* counted against an allowance whose only load-dependent term is OUR OWN
|
|
243
|
+
* queue — and that queue is empty by construction, because it drains the moment
|
|
244
|
+
* libdatachannel accepts the bytes, whether or not usrsctp then puts them on
|
|
245
|
+
* the wire. So a browser filling its cushion as fast as the link allows shows
|
|
246
|
+
* an empty queue, a small allowance and a large gap, which reads exactly like a
|
|
247
|
+
* stopped association. Measured 2026-08-28: four `association-stopped` in the
|
|
248
|
+
* first two minutes of a healthy session, on a connection whose every queue was
|
|
249
|
+
* at 0 B and whose viewer never saw the picture stop. Deepening the browser's
|
|
250
|
+
* cushion from 30 s to 120 s (roadmap item 4) made the burst four times longer
|
|
251
|
+
* and the false verdict correspondingly likelier.
|
|
252
|
+
*
|
|
253
|
+
* What separates the two is not the size of the backlog — it is large in both —
|
|
254
|
+
* but whether bytes are still arriving. The browser reports its own
|
|
255
|
+
* transport-level received total with every echo, so the question is answered
|
|
256
|
+
* by a counter rather than by a threshold: while that total is advancing, bytes
|
|
257
|
+
* ARE crossing and no verdict of a stopped association can stand, however far
|
|
258
|
+
* behind the probes are. `null` where the far end does not report it, and then
|
|
259
|
+
* the rule falls back to what it says without the term.
|
|
260
|
+
*
|
|
261
|
+
* `peerLoopLagMs` and `peerVisibility` are printed but never compared against
|
|
262
|
+
* anything here: they are what made the allowance as wide as it is, and a
|
|
263
|
+
* reader of the log cannot check that arithmetic unless the terms are on the
|
|
264
|
+
* same line as the result.
|
|
265
|
+
*
|
|
266
|
+
* @param {{ seq: number, seen: Map<string, number> | Record<string, number>, labels: string[], echoes: number, echoAgeMs: number | null, allowed?: Map<string, number | null> | Record<string, number | null>, echoStaleMs?: number, peerBytesAdvancing?: boolean | null, peerLoopLagMs?: number | null, peerVisibility?: string | null, peerPending?: number | null }} state
|
|
267
|
+
* @returns {{ verdict: string, detail: string }}
|
|
268
|
+
*/
|
|
269
|
+
export function readProbeState(state) {
|
|
270
|
+
const seenOf = (label) =>
|
|
271
|
+
state.seen instanceof Map ? state.seen.get(label) : state.seen?.[label];
|
|
272
|
+
/**
|
|
273
|
+
* @param {unknown} source
|
|
274
|
+
* @param {string} label
|
|
275
|
+
* @returns {number | null}
|
|
276
|
+
*/
|
|
277
|
+
const timeOf = (source, label) => {
|
|
278
|
+
const value = source instanceof Map ? source.get(label) : /** @type {any} */ (source)?.[label];
|
|
279
|
+
return Number.isFinite(value) ? Number(value) : null;
|
|
280
|
+
};
|
|
281
|
+
const allowedOf = (label) => {
|
|
282
|
+
const source = state.allowed;
|
|
283
|
+
const value = source instanceof Map ? source.get(label) : source?.[label];
|
|
284
|
+
return Number.isInteger(value) ? Number(value) : null;
|
|
285
|
+
};
|
|
286
|
+
const parts = [];
|
|
287
|
+
let isOrderedBehind = false;
|
|
288
|
+
let isUnreliableBehind = false;
|
|
289
|
+
let isUnreliableKnown = false;
|
|
290
|
+
let isJudgeable = false;
|
|
291
|
+
for (const label of state.labels) {
|
|
292
|
+
const seen = seenOf(label);
|
|
293
|
+
const gap = Number.isInteger(seen) ? state.seq - Number(seen) : null;
|
|
294
|
+
const allowance = allowedOf(label);
|
|
295
|
+
// BY TIME WHERE IT IS KNOWN. The count is what the line prints, because it
|
|
296
|
+
// is what a reader recognises; what decides is a time.
|
|
297
|
+
//
|
|
298
|
+
// The measured one-way delay first: it is the forward direction itself,
|
|
299
|
+
// with the two clocks reconciled from the exchange. The age of the newest
|
|
300
|
+
// reported probe is the fallback, and it is larger than the thing itself by
|
|
301
|
+
// the peer's reporting cadence and the way back — which is why its own
|
|
302
|
+
// allowance carries both of those and the one-way allowance carries
|
|
303
|
+
// neither.
|
|
304
|
+
const oneWay = timeOf(state.oneWayMs, label);
|
|
305
|
+
const oneWayAllowed = timeOf(state.allowedOneWayMs, label);
|
|
306
|
+
const lagMs = oneWay !== null && oneWayAllowed !== null ? oneWay : timeOf(state.behindMs, label);
|
|
307
|
+
const mayWaitMs = oneWay !== null && oneWayAllowed !== null
|
|
308
|
+
? oneWayAllowed
|
|
309
|
+
: timeOf(state.allowedWaitMs, label);
|
|
310
|
+
const lagText = lagMs === null || mayWaitMs === null
|
|
311
|
+
? ""
|
|
312
|
+
: ` ${Math.round(lagMs)}ms of ${Math.round(mayWaitMs)}ms${oneWay !== null ? " one way" : ""}`;
|
|
313
|
+
parts.push(`${label}=${seen ?? "?"}(gap ${gap ?? "?"} of ${allowance ?? "?"}${lagText})`);
|
|
314
|
+
if (allowance === null) {
|
|
315
|
+
continue;
|
|
316
|
+
}
|
|
317
|
+
isJudgeable = true;
|
|
318
|
+
const isBehind = lagMs !== null && mayWaitMs !== null
|
|
319
|
+
? lagMs > mayWaitMs
|
|
320
|
+
: gap === null || gap > allowance;
|
|
321
|
+
if (label === UNRELIABLE_LABEL) {
|
|
322
|
+
isUnreliableKnown = true;
|
|
323
|
+
isUnreliableBehind = isBehind;
|
|
324
|
+
} else if (isBehind) {
|
|
325
|
+
isOrderedBehind = true;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
const isPeerAdvancing = state.peerBytesAdvancing === true;
|
|
329
|
+
const detail =
|
|
330
|
+
`sent=${state.seq} ${parts.join(" ")} ` +
|
|
331
|
+
`echoAge=${state.echoAgeMs === null ? "never" : `${state.echoAgeMs}ms`}` +
|
|
332
|
+
(state.peerBytesAdvancing === null || state.peerBytesAdvancing === undefined
|
|
333
|
+
? ""
|
|
334
|
+
: ` peerBytes=${isPeerAdvancing ? "advancing" : "still"}`) +
|
|
335
|
+
(Number.isFinite(state.peerLoopLagMs) ? ` peerLoopLag=${Math.round(Number(state.peerLoopLagMs))}ms` : "") +
|
|
336
|
+
(state.peerVisibility ? ` peerTab=${state.peerVisibility}` : "") +
|
|
337
|
+
// WHAT THE PEER SAYS IT IS STILL WAITING FOR. The far end has sent this
|
|
338
|
+
// twice a second since the probe echo was built and nothing read it, while
|
|
339
|
+
// every verdict here was reasoned out from this side alone. It is evidence
|
|
340
|
+
// and not a verdict: the receiver declares its own transport lost, because
|
|
341
|
+
// it knows first — on 2026-09-06 the browser said so sixteen minutes before
|
|
342
|
+
// this end did.
|
|
343
|
+
(Number.isFinite(state.peerPending) ? ` peerWaiting=${Number(state.peerPending)}` : "");
|
|
344
|
+
|
|
345
|
+
if (state.echoes === 0) {
|
|
346
|
+
return { verdict: "no-echo-yet", detail };
|
|
347
|
+
}
|
|
348
|
+
if (!isJudgeable) {
|
|
349
|
+
return { verdict: "no-rate-yet", detail };
|
|
350
|
+
}
|
|
351
|
+
const staleAfterMs = Number.isFinite(state.echoStaleMs) && state.echoStaleMs > 0
|
|
352
|
+
? state.echoStaleMs
|
|
353
|
+
: ECHO_STALE_FALLBACK_MS;
|
|
354
|
+
if (state.echoAgeMs !== null && state.echoAgeMs > staleAfterMs) {
|
|
355
|
+
return { verdict: "reverse-direction-gone", detail };
|
|
356
|
+
}
|
|
357
|
+
if (!isOrderedBehind && !(isUnreliableKnown && isUnreliableBehind)) {
|
|
358
|
+
return { verdict: "flowing", detail };
|
|
359
|
+
}
|
|
360
|
+
// Bytes are still arriving at the far end. Whatever the probe gaps say, this
|
|
361
|
+
// association has not stopped — the probes are behind a backlog, which is
|
|
362
|
+
// what filling a cushion looks like from here.
|
|
363
|
+
if (isPeerAdvancing) {
|
|
364
|
+
return { verdict: "flowing", detail };
|
|
365
|
+
}
|
|
366
|
+
if (isOrderedBehind && isUnreliableKnown && !isUnreliableBehind) {
|
|
367
|
+
return { verdict: "stream-stuck", detail };
|
|
368
|
+
}
|
|
369
|
+
if (isOrderedBehind) {
|
|
370
|
+
return {
|
|
371
|
+
verdict: isUnreliableKnown ? "association-stopped" : "ordered-behind-no-comparison",
|
|
372
|
+
detail
|
|
373
|
+
};
|
|
374
|
+
}
|
|
375
|
+
return { verdict: "unreliable-behind-only", detail };
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
/**
|
|
379
|
+
* Create the probe service. One instance serves every session.
|
|
380
|
+
*
|
|
381
|
+
* @param {Object} options
|
|
382
|
+
* @param {(message: string) => void} options.log
|
|
383
|
+
* @param {number} [options.intervalMs]
|
|
384
|
+
* @param {(sessionId: string) => object | null} [options.getTransportSnapshot]
|
|
385
|
+
* Needed only to hand the witness a remote endpoint when this probe is the
|
|
386
|
+
* one declaring a wedge.
|
|
387
|
+
* @param {{ maybeCapture: (trigger: object) => boolean }} [options.witness]
|
|
388
|
+
* @param {{ maybeRead: (reasonText: string) => boolean }} [options.usrsctpState]
|
|
389
|
+
* @returns {{
|
|
390
|
+
* attach: (sessionId: string, tag: string, label: string, channel: import('node-datachannel').DataChannel) => void,
|
|
391
|
+
* detach: (sessionId: string, channel: import('node-datachannel').DataChannel) => void,
|
|
392
|
+
* noteEcho: (sessionId: string, echo: object) => void,
|
|
393
|
+
* dispose: () => void
|
|
394
|
+
* }}
|
|
395
|
+
*/
|
|
396
|
+
export function createDeliveryProbe({
|
|
397
|
+
log,
|
|
398
|
+
intervalMs = PROBE_INTERVAL_MS,
|
|
399
|
+
readDelivery,
|
|
400
|
+
getTransportSnapshot,
|
|
401
|
+
witness,
|
|
402
|
+
usrsctpState
|
|
403
|
+
}) {
|
|
404
|
+
/** @type {Map<string, ProbeConnection>} */
|
|
405
|
+
const connections = new Map();
|
|
406
|
+
|
|
407
|
+
/**
|
|
408
|
+
* Send this tick's probe on every channel of one connection, then report.
|
|
409
|
+
*
|
|
410
|
+
* @param {ProbeConnection} connection
|
|
411
|
+
* @returns {void}
|
|
412
|
+
*/
|
|
413
|
+
function tick(connection) {
|
|
414
|
+
const now = Date.now();
|
|
415
|
+
connection.seq += 1;
|
|
416
|
+
connection.sentAt = now;
|
|
417
|
+
// WHEN each probe went out, so that what the peer reports can be read as a
|
|
418
|
+
// time rather than as a count of probes. Pruned to the oldest probe any
|
|
419
|
+
// judgement could still be about: once a probe is older than the point at
|
|
420
|
+
// which the reverse direction is called gone, its age says nothing further.
|
|
421
|
+
connection.sentAtBySeq.set(connection.seq, now);
|
|
422
|
+
for (const [probeNumber, sentAt] of connection.sentAtBySeq) {
|
|
423
|
+
if (now - sentAt > PROBE_HISTORY_MS) {
|
|
424
|
+
connection.sentAtBySeq.delete(probeNumber);
|
|
425
|
+
} else {
|
|
426
|
+
break;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
const message = JSON.stringify({ type: "probe", seq: connection.seq, sentAt: now });
|
|
430
|
+
for (const channel of connection.channels.keys()) {
|
|
431
|
+
try {
|
|
432
|
+
channel.sendMessage(message);
|
|
433
|
+
} catch {
|
|
434
|
+
// A channel closing between the check and the send is ordinary.
|
|
435
|
+
}
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// What this connection is getting away, and how far behind it therefore
|
|
439
|
+
// sits. Both come from the send-queue watcher, which measures them anyway.
|
|
440
|
+
const delivery = typeof readDelivery === "function" ? readDelivery(connection.id) : null;
|
|
441
|
+
const bytesPerSecond = Number(delivery?.bytesPerSecond) || 0;
|
|
442
|
+
const rttMs = Number(delivery?.rttMs) || 0;
|
|
443
|
+
/** @type {Map<string, number | null>} */
|
|
444
|
+
const allowed = new Map();
|
|
445
|
+
/** How long each channel's newest unreported probe may legitimately be. */
|
|
446
|
+
const allowedWait = new Map();
|
|
447
|
+
/** The same, for the measured one-way time where the clocks are reconciled. */
|
|
448
|
+
const allowedOneWay = new Map();
|
|
449
|
+
for (const [channel, label] of connection.channels) {
|
|
450
|
+
let queuedBytes = 0;
|
|
451
|
+
try {
|
|
452
|
+
queuedBytes = typeof channel.bufferedAmount === "function" ? channel.bufferedAmount() : 0;
|
|
453
|
+
} catch {
|
|
454
|
+
queuedBytes = 0;
|
|
455
|
+
}
|
|
456
|
+
const allowance = allowedGap({
|
|
457
|
+
queuedBytes,
|
|
458
|
+
bytesPerSecond,
|
|
459
|
+
rttMs,
|
|
460
|
+
echoIntervalMs: connection.echoIntervalMs,
|
|
461
|
+
peerLoopLagMs: connection.peerLoopLagMs ?? 0,
|
|
462
|
+
intervalMs
|
|
463
|
+
});
|
|
464
|
+
const waitMs = allowedWaitMs({
|
|
465
|
+
queuedBytes,
|
|
466
|
+
bytesPerSecond,
|
|
467
|
+
rttMs,
|
|
468
|
+
echoIntervalMs: connection.echoIntervalMs,
|
|
469
|
+
peerLoopLagMs: connection.peerLoopLagMs ?? 0
|
|
470
|
+
});
|
|
471
|
+
const heldWait = allowedWait.get(label);
|
|
472
|
+
if (!Number.isFinite(heldWait) || waitMs > heldWait) {
|
|
473
|
+
allowedWait.set(label, waitMs);
|
|
474
|
+
}
|
|
475
|
+
// What a probe may take ONE WAY: the queue's own drain time and half the
|
|
476
|
+
// crossing. Neither the peer's reporting cadence nor its loop delay
|
|
477
|
+
// belongs here — with the clocks reconciled they are no longer in the
|
|
478
|
+
// measurement they would have to be allowed for.
|
|
479
|
+
const oneWayAllowed = allowedWaitMs({ queuedBytes, bytesPerSecond, rttMs: rttMs / 2 });
|
|
480
|
+
const heldOneWay = allowedOneWay.get(label);
|
|
481
|
+
if (!Number.isFinite(heldOneWay) || oneWayAllowed > heldOneWay) {
|
|
482
|
+
allowedOneWay.set(label, oneWayAllowed);
|
|
483
|
+
}
|
|
484
|
+
// Several channels can carry one label only in malformed cases; the
|
|
485
|
+
// larger allowance is the safer of the two.
|
|
486
|
+
const held = allowed.get(label);
|
|
487
|
+
if (allowance !== null && (!Number.isInteger(held) || allowance > held)) {
|
|
488
|
+
allowed.set(label, allowance);
|
|
489
|
+
} else if (!allowed.has(label)) {
|
|
490
|
+
allowed.set(label, allowance);
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
const widest = [...allowed.values()].reduce(
|
|
494
|
+
(most, value) => (Number.isInteger(value) && value > most ? value : most),
|
|
495
|
+
0
|
|
496
|
+
);
|
|
497
|
+
// Advancing SINCE THE PREVIOUS TICK, not since the connection began: the
|
|
498
|
+
// question is whether bytes are crossing now. Prefer channel bytes —
|
|
499
|
+
// transport's bytesReceived advances on SACKs (140 B/s on a wedge) and
|
|
500
|
+
// gives a false "advancing" every other tick, while channel bytes are flat.
|
|
501
|
+
let peerBytesAdvancing = null;
|
|
502
|
+
if (connection.peerChannelBytes !== null) {
|
|
503
|
+
peerBytesAdvancing =
|
|
504
|
+
connection.peerChannelBytesAtTick === null ||
|
|
505
|
+
connection.peerChannelBytes > connection.peerChannelBytesAtTick;
|
|
506
|
+
connection.peerChannelBytesAtTick = connection.peerChannelBytes;
|
|
507
|
+
// keep transport's tick in sync for fallback logging
|
|
508
|
+
connection.peerBytesAtTick = connection.peerBytes;
|
|
509
|
+
} else if (connection.peerBytes !== null) {
|
|
510
|
+
peerBytesAdvancing =
|
|
511
|
+
connection.peerBytesAtTick === null || connection.peerBytes > connection.peerBytesAtTick;
|
|
512
|
+
connection.peerBytesAtTick = connection.peerBytes;
|
|
513
|
+
}
|
|
514
|
+
const { verdict, detail } = readProbeState({
|
|
515
|
+
seq: connection.seq,
|
|
516
|
+
seen: connection.seen,
|
|
517
|
+
peerBytesAdvancing,
|
|
518
|
+
peerPending: connection.peerPending,
|
|
519
|
+
labels: [...new Set(connection.channels.values())],
|
|
520
|
+
echoes: connection.echoes,
|
|
521
|
+
echoAgeMs: connection.echoAt === 0 ? null : now - connection.echoAt,
|
|
522
|
+
peerLoopLagMs: connection.peerLoopLagMs,
|
|
523
|
+
peerVisibility: connection.peerVisibility,
|
|
524
|
+
allowed,
|
|
525
|
+
// WHAT IS BEHIND, IN TIME. For each channel: how old the newest probe the
|
|
526
|
+
// peer has reported seeing is. The same probe goes down every channel
|
|
527
|
+
// including the one carrying the film, and SCTP schedules per
|
|
528
|
+
// association, so a probe waits behind queued video exactly as a segment
|
|
529
|
+
// does — which means a count of outstanding probes measures the queue,
|
|
530
|
+
// not the association. This is the queue's own time, against the time the
|
|
531
|
+
// queue is allowed to take.
|
|
532
|
+
behindMs: Object.fromEntries(
|
|
533
|
+
[...connection.seen.entries()].map(([label, newestSeenNumber]) => {
|
|
534
|
+
const sentAt = connection.sentAtBySeq.get(newestSeenNumber);
|
|
535
|
+
return [label, Number.isFinite(sentAt) ? now - sentAt : null];
|
|
536
|
+
})
|
|
537
|
+
),
|
|
538
|
+
allowedWaitMs: Object.fromEntries(allowedWait.entries()),
|
|
539
|
+
// The measured forward delay, where the two clocks have been reconciled.
|
|
540
|
+
// Preferred over the age above because it is the thing itself: the age
|
|
541
|
+
// also carries the peer's reporting cadence and the way back.
|
|
542
|
+
oneWayMs: Object.fromEntries(connection.oneWayMs.entries()),
|
|
543
|
+
allowedOneWayMs: Object.fromEntries(allowedOneWay.entries()),
|
|
544
|
+
// Same arithmetic for the echo's own age: the peer cannot answer sooner
|
|
545
|
+
// than its own cadence allows, nor sooner than its own event loop runs.
|
|
546
|
+
echoStaleMs:
|
|
547
|
+
widest > 0
|
|
548
|
+
? widest * intervalMs +
|
|
549
|
+
rttMs +
|
|
550
|
+
connection.echoIntervalMs +
|
|
551
|
+
Math.max(connection.peerLoopLagMs ?? 0, 0)
|
|
552
|
+
: 0
|
|
553
|
+
});
|
|
554
|
+
if (verdict !== connection.verdict || now - connection.reportedAt >= REPORT_INTERVAL_MS) {
|
|
555
|
+
connection.verdict = verdict;
|
|
556
|
+
connection.reportedAt = now;
|
|
557
|
+
log(`[dc-probe] ${connection.tag} ${verdict} — ${detail} at=${new Date(now).toISOString()}`);
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
// `association-stopped` alone is not certainty — see probeWedgeIsCertain.
|
|
561
|
+
// A connection that is merely lagging by a bounded amount reaches this
|
|
562
|
+
// verdict too (a hidden tab's own echo cadence, measured 2026-08-28), and
|
|
563
|
+
// `seen` keeps advancing right along with it. Only a `seen` value that has
|
|
564
|
+
// stopped moving ENTIRELY, for longer than this connection has ever shown
|
|
565
|
+
// as a legitimate gap, is the wedge this exists to catch.
|
|
566
|
+
const stuckForMs = connection.lastSeenAdvanceAt === 0 ? 0 : now - connection.lastSeenAdvanceAt;
|
|
567
|
+
if (verdict === "association-stopped") {
|
|
568
|
+
const { isCertain, needMs } = probeWedgeIsCertain({
|
|
569
|
+
stuckForMs,
|
|
570
|
+
longestHealthySeenGapMs: connection.longestHealthySeenGapMs,
|
|
571
|
+
// The same sum the echo's own staleness is judged by: the peer reports
|
|
572
|
+
// twice a second, the report has to cross, and the peer's loop may be
|
|
573
|
+
// late. Nothing shorter than that says anything.
|
|
574
|
+
// BOTH cadences, because they beat against each other: a probe sent
|
|
575
|
+
// just after the peer composed a report shows up only in the NEXT one.
|
|
576
|
+
// So the longest legitimate gap between two advances is one probe
|
|
577
|
+
// interval plus one report interval plus the crossing plus whatever
|
|
578
|
+
// the peer's own loop is behind by.
|
|
579
|
+
legitimateReportMs:
|
|
580
|
+
intervalMs +
|
|
581
|
+
connection.echoIntervalMs +
|
|
582
|
+
rttMs +
|
|
583
|
+
Math.max(connection.peerLoopLagMs ?? 0, 0)
|
|
584
|
+
});
|
|
585
|
+
if (isCertain && !connection.probeCaptureStarted) {
|
|
586
|
+
connection.probeCaptureStarted = true;
|
|
587
|
+
const reasonText =
|
|
588
|
+
`probe seen-counter unmoved ${Math.round(stuckForMs / 1000)}s against the ` +
|
|
589
|
+
`${(needMs / 1000).toFixed(1)}s this connection's own history says is legitimate`;
|
|
590
|
+
if (witness) {
|
|
591
|
+
const snapshot = getTransportSnapshot?.(connection.id) ?? null;
|
|
592
|
+
const started = witness.maybeCapture({
|
|
593
|
+
sessionId: connection.id,
|
|
594
|
+
tag: connection.tag,
|
|
595
|
+
label: "probe",
|
|
596
|
+
remote: snapshot?.remote ?? null,
|
|
597
|
+
queuedBytes: 0,
|
|
598
|
+
stuckForMs
|
|
599
|
+
});
|
|
600
|
+
if (!started) {
|
|
601
|
+
connection.probeCaptureStarted = false;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
if (usrsctpState) {
|
|
605
|
+
usrsctpState.maybeRead(reasonText);
|
|
606
|
+
}
|
|
607
|
+
}
|
|
608
|
+
} else {
|
|
609
|
+
// Not association-stopped any more: whatever was flagged has cleared,
|
|
610
|
+
// and a later wedge on the same connection deserves its own attempt.
|
|
611
|
+
connection.probeCaptureStarted = false;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
return {
|
|
616
|
+
attach(sessionId, tag, label, channel) {
|
|
617
|
+
let connection = connections.get(sessionId);
|
|
618
|
+
if (!connection) {
|
|
619
|
+
connection = {
|
|
620
|
+
id: sessionId,
|
|
621
|
+
tag,
|
|
622
|
+
// The longest this peer has ever taken between two echoes. Its own
|
|
623
|
+
// schedule, measured rather than assumed - a hidden tab answers
|
|
624
|
+
// about once a second because the browser throttles the timer.
|
|
625
|
+
echoIntervalMs: 0,
|
|
626
|
+
channels: new Map(),
|
|
627
|
+
seq: 0,
|
|
628
|
+
sentAt: 0,
|
|
629
|
+
seen: new Map(),
|
|
630
|
+
echoAt: 0,
|
|
631
|
+
echoes: 0,
|
|
632
|
+
// The far end's own transport-level received total, and its value at
|
|
633
|
+
// the previous tick. Null until a browser that reports it has echoed.
|
|
634
|
+
/** @type {number | null} */
|
|
635
|
+
peerBytes: null,
|
|
636
|
+
// Requests the far end says it is still waiting for, as last reported.
|
|
637
|
+
peerPending: null,
|
|
638
|
+
/** @type {number | null} */
|
|
639
|
+
peerBytesAtTick: null,
|
|
640
|
+
// Sum of per-channel bytes (report.channels[].bytes). Transport's
|
|
641
|
+
// bytesReceived advances on SACKs (140 B/s on a wedge) and gives a
|
|
642
|
+
// false "advancing" every other tick — channel bytes are flat on a wedge.
|
|
643
|
+
/** @type {number | null} */
|
|
644
|
+
peerChannelBytes: null,
|
|
645
|
+
/** @type {number | null} */
|
|
646
|
+
peerChannelBytesAtTick: null,
|
|
647
|
+
// The far end's own event-loop delay and tab state, as last reported.
|
|
648
|
+
// The lag is a term in every allowance below; the tab state is only
|
|
649
|
+
// printed, so that a wide allowance can be read back to its cause.
|
|
650
|
+
/** @type {number | null} */
|
|
651
|
+
peerLoopLagMs: null,
|
|
652
|
+
/** @type {string | null} */
|
|
653
|
+
peerVisibility: null,
|
|
654
|
+
verdict: "",
|
|
655
|
+
reportedAt: 0,
|
|
656
|
+
lastSeenAdvanceAt: 0,
|
|
657
|
+
longestHealthySeenGapMs: 0,
|
|
658
|
+
sentAtBySeq: new Map(),
|
|
659
|
+
// THE TWO CLOCKS, separated by the exchange rather than assumed to
|
|
660
|
+
// agree. Every report carries when each channel's newest probe
|
|
661
|
+
// arrived at the peer and when the report itself left; with the time
|
|
662
|
+
// this proxy stamped into that probe, and the time the report
|
|
663
|
+
// arrives, that is the four timestamps an offset is computed from.
|
|
664
|
+
//
|
|
665
|
+
// The estimate is kept from the report with the SMALLEST round trip
|
|
666
|
+
// seen, because that is the one that queued the least — and the two
|
|
667
|
+
// directions here are as unequal as they get, film one way and almost
|
|
668
|
+
// nothing the other, which is exactly where an offset taken from a
|
|
669
|
+
// busy moment is wrong by half the difference.
|
|
670
|
+
/** @type {number | null} */
|
|
671
|
+
clockOffsetMs: null,
|
|
672
|
+
/** @type {number} */
|
|
673
|
+
offsetFromRoundTripMs: Number.POSITIVE_INFINITY,
|
|
674
|
+
/** How long the newest probe took to reach the peer, by channel. @type {Map<string, number>} */
|
|
675
|
+
oneWayMs: new Map(),
|
|
676
|
+
probeCaptureStarted: false,
|
|
677
|
+
timer: null
|
|
678
|
+
};
|
|
679
|
+
connections.set(sessionId, connection);
|
|
680
|
+
}
|
|
681
|
+
connection.channels.set(channel, label);
|
|
682
|
+
if (connection.timer === null) {
|
|
683
|
+
const held = connection;
|
|
684
|
+
connection.timer = setInterval(() => tick(held), intervalMs);
|
|
685
|
+
// The probe must never be the reason a process stays alive.
|
|
686
|
+
if (typeof connection.timer.unref === "function") {
|
|
687
|
+
connection.timer.unref();
|
|
688
|
+
}
|
|
689
|
+
}
|
|
690
|
+
},
|
|
691
|
+
|
|
692
|
+
detach(sessionId, channel) {
|
|
693
|
+
const connection = connections.get(sessionId);
|
|
694
|
+
if (!connection) {
|
|
695
|
+
return;
|
|
696
|
+
}
|
|
697
|
+
connection.channels.delete(channel);
|
|
698
|
+
if (connection.channels.size === 0) {
|
|
699
|
+
if (connection.timer !== null) {
|
|
700
|
+
clearInterval(connection.timer);
|
|
701
|
+
connection.timer = null;
|
|
702
|
+
}
|
|
703
|
+
if (connections.get(sessionId) === connection) {
|
|
704
|
+
connections.delete(sessionId);
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
},
|
|
708
|
+
|
|
709
|
+
noteEcho(sessionId, echo) {
|
|
710
|
+
const connection = connections.get(sessionId);
|
|
711
|
+
if (!connection || !echo || typeof echo !== "object") {
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
const now = Date.now();
|
|
715
|
+
const seen = echo.seen;
|
|
716
|
+
if (seen && typeof seen === "object") {
|
|
717
|
+
let advanced = false;
|
|
718
|
+
for (const [label, value] of Object.entries(seen)) {
|
|
719
|
+
if (!Number.isInteger(value)) {
|
|
720
|
+
continue;
|
|
721
|
+
}
|
|
722
|
+
const previous = connection.seen.get(label);
|
|
723
|
+
if (!Number.isInteger(previous) || value > previous) {
|
|
724
|
+
advanced = true;
|
|
725
|
+
}
|
|
726
|
+
connection.seen.set(label, value);
|
|
727
|
+
}
|
|
728
|
+
// What a wedge shows is this counter frozen, not merely behind — see
|
|
729
|
+
// probeWedgeIsCertain. The gap since the last time ANY label moved is
|
|
730
|
+
// this connection's own answer to "how long may a healthy report take
|
|
731
|
+
// to arrive", recorded only while nothing is currently flagged (the
|
|
732
|
+
// same guard `longestHealthyFlatMs` uses): a stretch already under
|
|
733
|
+
// suspicion must not teach the detector to tolerate it.
|
|
734
|
+
if (advanced) {
|
|
735
|
+
if (connection.lastSeenAdvanceAt !== 0 && !connection.probeCaptureStarted) {
|
|
736
|
+
const gap = now - connection.lastSeenAdvanceAt;
|
|
737
|
+
if (gap > connection.longestHealthySeenGapMs) {
|
|
738
|
+
connection.longestHealthySeenGapMs = gap;
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
connection.lastSeenAdvanceAt = now;
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
// THE FOUR TIMESTAMPS. `t1` is when this proxy sent the probe the peer
|
|
745
|
+
// is reporting, `t2` when the peer received it, `t3` when the peer sent
|
|
746
|
+
// this report, `t4` is now. From them the round trip is
|
|
747
|
+
// `(t4 - t1) - (t3 - t2)` — the peer's own thinking time removed — and
|
|
748
|
+
// the difference between the two clocks is `((t2 - t1) + (t3 - t4)) / 2`.
|
|
749
|
+
//
|
|
750
|
+
// The offset is kept from the report whose round trip was SMALLEST: that
|
|
751
|
+
// arithmetic assumes the two directions are equally quick, which here
|
|
752
|
+
// they are not — film one way, almost nothing the other — and the least
|
|
753
|
+
// queued sample is the one where the assumption costs least. A fresh
|
|
754
|
+
// minimum replaces the estimate, so a clock that jumps is followed within
|
|
755
|
+
// a few reports rather than believed for ever.
|
|
756
|
+
const peerSentAt = Number(echo?.sentAt);
|
|
757
|
+
const seenAt = echo?.seenAt;
|
|
758
|
+
if (Number.isFinite(peerSentAt) && seenAt && typeof seenAt === "object") {
|
|
759
|
+
for (const [label, peerSawAtRaw] of Object.entries(seenAt)) {
|
|
760
|
+
const peerSawAt = Number(peerSawAtRaw);
|
|
761
|
+
const newestSeenNumber = connection.seen.get(label);
|
|
762
|
+
const weSentAt = Number.isInteger(newestSeenNumber)
|
|
763
|
+
? connection.sentAtBySeq.get(newestSeenNumber)
|
|
764
|
+
: undefined;
|
|
765
|
+
if (!Number.isFinite(peerSawAt) || !Number.isFinite(weSentAt)) {
|
|
766
|
+
continue;
|
|
767
|
+
}
|
|
768
|
+
const roundTrip = (now - weSentAt) - (peerSentAt - peerSawAt);
|
|
769
|
+
if (roundTrip >= 0 && roundTrip < connection.offsetFromRoundTripMs) {
|
|
770
|
+
connection.offsetFromRoundTripMs = roundTrip;
|
|
771
|
+
connection.clockOffsetMs = ((peerSawAt - weSentAt) + (peerSentAt - now)) / 2;
|
|
772
|
+
}
|
|
773
|
+
if (connection.clockOffsetMs !== null) {
|
|
774
|
+
// What the probe itself took, one way, with the clocks reconciled.
|
|
775
|
+
// Negative only if the offset is stale, and then it says so rather
|
|
776
|
+
// than being hidden.
|
|
777
|
+
connection.oneWayMs.set(label, peerSawAt - weSentAt - connection.clockOffsetMs);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
// What the far end says it has received at the transport level. It is the
|
|
782
|
+
// one figure that separates a backlog from a stopped association, and it
|
|
783
|
+
// arrives on the direction that goes on working through a freeze.
|
|
784
|
+
// How many requests the far end says it is still waiting for. Theirs to
|
|
785
|
+
// count: this end knows what it has answered, not what reached them.
|
|
786
|
+
const peerPending = Number(echo?.report?.pending);
|
|
787
|
+
connection.peerPending = Number.isFinite(peerPending) && peerPending >= 0 ? peerPending : null;
|
|
788
|
+
const peerBytes = Number(echo?.report?.transportBytesReceived);
|
|
789
|
+
if (Number.isFinite(peerBytes) && peerBytes >= 0) {
|
|
790
|
+
connection.peerBytes = peerBytes;
|
|
791
|
+
}
|
|
792
|
+
// Sum of per-channel bytes. Transport's bytesReceived advances on SACKs
|
|
793
|
+
// (140 B/s on a wedge) and gives a false "advancing" every other tick —
|
|
794
|
+
// channel bytes are flat on a wedge, so they are the correct signal.
|
|
795
|
+
const peerChannels = echo?.report?.channels;
|
|
796
|
+
if (peerChannels && typeof peerChannels === "object") {
|
|
797
|
+
let bytesAcrossChannels = 0;
|
|
798
|
+
let hasChannelReport = false;
|
|
799
|
+
for (const channelReport of Object.values(peerChannels)) {
|
|
800
|
+
if (channelReport && Number.isFinite(channelReport.bytes)) {
|
|
801
|
+
bytesAcrossChannels += channelReport.bytes;
|
|
802
|
+
hasChannelReport = true;
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
if (hasChannelReport) {
|
|
806
|
+
connection.peerChannelBytes = bytesAcrossChannels;
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
// How far behind the far end's own event loop is running. A browser that
|
|
810
|
+
// cannot run its timers cannot answer a probe, and every allowance here
|
|
811
|
+
// is a wait the answer has to fit inside — so this is a term in the
|
|
812
|
+
// arithmetic, not a note. It is the peer's own measurement, taken on the
|
|
813
|
+
// peer, and it arrives on the direction that survives a freeze.
|
|
814
|
+
const peerLoopLag = Number(echo?.report?.loopLagMs);
|
|
815
|
+
if (Number.isFinite(peerLoopLag) && peerLoopLag >= 0) {
|
|
816
|
+
connection.peerLoopLagMs = peerLoopLag;
|
|
817
|
+
}
|
|
818
|
+
const peerVisibility = echo?.report?.visibility;
|
|
819
|
+
if (typeof peerVisibility === "string" && peerVisibility.length > 0) {
|
|
820
|
+
connection.peerVisibility = peerVisibility;
|
|
821
|
+
}
|
|
822
|
+
if (connection.echoAt !== 0) {
|
|
823
|
+
const sinceLast = now - connection.echoAt;
|
|
824
|
+
if (sinceLast > connection.echoIntervalMs) {
|
|
825
|
+
connection.echoIntervalMs = sinceLast;
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
connection.echoAt = now;
|
|
829
|
+
connection.echoes += 1;
|
|
830
|
+
},
|
|
831
|
+
|
|
832
|
+
dispose() {
|
|
833
|
+
for (const connection of connections.values()) {
|
|
834
|
+
if (connection.timer !== null) {
|
|
835
|
+
clearInterval(connection.timer);
|
|
836
|
+
connection.timer = null;
|
|
837
|
+
}
|
|
838
|
+
}
|
|
839
|
+
connections.clear();
|
|
840
|
+
}
|
|
841
|
+
};
|
|
842
|
+
}
|