@torrent-tv/proxy 2.89.7 → 2.89.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (498) hide show
  1. package/.claude/commands/opsx/apply.md +155 -155
  2. package/.claude/commands/opsx/archive.md +160 -160
  3. package/.claude/commands/opsx/explore.md +174 -174
  4. package/.claude/commands/opsx/propose.md +109 -109
  5. package/.claude/commands/opsx/sync.md +143 -143
  6. package/.claude/skills/openspec-apply-change/SKILL.md +159 -159
  7. package/.claude/skills/openspec-archive-change/SKILL.md +117 -117
  8. package/.claude/skills/openspec-explore/SKILL.md +289 -289
  9. package/.claude/skills/openspec-propose/SKILL.md +113 -113
  10. package/.claude/skills/openspec-sync-specs/SKILL.md +147 -147
  11. package/.dockerignore +5 -5
  12. package/AGENTS.md +363 -338
  13. package/CHANGELOG.md +2345 -2335
  14. package/Dockerfile +30 -30
  15. package/LICENSE +16 -16
  16. package/README.md +369 -369
  17. package/assets/calibration/NOTICE.md +100 -100
  18. package/assets/calibration/audio/NOTICE.md +19 -19
  19. package/assets/calibration/audio/manifest.json +98 -98
  20. package/assets/diagnostics/sctpstate.gdb +31 -31
  21. package/bin/cli.js +684 -684
  22. package/bin/thread-pool.js +30 -30
  23. package/biome.json +412 -412
  24. package/docs/browser-proxy-contract.md +155 -155
  25. package/docs/container-architecture.md +273 -273
  26. package/docs/download-architecture.md +261 -261
  27. package/docs/encode-architecture.md +765 -765
  28. package/docs/encode-run-state.md +96 -96
  29. package/docs/logs.md +97 -97
  30. package/docs/playback-start-forecast.md +113 -113
  31. package/docs/socks5-egress-research.md +114 -114
  32. package/docs/storage-architecture.md +273 -273
  33. package/knip.json +14 -14
  34. package/openspec/changes/adaptive-bitrate/design.md +92 -92
  35. package/openspec/changes/adaptive-bitrate/proposal.md +56 -56
  36. package/openspec/changes/adaptive-bitrate/specs/adaptive-bitrate/spec.md +51 -51
  37. package/openspec/changes/adaptive-bitrate/tasks.md +40 -40
  38. package/openspec/changes/binary-distribution/.openspec.yaml +2 -2
  39. package/openspec/changes/binary-distribution/proposal.md +53 -53
  40. package/openspec/changes/chunked-request-bodies/design.md +81 -81
  41. package/openspec/changes/chunked-request-bodies/proposal.md +53 -53
  42. package/openspec/changes/chunked-request-bodies/specs/chunked-request-bodies/spec.md +30 -30
  43. package/openspec/changes/chunked-request-bodies/tasks.md +40 -40
  44. package/openspec/changes/cold-start/design.md +98 -98
  45. package/openspec/changes/cold-start/proposal.md +62 -62
  46. package/openspec/changes/cold-start/specs/cold-start/spec.md +47 -47
  47. package/openspec/changes/cold-start/tasks.md +48 -48
  48. package/openspec/changes/disk-cap/.openspec.yaml +2 -2
  49. package/openspec/changes/disk-cap/proposal.md +37 -37
  50. package/openspec/changes/disk-cap/specs/disk-cap/spec.md +25 -25
  51. package/openspec/changes/disk-cap/tasks.md +19 -19
  52. package/openspec/changes/proxy-observability/.openspec.yaml +2 -2
  53. package/openspec/changes/proxy-observability/design.md +38 -38
  54. package/openspec/changes/proxy-observability/proposal.md +49 -49
  55. package/openspec/changes/proxy-observability/specs/observability/spec.md +33 -33
  56. package/openspec/changes/proxy-observability/tasks.md +21 -21
  57. package/openspec/changes/seek-debounce/design.md +118 -118
  58. package/openspec/changes/seek-debounce/proposal.md +59 -59
  59. package/openspec/changes/seek-debounce/specs/seek-debounce/spec.md +41 -41
  60. package/openspec/changes/seek-debounce/tasks.md +33 -33
  61. package/openspec/changes/subtitle-language/.openspec.yaml +2 -2
  62. package/openspec/changes/subtitle-language/proposal.md +50 -50
  63. package/openspec/changes/subtitle-language/specs/subtitle-language/spec.md +34 -34
  64. package/openspec/changes/subtitle-language/tasks.md +26 -26
  65. package/openspec/changes/track-selection/.openspec.yaml +2 -2
  66. package/openspec/changes/track-selection/design.md +47 -47
  67. package/openspec/changes/track-selection/proposal.md +45 -45
  68. package/openspec/changes/track-selection/specs/track-selection/spec.md +41 -41
  69. package/openspec/changes/track-selection/tasks.md +20 -20
  70. package/openspec/changes/transcode-quality/.openspec.yaml +2 -2
  71. package/openspec/changes/transcode-quality/proposal.md +51 -51
  72. package/openspec/changes/transcode-quality/specs/transcode-quality/spec.md +139 -139
  73. package/openspec/changes/transcode-quality/tasks.md +63 -63
  74. package/openspec/config.yaml +38 -38
  75. package/package.json +51 -50
  76. package/public/.well-known/appspecific/com.chrome.devtools.json +1 -1
  77. package/research/delivery-wedge-stand-ladder-2026-09-13.md +329 -329
  78. package/research/double-spawn-2026-09-10.md +171 -171
  79. package/research/handover-2026-09-13.md +209 -209
  80. package/research/handover-reader-claims-removal-2026-09-12.md +166 -166
  81. package/research/piece-withdrawn-but-still-claimed-2026-09-12.md +175 -175
  82. package/research/preset-benchmark-size-scaling-2026-08-22.md +55 -55
  83. package/research/priority-map-is-the-truth-2026-09-12.md +225 -225
  84. package/routes/api/client-logs/post.js +86 -86
  85. package/routes/api/delivery-sink/get.js +72 -72
  86. package/routes/api/link-probe/get.js +51 -51
  87. package/routes/api/playback-plan/audio-tracks/post.js +29 -29
  88. package/routes/api/playback-plan/post.js +53 -53
  89. package/routes/api/sources/files/get.js +106 -106
  90. package/routes/api/sources/post.js +36 -36
  91. package/routes/api/sources/stats/get.js +132 -132
  92. package/routes/api/sources/warm/post.js +217 -217
  93. package/routes/api/subtitles/get.js +230 -230
  94. package/routes/api/transcode-sessions/fragment-far/post.js +60 -60
  95. package/routes/api/transcode-sessions/net-report/post.js +138 -138
  96. package/routes/api/transcode-sessions/post.js +178 -178
  97. package/routes/api/transcode-sessions/progress/get.js +27 -27
  98. package/routes/api/transcode-sessions/release/post.js +41 -41
  99. package/routes/api/transcode-sessions/seek/post.js +63 -63
  100. package/routes/health/get.js +13 -13
  101. package/routes/healthz/get.js +13 -13
  102. package/routes/stream/get.js +416 -416
  103. package/routes/transcode/audio-file/get.js +64 -64
  104. package/routes/transcode/audio-warm/get.js +95 -95
  105. package/routes/transcode/session-file/get.js +52 -52
  106. package/routes/transcode/variant-file/get.js +95 -95
  107. package/routes/transcode/variant-warm/get.js +84 -84
  108. package/scripts/build-audio-samples.mjs +31 -31
  109. package/scripts/render-run-graph.js +173 -173
  110. package/server.js +748 -748
  111. package/services/encode/CoverageMap.js +428 -428
  112. package/services/encode/CushionReport.js +345 -345
  113. package/services/encode/EncodeAdmission.js +376 -376
  114. package/services/encode/EncodeOrchestrator.js +1192 -1192
  115. package/services/encode/EncodePlan.js +1368 -1368
  116. package/services/encode/EncodeRun.js +36 -12
  117. package/services/encode/Encoder.js +142 -142
  118. package/services/encode/EncoderSelection.js +38 -38
  119. package/services/encode/LocalObservations.js +359 -359
  120. package/services/encode/NvencEncoder.js +50 -50
  121. package/services/encode/OutputOpening.js +784 -784
  122. package/services/encode/OutputTimes.js +579 -579
  123. package/services/encode/QsvEncoder.js +58 -58
  124. package/services/encode/Renditions.js +2283 -2283
  125. package/services/encode/RunClock.js +87 -87
  126. package/services/encode/RunProgress.js +79 -79
  127. package/services/encode/SegmentDemand.js +95 -95
  128. package/services/encode/SoftwareEncoder.js +121 -121
  129. package/services/encode/V4l2m2mEncoder.js +53 -53
  130. package/services/encode/VaapiEncoder.js +66 -66
  131. package/services/encode/args.js +418 -418
  132. package/services/encode/audio-calibration.js +67 -67
  133. package/services/encode/audio-work-rate.js +15 -15
  134. package/services/encode/available-share.js +89 -89
  135. package/services/encode/calibration.js +361 -361
  136. package/services/encode/contention.js +128 -128
  137. package/services/encode/decode-cost-fit.js +363 -363
  138. package/services/encode/encode-exit.js +116 -116
  139. package/services/encode/encode-run-state.js +551 -551
  140. package/services/encode/encoder-readings.js +43 -43
  141. package/services/encode/fingerprint.js +322 -322
  142. package/services/encode/host-load.js +174 -174
  143. package/services/encode/hwaccel.js +1894 -1894
  144. package/services/encode/index.js +9 -9
  145. package/services/encode/learned-median.js +90 -90
  146. package/services/encode/output/EncodedOutput.js +57 -57
  147. package/services/encode/output/Output.js +131 -131
  148. package/services/encode/output/OutputCatalog.js +414 -414
  149. package/services/encode/output/OutputSpec.js +390 -390
  150. package/services/encode/output/Timeline.js +308 -308
  151. package/services/encode/output/cut-grid.js +107 -107
  152. package/services/encode/output/index.js +24 -24
  153. package/services/encode/output/ladder.js +32 -32
  154. package/services/encode/output/playlists.js +284 -284
  155. package/services/encode/output/rates.js +125 -125
  156. package/services/encode/output-key-format.js +21 -21
  157. package/services/encode/quality/EncodeCost.js +1043 -1043
  158. package/services/encode/quality/HostLoad.js +525 -525
  159. package/services/encode/quality/HostTimings.js +249 -249
  160. package/services/encode/quality/OutputQualityState.js +24 -24
  161. package/services/encode/quality/QualityController.js +758 -758
  162. package/services/encode/quality/QualityOffer.js +416 -416
  163. package/services/encode/quality/drain-threat.js +47 -47
  164. package/services/encode/quality/link-budget.js +337 -337
  165. package/services/encode/quality/output-format.js +319 -319
  166. package/services/encode/quality/serving-output.js +216 -216
  167. package/services/encode/quality/visible-rung.js +57 -57
  168. package/services/encode/run-budget.js +97 -97
  169. package/services/encode/run-command.js +757 -757
  170. package/services/encode/run-costs.js +155 -155
  171. package/services/encode/run-inputs.js +53 -53
  172. package/services/encode/segment-formats/fmp4.js +476 -476
  173. package/services/encode/segment-formats/init-compat.js +324 -324
  174. package/services/encode/segment-formats/mpegts.js +134 -134
  175. package/services/encode/start-stop-cost.js +178 -178
  176. package/services/encode/throughput.js +97 -97
  177. package/services/encode/torrent-cost.js +88 -88
  178. package/services/identity/installation-id.js +38 -38
  179. package/services/media/ContainerOrchestrator.js +202 -202
  180. package/services/media/KeyframeTables.js +356 -356
  181. package/services/media/SourceFile.js +298 -298
  182. package/services/media/SubtitleCues.js +592 -592
  183. package/services/media/SubtitleOrchestrator.js +167 -167
  184. package/services/media/audio-inventory.js +299 -299
  185. package/services/media/container/AviContainer.js +268 -268
  186. package/services/media/container/Container.js +632 -632
  187. package/services/media/container/ContainerFactory.js +38 -38
  188. package/services/media/container/KeyframeTable.js +272 -272
  189. package/services/media/container/MatroskaContainer.js +1031 -1031
  190. package/services/media/container/Mp4Container.js +1354 -1354
  191. package/services/media/container/SubtitleFileContainer.js +339 -339
  192. package/services/media/container/ebml-reader.js +159 -159
  193. package/services/media/container/ebml-stream.js +174 -174
  194. package/services/media/container/matroska-clusters.js +1165 -1165
  195. package/services/media/container/unavailable.js +77 -77
  196. package/services/media/ffmpeg-banner.js +260 -260
  197. package/services/media/keyframe-probe.js +157 -157
  198. package/services/media/media-info-probe.js +93 -93
  199. package/services/media/playback-planner.js +931 -931
  200. package/services/media/tracks/AudioTrack.js +143 -143
  201. package/services/media/tracks/ContainerTrack.js +84 -84
  202. package/services/media/tracks/ImageSubtitleTrack.js +19 -19
  203. package/services/media/tracks/SubtitleTrack.js +38 -38
  204. package/services/media/tracks/TextSubtitleTrack.js +295 -295
  205. package/services/media/tracks/VideoTrack.js +33 -33
  206. package/services/media/tracks/audio-bound.js +233 -233
  207. package/services/media/tracks/index.js +15 -15
  208. package/services/media/tracks/language-detect.js +174 -174
  209. package/services/server/OutputLifecycle.js +418 -418
  210. package/services/server/SegmentServing.js +1152 -1152
  211. package/services/server/ViewerRequests.js +785 -785
  212. package/services/server/WaitLedger.js +142 -142
  213. package/services/server/controllers/PlaybackController.js +40 -40
  214. package/services/server/controllers/SubtitleController.js +109 -109
  215. package/services/server/http-responses.js +60 -60
  216. package/services/server/transcode-session-files.js +325 -325
  217. package/services/server/wire-outputs.js +1029 -1029
  218. package/services/storage/Diagnostics.js +202 -202
  219. package/services/storage/MachineBudget.js +235 -235
  220. package/services/storage/allowance.js +107 -107
  221. package/services/storage/core-dumps.js +106 -106
  222. package/services/storage/files/CompletedFiles.js +391 -391
  223. package/services/storage/files/piece-from-whole-file.js +118 -118
  224. package/services/storage/free.js +71 -71
  225. package/services/storage/keep.js +48 -48
  226. package/services/storage/machine-memory.js +56 -56
  227. package/services/storage/memory-report.js +575 -575
  228. package/services/storage/piece-store/piece-disk-store.js +539 -539
  229. package/services/storage/piece-store/piece-lru.js +495 -495
  230. package/services/storage/piece-store/shared-piece-store.js +1937 -1937
  231. package/services/storage/returns.js +103 -103
  232. package/services/storage/segment-store/SegmentStore.js +1294 -1294
  233. package/services/storage/wire.js +175 -175
  234. package/services/torrent/Contents.js +382 -382
  235. package/services/torrent/demand/DemandRegister.js +202 -202
  236. package/services/torrent/demand/Urgency.js +137 -137
  237. package/services/torrent/demand/Window.js +118 -118
  238. package/services/torrent/demand/index.js +8 -8
  239. package/services/torrent/demand/pieces.js +140 -140
  240. package/services/torrent/download/SwarmSelection.js +386 -386
  241. package/services/torrent/download/registry.js +102 -102
  242. package/services/torrent/download/withdraw-claim.js +80 -80
  243. package/services/torrent/episode-naming.js +182 -182
  244. package/services/torrent/files.js +455 -455
  245. package/services/torrent/naming.js +619 -619
  246. package/services/torrent/piece-store-of.js +29 -29
  247. package/services/torrent/supply-margin.js +264 -264
  248. package/services/torrent/torrent-pool.js +2688 -2688
  249. package/services/torrent/worker/background-fill.js +213 -213
  250. package/services/torrent/worker/channel.js +271 -271
  251. package/services/torrent/worker/client.js +836 -836
  252. package/services/torrent/worker/fastest-wires.js +342 -342
  253. package/services/torrent/worker/handle-state.js +23 -23
  254. package/services/torrent/worker/held-bytes.js +231 -231
  255. package/services/torrent/worker/install-webrtc-shim.js +32 -32
  256. package/services/torrent/worker/piece-reader.js +1078 -1078
  257. package/services/torrent/worker/pool-adapter.js +371 -371
  258. package/services/torrent/worker/protocol.js +184 -184
  259. package/services/torrent/worker/resume-warm.js +107 -107
  260. package/services/torrent/worker/webrtc-shim.js +219 -219
  261. package/services/torrent/worker/worker.js +1137 -1137
  262. package/services/transport/data-channel-handler.js +1406 -1373
  263. package/services/transport/delivery-probe.js +798 -842
  264. package/services/transport/health-collector.js +53 -53
  265. package/services/transport/nat-classifier.js +231 -231
  266. package/services/transport/packet-witness.js +808 -808
  267. package/services/transport/port-mapper.js +235 -235
  268. package/services/transport/registry-api.js +81 -81
  269. package/services/transport/tunnel-client.js +447 -447
  270. package/services/transport/usrsctp-state.js +175 -175
  271. package/services/transport/webrtc-manager.js +484 -484
  272. package/services/viewer/Assignments.js +340 -340
  273. package/services/viewer/PriorityMap.js +421 -421
  274. package/services/viewer/PriorityOrchestrator.js +325 -325
  275. package/services/viewer/Viewer.js +691 -691
  276. package/services/viewer/Viewers.js +368 -368
  277. package/services/viewer/active-output.js +37 -37
  278. package/services/viewer/audio-start.js +46 -46
  279. package/services/viewer/buffer-trend.js +81 -81
  280. package/services/viewer/choices.js +647 -647
  281. package/services/viewer/media-time.js +355 -355
  282. package/services/viewer/playback-readiness.js +635 -635
  283. package/services/viewer/positions.js +126 -126
  284. package/services/viewer/report-intake.js +46 -46
  285. package/store/source-registry.js +71 -71
  286. package/test/asset-paths.test.js +28 -28
  287. package/test/audio-buffer-continuity.test.js +77 -77
  288. package/test/audio-calibration.test.js +23 -23
  289. package/test/audio-inventory.test.js +212 -212
  290. package/test/audio-track-bitrate.test.js +40 -40
  291. package/test/auto-quality-step.test.js +762 -762
  292. package/test/available-share.test.js +76 -76
  293. package/test/background-fill.test.js +190 -190
  294. package/test/client-log-file.test.js +110 -110
  295. package/test/coalesce.test.js +39 -39
  296. package/test/completed-files.test.js +115 -115
  297. package/test/concurrent-cost.test.js +148 -148
  298. package/test/container-media-info.test.js +228 -228
  299. package/test/contention.test.js +76 -76
  300. package/test/core-dumps.test.js +57 -57
  301. package/test/coverage-follows-the-disk.test.js +233 -228
  302. package/test/coverage-map.test.js +195 -195
  303. package/test/cue-push-carries-what-the-walk-found.test.js +58 -58
  304. package/test/cut-times-timeline.test.js +65 -65
  305. package/test/cuts-follow-published-grid.test.js +109 -109
  306. package/test/data-channel-frame.test.js +63 -63
  307. package/test/data-channel-viewer-lifecycle.test.js +162 -92
  308. package/test/dead-torrent-handle.test.js +52 -52
  309. package/test/declared-rates.test.js +116 -116
  310. package/test/declared-tracks.test.js +43 -43
  311. package/test/decode-cost-fit.test.js +202 -202
  312. package/test/decode-cost.test.js +475 -475
  313. package/test/delivery-probe-field.test.js +173 -0
  314. package/test/delivery-probe.test.js +382 -393
  315. package/test/demand-register.test.js +206 -206
  316. package/test/dht-bootstrap.test.js +72 -72
  317. package/test/drain-threat.test.js +65 -65
  318. package/test/duplicate-add.test.js +48 -48
  319. package/test/encode-admission.test.js +146 -146
  320. package/test/encode-cost.test.js +275 -275
  321. package/test/encode-exit.test.js +111 -111
  322. package/test/encode-orchestrator.test.js +426 -426
  323. package/test/encode-plan-viewers.test.js +725 -725
  324. package/test/encode-plan.test.js +545 -545
  325. package/test/encode-run-state.test.js +314 -314
  326. package/test/encode-run.test.js +20 -0
  327. package/test/encode-slope.test.js +46 -46
  328. package/test/encoded-output.test.js +39 -39
  329. package/test/encoder-kinds.test.js +122 -122
  330. package/test/encoder-readings.test.js +68 -68
  331. package/test/episode-naming.test.js +131 -131
  332. package/test/every-claimant-is-told-its-share.test.js +152 -152
  333. package/test/failed-encoder-still-serves-what-exists.test.js +354 -354
  334. package/test/failed-start-breaker.test.js +202 -202
  335. package/test/fastest-wires.test.js +143 -143
  336. package/test/ffmpeg-stream-counts.test.js +78 -78
  337. package/test/file-edges.test.js +191 -191
  338. package/test/fixtures/audio-frame-boundary.json +1 -1
  339. package/test/fmp4-self-contained.test.js +80 -80
  340. package/test/fmp4-timestamps.test.js +140 -140
  341. package/test/fmp4-track-completeness.test.js +41 -41
  342. package/test/format-keyed-outputs.test.js +78 -78
  343. package/test/health-metrics.test.js +36 -38
  344. package/test/held-read-ownership.test.js +77 -77
  345. package/test/held-request-width.test.js +157 -157
  346. package/test/helpers/encode-run.js +248 -240
  347. package/test/helpers/held-file.js +77 -77
  348. package/test/helpers/manager.js +62 -62
  349. package/test/helpers/matroska-file.js +296 -296
  350. package/test/helpers/output-spec.js +63 -63
  351. package/test/host-load.test.js +68 -68
  352. package/test/index-accuracy-is-about-the-file.test.js +128 -128
  353. package/test/init-compat.test.js +219 -219
  354. package/test/input-lost-quiets-the-plan.test.js +261 -261
  355. package/test/input-unavailable.test.js +48 -48
  356. package/test/installation-id.test.js +39 -39
  357. package/test/keeping-period.test.js +83 -83
  358. package/test/keyframe-index-accuracy.test.js +188 -188
  359. package/test/keyframe-table-budget.test.js +123 -123
  360. package/test/keyframes-belong-to-the-file.test.js +249 -249
  361. package/test/learned-median.test.js +37 -37
  362. package/test/letting-go-asks-who-is-watching.test.js +191 -191
  363. package/test/limit-row-by-area.test.js +60 -60
  364. package/test/link-budget.test.js +111 -111
  365. package/test/link-probe.test.js +56 -56
  366. package/test/local-encoding-data.test.js +273 -273
  367. package/test/logger-repeats.test.js +134 -134
  368. package/test/look-ahead-coverage.test.js +56 -56
  369. package/test/manager-surface.test.js +291 -291
  370. package/test/matroska-cluster-discovery.test.js +323 -323
  371. package/test/matroska-cues-track.test.js +192 -192
  372. package/test/media-time.test.js +62 -62
  373. package/test/memory-budget.test.js +390 -390
  374. package/test/move-cost.test.js +216 -216
  375. package/test/mp4-subtitles.test.js +173 -173
  376. package/test/one-authority.test.js +285 -285
  377. package/test/one-budget.test.js +149 -149
  378. package/test/one-output-one-session.test.js +273 -273
  379. package/test/one-resource-divided.test.js +185 -185
  380. package/test/one-serving-output-per-height.test.js +458 -458
  381. package/test/orchestrator-wired.test.js +207 -207
  382. package/test/output-catalog.test.js +39 -39
  383. package/test/output-shape.test.js +21 -21
  384. package/test/output-spec.test.js +251 -251
  385. package/test/output-speed.test.js +92 -87
  386. package/test/outputs-being-prepared.test.js +40 -40
  387. package/test/packet-witness-ring.test.js +236 -236
  388. package/test/packet-witness.test.js +148 -148
  389. package/test/piece-disk-store.test.js +401 -401
  390. package/test/piece-from-whole-file.test.js +129 -129
  391. package/test/piece-lru.test.js +319 -319
  392. package/test/piece-reader.test.js +162 -162
  393. package/test/piece-selection-offset.test.js +85 -85
  394. package/test/piece-store-eviction.test.js +595 -595
  395. package/test/piece-store-never-refuses.test.js +153 -153
  396. package/test/piece-store-reservations.test.js +403 -403
  397. package/test/piece-store-slow-disk.test.js +159 -159
  398. package/test/piece-tail.test.js +97 -97
  399. package/test/plan-host-timings.test.js +68 -68
  400. package/test/plan-is-steady.test.js +225 -225
  401. package/test/playback-readiness.test.js +518 -518
  402. package/test/playback-start-regressions.test.js +101 -101
  403. package/test/priority-map-download.test.js +174 -174
  404. package/test/priority-map-emptied.test.js +204 -204
  405. package/test/priority-map-per-output.test.js +215 -215
  406. package/test/priority-map.test.js +163 -163
  407. package/test/probe-wedge-certainty.test.js +60 -60
  408. package/test/produced-copy-choice.test.js +213 -213
  409. package/test/published-timeline.test.js +83 -83
  410. package/test/quality-offer.test.js +160 -160
  411. package/test/quality-variants.test.js +1274 -1270
  412. package/test/rate-control-in-the-output.test.js +181 -180
  413. package/test/read-survives-withdrawal.test.js +164 -164
  414. package/test/read-window.test.js +365 -365
  415. package/test/reads-not-yet-arrived.test.js +218 -218
  416. package/test/resume-warm.test.js +39 -39
  417. package/test/run-graph-drift.test.js +37 -37
  418. package/test/run-inputs.test.js +80 -80
  419. package/test/run-intervals.test.js +100 -100
  420. package/test/run-position-follows-published-grid.test.js +89 -89
  421. package/test/run-progress.test.js +64 -64
  422. package/test/same-height-switch.test.js +663 -663
  423. package/test/seek-frees-the-viewers-requests.test.js +217 -217
  424. package/test/seek-landing.test.js +109 -109
  425. package/test/seek-target-not-superseded.test.js +195 -195
  426. package/test/segment-closed-on-both-branches.test.js +114 -114
  427. package/test/segment-cut-times.test.js +63 -63
  428. package/test/segment-demand.test.js +80 -80
  429. package/test/segment-hold-supersede.test.js +91 -91
  430. package/test/segment-judged-by-its-own-header.test.js +222 -222
  431. package/test/segment-serve-wiring.test.js +481 -481
  432. package/test/segment-store-eviction.test.js +247 -247
  433. package/test/segment-store.test.js +352 -352
  434. package/test/segment-true-start.test.js +86 -86
  435. package/test/segments-are-shared.test.js +176 -176
  436. package/test/send-chunk-bytes.test.js +86 -86
  437. package/test/serving-existing-output.test.js +111 -111
  438. package/test/serving-output.test.js +102 -102
  439. package/test/shared-piece-store.test.js +310 -310
  440. package/test/sidecar-naming.test.js +142 -142
  441. package/test/soundtrack-rate-bound.test.js +289 -289
  442. package/test/source-file.test.js +133 -133
  443. package/test/source-files-route.test.js +113 -113
  444. package/test/source-registry.test.js +18 -18
  445. package/test/split-a-stretch.test.js +139 -139
  446. package/test/stale-request-after-seek.test.js +158 -158
  447. package/test/startup-readings.test.js +124 -124
  448. package/test/stream-route.test.js +237 -237
  449. package/test/subtitle-cue-framing.test.js +200 -200
  450. package/test/subtitle-cue-source.test.js +102 -102
  451. package/test/subtitle-cue-walk.test.js +385 -385
  452. package/test/subtitle-cursor.test.js +63 -63
  453. package/test/subtitle-defaults.test.js +118 -118
  454. package/test/subtitle-language.test.js +252 -252
  455. package/test/subtitle-metadata.test.js +15 -15
  456. package/test/subtitle-plan-waits.test.js +125 -125
  457. package/test/subtitle-track-numbering.test.js +391 -391
  458. package/test/supply-margin.test.js +190 -190
  459. package/test/swarm-follows-readers.test.js +174 -174
  460. package/test/swarm-reach.test.js +130 -130
  461. package/test/swarm-selection.test.js +220 -220
  462. package/test/tail-duplication.test.js +167 -167
  463. package/test/the-channel-has-both-ends.test.js +91 -91
  464. package/test/the-torrent-thread-serves-bytes.test.js +107 -107
  465. package/test/thread-pool.test.js +53 -53
  466. package/test/timeline.test.js +84 -84
  467. package/test/torrent-contents.test.js +235 -235
  468. package/test/torrent-cost.test.js +71 -71
  469. package/test/torrent-files.test.js +221 -221
  470. package/test/torrent-naming.test.js +255 -255
  471. package/test/tracks-begin-together.test.js +209 -209
  472. package/test/tunnel-renewal.test.js +167 -167
  473. package/test/two-viewers-one-picture.test.js +508 -508
  474. package/test/upload-hurry.test.js +186 -186
  475. package/test/usrsctp-state.test.js +131 -131
  476. package/test/video-facts.test.js +102 -102
  477. package/test/viewer-assignments.test.js +143 -143
  478. package/test/viewer-outputs.test.js +285 -285
  479. package/test/viewer-presence.test.js +116 -116
  480. package/test/viewer-requests-admission.test.js +92 -92
  481. package/test/viewer-starts-on-map.test.js +27 -27
  482. package/test/viewer-states.test.js +192 -189
  483. package/test/viewer-subtitle-subscription.test.js +96 -96
  484. package/test/viewer.test.js +139 -139
  485. package/test/viewing-generations.test.js +390 -390
  486. package/test/visible-rung.test.js +37 -37
  487. package/test/wait-ledger.test.js +100 -100
  488. package/test/webrtc-shim.test.js +149 -149
  489. package/test/wedge-certainty.test.js +131 -131
  490. package/test/whole-file-held-read.test.js +30 -30
  491. package/test/withdraw-piece-claim.test.js +212 -212
  492. package/test/worker-channel.test.js +158 -158
  493. package/utils/client-log-file.js +151 -151
  494. package/utils/coalesce.js +41 -41
  495. package/utils/logger.js +312 -312
  496. package/utils/parse-range.js +25 -25
  497. package/utils/perf.js +59 -59
  498. package/utils/torrent-source-key.js +38 -38
@@ -1,808 +1,808 @@
1
- /**
2
- * @file Packet witness for the delivery freeze: a ring that is always running,
3
- * and a tail that outlasts the sender's own retransmission timeout.
4
- *
5
- * Roadmap item 11. Two captures taken by the earlier version of this file, read
6
- * on 2026-08-26 (`research/delivery-freeze-sender-silent-2026-08-26.md`), place
7
- * the failure: while a channel held 67 MB, the proxy put NO packet larger than
8
- * 89 bytes of UDP payload on the wire for 28 s, twice, in two separate
9
- * episodes. The receiver never had anything to drop, the path carried STUN and
10
- * the browser's own requests throughout, and `usrsctp_sendv` refused every byte
11
- * for 54 minutes. What those captures could NOT show is what happened at the
12
- * ONSET, because they began 30 s after the queue was already declared stuck,
13
- * and what happens beyond 28 s, because that is where they stopped.
14
- *
15
- * So the capture is in two halves now.
16
- *
17
- * The RING runs the whole time a data channel is open, size-bounded and
18
- * wrapping, filtered to the WebRTC UDP port. It costs a rotating file on disk
19
- * and nothing else, and it means the seconds BEFORE a freeze are already
20
- * recorded when the freeze is noticed. On a wedge its files are copied aside
21
- * before the wrap can reach them.
22
- *
23
- * The TAIL then records the wedged session's own 5-tuple for
24
- * {@linkcode WITNESS_TAIL_SECONDS}, which is three times usrsctp's maximum
25
- * retransmission timeout. That length is what turns "no zero-window probe in
26
- * the 28 s we watched" into a statement about the sender: a stalled SCTP sender
27
- * whose peer advertises a zero window MUST probe once per timeout, and the
28
- * timeout cannot exceed {@linkcode WITNESS_RTO_CEILING_SECONDS}. Silence across
29
- * three of those is not a sampling gap.
30
- *
31
- * Bounded so it can never cost more than an episode is worth: the ring is
32
- * capped in bytes and deleted when the last channel closes, the tail process is
33
- * killed after its window plus grace, one tail runs at a time with a cooldown,
34
- * and old captures are pruned at startup like core dumps.
35
- */
36
-
37
- import { spawn } from "node:child_process";
38
- import { copyFile, readdir, rename, rm, stat } from "node:fs/promises";
39
- import os from "node:os";
40
- import path from "node:path";
41
-
42
- /** Per-packet capture length. Small: headers are what the signatures need. */
43
- export const WITNESS_SNAPLEN_BYTES = 128;
44
-
45
- /**
46
- * usrsctp's maximum retransmission timeout, seconds (its `RTO.max` default).
47
- *
48
- * Not ours to choose: it is the longest a stalled SCTP sender may wait before
49
- * it must try again, so it is the shortest window in which silence means
50
- * anything at all.
51
- */
52
- export const WITNESS_RTO_CEILING_SECONDS = 60;
53
-
54
- /**
55
- * How long the tail capture records the wedged session, seconds.
56
- *
57
- * Three retransmission timeouts. One would leave the answer to a single
58
- * scheduling accident; three is silence that has had three chances to break.
59
- */
60
- export const WITNESS_TAIL_SECONDS = WITNESS_RTO_CEILING_SECONDS * 3;
61
-
62
- /** Megabytes per ring file before tcpdump rotates to the next. */
63
- export const WITNESS_RING_FILE_MB = 16;
64
-
65
- /** How many ring files wrap around (16 MB × 4 = 64 MB of history). */
66
- export const WITNESS_RING_FILES = 4;
67
-
68
- /** Megabytes per tail file. A wedged session emits a few packets a second. */
69
- export const WITNESS_TAIL_FILE_MB = 8;
70
-
71
- /** How many tail files may be written before tcpdump stops on its own. */
72
- export const WITNESS_TAIL_FILES = 2;
73
-
74
- /** Extra time before the SIGKILL fallback lands on a hanging tcpdump. */
75
- export const WITNESS_KILL_GRACE_MS = 5_000;
76
-
77
- /** Minimum spacing between tail captures, whatever the reason for them. */
78
- export const WITNESS_COOLDOWN_MS = 10 * 60_000;
79
-
80
- /** How many old captures survive at startup, newest first. */
81
- export const WITNESS_CAPTURES_KEPT = 8;
82
-
83
- /** Base name of the rolling ring, before tcpdump appends its file number. */
84
- export const WITNESS_RING_BASENAME = "packet-witness-ring.pcap";
85
-
86
- /**
87
- * The remote endpoint of a transport snapshot, already structured.
88
- *
89
- * @typedef {Object} WitnessRemoteEndpoint
90
- * @property {string} address - IP literal (may carry a `%zone` suffix).
91
- * @property {number} port
92
- */
93
-
94
- /**
95
- * What the watcher hands over when a wedge crosses the capture threshold.
96
- *
97
- * @typedef {Object} WitnessTrigger
98
- * @property {string} sessionId
99
- * @property {string} tag - Session id, first 8 characters.
100
- * @property {string} label - Data channel label ("proxy", "proxy-control").
101
- * @property {WitnessRemoteEndpoint | null} remote
102
- * @property {number} queuedBytes
103
- * @property {number} stuckForMs
104
- */
105
-
106
- /**
107
- * Strip a zone suffix and keep only IPv4/IPv6 literals.
108
- *
109
- * The address travels into a tcpdump BPF filter argument. The child is spawned
110
- * as an argv array (no shell), so injection is not reachable, but a garbage or
111
- * hostname value would produce a capture of nothing — rejected instead, so the
112
- * log can say why nothing was written.
113
- *
114
- * @param {unknown} raw - Address as libdatachannel reports it.
115
- * @returns {string | null} A clean literal, or null when not usable.
116
- */
117
- export function normalizeRemoteAddress(raw) {
118
- if (typeof raw !== "string") {
119
- return null;
120
- }
121
- const zoneFree = raw.replace(/%.*$/, "");
122
- if (zoneFree.length === 0 || zoneFree.length > 45) {
123
- return null;
124
- }
125
- const ipv4 = /^(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}$/;
126
- // Hex groups and at most one "::" compression — enough to keep hostnames,
127
- // decimal-octet junk and shell metacharacters out of the filter.
128
- const ipv6 = /^(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}$/;
129
- if (ipv4.test(zoneFree) || (zoneFree.includes(":") && ipv6.test(zoneFree))) {
130
- return zoneFree;
131
- }
132
- return null;
133
- }
134
-
135
- /**
136
- * The tcpdump command line for a size-rotated capture.
137
- *
138
- * `-i any` covers hosts where the UDP mux socket is not on a named interface;
139
- * `-n` keeps DNS out of the hot path; `-S` prints absolute sequence numbers so
140
- * duplicate SACKs across file rotations compare equal by eye.
141
- *
142
- * Rotation is by SIZE, not by time, and that is a correction rather than a
143
- * preference. `-G <seconds>` with `-W <count>` and a file name carrying no
144
- * strftime field makes every rotation write the SAME name: the four files the
145
- * previous version believed it was keeping were one file overwritten four
146
- * times, which is why both field captures hold 28 s and not the intended 120.
147
- * `-C <megabytes>` with `-W <count>` appends a number to the name and wraps
148
- * around, which is the ring this needs — and a byte bound is the right bound
149
- * anyway, because the rate varies by two orders of magnitude between a wedged
150
- * session and a healthy burst.
151
- *
152
- * @param {Object} parts
153
- * @param {string} [parts.host] - Validated IP literal; omitted captures every peer.
154
- * @param {number} parts.port - The WebRTC UDP port.
155
- * @param {string} parts.filePrefix - Path prefix; tcpdump appends the file number.
156
- * @param {number} [parts.snaplen] - Bytes per packet to store.
157
- * @param {number} [parts.fileMegabytes]
158
- * @param {number} [parts.files]
159
- * @returns {string[]}
160
- */
161
- export function buildTcpdumpArgs({
162
- host = "",
163
- port,
164
- filePrefix,
165
- snaplen = WITNESS_SNAPLEN_BYTES,
166
- fileMegabytes = WITNESS_RING_FILE_MB,
167
- files = WITNESS_RING_FILES
168
- }) {
169
- const filter = ["udp", "and", "port", String(port)];
170
- if (host) {
171
- filter.push("and", "host", host);
172
- }
173
- return [
174
- "-n",
175
- "-S",
176
- "-s",
177
- String(snaplen),
178
- "-i",
179
- "any",
180
- "-w",
181
- filePrefix,
182
- "-C",
183
- String(fileMegabytes),
184
- "-W",
185
- String(files),
186
- ...filter
187
- ];
188
- }
189
-
190
- /**
191
- * Whether a new capture may start right now.
192
- *
193
- * Pure, so the gating rule is testable without clocks or processes: never two
194
- * at once, and none within the cooldown of the previous one. `lastStartedAt`
195
- * of 0 means this process has not captured yet.
196
- *
197
- * @param {{ running: boolean, lastStartedAt: number, now?: number, cooldownMs?: number }} state
198
- * @returns {boolean}
199
- */
200
- export function shouldStartCapture({ running, lastStartedAt, now = Date.now(), cooldownMs = WITNESS_COOLDOWN_MS }) {
201
- if (running) {
202
- return false;
203
- }
204
- return lastStartedAt <= 0 || now - lastStartedAt >= cooldownMs;
205
- }
206
-
207
- /**
208
- * Whether a file name is a capture this module wrote (including the
209
- * timestamp-suffixed rotations tcpdump appends under `-G`).
210
- *
211
- * @param {string} name
212
- * @returns {boolean}
213
- */
214
- export function isWitnessCapture(name) {
215
- return typeof name === "string" && /^packet-witness\.[^/\\]+\.pcap/.test(name);
216
- }
217
-
218
- /**
219
- * Whether a file name is one of the rolling ring's own files.
220
- *
221
- * The ring is scratch, not evidence: it wraps, and it is deleted when the last
222
- * channel closes. Only the copies taken at a wedge are kept, and those are
223
- * named so that {@link isWitnessCapture} — and therefore the pruner — sees
224
- * them and the ring's own files are left alone.
225
- *
226
- * @param {string} name
227
- * @returns {boolean}
228
- */
229
- export function isWitnessRingFile(name) {
230
- return typeof name === "string" && name.startsWith(WITNESS_RING_BASENAME) && !name.includes("/") && !name.includes("\\");
231
- }
232
-
233
- /**
234
- * Delete all but the newest few captures in `dir`.
235
- *
236
- * Mirrors `pruneCoreDumps`: best-effort, never fatal, one summary line.
237
- *
238
- * @param {string} dir
239
- * @param {(files: Array<{ name: string, writtenAt: number }>, keep: number) => string[]} chooseRemovals -
240
- * Which files go, given what is there: the storage component's rule, handed in.
241
- * @param {number} [keep]
242
- * @returns {Promise<void>}
243
- */
244
- export async function pruneWitnessCaptures(dir, chooseRemovals, keep = WITNESS_CAPTURES_KEPT) {
245
- if (typeof dir !== "string" || dir.length === 0 || typeof chooseRemovals !== "function") {
246
- return;
247
- }
248
- /** @type {Array<{ name: string, writtenAt: number, bytes: number }>} */
249
- const captures = [];
250
- try {
251
- for (const name of await readdir(dir)) {
252
- if (!isWitnessCapture(name)) {
253
- continue;
254
- }
255
- try {
256
- const info = await stat(path.join(dir, name));
257
- if (info.isFile()) {
258
- captures.push({ name, writtenAt: info.mtimeMs, bytes: info.size });
259
- }
260
- } catch {
261
- // silent-ok: vanished between listing and reading.
262
- }
263
- }
264
- } catch {
265
- return; // No such directory, or unreadable. Nothing to tidy.
266
- }
267
- if (captures.length === 0) {
268
- return;
269
- }
270
- const doomed = chooseRemovals(captures, keep);
271
- const freed = captures
272
- .filter((capture) => doomed.includes(capture.name))
273
- .reduce((total, capture) => total + capture.bytes, 0);
274
- for (const name of doomed) {
275
- try {
276
- await rm(path.join(dir, name), { force: true });
277
- } catch {
278
- // silent-ok: best effort, retried at the next start.
279
- }
280
- }
281
- const keptBytes = captures
282
- .filter((capture) => !doomed.includes(capture.name))
283
- .reduce((total, capture) => total + capture.bytes, 0);
284
- if (freed > 0 || captures.length > keep) {
285
- logLine(
286
- `packet witness: ${captures.length} capture(s) present, keeping the newest ${Math.min(keep, captures.length)}, ` +
287
- `removed ${doomed.length} (${(freed / 1024).toFixed(1)} KB), keeping ${(keptBytes / 1024).toFixed(1)} KB`
288
- );
289
- }
290
- }
291
-
292
- /**
293
- * Turn ring files left by a previous process into evidence, at startup.
294
- *
295
- * The ring is scratch while a process lives, but a process that was KILLED —
296
- * which the crash family of roadmap item 1 does regularly, seven times in a
297
- * fortnight — leaves behind the last seconds it recorded, and those seconds
298
- * contain the death. Renaming them under a name the pruner recognises keeps
299
- * them; without this the next viewer's first channel would delete them before
300
- * anyone had looked.
301
- *
302
- * @param {string} dir
303
- * @returns {Promise<string[]>} Names of the files adopted.
304
- */
305
- export async function adoptOrphanRingFiles(dir) {
306
- if (typeof dir !== "string" || dir.length === 0) {
307
- return [];
308
- }
309
- /** @type {string[]} */
310
- const adopted = [];
311
- let names;
312
- try {
313
- names = (await readdir(dir)).filter((name) => isWitnessRingFile(name)).sort();
314
- } catch {
315
- return adopted;
316
- }
317
- if (names.length === 0) {
318
- return adopted;
319
- }
320
- const stampSeconds = Math.floor(Date.now() / 1000);
321
- for (const name of names) {
322
- const suffix = name.slice(WITNESS_RING_BASENAME.length) || "0";
323
- const target = `packet-witness.orphan.${stampSeconds}.before${suffix}.pcap`;
324
- try {
325
- await rename(path.join(dir, name), path.join(dir, target));
326
- adopted.push(target);
327
- } catch {
328
- // silent-ok: unreadable or already gone.
329
- }
330
- }
331
- if (adopted.length > 0) {
332
- logLine(
333
- `packet witness: ${adopted.length} ring file(s) survived the previous process and were kept ` +
334
- `as ${adopted.join(", ")} — whatever ended it is in them`
335
- );
336
- }
337
- return adopted;
338
- }
339
-
340
- /**
341
- * The logger handed to {@linkcode createPacketWitness}; module-level so the
342
- * pruner can speak without an options bag threaded everywhere.
343
- *
344
- * @type {(message: string) => void}
345
- */
346
- let logLine = () => {};
347
-
348
- /**
349
- * Create the witness.
350
- *
351
- * @param {Object} options
352
- * @param {(message: string) => void} options.log - Log sink (the shared logger).
353
- * @param {string} [options.dir] - Where captures go; empty means os.tmpdir().
354
- * @param {number} options.port - The WebRTC UDP port to filter on.
355
- * @param {typeof spawn} [options.spawnProcess] - Seam for tests; defaults to node's spawn.
356
- * @returns {{
357
- * dir: string,
358
- * maybeCapture: (trigger: WitnessTrigger) => boolean,
359
- * holdRing: () => void,
360
- * releaseRing: () => void
361
- * }}
362
- */
363
- export function createPacketWitness({ log, dir = "", port, spawnProcess = spawn, mayKeep = null, chooseRemovals = null }) {
364
- logLine = typeof log === "function" ? log : logLine;
365
- const resolvedDir = typeof dir === "string" && dir.length > 0 ? dir : os.tmpdir();
366
-
367
- /** @type {{ running: boolean, lastStartedAt: number, availability: "unknown" | "yes" | "no" }} */
368
- const state = { running: false, lastStartedAt: 0, availability: "unknown" };
369
-
370
- /**
371
- * Connections already captured, so one wedge is recorded once.
372
- *
373
- * A capture of a connection already captured adds nothing: the wedge does not
374
- * un-wedge, and the second recording of it is the same silence as the first.
375
- * Field 2026-09-06: thirteen captures of ONE wedge over six hours, 74 MB, all
376
- * saying what the first one said.
377
- *
378
- * @type {Set<string>}
379
- */
380
- const captured = new Set();
381
-
382
- /**
383
- * Ask whether tcpdump exists at all — once per process, whichever way it
384
- * answers. A host without it gets exactly one log line, ever.
385
- *
386
- * @returns {Promise<boolean>}
387
- */
388
- const probeAvailability = () =>
389
- new Promise((resolve) => {
390
- let settled = false;
391
- const done = (value) => {
392
- if (!settled) {
393
- settled = true;
394
- resolve(value);
395
- }
396
- };
397
- try {
398
- const probe = spawnProcess("tcpdump", ["--version"], { stdio: "ignore" });
399
- probe.on("error", () => done(false));
400
- probe.on("spawn", () => done(true));
401
- probe.on("close", () => done(true));
402
- } catch {
403
- done(false);
404
- }
405
- });
406
-
407
- /**
408
- * Keep the ring's history: stop it, copy its files, start it again.
409
- *
410
- * This is the half of the evidence the earlier version never had — the
411
- * seconds BEFORE the freeze. Stopping first is not tidiness: tcpdump buffers,
412
- * so the newest packets are only on disk once it has been asked to finish,
413
- * and the file it is currently writing is the one that holds the onset. The
414
- * gap this leaves is a fraction of a second, and the tail capture that
415
- * follows covers everything after it.
416
- *
417
- * @param {WitnessTrigger} trigger
418
- * @param {number} stampSeconds - Shared with the tail, so the pair sorts together.
419
- * @returns {Promise<string[]>} Names of the copies that landed.
420
- */
421
- const preserveRing = async (trigger, stampSeconds) => {
422
- /** @type {string[]} */
423
- const kept = [];
424
- ring.preserving = true;
425
- try {
426
- await stopRingProcess();
427
- let names;
428
- try {
429
- names = (await readdir(resolvedDir)).filter((name) => isWitnessRingFile(name)).sort();
430
- } catch {
431
- return kept;
432
- }
433
- for (const name of names) {
434
- const suffix = name.slice(WITNESS_RING_BASENAME.length) || "0";
435
- const target = `packet-witness.${trigger.tag}.${stampSeconds}.before${suffix}.pcap`;
436
- try {
437
- await copyFile(path.join(resolvedDir, name), path.join(resolvedDir, target));
438
- kept.push(target);
439
- } catch {
440
- // silent-ok: nothing to copy, or it vanished under us.
441
- }
442
- }
443
- return kept;
444
- } finally {
445
- ring.preserving = false;
446
- // Back to recording, unless everyone has gone in the meantime.
447
- if (ring.holders > 0) {
448
- await startRingProcess();
449
- } else {
450
- await clearRingFiles();
451
- }
452
- }
453
- };
454
-
455
- /**
456
- * Run one bounded capture and report what it wrote.
457
- *
458
- * @param {WitnessTrigger} trigger
459
- * @param {string} address - Validated remote IP literal.
460
- * @param {number} stampSeconds
461
- * @returns {Promise<void>}
462
- */
463
- const runCapture = async (trigger, address, stampSeconds) => {
464
- const prefix = path.join(
465
- resolvedDir,
466
- `packet-witness.${trigger.tag}.${stampSeconds}.tail.pcap`
467
- );
468
- const args = buildTcpdumpArgs({
469
- host: address,
470
- port,
471
- filePrefix: prefix,
472
- fileMegabytes: WITNESS_TAIL_FILE_MB,
473
- files: WITNESS_TAIL_FILES
474
- });
475
- logLine(
476
- `packet witness: capturing ${WITNESS_TAIL_SECONDS}s of udp port ${port} ↔ ${address} ` +
477
- `(session ${trigger.tag}, "${trigger.label}" queue ${trigger.queuedBytes}B wedged ` +
478
- `${Math.round(trigger.stuckForMs / 1000)}s; ${WITNESS_TAIL_SECONDS}s is three times ` +
479
- `usrsctp's ${WITNESS_RTO_CEILING_SECONDS}s retransmission ceiling, so a zero-window ` +
480
- `probe cannot hide inside it) → ${prefix}`
481
- );
482
- await new Promise((resolve) => {
483
- /** @type {ReturnType<typeof setTimeout> | null} */
484
- let killer = null;
485
- /** @type {NodeJS.Timeout | null} */
486
- let hardKill = null;
487
- let child;
488
- try {
489
- child = spawnProcess("tcpdump", args, { stdio: "ignore" });
490
- } catch (error) {
491
- logLine(`packet witness: could not start tcpdump: ${error?.message ?? error}`);
492
- resolve();
493
- return;
494
- }
495
- const finish = () => {
496
- if (killer) {
497
- clearTimeout(killer);
498
- killer = null;
499
- }
500
- if (hardKill) {
501
- clearTimeout(hardKill);
502
- hardKill = null;
503
- }
504
- resolve();
505
- };
506
- child.on("error", (error) => {
507
- state.availability = "no";
508
- logLine(`packet witness: tcpdump failed to run: ${error?.message ?? error}`);
509
- finish();
510
- });
511
- child.on("close", (code, signal) => {
512
- logLine(
513
- `packet witness: capture ended${signal ? ` (${signal})` : ` (exit ${code ?? "?"})`}`
514
- );
515
- finish();
516
- });
517
- killer = setTimeout(() => {
518
- try { child.kill("SIGTERM"); } catch { /* already gone */ }
519
- hardKill = setTimeout(() => {
520
- try { child.kill("SIGKILL"); } catch { /* already gone */ }
521
- }, WITNESS_KILL_GRACE_MS);
522
- }, WITNESS_TAIL_SECONDS * 1000);
523
- });
524
- // Say what landed on disk, so whoever reads the log later knows whether
525
- // the evidence exists without listing the directory themselves.
526
- try {
527
- const names = (await readdir(resolvedDir)).filter((name) => isWitnessCapture(name));
528
- const mine = [];
529
- for (const name of names) {
530
- if (!name.startsWith(path.basename(prefix))) {
531
- continue;
532
- }
533
- try {
534
- const info = await stat(path.join(resolvedDir, name));
535
- mine.push(`${name} ${(info.size / 1024).toFixed(1)} KB`);
536
- } catch { /* gone between listing and reading */ }
537
- }
538
- logLine(`packet witness: wrote ${mine.length} file(s): ${mine.join(", ") || "none"}`);
539
- } catch {
540
- logLine("packet witness: could not list the capture directory");
541
- }
542
- };
543
-
544
- /**
545
- * Start a capture for this trigger, if the rules allow one.
546
- *
547
- * @param {WitnessTrigger} trigger
548
- * @returns {boolean} True when a capture actually started.
549
- */
550
- const maybeCapture = (trigger) => {
551
- const address = normalizeRemoteAddress(trigger?.remote?.address);
552
- const portNumber = trigger?.remote?.port;
553
- if (!address || !Number.isInteger(portNumber) || portNumber < 1 || portNumber > 65535) {
554
- return false;
555
- }
556
- if (!shouldStartCapture(state)) {
557
- return false;
558
- }
559
- const connection = `${address}:${portNumber}`;
560
- if (captured.has(connection)) {
561
- // Not evidence twice.
562
- return false;
563
- }
564
- // The evidence takes room from the product, so it is asked for. Refused, it
565
- // is a line rather than a silence — see `storage/Diagnostics.js`.
566
- const expectedBytes = (WITNESS_RING_FILES + 1) * WITNESS_RING_FILE_MB * 1024 * 1024;
567
- if (typeof mayKeep === "function" && !mayKeep(expectedBytes)) {
568
- return false;
569
- }
570
- captured.add(connection);
571
- state.running = true;
572
- state.lastStartedAt = Date.now();
573
- const startedAt = state.lastStartedAt;
574
- void (async () => {
575
- try {
576
- if (state.availability === "unknown") {
577
- state.availability = (await probeAvailability()) ? "yes" : "no";
578
- }
579
- if (state.availability === "no") {
580
- logLine(
581
- "packet witness: unavailable — no tcpdump on this host; " +
582
- "the stuck-queue log lines remain the only evidence here"
583
- );
584
- return;
585
- }
586
- const stampSeconds = Math.floor(Date.now() / 1000);
587
- const before = await preserveRing(trigger, stampSeconds);
588
- logLine(
589
- before.length > 0
590
- ? `packet witness: kept ${before.length} ring file(s) from before the wedge: ${before.join(", ")}`
591
- : "packet witness: no ring history to keep — the ring was not running"
592
- );
593
- await runCapture(trigger, address, stampSeconds);
594
- // The disk bound has to hold between restarts too: one episode writes
595
- // up to four ring copies and two tail files, and the cooldown allows
596
- // six episodes an hour. Pruning only at startup let that accumulate.
597
- await pruneWitnessCaptures(resolvedDir, chooseRemovals);
598
- } finally {
599
- state.running = false;
600
- // Keep the requested spacing honest even when the capture ended early.
601
- const earliestNext = startedAt + WITNESS_COOLDOWN_MS;
602
- if (state.lastStartedAt < earliestNext) {
603
- state.lastStartedAt = earliestNext;
604
- }
605
- }
606
- })();
607
- return true;
608
- };
609
-
610
- /**
611
- * The rolling ring: one tcpdump, reference-counted by open data channels.
612
- *
613
- * @type {{
614
- * child: import('node:child_process').ChildProcess | null,
615
- * holders: number,
616
- * starting: boolean,
617
- * preserving: boolean,
618
- * failed: boolean
619
- * }}
620
- */
621
- const ring = { child: null, holders: 0, starting: false, preserving: false, failed: false };
622
-
623
- /**
624
- * Delete the ring's own files.
625
- *
626
- * Refused while a wedge is copying them out: the copies are the only reason
627
- * the ring exists, and the viewer closing the tab is exactly when both happen
628
- * at once. Deferred until the copy is done instead.
629
- *
630
- * @returns {Promise<void>}
631
- */
632
- const clearRingFiles = async () => {
633
- if (ring.preserving) {
634
- return;
635
- }
636
- try {
637
- for (const name of await readdir(resolvedDir)) {
638
- if (isWitnessRingFile(name)) {
639
- await rm(path.join(resolvedDir, name), { force: true });
640
- }
641
- }
642
- } catch {
643
- // silent-ok: best effort, retried when the ring next stops.
644
- }
645
- };
646
-
647
- /**
648
- * Stop the ring's tcpdump and wait for it to flush and exit.
649
- *
650
- * The wait matters. tcpdump buffers its output, so the newest packets — the
651
- * ONSET, which is the whole reason the ring runs — sit in that buffer until
652
- * the process is asked to finish. `-U` would flush per packet instead, at one
653
- * write syscall per packet, and at the measured 150 Mbps that is twelve
654
- * thousand a second on a machine whose spare capacity is already the binding
655
- * constraint. Stopping costs nothing while nothing is wrong.
656
- *
657
- * @returns {Promise<void>}
658
- */
659
- const stopRingProcess = () => {
660
- const child = ring.child;
661
- ring.child = null;
662
- if (!child) {
663
- return Promise.resolve();
664
- }
665
- return new Promise((resolve) => {
666
- let settled = false;
667
- const done = () => {
668
- if (!settled) {
669
- settled = true;
670
- clearTimeout(hardKill);
671
- resolve();
672
- }
673
- };
674
- const hardKill = setTimeout(() => {
675
- try { child.kill("SIGKILL"); } catch { /* already gone */ }
676
- done();
677
- }, WITNESS_KILL_GRACE_MS);
678
- if (typeof hardKill.unref === "function") {
679
- hardKill.unref();
680
- }
681
- child.on("close", done);
682
- child.on("error", done);
683
- try {
684
- child.kill("SIGTERM");
685
- } catch {
686
- done();
687
- }
688
- });
689
- };
690
-
691
- /**
692
- * Start the ring's tcpdump, if it is wanted and not already running.
693
- *
694
- * @returns {Promise<void>}
695
- */
696
- const startRingProcess = async () => {
697
- if (ring.child !== null || ring.starting || ring.holders === 0) {
698
- return;
699
- }
700
- ring.starting = true;
701
- try {
702
- if (state.availability === "unknown") {
703
- state.availability = (await probeAvailability()) ? "yes" : "no";
704
- }
705
- // Everything above suspended; the last channel may have closed meanwhile.
706
- // Without this second look the ring starts with nobody holding it and
707
- // nobody left to stop it.
708
- if (state.availability === "no" || ring.holders === 0) {
709
- return;
710
- }
711
- const prefix = path.join(resolvedDir, WITNESS_RING_BASENAME);
712
- const args = buildTcpdumpArgs({ port, filePrefix: prefix });
713
- let child;
714
- try {
715
- child = spawnProcess("tcpdump", args, { stdio: "ignore" });
716
- } catch (error) {
717
- ring.failed = true;
718
- logLine(`packet witness: the ring could not start: ${error?.message ?? error}`);
719
- return;
720
- }
721
- ring.child = child;
722
- child.on("error", (error) => {
723
- ring.failed = true;
724
- if (ring.child === child) {
725
- ring.child = null;
726
- }
727
- logLine(`packet witness: the ring stopped: ${error?.message ?? error}`);
728
- });
729
- child.on("close", (code, signal) => {
730
- if (ring.child !== child) {
731
- return; // An orderly stop; it has already said what it needed to.
732
- }
733
- ring.child = null;
734
- ring.failed = true;
735
- // Ending on its own means it never recorded what it claimed to. A host
736
- // with tcpdump installed but without the capability to capture answers
737
- // `--version` happily and dies here, which would otherwise leave a
738
- // "ring recording" line in the log and no history behind it.
739
- logLine(
740
- `packet witness: the ring ended on its own${signal ? ` (${signal})` : ` (exit ${code ?? "?"})`}` +
741
- `${ring.holders > 0 ? " while a viewer was still being served — no history is being kept" : ""}`
742
- );
743
- });
744
- logLine(
745
- `packet witness: ring recording udp port ${port}, ` +
746
- `${WITNESS_RING_FILES} × ${WITNESS_RING_FILE_MB} MB wrapping — ` +
747
- "the seconds before a freeze, kept in advance"
748
- );
749
- } finally {
750
- ring.starting = false;
751
- // Holders may have gone while the probe or the spawn was in flight.
752
- if (ring.holders === 0 && ring.child !== null) {
753
- await stopRingProcess();
754
- await clearRingFiles();
755
- }
756
- }
757
- };
758
-
759
- /**
760
- * Begin recording, or note one more reason to keep recording.
761
- *
762
- * Reference-counted by open data channels, so the ring runs exactly while a
763
- * viewer is being served and costs nothing on an idle proxy. Starting it is
764
- * what makes the ONSET of a freeze readable: by the time a wedge is declared,
765
- * the seconds before it are already on disk.
766
- *
767
- * @returns {void}
768
- */
769
- const holdRing = () => {
770
- ring.holders += 1;
771
- if (ring.holders !== 1) {
772
- return;
773
- }
774
- void startRingProcess();
775
- };
776
-
777
- /**
778
- * Release one reason to keep recording; the last one stops the ring.
779
- *
780
- * @returns {void}
781
- */
782
- const releaseRing = () => {
783
- if (ring.holders > 0) {
784
- ring.holders -= 1;
785
- }
786
- if (ring.holders > 0 || ring.starting) {
787
- return;
788
- }
789
- void (async () => {
790
- await stopRingProcess();
791
- await clearRingFiles();
792
- })();
793
- };
794
-
795
- /**
796
- * Stop everything this witness owns. For process shutdown.
797
- *
798
- * @returns {Promise<void>}
799
- */
800
- const dispose = async () => {
801
- ring.holders = 0;
802
- await stopRingProcess();
803
- await clearRingFiles();
804
- };
805
-
806
- return { dir: resolvedDir, maybeCapture, holdRing, releaseRing, dispose, ring };
807
-
808
- }
1
+ /**
2
+ * @file Packet witness for the delivery freeze: a ring that is always running,
3
+ * and a tail that outlasts the sender's own retransmission timeout.
4
+ *
5
+ * Roadmap item 11. Two captures taken by the earlier version of this file, read
6
+ * on 2026-08-26 (`research/delivery-freeze-sender-silent-2026-08-26.md`), place
7
+ * the failure: while a channel held 67 MB, the proxy put NO packet larger than
8
+ * 89 bytes of UDP payload on the wire for 28 s, twice, in two separate
9
+ * episodes. The receiver never had anything to drop, the path carried STUN and
10
+ * the browser's own requests throughout, and `usrsctp_sendv` refused every byte
11
+ * for 54 minutes. What those captures could NOT show is what happened at the
12
+ * ONSET, because they began 30 s after the queue was already declared stuck,
13
+ * and what happens beyond 28 s, because that is where they stopped.
14
+ *
15
+ * So the capture is in two halves now.
16
+ *
17
+ * The RING runs the whole time a data channel is open, size-bounded and
18
+ * wrapping, filtered to the WebRTC UDP port. It costs a rotating file on disk
19
+ * and nothing else, and it means the seconds BEFORE a freeze are already
20
+ * recorded when the freeze is noticed. On a wedge its files are copied aside
21
+ * before the wrap can reach them.
22
+ *
23
+ * The TAIL then records the wedged session's own 5-tuple for
24
+ * {@linkcode WITNESS_TAIL_SECONDS}, which is three times usrsctp's maximum
25
+ * retransmission timeout. That length is what turns "no zero-window probe in
26
+ * the 28 s we watched" into a statement about the sender: a stalled SCTP sender
27
+ * whose peer advertises a zero window MUST probe once per timeout, and the
28
+ * timeout cannot exceed {@linkcode WITNESS_RTO_CEILING_SECONDS}. Silence across
29
+ * three of those is not a sampling gap.
30
+ *
31
+ * Bounded so it can never cost more than an episode is worth: the ring is
32
+ * capped in bytes and deleted when the last channel closes, the tail process is
33
+ * killed after its window plus grace, one tail runs at a time with a cooldown,
34
+ * and old captures are pruned at startup like core dumps.
35
+ */
36
+
37
+ import { spawn } from "node:child_process";
38
+ import { copyFile, readdir, rename, rm, stat } from "node:fs/promises";
39
+ import os from "node:os";
40
+ import path from "node:path";
41
+
42
+ /** Per-packet capture length. Small: headers are what the signatures need. */
43
+ export const WITNESS_SNAPLEN_BYTES = 128;
44
+
45
+ /**
46
+ * usrsctp's maximum retransmission timeout, seconds (its `RTO.max` default).
47
+ *
48
+ * Not ours to choose: it is the longest a stalled SCTP sender may wait before
49
+ * it must try again, so it is the shortest window in which silence means
50
+ * anything at all.
51
+ */
52
+ export const WITNESS_RTO_CEILING_SECONDS = 60;
53
+
54
+ /**
55
+ * How long the tail capture records the wedged session, seconds.
56
+ *
57
+ * Three retransmission timeouts. One would leave the answer to a single
58
+ * scheduling accident; three is silence that has had three chances to break.
59
+ */
60
+ export const WITNESS_TAIL_SECONDS = WITNESS_RTO_CEILING_SECONDS * 3;
61
+
62
+ /** Megabytes per ring file before tcpdump rotates to the next. */
63
+ export const WITNESS_RING_FILE_MB = 16;
64
+
65
+ /** How many ring files wrap around (16 MB × 4 = 64 MB of history). */
66
+ export const WITNESS_RING_FILES = 4;
67
+
68
+ /** Megabytes per tail file. A wedged session emits a few packets a second. */
69
+ export const WITNESS_TAIL_FILE_MB = 8;
70
+
71
+ /** How many tail files may be written before tcpdump stops on its own. */
72
+ export const WITNESS_TAIL_FILES = 2;
73
+
74
+ /** Extra time before the SIGKILL fallback lands on a hanging tcpdump. */
75
+ export const WITNESS_KILL_GRACE_MS = 5_000;
76
+
77
+ /** Minimum spacing between tail captures, whatever the reason for them. */
78
+ export const WITNESS_COOLDOWN_MS = 10 * 60_000;
79
+
80
+ /** How many old captures survive at startup, newest first. */
81
+ export const WITNESS_CAPTURES_KEPT = 8;
82
+
83
+ /** Base name of the rolling ring, before tcpdump appends its file number. */
84
+ export const WITNESS_RING_BASENAME = "packet-witness-ring.pcap";
85
+
86
+ /**
87
+ * The remote endpoint of a transport snapshot, already structured.
88
+ *
89
+ * @typedef {Object} WitnessRemoteEndpoint
90
+ * @property {string} address - IP literal (may carry a `%zone` suffix).
91
+ * @property {number} port
92
+ */
93
+
94
+ /**
95
+ * What the watcher hands over when a wedge crosses the capture threshold.
96
+ *
97
+ * @typedef {Object} WitnessTrigger
98
+ * @property {string} sessionId
99
+ * @property {string} tag - Session id, first 8 characters.
100
+ * @property {string} label - Data channel label ("proxy", "proxy-control").
101
+ * @property {WitnessRemoteEndpoint | null} remote
102
+ * @property {number} queuedBytes
103
+ * @property {number} stuckForMs
104
+ */
105
+
106
+ /**
107
+ * Strip a zone suffix and keep only IPv4/IPv6 literals.
108
+ *
109
+ * The address travels into a tcpdump BPF filter argument. The child is spawned
110
+ * as an argv array (no shell), so injection is not reachable, but a garbage or
111
+ * hostname value would produce a capture of nothing — rejected instead, so the
112
+ * log can say why nothing was written.
113
+ *
114
+ * @param {unknown} raw - Address as libdatachannel reports it.
115
+ * @returns {string | null} A clean literal, or null when not usable.
116
+ */
117
+ export function normalizeRemoteAddress(raw) {
118
+ if (typeof raw !== "string") {
119
+ return null;
120
+ }
121
+ const zoneFree = raw.replace(/%.*$/, "");
122
+ if (zoneFree.length === 0 || zoneFree.length > 45) {
123
+ return null;
124
+ }
125
+ const ipv4 = /^(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}$/;
126
+ // Hex groups and at most one "::" compression — enough to keep hostnames,
127
+ // decimal-octet junk and shell metacharacters out of the filter.
128
+ const ipv6 = /^(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}$/;
129
+ if (ipv4.test(zoneFree) || (zoneFree.includes(":") && ipv6.test(zoneFree))) {
130
+ return zoneFree;
131
+ }
132
+ return null;
133
+ }
134
+
135
+ /**
136
+ * The tcpdump command line for a size-rotated capture.
137
+ *
138
+ * `-i any` covers hosts where the UDP mux socket is not on a named interface;
139
+ * `-n` keeps DNS out of the hot path; `-S` prints absolute sequence numbers so
140
+ * duplicate SACKs across file rotations compare equal by eye.
141
+ *
142
+ * Rotation is by SIZE, not by time, and that is a correction rather than a
143
+ * preference. `-G <seconds>` with `-W <count>` and a file name carrying no
144
+ * strftime field makes every rotation write the SAME name: the four files the
145
+ * previous version believed it was keeping were one file overwritten four
146
+ * times, which is why both field captures hold 28 s and not the intended 120.
147
+ * `-C <megabytes>` with `-W <count>` appends a number to the name and wraps
148
+ * around, which is the ring this needs — and a byte bound is the right bound
149
+ * anyway, because the rate varies by two orders of magnitude between a wedged
150
+ * session and a healthy burst.
151
+ *
152
+ * @param {Object} parts
153
+ * @param {string} [parts.host] - Validated IP literal; omitted captures every peer.
154
+ * @param {number} parts.port - The WebRTC UDP port.
155
+ * @param {string} parts.filePrefix - Path prefix; tcpdump appends the file number.
156
+ * @param {number} [parts.snaplen] - Bytes per packet to store.
157
+ * @param {number} [parts.fileMegabytes]
158
+ * @param {number} [parts.files]
159
+ * @returns {string[]}
160
+ */
161
+ export function buildTcpdumpArgs({
162
+ host = "",
163
+ port,
164
+ filePrefix,
165
+ snaplen = WITNESS_SNAPLEN_BYTES,
166
+ fileMegabytes = WITNESS_RING_FILE_MB,
167
+ files = WITNESS_RING_FILES
168
+ }) {
169
+ const filter = ["udp", "and", "port", String(port)];
170
+ if (host) {
171
+ filter.push("and", "host", host);
172
+ }
173
+ return [
174
+ "-n",
175
+ "-S",
176
+ "-s",
177
+ String(snaplen),
178
+ "-i",
179
+ "any",
180
+ "-w",
181
+ filePrefix,
182
+ "-C",
183
+ String(fileMegabytes),
184
+ "-W",
185
+ String(files),
186
+ ...filter
187
+ ];
188
+ }
189
+
190
+ /**
191
+ * Whether a new capture may start right now.
192
+ *
193
+ * Pure, so the gating rule is testable without clocks or processes: never two
194
+ * at once, and none within the cooldown of the previous one. `lastStartedAt`
195
+ * of 0 means this process has not captured yet.
196
+ *
197
+ * @param {{ running: boolean, lastStartedAt: number, now?: number, cooldownMs?: number }} state
198
+ * @returns {boolean}
199
+ */
200
+ export function shouldStartCapture({ running, lastStartedAt, now = Date.now(), cooldownMs = WITNESS_COOLDOWN_MS }) {
201
+ if (running) {
202
+ return false;
203
+ }
204
+ return lastStartedAt <= 0 || now - lastStartedAt >= cooldownMs;
205
+ }
206
+
207
+ /**
208
+ * Whether a file name is a capture this module wrote (including the
209
+ * timestamp-suffixed rotations tcpdump appends under `-G`).
210
+ *
211
+ * @param {string} name
212
+ * @returns {boolean}
213
+ */
214
+ export function isWitnessCapture(name) {
215
+ return typeof name === "string" && /^packet-witness\.[^/\\]+\.pcap/.test(name);
216
+ }
217
+
218
+ /**
219
+ * Whether a file name is one of the rolling ring's own files.
220
+ *
221
+ * The ring is scratch, not evidence: it wraps, and it is deleted when the last
222
+ * channel closes. Only the copies taken at a wedge are kept, and those are
223
+ * named so that {@link isWitnessCapture} — and therefore the pruner — sees
224
+ * them and the ring's own files are left alone.
225
+ *
226
+ * @param {string} name
227
+ * @returns {boolean}
228
+ */
229
+ export function isWitnessRingFile(name) {
230
+ return typeof name === "string" && name.startsWith(WITNESS_RING_BASENAME) && !name.includes("/") && !name.includes("\\");
231
+ }
232
+
233
+ /**
234
+ * Delete all but the newest few captures in `dir`.
235
+ *
236
+ * Mirrors `pruneCoreDumps`: best-effort, never fatal, one summary line.
237
+ *
238
+ * @param {string} dir
239
+ * @param {(files: Array<{ name: string, writtenAt: number }>, keep: number) => string[]} chooseRemovals -
240
+ * Which files go, given what is there: the storage component's rule, handed in.
241
+ * @param {number} [keep]
242
+ * @returns {Promise<void>}
243
+ */
244
+ export async function pruneWitnessCaptures(dir, chooseRemovals, keep = WITNESS_CAPTURES_KEPT) {
245
+ if (typeof dir !== "string" || dir.length === 0 || typeof chooseRemovals !== "function") {
246
+ return;
247
+ }
248
+ /** @type {Array<{ name: string, writtenAt: number, bytes: number }>} */
249
+ const captures = [];
250
+ try {
251
+ for (const name of await readdir(dir)) {
252
+ if (!isWitnessCapture(name)) {
253
+ continue;
254
+ }
255
+ try {
256
+ const info = await stat(path.join(dir, name));
257
+ if (info.isFile()) {
258
+ captures.push({ name, writtenAt: info.mtimeMs, bytes: info.size });
259
+ }
260
+ } catch {
261
+ // silent-ok: vanished between listing and reading.
262
+ }
263
+ }
264
+ } catch {
265
+ return; // No such directory, or unreadable. Nothing to tidy.
266
+ }
267
+ if (captures.length === 0) {
268
+ return;
269
+ }
270
+ const doomed = chooseRemovals(captures, keep);
271
+ const freed = captures
272
+ .filter((capture) => doomed.includes(capture.name))
273
+ .reduce((total, capture) => total + capture.bytes, 0);
274
+ for (const name of doomed) {
275
+ try {
276
+ await rm(path.join(dir, name), { force: true });
277
+ } catch {
278
+ // silent-ok: best effort, retried at the next start.
279
+ }
280
+ }
281
+ const keptBytes = captures
282
+ .filter((capture) => !doomed.includes(capture.name))
283
+ .reduce((total, capture) => total + capture.bytes, 0);
284
+ if (freed > 0 || captures.length > keep) {
285
+ logLine(
286
+ `packet witness: ${captures.length} capture(s) present, keeping the newest ${Math.min(keep, captures.length)}, ` +
287
+ `removed ${doomed.length} (${(freed / 1024).toFixed(1)} KB), keeping ${(keptBytes / 1024).toFixed(1)} KB`
288
+ );
289
+ }
290
+ }
291
+
292
+ /**
293
+ * Turn ring files left by a previous process into evidence, at startup.
294
+ *
295
+ * The ring is scratch while a process lives, but a process that was KILLED —
296
+ * which the crash family of roadmap item 1 does regularly, seven times in a
297
+ * fortnight — leaves behind the last seconds it recorded, and those seconds
298
+ * contain the death. Renaming them under a name the pruner recognises keeps
299
+ * them; without this the next viewer's first channel would delete them before
300
+ * anyone had looked.
301
+ *
302
+ * @param {string} dir
303
+ * @returns {Promise<string[]>} Names of the files adopted.
304
+ */
305
+ export async function adoptOrphanRingFiles(dir) {
306
+ if (typeof dir !== "string" || dir.length === 0) {
307
+ return [];
308
+ }
309
+ /** @type {string[]} */
310
+ const adopted = [];
311
+ let names;
312
+ try {
313
+ names = (await readdir(dir)).filter((name) => isWitnessRingFile(name)).sort();
314
+ } catch {
315
+ return adopted;
316
+ }
317
+ if (names.length === 0) {
318
+ return adopted;
319
+ }
320
+ const stampSeconds = Math.floor(Date.now() / 1000);
321
+ for (const name of names) {
322
+ const suffix = name.slice(WITNESS_RING_BASENAME.length) || "0";
323
+ const target = `packet-witness.orphan.${stampSeconds}.before${suffix}.pcap`;
324
+ try {
325
+ await rename(path.join(dir, name), path.join(dir, target));
326
+ adopted.push(target);
327
+ } catch {
328
+ // silent-ok: unreadable or already gone.
329
+ }
330
+ }
331
+ if (adopted.length > 0) {
332
+ logLine(
333
+ `packet witness: ${adopted.length} ring file(s) survived the previous process and were kept ` +
334
+ `as ${adopted.join(", ")} — whatever ended it is in them`
335
+ );
336
+ }
337
+ return adopted;
338
+ }
339
+
340
+ /**
341
+ * The logger handed to {@linkcode createPacketWitness}; module-level so the
342
+ * pruner can speak without an options bag threaded everywhere.
343
+ *
344
+ * @type {(message: string) => void}
345
+ */
346
+ let logLine = () => {};
347
+
348
+ /**
349
+ * Create the witness.
350
+ *
351
+ * @param {Object} options
352
+ * @param {(message: string) => void} options.log - Log sink (the shared logger).
353
+ * @param {string} [options.dir] - Where captures go; empty means os.tmpdir().
354
+ * @param {number} options.port - The WebRTC UDP port to filter on.
355
+ * @param {typeof spawn} [options.spawnProcess] - Seam for tests; defaults to node's spawn.
356
+ * @returns {{
357
+ * dir: string,
358
+ * maybeCapture: (trigger: WitnessTrigger) => boolean,
359
+ * holdRing: () => void,
360
+ * releaseRing: () => void
361
+ * }}
362
+ */
363
+ export function createPacketWitness({ log, dir = "", port, spawnProcess = spawn, mayKeep = null, chooseRemovals = null }) {
364
+ logLine = typeof log === "function" ? log : logLine;
365
+ const resolvedDir = typeof dir === "string" && dir.length > 0 ? dir : os.tmpdir();
366
+
367
+ /** @type {{ running: boolean, lastStartedAt: number, availability: "unknown" | "yes" | "no" }} */
368
+ const state = { running: false, lastStartedAt: 0, availability: "unknown" };
369
+
370
+ /**
371
+ * Connections already captured, so one wedge is recorded once.
372
+ *
373
+ * A capture of a connection already captured adds nothing: the wedge does not
374
+ * un-wedge, and the second recording of it is the same silence as the first.
375
+ * Field 2026-09-06: thirteen captures of ONE wedge over six hours, 74 MB, all
376
+ * saying what the first one said.
377
+ *
378
+ * @type {Set<string>}
379
+ */
380
+ const captured = new Set();
381
+
382
+ /**
383
+ * Ask whether tcpdump exists at all — once per process, whichever way it
384
+ * answers. A host without it gets exactly one log line, ever.
385
+ *
386
+ * @returns {Promise<boolean>}
387
+ */
388
+ const probeAvailability = () =>
389
+ new Promise((resolve) => {
390
+ let settled = false;
391
+ const done = (value) => {
392
+ if (!settled) {
393
+ settled = true;
394
+ resolve(value);
395
+ }
396
+ };
397
+ try {
398
+ const probe = spawnProcess("tcpdump", ["--version"], { stdio: "ignore" });
399
+ probe.on("error", () => done(false));
400
+ probe.on("spawn", () => done(true));
401
+ probe.on("close", () => done(true));
402
+ } catch {
403
+ done(false);
404
+ }
405
+ });
406
+
407
+ /**
408
+ * Keep the ring's history: stop it, copy its files, start it again.
409
+ *
410
+ * This is the half of the evidence the earlier version never had — the
411
+ * seconds BEFORE the freeze. Stopping first is not tidiness: tcpdump buffers,
412
+ * so the newest packets are only on disk once it has been asked to finish,
413
+ * and the file it is currently writing is the one that holds the onset. The
414
+ * gap this leaves is a fraction of a second, and the tail capture that
415
+ * follows covers everything after it.
416
+ *
417
+ * @param {WitnessTrigger} trigger
418
+ * @param {number} stampSeconds - Shared with the tail, so the pair sorts together.
419
+ * @returns {Promise<string[]>} Names of the copies that landed.
420
+ */
421
+ const preserveRing = async (trigger, stampSeconds) => {
422
+ /** @type {string[]} */
423
+ const kept = [];
424
+ ring.preserving = true;
425
+ try {
426
+ await stopRingProcess();
427
+ let names;
428
+ try {
429
+ names = (await readdir(resolvedDir)).filter((name) => isWitnessRingFile(name)).sort();
430
+ } catch {
431
+ return kept;
432
+ }
433
+ for (const name of names) {
434
+ const suffix = name.slice(WITNESS_RING_BASENAME.length) || "0";
435
+ const target = `packet-witness.${trigger.tag}.${stampSeconds}.before${suffix}.pcap`;
436
+ try {
437
+ await copyFile(path.join(resolvedDir, name), path.join(resolvedDir, target));
438
+ kept.push(target);
439
+ } catch {
440
+ // silent-ok: nothing to copy, or it vanished under us.
441
+ }
442
+ }
443
+ return kept;
444
+ } finally {
445
+ ring.preserving = false;
446
+ // Back to recording, unless everyone has gone in the meantime.
447
+ if (ring.holders > 0) {
448
+ await startRingProcess();
449
+ } else {
450
+ await clearRingFiles();
451
+ }
452
+ }
453
+ };
454
+
455
+ /**
456
+ * Run one bounded capture and report what it wrote.
457
+ *
458
+ * @param {WitnessTrigger} trigger
459
+ * @param {string} address - Validated remote IP literal.
460
+ * @param {number} stampSeconds
461
+ * @returns {Promise<void>}
462
+ */
463
+ const runCapture = async (trigger, address, stampSeconds) => {
464
+ const prefix = path.join(
465
+ resolvedDir,
466
+ `packet-witness.${trigger.tag}.${stampSeconds}.tail.pcap`
467
+ );
468
+ const args = buildTcpdumpArgs({
469
+ host: address,
470
+ port,
471
+ filePrefix: prefix,
472
+ fileMegabytes: WITNESS_TAIL_FILE_MB,
473
+ files: WITNESS_TAIL_FILES
474
+ });
475
+ logLine(
476
+ `packet witness: capturing ${WITNESS_TAIL_SECONDS}s of udp port ${port} ↔ ${address} ` +
477
+ `(session ${trigger.tag}, "${trigger.label}" queue ${trigger.queuedBytes}B wedged ` +
478
+ `${Math.round(trigger.stuckForMs / 1000)}s; ${WITNESS_TAIL_SECONDS}s is three times ` +
479
+ `usrsctp's ${WITNESS_RTO_CEILING_SECONDS}s retransmission ceiling, so a zero-window ` +
480
+ `probe cannot hide inside it) → ${prefix}`
481
+ );
482
+ await new Promise((resolve) => {
483
+ /** @type {ReturnType<typeof setTimeout> | null} */
484
+ let killer = null;
485
+ /** @type {NodeJS.Timeout | null} */
486
+ let hardKill = null;
487
+ let child;
488
+ try {
489
+ child = spawnProcess("tcpdump", args, { stdio: "ignore" });
490
+ } catch (error) {
491
+ logLine(`packet witness: could not start tcpdump: ${error?.message ?? error}`);
492
+ resolve();
493
+ return;
494
+ }
495
+ const finish = () => {
496
+ if (killer) {
497
+ clearTimeout(killer);
498
+ killer = null;
499
+ }
500
+ if (hardKill) {
501
+ clearTimeout(hardKill);
502
+ hardKill = null;
503
+ }
504
+ resolve();
505
+ };
506
+ child.on("error", (error) => {
507
+ state.availability = "no";
508
+ logLine(`packet witness: tcpdump failed to run: ${error?.message ?? error}`);
509
+ finish();
510
+ });
511
+ child.on("close", (code, signal) => {
512
+ logLine(
513
+ `packet witness: capture ended${signal ? ` (${signal})` : ` (exit ${code ?? "?"})`}`
514
+ );
515
+ finish();
516
+ });
517
+ killer = setTimeout(() => {
518
+ try { child.kill("SIGTERM"); } catch { /* already gone */ }
519
+ hardKill = setTimeout(() => {
520
+ try { child.kill("SIGKILL"); } catch { /* already gone */ }
521
+ }, WITNESS_KILL_GRACE_MS);
522
+ }, WITNESS_TAIL_SECONDS * 1000);
523
+ });
524
+ // Say what landed on disk, so whoever reads the log later knows whether
525
+ // the evidence exists without listing the directory themselves.
526
+ try {
527
+ const names = (await readdir(resolvedDir)).filter((name) => isWitnessCapture(name));
528
+ const mine = [];
529
+ for (const name of names) {
530
+ if (!name.startsWith(path.basename(prefix))) {
531
+ continue;
532
+ }
533
+ try {
534
+ const info = await stat(path.join(resolvedDir, name));
535
+ mine.push(`${name} ${(info.size / 1024).toFixed(1)} KB`);
536
+ } catch { /* gone between listing and reading */ }
537
+ }
538
+ logLine(`packet witness: wrote ${mine.length} file(s): ${mine.join(", ") || "none"}`);
539
+ } catch {
540
+ logLine("packet witness: could not list the capture directory");
541
+ }
542
+ };
543
+
544
+ /**
545
+ * Start a capture for this trigger, if the rules allow one.
546
+ *
547
+ * @param {WitnessTrigger} trigger
548
+ * @returns {boolean} True when a capture actually started.
549
+ */
550
+ const maybeCapture = (trigger) => {
551
+ const address = normalizeRemoteAddress(trigger?.remote?.address);
552
+ const portNumber = trigger?.remote?.port;
553
+ if (!address || !Number.isInteger(portNumber) || portNumber < 1 || portNumber > 65535) {
554
+ return false;
555
+ }
556
+ if (!shouldStartCapture(state)) {
557
+ return false;
558
+ }
559
+ const connection = `${address}:${portNumber}`;
560
+ if (captured.has(connection)) {
561
+ // Not evidence twice.
562
+ return false;
563
+ }
564
+ // The evidence takes room from the product, so it is asked for. Refused, it
565
+ // is a line rather than a silence — see `storage/Diagnostics.js`.
566
+ const expectedBytes = (WITNESS_RING_FILES + 1) * WITNESS_RING_FILE_MB * 1024 * 1024;
567
+ if (typeof mayKeep === "function" && !mayKeep(expectedBytes)) {
568
+ return false;
569
+ }
570
+ captured.add(connection);
571
+ state.running = true;
572
+ state.lastStartedAt = Date.now();
573
+ const startedAt = state.lastStartedAt;
574
+ void (async () => {
575
+ try {
576
+ if (state.availability === "unknown") {
577
+ state.availability = (await probeAvailability()) ? "yes" : "no";
578
+ }
579
+ if (state.availability === "no") {
580
+ logLine(
581
+ "packet witness: unavailable — no tcpdump on this host; " +
582
+ "the stuck-queue log lines remain the only evidence here"
583
+ );
584
+ return;
585
+ }
586
+ const stampSeconds = Math.floor(Date.now() / 1000);
587
+ const before = await preserveRing(trigger, stampSeconds);
588
+ logLine(
589
+ before.length > 0
590
+ ? `packet witness: kept ${before.length} ring file(s) from before the wedge: ${before.join(", ")}`
591
+ : "packet witness: no ring history to keep — the ring was not running"
592
+ );
593
+ await runCapture(trigger, address, stampSeconds);
594
+ // The disk bound has to hold between restarts too: one episode writes
595
+ // up to four ring copies and two tail files, and the cooldown allows
596
+ // six episodes an hour. Pruning only at startup let that accumulate.
597
+ await pruneWitnessCaptures(resolvedDir, chooseRemovals);
598
+ } finally {
599
+ state.running = false;
600
+ // Keep the requested spacing honest even when the capture ended early.
601
+ const earliestNext = startedAt + WITNESS_COOLDOWN_MS;
602
+ if (state.lastStartedAt < earliestNext) {
603
+ state.lastStartedAt = earliestNext;
604
+ }
605
+ }
606
+ })();
607
+ return true;
608
+ };
609
+
610
+ /**
611
+ * The rolling ring: one tcpdump, reference-counted by open data channels.
612
+ *
613
+ * @type {{
614
+ * child: import('node:child_process').ChildProcess | null,
615
+ * holders: number,
616
+ * starting: boolean,
617
+ * preserving: boolean,
618
+ * failed: boolean
619
+ * }}
620
+ */
621
+ const ring = { child: null, holders: 0, starting: false, preserving: false, failed: false };
622
+
623
+ /**
624
+ * Delete the ring's own files.
625
+ *
626
+ * Refused while a wedge is copying them out: the copies are the only reason
627
+ * the ring exists, and the viewer closing the tab is exactly when both happen
628
+ * at once. Deferred until the copy is done instead.
629
+ *
630
+ * @returns {Promise<void>}
631
+ */
632
+ const clearRingFiles = async () => {
633
+ if (ring.preserving) {
634
+ return;
635
+ }
636
+ try {
637
+ for (const name of await readdir(resolvedDir)) {
638
+ if (isWitnessRingFile(name)) {
639
+ await rm(path.join(resolvedDir, name), { force: true });
640
+ }
641
+ }
642
+ } catch {
643
+ // silent-ok: best effort, retried when the ring next stops.
644
+ }
645
+ };
646
+
647
+ /**
648
+ * Stop the ring's tcpdump and wait for it to flush and exit.
649
+ *
650
+ * The wait matters. tcpdump buffers its output, so the newest packets — the
651
+ * ONSET, which is the whole reason the ring runs — sit in that buffer until
652
+ * the process is asked to finish. `-U` would flush per packet instead, at one
653
+ * write syscall per packet, and at the measured 150 Mbps that is twelve
654
+ * thousand a second on a machine whose spare capacity is already the binding
655
+ * constraint. Stopping costs nothing while nothing is wrong.
656
+ *
657
+ * @returns {Promise<void>}
658
+ */
659
+ const stopRingProcess = () => {
660
+ const child = ring.child;
661
+ ring.child = null;
662
+ if (!child) {
663
+ return Promise.resolve();
664
+ }
665
+ return new Promise((resolve) => {
666
+ let settled = false;
667
+ const done = () => {
668
+ if (!settled) {
669
+ settled = true;
670
+ clearTimeout(hardKill);
671
+ resolve();
672
+ }
673
+ };
674
+ const hardKill = setTimeout(() => {
675
+ try { child.kill("SIGKILL"); } catch { /* already gone */ }
676
+ done();
677
+ }, WITNESS_KILL_GRACE_MS);
678
+ if (typeof hardKill.unref === "function") {
679
+ hardKill.unref();
680
+ }
681
+ child.on("close", done);
682
+ child.on("error", done);
683
+ try {
684
+ child.kill("SIGTERM");
685
+ } catch {
686
+ done();
687
+ }
688
+ });
689
+ };
690
+
691
+ /**
692
+ * Start the ring's tcpdump, if it is wanted and not already running.
693
+ *
694
+ * @returns {Promise<void>}
695
+ */
696
+ const startRingProcess = async () => {
697
+ if (ring.child !== null || ring.starting || ring.holders === 0) {
698
+ return;
699
+ }
700
+ ring.starting = true;
701
+ try {
702
+ if (state.availability === "unknown") {
703
+ state.availability = (await probeAvailability()) ? "yes" : "no";
704
+ }
705
+ // Everything above suspended; the last channel may have closed meanwhile.
706
+ // Without this second look the ring starts with nobody holding it and
707
+ // nobody left to stop it.
708
+ if (state.availability === "no" || ring.holders === 0) {
709
+ return;
710
+ }
711
+ const prefix = path.join(resolvedDir, WITNESS_RING_BASENAME);
712
+ const args = buildTcpdumpArgs({ port, filePrefix: prefix });
713
+ let child;
714
+ try {
715
+ child = spawnProcess("tcpdump", args, { stdio: "ignore" });
716
+ } catch (error) {
717
+ ring.failed = true;
718
+ logLine(`packet witness: the ring could not start: ${error?.message ?? error}`);
719
+ return;
720
+ }
721
+ ring.child = child;
722
+ child.on("error", (error) => {
723
+ ring.failed = true;
724
+ if (ring.child === child) {
725
+ ring.child = null;
726
+ }
727
+ logLine(`packet witness: the ring stopped: ${error?.message ?? error}`);
728
+ });
729
+ child.on("close", (code, signal) => {
730
+ if (ring.child !== child) {
731
+ return; // An orderly stop; it has already said what it needed to.
732
+ }
733
+ ring.child = null;
734
+ ring.failed = true;
735
+ // Ending on its own means it never recorded what it claimed to. A host
736
+ // with tcpdump installed but without the capability to capture answers
737
+ // `--version` happily and dies here, which would otherwise leave a
738
+ // "ring recording" line in the log and no history behind it.
739
+ logLine(
740
+ `packet witness: the ring ended on its own${signal ? ` (${signal})` : ` (exit ${code ?? "?"})`}` +
741
+ `${ring.holders > 0 ? " while a viewer was still being served — no history is being kept" : ""}`
742
+ );
743
+ });
744
+ logLine(
745
+ `packet witness: ring recording udp port ${port}, ` +
746
+ `${WITNESS_RING_FILES} × ${WITNESS_RING_FILE_MB} MB wrapping — ` +
747
+ "the seconds before a freeze, kept in advance"
748
+ );
749
+ } finally {
750
+ ring.starting = false;
751
+ // Holders may have gone while the probe or the spawn was in flight.
752
+ if (ring.holders === 0 && ring.child !== null) {
753
+ await stopRingProcess();
754
+ await clearRingFiles();
755
+ }
756
+ }
757
+ };
758
+
759
+ /**
760
+ * Begin recording, or note one more reason to keep recording.
761
+ *
762
+ * Reference-counted by open data channels, so the ring runs exactly while a
763
+ * viewer is being served and costs nothing on an idle proxy. Starting it is
764
+ * what makes the ONSET of a freeze readable: by the time a wedge is declared,
765
+ * the seconds before it are already on disk.
766
+ *
767
+ * @returns {void}
768
+ */
769
+ const holdRing = () => {
770
+ ring.holders += 1;
771
+ if (ring.holders !== 1) {
772
+ return;
773
+ }
774
+ void startRingProcess();
775
+ };
776
+
777
+ /**
778
+ * Release one reason to keep recording; the last one stops the ring.
779
+ *
780
+ * @returns {void}
781
+ */
782
+ const releaseRing = () => {
783
+ if (ring.holders > 0) {
784
+ ring.holders -= 1;
785
+ }
786
+ if (ring.holders > 0 || ring.starting) {
787
+ return;
788
+ }
789
+ void (async () => {
790
+ await stopRingProcess();
791
+ await clearRingFiles();
792
+ })();
793
+ };
794
+
795
+ /**
796
+ * Stop everything this witness owns. For process shutdown.
797
+ *
798
+ * @returns {Promise<void>}
799
+ */
800
+ const dispose = async () => {
801
+ ring.holders = 0;
802
+ await stopRingProcess();
803
+ await clearRingFiles();
804
+ };
805
+
806
+ return { dir: resolvedDir, maybeCapture, holdRing, releaseRing, dispose, ring };
807
+
808
+ }