@torrent-tv/proxy 2.88.2 → 2.88.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/opsx/apply.md +155 -155
- package/.claude/commands/opsx/archive.md +160 -160
- package/.claude/commands/opsx/explore.md +174 -174
- package/.claude/commands/opsx/propose.md +109 -109
- package/.claude/commands/opsx/sync.md +143 -143
- package/.claude/skills/openspec-apply-change/SKILL.md +159 -159
- package/.claude/skills/openspec-archive-change/SKILL.md +117 -117
- package/.claude/skills/openspec-explore/SKILL.md +289 -289
- package/.claude/skills/openspec-propose/SKILL.md +113 -113
- package/.claude/skills/openspec-sync-specs/SKILL.md +147 -147
- package/.dockerignore +5 -5
- package/AGENTS.md +322 -322
- package/CHANGELOG.md +2259 -2246
- package/Dockerfile +30 -30
- package/LICENSE +16 -16
- package/README.md +369 -369
- package/assets/calibration/NOTICE.md +100 -100
- package/assets/diagnostics/sctpstate.gdb +31 -31
- package/bin/cli.js +684 -684
- package/bin/thread-pool.js +30 -30
- package/biome.json +411 -411
- package/docs/browser-proxy-contract.md +23 -4
- package/docs/container-architecture.md +218 -218
- package/docs/download-architecture.md +261 -261
- package/docs/encode-architecture.md +729 -729
- package/docs/encode-run-state.md +96 -96
- package/docs/logs.md +97 -97
- package/docs/socks5-egress-research.md +114 -114
- package/docs/storage-architecture.md +267 -267
- package/knip.json +14 -14
- package/openspec/changes/adaptive-bitrate/design.md +92 -92
- package/openspec/changes/adaptive-bitrate/proposal.md +56 -56
- package/openspec/changes/adaptive-bitrate/specs/adaptive-bitrate/spec.md +51 -51
- package/openspec/changes/adaptive-bitrate/tasks.md +40 -40
- package/openspec/changes/binary-distribution/.openspec.yaml +2 -2
- package/openspec/changes/binary-distribution/proposal.md +53 -53
- package/openspec/changes/chunked-request-bodies/design.md +81 -81
- package/openspec/changes/chunked-request-bodies/proposal.md +53 -53
- package/openspec/changes/chunked-request-bodies/specs/chunked-request-bodies/spec.md +30 -30
- package/openspec/changes/chunked-request-bodies/tasks.md +40 -40
- package/openspec/changes/cold-start/design.md +98 -98
- package/openspec/changes/cold-start/proposal.md +62 -62
- package/openspec/changes/cold-start/specs/cold-start/spec.md +47 -47
- package/openspec/changes/cold-start/tasks.md +48 -48
- package/openspec/changes/disk-cap/.openspec.yaml +2 -2
- package/openspec/changes/disk-cap/proposal.md +37 -37
- package/openspec/changes/disk-cap/specs/disk-cap/spec.md +25 -25
- package/openspec/changes/disk-cap/tasks.md +19 -19
- package/openspec/changes/proxy-observability/.openspec.yaml +2 -2
- package/openspec/changes/proxy-observability/design.md +38 -38
- package/openspec/changes/proxy-observability/proposal.md +49 -49
- package/openspec/changes/proxy-observability/specs/observability/spec.md +33 -33
- package/openspec/changes/proxy-observability/tasks.md +21 -21
- package/openspec/changes/seek-debounce/design.md +118 -118
- package/openspec/changes/seek-debounce/proposal.md +59 -59
- package/openspec/changes/seek-debounce/specs/seek-debounce/spec.md +41 -41
- package/openspec/changes/seek-debounce/tasks.md +33 -33
- package/openspec/changes/subtitle-language/.openspec.yaml +2 -2
- package/openspec/changes/subtitle-language/proposal.md +50 -50
- package/openspec/changes/subtitle-language/specs/subtitle-language/spec.md +34 -34
- package/openspec/changes/subtitle-language/tasks.md +26 -26
- package/openspec/changes/track-selection/.openspec.yaml +2 -2
- package/openspec/changes/track-selection/design.md +47 -47
- package/openspec/changes/track-selection/proposal.md +45 -45
- package/openspec/changes/track-selection/specs/track-selection/spec.md +41 -41
- package/openspec/changes/track-selection/tasks.md +20 -20
- package/openspec/changes/transcode-quality/.openspec.yaml +2 -2
- package/openspec/changes/transcode-quality/proposal.md +51 -51
- package/openspec/changes/transcode-quality/specs/transcode-quality/spec.md +139 -139
- package/openspec/changes/transcode-quality/tasks.md +63 -63
- package/openspec/config.yaml +38 -38
- package/package.json +1 -1
- package/public/.well-known/appspecific/com.chrome.devtools.json +1 -1
- package/research/delivery-wedge-stand-ladder-2026-09-13.md +329 -329
- package/research/double-spawn-2026-09-10.md +171 -171
- package/research/handover-2026-09-13.md +209 -209
- package/research/handover-reader-claims-removal-2026-09-12.md +166 -166
- package/research/piece-withdrawn-but-still-claimed-2026-09-12.md +175 -175
- package/research/preset-benchmark-size-scaling-2026-08-22.md +55 -55
- package/research/priority-map-is-the-truth-2026-09-12.md +225 -225
- package/routes/api/client-logs/post.js +86 -86
- package/routes/api/delivery-sink/get.js +72 -72
- package/routes/api/link-probe/get.js +51 -51
- package/routes/api/playback-plan/audio-tracks/post.js +29 -29
- package/routes/api/playback-plan/post.js +53 -53
- package/routes/api/sources/post.js +36 -36
- package/routes/api/sources/stats/get.js +132 -132
- package/routes/api/sources/warm/post.js +217 -217
- package/routes/api/subtitles/get.js +230 -230
- package/routes/api/transcode-sessions/fragment-far/post.js +60 -60
- package/routes/api/transcode-sessions/net-report/post.js +138 -138
- package/routes/api/transcode-sessions/post.js +178 -173
- package/routes/api/transcode-sessions/progress/get.js +27 -27
- package/routes/api/transcode-sessions/release/post.js +41 -41
- package/routes/api/transcode-sessions/seek/post.js +63 -63
- package/routes/health/get.js +13 -13
- package/routes/healthz/get.js +13 -13
- package/routes/stream/get.js +380 -380
- package/routes/transcode/audio-file/get.js +64 -64
- package/routes/transcode/audio-warm/get.js +95 -84
- package/routes/transcode/session-file/get.js +52 -52
- package/routes/transcode/variant-file/get.js +95 -95
- package/routes/transcode/variant-warm/get.js +84 -84
- package/scripts/render-run-graph.js +173 -173
- package/server.js +725 -725
- package/services/encode/CoverageMap.js +428 -428
- package/services/encode/CushionReport.js +345 -345
- package/services/encode/EncodeAdmission.js +376 -376
- package/services/encode/EncodeOrchestrator.js +1217 -1217
- package/services/encode/EncodePlan.js +1366 -1366
- package/services/encode/EncodeRun.js +813 -813
- package/services/encode/EncodeRuns.js +1235 -1235
- package/services/encode/Encoder.js +142 -142
- package/services/encode/EncoderSelection.js +38 -38
- package/services/encode/LocalObservations.js +359 -359
- package/services/encode/NvencEncoder.js +50 -50
- package/services/encode/OutputOpening.js +783 -758
- package/services/encode/OutputTimes.js +579 -579
- package/services/encode/QsvEncoder.js +58 -58
- package/services/encode/Renditions.js +2282 -2184
- package/services/encode/RunProgress.js +84 -84
- package/services/encode/SegmentDemand.js +95 -95
- package/services/encode/SoftwareEncoder.js +121 -121
- package/services/encode/V4l2m2mEncoder.js +53 -53
- package/services/encode/VaapiEncoder.js +66 -66
- package/services/encode/args.js +418 -416
- package/services/encode/available-share.js +89 -89
- package/services/encode/calibration.js +361 -361
- package/services/encode/contention.js +128 -128
- package/services/encode/decode-cost-fit.js +363 -363
- package/services/encode/encode-exit.js +114 -114
- package/services/encode/encode-run-state.js +551 -551
- package/services/encode/encoder-readings.js +45 -45
- package/services/encode/fingerprint.js +322 -322
- package/services/encode/host-load.js +174 -174
- package/services/encode/hwaccel.js +1894 -1894
- package/services/encode/index.js +9 -9
- package/services/encode/learned-median.js +90 -90
- package/services/encode/output/EncodedOutput.js +57 -57
- package/services/encode/output/Output.js +131 -131
- package/services/encode/output/OutputCatalog.js +414 -414
- package/services/encode/output/OutputSpec.js +390 -390
- package/services/encode/output/Timeline.js +308 -308
- package/services/encode/output/cut-grid.js +107 -107
- package/services/encode/output/index.js +20 -20
- package/services/encode/output/ladder.js +32 -32
- package/services/encode/output/playlists.js +284 -284
- package/services/encode/output/rates.js +125 -125
- package/services/encode/output-key-format.js +21 -21
- package/services/encode/quality/EncodeCost.js +1150 -1141
- package/services/encode/quality/HostLoad.js +525 -525
- package/services/encode/quality/HostTimings.js +249 -249
- package/services/encode/quality/OutputQualityState.js +28 -28
- package/services/encode/quality/QualityController.js +935 -935
- package/services/encode/quality/QualityOffer.js +416 -416
- package/services/encode/quality/drain-threat.js +47 -47
- package/services/encode/quality/link-budget.js +349 -254
- package/services/encode/quality/output-format.js +319 -315
- package/services/encode/quality/serving-output.js +216 -216
- package/services/encode/quality/visible-rung.js +57 -57
- package/services/encode/run-budget.js +97 -97
- package/services/encode/run-command.js +754 -754
- package/services/encode/run-costs.js +155 -155
- package/services/encode/run-inputs.js +42 -42
- package/services/encode/segment-formats/fmp4.js +456 -456
- package/services/encode/segment-formats/index.js +85 -85
- package/services/encode/segment-formats/init-compat.js +324 -324
- package/services/encode/segment-formats/mp4-boxes.js +21 -6
- package/services/encode/segment-formats/mpegts.js +134 -134
- package/services/encode/start-stop-cost.js +178 -178
- package/services/encode/throughput.js +97 -97
- package/services/encode/torrent-cost.js +88 -88
- package/services/identity/installation-id.js +38 -38
- package/services/media/ContainerOrchestrator.js +149 -149
- package/services/media/KeyframeTables.js +316 -316
- package/services/media/SourceFile.js +298 -298
- package/services/media/SubtitleCues.js +520 -520
- package/services/media/SubtitleOrchestrator.js +167 -167
- package/services/media/audio-inventory.js +298 -245
- package/services/media/container/AviContainer.js +266 -266
- package/services/media/container/Container.js +609 -603
- package/services/media/container/ContainerFactory.js +56 -56
- package/services/media/container/KeyframeTable.js +272 -272
- package/services/media/container/MatroskaContainer.js +1494 -1494
- package/services/media/container/Mp4Container.js +1344 -1244
- package/services/media/container/ebml-reader.js +159 -159
- package/services/media/ffmpeg-banner.js +260 -260
- package/services/media/keyframe-probe.js +157 -157
- package/services/media/media-info-probe.js +93 -93
- package/services/media/playback-planner.js +931 -931
- package/services/media/tracks/AudioTrack.js +143 -131
- package/services/media/tracks/ContainerTrack.js +75 -75
- package/services/media/tracks/ImageSubtitleTrack.js +19 -19
- package/services/media/tracks/SubtitleTrack.js +38 -38
- package/services/media/tracks/TextSubtitleTrack.js +287 -287
- package/services/media/tracks/VideoTrack.js +33 -33
- package/services/media/tracks/audio-bound.js +233 -0
- package/services/media/tracks/index.js +15 -15
- package/services/media/tracks/language-detect.js +174 -174
- package/services/server/OutputLifecycle.js +418 -418
- package/services/server/SegmentServing.js +1150 -1150
- package/services/server/ViewerRequests.js +760 -760
- package/services/server/WaitLedger.js +142 -142
- package/services/server/controllers/PlaybackController.js +40 -40
- package/services/server/controllers/SubtitleController.js +109 -109
- package/services/server/http-responses.js +60 -60
- package/services/server/playback-readiness.js +13 -8
- package/services/server/transcode-session-files.js +325 -325
- package/services/server/wire-outputs.js +995 -992
- package/services/storage/Diagnostics.js +202 -202
- package/services/storage/MachineBudget.js +235 -235
- package/services/storage/allowance.js +107 -107
- package/services/storage/core-dumps.js +106 -106
- package/services/storage/files/CompletedFiles.js +391 -391
- package/services/storage/files/piece-from-whole-file.js +118 -118
- package/services/storage/free.js +71 -71
- package/services/storage/keep.js +48 -48
- package/services/storage/machine-memory.js +56 -56
- package/services/storage/memory-report.js +575 -575
- package/services/storage/piece-store/piece-disk-store.js +539 -539
- package/services/storage/piece-store/piece-lru.js +495 -495
- package/services/storage/piece-store/shared-piece-store.js +1931 -1931
- package/services/storage/returns.js +103 -103
- package/services/storage/segment-store/SegmentStore.js +1240 -1240
- package/services/storage/wire.js +175 -175
- package/services/torrent/demand/DemandRegister.js +202 -202
- package/services/torrent/demand/Urgency.js +137 -137
- package/services/torrent/demand/Window.js +118 -118
- package/services/torrent/demand/index.js +8 -8
- package/services/torrent/demand/pieces.js +140 -140
- package/services/torrent/download/SwarmSelection.js +386 -386
- package/services/torrent/download/registry.js +102 -102
- package/services/torrent/download/withdraw-claim.js +80 -80
- package/services/torrent/episode-naming.js +182 -182
- package/services/torrent/files.js +455 -455
- package/services/torrent/naming.js +619 -619
- package/services/torrent/piece-store-of.js +29 -29
- package/services/torrent/supply-margin.js +262 -262
- package/services/torrent/torrent-pool.js +2675 -2675
- package/services/torrent/worker/background-fill.js +213 -213
- package/services/torrent/worker/channel.js +271 -271
- package/services/torrent/worker/client.js +765 -765
- package/services/torrent/worker/fastest-wires.js +342 -342
- package/services/torrent/worker/handle-state.js +23 -23
- package/services/torrent/worker/held-bytes.js +155 -155
- package/services/torrent/worker/install-webrtc-shim.js +32 -32
- package/services/torrent/worker/piece-reader.js +1078 -1078
- package/services/torrent/worker/pool-adapter.js +371 -371
- package/services/torrent/worker/protocol.js +184 -184
- package/services/torrent/worker/resume-warm.js +107 -107
- package/services/torrent/worker/webrtc-shim.js +219 -219
- package/services/torrent/worker/worker.js +1129 -1129
- package/services/transport/data-channel-handler.js +1369 -1369
- package/services/transport/delivery-probe.js +842 -842
- package/services/transport/health-collector.js +53 -53
- package/services/transport/nat-classifier.js +231 -231
- package/services/transport/packet-witness.js +808 -808
- package/services/transport/port-mapper.js +235 -235
- package/services/transport/registry-api.js +81 -81
- package/services/transport/tunnel-client.js +447 -447
- package/services/transport/usrsctp-state.js +175 -175
- package/services/transport/webrtc-manager.js +484 -484
- package/services/viewer/Assignments.js +340 -340
- package/services/viewer/PriorityMap.js +390 -390
- package/services/viewer/PriorityOrchestrator.js +325 -325
- package/services/viewer/Viewer.js +691 -691
- package/services/viewer/Viewers.js +368 -368
- package/services/viewer/active-output.js +37 -37
- package/services/viewer/audio-start.js +46 -46
- package/services/viewer/buffer-trend.js +81 -81
- package/services/viewer/choices.js +647 -647
- package/services/viewer/positions.js +126 -126
- package/services/viewer/report-intake.js +46 -46
- package/store/source-registry.js +71 -71
- package/test/asset-paths.test.js +28 -28
- package/test/audio-buffer-continuity.test.js +70 -0
- package/test/audio-inventory.test.js +212 -212
- package/test/audio-track-bitrate.test.js +40 -40
- package/test/auto-quality-step.test.js +707 -707
- package/test/available-share.test.js +76 -76
- package/test/background-fill.test.js +190 -190
- package/test/client-log-file.test.js +110 -110
- package/test/completed-files.test.js +115 -115
- package/test/concurrent-cost.test.js +148 -148
- package/test/container-media-info.test.js +228 -228
- package/test/contention.test.js +76 -76
- package/test/core-dumps.test.js +57 -57
- package/test/coverage-follows-the-disk.test.js +228 -228
- package/test/coverage-map.test.js +195 -195
- package/test/cue-push-carries-what-the-walk-found.test.js +58 -58
- package/test/cut-times-timeline.test.js +65 -65
- package/test/cuts-follow-published-grid.test.js +109 -109
- package/test/data-channel-frame.test.js +63 -63
- package/test/data-channel-viewer-lifecycle.test.js +92 -92
- package/test/dead-torrent-handle.test.js +52 -52
- package/test/declared-rates.test.js +116 -116
- package/test/declared-tracks.test.js +43 -43
- package/test/decode-cost-fit.test.js +202 -202
- package/test/decode-cost.test.js +462 -462
- package/test/delivery-probe.test.js +393 -393
- package/test/demand-register.test.js +206 -206
- package/test/dht-bootstrap.test.js +72 -72
- package/test/drain-threat.test.js +65 -65
- package/test/duplicate-add.test.js +48 -48
- package/test/encode-admission.test.js +146 -146
- package/test/encode-cost.test.js +283 -283
- package/test/encode-exit.test.js +111 -111
- package/test/encode-orchestrator.test.js +419 -419
- package/test/encode-plan-viewers.test.js +718 -718
- package/test/encode-plan.test.js +545 -545
- package/test/encode-run-state.test.js +314 -314
- package/test/encode-run.test.js +162 -162
- package/test/encode-slope.test.js +46 -46
- package/test/encoded-output.test.js +39 -39
- package/test/encoder-kinds.test.js +122 -122
- package/test/encoder-readings.test.js +46 -46
- package/test/episode-naming.test.js +131 -131
- package/test/every-claimant-is-told-its-share.test.js +152 -152
- package/test/failed-encoder-still-serves-what-exists.test.js +354 -354
- package/test/failed-start-breaker.test.js +202 -202
- package/test/fastest-wires.test.js +143 -143
- package/test/ffmpeg-stream-counts.test.js +78 -78
- package/test/file-edges.test.js +191 -191
- package/test/fixtures/audio-frame-boundary.json +1 -0
- package/test/fmp4-presentation-clock.test.js +147 -147
- package/test/fmp4-self-contained.test.js +80 -80
- package/test/fmp4-timestamps.test.js +140 -140
- package/test/fmp4-track-completeness.test.js +41 -41
- package/test/format-keyed-outputs.test.js +78 -78
- package/test/health-metrics.test.js +38 -38
- package/test/held-request-width.test.js +157 -157
- package/test/helpers/encode-run.js +136 -136
- package/test/helpers/held-file.js +29 -29
- package/test/helpers/manager.js +57 -57
- package/test/helpers/output-spec.js +63 -63
- package/test/host-load.test.js +68 -68
- package/test/index-accuracy-is-about-the-file.test.js +128 -128
- package/test/init-compat.test.js +219 -219
- package/test/input-lost-quiets-the-plan.test.js +261 -261
- package/test/input-unavailable.test.js +48 -48
- package/test/installation-id.test.js +39 -39
- package/test/keeping-period.test.js +83 -83
- package/test/keyframe-index-accuracy.test.js +188 -188
- package/test/keyframe-table-budget.test.js +102 -102
- package/test/keyframes-belong-to-the-file.test.js +249 -249
- package/test/learned-median.test.js +37 -37
- package/test/letting-go-asks-who-is-watching.test.js +191 -191
- package/test/limit-row-by-area.test.js +60 -60
- package/test/link-budget.test.js +112 -111
- package/test/link-probe.test.js +56 -56
- package/test/local-encoding-data.test.js +273 -273
- package/test/logger-repeats.test.js +134 -134
- package/test/look-ahead-coverage.test.js +56 -56
- package/test/manager-surface.test.js +292 -292
- package/test/matroska-cues-track.test.js +192 -192
- package/test/memory-budget.test.js +390 -390
- package/test/move-cost.test.js +216 -216
- package/test/mp4-subtitles.test.js +173 -173
- package/test/one-authority.test.js +285 -285
- package/test/one-budget.test.js +149 -149
- package/test/one-output-one-session.test.js +270 -204
- package/test/one-resource-divided.test.js +185 -185
- package/test/one-serving-output-per-height.test.js +458 -458
- package/test/orchestrator-wired.test.js +206 -206
- package/test/output-catalog.test.js +39 -39
- package/test/output-shape.test.js +21 -21
- package/test/output-spec.test.js +251 -251
- package/test/output-speed.test.js +88 -88
- package/test/outputs-being-prepared.test.js +40 -40
- package/test/packet-witness-ring.test.js +236 -236
- package/test/packet-witness.test.js +148 -148
- package/test/piece-disk-store.test.js +401 -401
- package/test/piece-from-whole-file.test.js +129 -129
- package/test/piece-lru.test.js +319 -319
- package/test/piece-reader.test.js +162 -162
- package/test/piece-selection-offset.test.js +85 -85
- package/test/piece-store-eviction.test.js +595 -595
- package/test/piece-store-never-refuses.test.js +153 -153
- package/test/piece-store-reservations.test.js +403 -403
- package/test/piece-store-slow-disk.test.js +159 -159
- package/test/piece-tail.test.js +97 -97
- package/test/plan-host-timings.test.js +68 -68
- package/test/plan-is-steady.test.js +225 -225
- package/test/playback-readiness.test.js +453 -453
- package/test/playback-start-regressions.test.js +99 -99
- package/test/priority-map-download.test.js +174 -174
- package/test/priority-map-emptied.test.js +204 -204
- package/test/priority-map-per-output.test.js +215 -215
- package/test/priority-map.test.js +163 -163
- package/test/probe-wedge-certainty.test.js +60 -60
- package/test/produced-copy-choice.test.js +213 -213
- package/test/published-timeline.test.js +83 -83
- package/test/quality-offer.test.js +162 -162
- package/test/quality-variants.test.js +1257 -1229
- package/test/rate-control-in-the-output.test.js +180 -180
- package/test/read-survives-withdrawal.test.js +164 -164
- package/test/read-window.test.js +365 -365
- package/test/resume-warm.test.js +39 -39
- package/test/run-graph-drift.test.js +37 -37
- package/test/run-inputs.test.js +80 -80
- package/test/run-intervals.test.js +100 -100
- package/test/run-position-follows-published-grid.test.js +89 -89
- package/test/run-progress.test.js +65 -65
- package/test/same-height-switch.test.js +664 -664
- package/test/seek-frees-the-viewers-requests.test.js +217 -217
- package/test/seek-landing.test.js +109 -109
- package/test/seek-target-not-superseded.test.js +195 -195
- package/test/segment-closed-on-both-branches.test.js +96 -96
- package/test/segment-cut-times.test.js +63 -63
- package/test/segment-demand.test.js +80 -80
- package/test/segment-hold-supersede.test.js +91 -91
- package/test/segment-judged-by-its-own-header.test.js +221 -221
- package/test/segment-serve-wiring.test.js +475 -475
- package/test/segment-store-eviction.test.js +247 -247
- package/test/segment-store.test.js +323 -323
- package/test/segment-true-start.test.js +86 -86
- package/test/segments-are-shared.test.js +176 -176
- package/test/send-chunk-bytes.test.js +86 -86
- package/test/serving-existing-output.test.js +105 -102
- package/test/serving-output.test.js +102 -102
- package/test/shared-piece-store.test.js +310 -310
- package/test/sidecar-naming.test.js +142 -142
- package/test/soundtrack-rate-bound.test.js +289 -0
- package/test/source-file.test.js +133 -133
- package/test/source-files-route.test.js +113 -113
- package/test/source-registry.test.js +18 -18
- package/test/split-a-stretch.test.js +139 -139
- package/test/stale-request-after-seek.test.js +158 -158
- package/test/startup-readings.test.js +124 -124
- package/test/stream-route.test.js +209 -209
- package/test/subtitle-cue-framing.test.js +200 -200
- package/test/subtitle-cue-source.test.js +102 -102
- package/test/subtitle-cue-walk.test.js +370 -370
- package/test/subtitle-cursor.test.js +63 -63
- package/test/subtitle-defaults.test.js +118 -118
- package/test/subtitle-language.test.js +252 -252
- package/test/subtitle-track-numbering.test.js +386 -386
- package/test/supply-margin.test.js +177 -177
- package/test/swarm-follows-readers.test.js +174 -174
- package/test/swarm-reach.test.js +130 -130
- package/test/swarm-selection.test.js +220 -220
- package/test/tail-duplication.test.js +167 -167
- package/test/the-channel-has-both-ends.test.js +91 -91
- package/test/the-torrent-thread-serves-bytes.test.js +107 -107
- package/test/thread-pool.test.js +53 -53
- package/test/timeline.test.js +84 -84
- package/test/torrent-contents.test.js +235 -235
- package/test/torrent-cost.test.js +71 -71
- package/test/torrent-files.test.js +221 -221
- package/test/torrent-naming.test.js +255 -255
- package/test/tracks-begin-together.test.js +209 -209
- package/test/tunnel-renewal.test.js +167 -167
- package/test/two-viewers-one-picture.test.js +504 -407
- package/test/upload-hurry.test.js +186 -186
- package/test/usrsctp-state.test.js +131 -131
- package/test/video-facts.test.js +102 -102
- package/test/viewer-assignments.test.js +143 -143
- package/test/viewer-outputs.test.js +285 -285
- package/test/viewer-presence.test.js +116 -116
- package/test/viewer-requests-admission.test.js +89 -89
- package/test/viewer-states.test.js +189 -189
- package/test/viewer-subtitle-subscription.test.js +96 -96
- package/test/viewer.test.js +139 -139
- package/test/viewing-generations.test.js +390 -390
- package/test/visible-rung.test.js +37 -37
- package/test/wait-ledger.test.js +100 -100
- package/test/webrtc-shim.test.js +149 -149
- package/test/wedge-certainty.test.js +131 -131
- package/test/withdraw-piece-claim.test.js +212 -212
- package/test/worker-channel.test.js +158 -158
- package/utils/client-log-file.js +151 -151
- package/utils/logger.js +312 -312
- package/utils/parse-range.js +25 -25
- package/utils/perf.js +59 -59
- package/utils/torrent-source-key.js +38 -38
|
@@ -1,1129 +1,1129 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file The torrent thread: WebTorrent and nothing else.
|
|
3
|
-
*
|
|
4
|
-
* Everything that made the main thread unresponsive lives here now — peer
|
|
5
|
-
* connections, buffer concatenation, piece bookkeeping, garbage collection from
|
|
6
|
-
* all of it. The main thread keeps only what owes a viewer a prompt answer.
|
|
7
|
-
*
|
|
8
|
-
* This file deliberately holds no HTTP, no session logic and no knowledge of
|
|
9
|
-
* HLS: it answers the commands in `protocol.js` and streams bytes back. That
|
|
10
|
-
* boundary is what keeps the split honest — anything added here will compete
|
|
11
|
-
* with the torrent for this thread, which is exactly the problem being solved.
|
|
12
|
-
*
|
|
13
|
-
* The existing `TorrentPool` is reused wholesale rather than reimplemented. It
|
|
14
|
-
* already carries the parts that took field failures to get right — idle
|
|
15
|
-
* removal, the global disk cap with LRU eviction, seek-aware piece
|
|
16
|
-
* prioritisation, adaptive upload — and none of that changes by moving threads.
|
|
17
|
-
*/
|
|
18
|
-
|
|
19
|
-
// MUST stay first: it redirects `webrtc-polyfill` to a JavaScript WebRTC stack
|
|
20
|
-
// before WebTorrent can reach the native one. Two isolates using
|
|
21
|
-
// node-datachannel at once abort the process, and the torrent's wss trackers
|
|
22
|
-
// create peer connections of their own.
|
|
23
|
-
import { isUsableTorrentHandle } from "./handle-state.js";
|
|
24
|
-
import "./install-webrtc-shim.js";
|
|
25
|
-
import { parentPort, workerData } from "node:worker_threads";
|
|
26
|
-
import { createSendStream } from "./channel.js";
|
|
27
|
-
import { readFragments, supplyFiguresFor } from "./piece-reader.js";
|
|
28
|
-
import {
|
|
29
|
-
warmResumePosition
|
|
30
|
-
} from "./resume-warm.js";
|
|
31
|
-
import { fillFileInBackground } from "./background-fill.js";
|
|
32
|
-
import { demandFor } from "../download/registry.js";
|
|
33
|
-
import { heldRangesOf, readHeldBytes } from "./held-bytes.js";
|
|
34
|
-
import { CompletedFiles, completedFilesRoot } from "../../storage/files/CompletedFiles.js";
|
|
35
|
-
import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../../storage/files/piece-from-whole-file.js";
|
|
36
|
-
import { Command, Event } from "./protocol.js";
|
|
37
|
-
import { startMemoryReport, WORKER_MEMORY_SAMPLE_MS } from "../../storage/memory-report.js";
|
|
38
|
-
import { forwardLogsTo, logger } from "../../../utils/logger.js";
|
|
39
|
-
|
|
40
|
-
/**
|
|
41
|
-
* Everything this thread logs goes to the main thread, which owns the file.
|
|
42
|
-
*
|
|
43
|
-
* Two threads cannot both write it — they would race on the rotation and could
|
|
44
|
-
* interleave mid-line — so there is one writer and this is how everyone else
|
|
45
|
-
* reaches it. Set before anything else runs, because a line written before this
|
|
46
|
-
* point reaches the console only, and the console is destroyed by every
|
|
47
|
-
* release.
|
|
48
|
-
*
|
|
49
|
-
* Until 2026-09-02 only the `log` function below took this route. Modules that
|
|
50
|
-
* called `logger.*` directly — the piece reader, the torrent pool, the
|
|
51
|
-
* background fill, the container track reader, the subtitle walk — wrote into a
|
|
52
|
-
* copy of the logger that had no file, and every one of their lines was lost:
|
|
53
|
-
* measured over a whole 49 938-line file, not one of them was in it.
|
|
54
|
-
*/
|
|
55
|
-
forwardLogsTo((level, message) => {
|
|
56
|
-
// THE LEVEL TRAVELS TOO. It was dropped here, so every line this thread wrote
|
|
57
|
-
// — a warning about a spill that failed, an error about a torrent that went
|
|
58
|
-
// away — arrived on the other side as information and was coloured and
|
|
59
|
-
// recorded as such.
|
|
60
|
-
parentPort.postMessage({ type: Event.LOG, level, message });
|
|
61
|
-
});
|
|
62
|
-
|
|
63
|
-
// Imported dynamically, and that is load-bearing: static imports are RESOLVED
|
|
64
|
-
// during linking, before any module body runs, so a statically imported pool
|
|
65
|
-
// would drag in WebTorrent — and with it the real `webrtc-polyfill` — before
|
|
66
|
-
// the hook above had a chance to register. Verified the hard way: with a static
|
|
67
|
-
// import the process still aborted, and the stack named the genuine polyfill.
|
|
68
|
-
const { TorrentPool, resolveDhtBootstrap } = await import("../torrent-pool.js");
|
|
69
|
-
const { SharedPieceStore, collectStoreStats, findSharedStore, machineReserveBytes, memoryClaim, pieceBufferCollection, reviseSpillBudgets, reviseStoreBudgets } =
|
|
70
|
-
await import("../../storage/piece-store/shared-piece-store.js");
|
|
71
|
-
|
|
72
|
-
// Resolved before the client exists, because the client builds its DHT in its
|
|
73
|
-
// own constructor and the addresses have to be in hand by then. Awaiting here
|
|
74
|
-
// costs the few milliseconds of a DNS answer, once, on a thread that has not
|
|
75
|
-
// been asked for anything yet.
|
|
76
|
-
const dhtBootstrap = await resolveDhtBootstrap();
|
|
77
|
-
|
|
78
|
-
const pool = new TorrentPool({
|
|
79
|
-
memoryBytes: workerData?.memoryBytes,
|
|
80
|
-
dhtBootstrap,
|
|
81
|
-
pieceStore: SharedPieceStore
|
|
82
|
-
});
|
|
83
|
-
|
|
84
|
-
/** Torrents by sourceKey — the main thread names them, this thread owns them. */
|
|
85
|
-
const torrentsByKey = new Map();
|
|
86
|
-
|
|
87
|
-
/**
|
|
88
|
-
* How each source was named when it was added, so a torrent that has since been
|
|
89
|
-
* destroyed can be added again. Kept separately from {@link torrentsByKey}
|
|
90
|
-
* because that map holds the promise, not the recipe.
|
|
91
|
-
*
|
|
92
|
-
* @type {Map<string, { sourceType: string, source: string }>}
|
|
93
|
-
*/
|
|
94
|
-
const sourceRecipes = new Map();
|
|
95
|
-
/** In-flight reads, so a cancel can stop one mid-body. */
|
|
96
|
-
const readsById = new Map();
|
|
97
|
-
|
|
98
|
-
/**
|
|
99
|
-
* Shorthand for this file. The same path as `logger.info` anywhere else in the
|
|
100
|
-
* thread — kept only because it reads better at the hundred call sites here.
|
|
101
|
-
*
|
|
102
|
-
* @param {string} message
|
|
103
|
-
* @returns {void}
|
|
104
|
-
*/
|
|
105
|
-
function log(message) {
|
|
106
|
-
logger.info(message);
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/**
|
|
110
|
-
* The torrent for a sourceKey, waiting for it if it is still being added.
|
|
111
|
-
*
|
|
112
|
-
* The map holds a PROMISE, registered the moment the add begins rather than
|
|
113
|
-
* when it finishes. That distinction is the whole fix: adding a magnet takes as
|
|
114
|
-
* long as its metadata does — seconds to tens of seconds — and until 2.9.77
|
|
115
|
-
* everything naming that source in the meantime was told `Unknown source`,
|
|
116
|
-
* which is false. The source exists; it is not ready. Reproduced with a magnet
|
|
117
|
-
* nobody seeds: stats, the file listing and a read all failed instantly while
|
|
118
|
-
* the add was still in flight, which on the loading screen shows up as no
|
|
119
|
-
* peers, no progress, and a plan request that fails before the torrent has had
|
|
120
|
-
* a chance to start.
|
|
121
|
-
*
|
|
122
|
-
* A source that was never added still throws, which is the honest answer.
|
|
123
|
-
*
|
|
124
|
-
* @param {string} sourceKey
|
|
125
|
-
* @returns {Promise<import("webtorrent").Torrent>}
|
|
126
|
-
*/
|
|
127
|
-
/**
|
|
128
|
-
* The torrent for a sourceKey IF there is one, and never one that has to be
|
|
129
|
-
* built to answer.
|
|
130
|
-
*
|
|
131
|
-
* The other half of the pair above, for the questions that are about something
|
|
132
|
-
* going away: a departure is not a reason to add anything, and the answer
|
|
133
|
-
* "there is nothing here" is a complete answer to them.
|
|
134
|
-
*
|
|
135
|
-
* @param {string} sourceKey
|
|
136
|
-
* @returns {Promise<import("webtorrent").Torrent | null>}
|
|
137
|
-
*/
|
|
138
|
-
async function knownTorrent(sourceKey) {
|
|
139
|
-
const pending = torrentsByKey.get(sourceKey);
|
|
140
|
-
if (!pending) {
|
|
141
|
-
return null;
|
|
142
|
-
}
|
|
143
|
-
const torrent = await pending.catch(() => null);
|
|
144
|
-
return isUsableTorrentHandle(torrent) ? torrent : null;
|
|
145
|
-
}
|
|
146
|
-
|
|
147
|
-
async function requireTorrent(sourceKey) {
|
|
148
|
-
const pending = torrentsByKey.get(sourceKey);
|
|
149
|
-
if (!pending) {
|
|
150
|
-
throw new Error(`Unknown source ${sourceKey}.`);
|
|
151
|
-
}
|
|
152
|
-
const torrent = await pending;
|
|
153
|
-
if (isUsableTorrentHandle(torrent)) {
|
|
154
|
-
return torrent;
|
|
155
|
-
}
|
|
156
|
-
// The pool destroys a torrent that has gone unread for a quarter of an hour,
|
|
157
|
-
// and under disk pressure. It clears its OWN map when it does; this one it
|
|
158
|
-
// knows nothing about, so the promise here went on resolving to a corpse: a
|
|
159
|
-
// destroyed torrent keeps its object but loses its files. Every later session
|
|
160
|
-
// for that source then failed the same way — the plan and the codec probe
|
|
161
|
-
// answered from cache in milliseconds, nothing waited for metadata because
|
|
162
|
-
// everything believed the torrent was known, and ffmpeg's first read died on
|
|
163
|
-
// `File N not found` 130 ms in, after which the session answered 500 for
|
|
164
|
-
// ever. Measured 2026-08-06 on two sessions in a row, both from a phone,
|
|
165
|
-
// which is what made it look like a mobile problem.
|
|
166
|
-
const recipe = sourceRecipes.get(sourceKey);
|
|
167
|
-
if (!recipe) {
|
|
168
|
-
torrentsByKey.delete(sourceKey);
|
|
169
|
-
throw new Error(`Source ${sourceKey} is gone and cannot be re-added.`);
|
|
170
|
-
}
|
|
171
|
-
const revived = pool.getTorrent(recipe.sourceType, recipe.source);
|
|
172
|
-
torrentsByKey.set(sourceKey, revived);
|
|
173
|
-
revived.catch(() => {
|
|
174
|
-
if (torrentsByKey.get(sourceKey) === revived) {
|
|
175
|
-
torrentsByKey.delete(sourceKey);
|
|
176
|
-
}
|
|
177
|
-
});
|
|
178
|
-
return revived;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
/**
|
|
183
|
-
* Fragments waiting for the main thread to say it has finished reading them,
|
|
184
|
-
* keyed by request id. One per read, because only one fragment is in flight.
|
|
185
|
-
*
|
|
186
|
-
* @type {Map<number, () => void>}
|
|
187
|
-
*/
|
|
188
|
-
const fragmentWaiters = new Map();
|
|
189
|
-
|
|
190
|
-
/**
|
|
191
|
-
* Wake a read that is waiting for a fragment to be confirmed.
|
|
192
|
-
*
|
|
193
|
-
* Used both by the confirmation itself and by cancellation — a cancelled read
|
|
194
|
-
* will never be confirmed, and without this it would wait forever holding a pin.
|
|
195
|
-
*
|
|
196
|
-
* @param {number} id
|
|
197
|
-
* @returns {void}
|
|
198
|
-
*/
|
|
199
|
-
function settleFragment(id) {
|
|
200
|
-
const done = fragmentWaiters.get(id);
|
|
201
|
-
if (done) {
|
|
202
|
-
fragmentWaiters.delete(id);
|
|
203
|
-
done();
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
/**
|
|
208
|
-
* Send one fragment's position and wait until the main thread is done with it.
|
|
209
|
-
*
|
|
210
|
-
* The pin is dropped only after the confirmation, because until then the other
|
|
211
|
-
* thread may still be reading those exact bytes.
|
|
212
|
-
*
|
|
213
|
-
* @param {number} id
|
|
214
|
-
* @param {import("./piece-reader.js").PieceFragment} fragment
|
|
215
|
-
* @returns {Promise<void>}
|
|
216
|
-
*/
|
|
217
|
-
function sendFragment(id, fragment) {
|
|
218
|
-
return new Promise((resolve) => {
|
|
219
|
-
fragmentWaiters.set(id, () => {
|
|
220
|
-
fragment.release();
|
|
221
|
-
resolve();
|
|
222
|
-
});
|
|
223
|
-
parentPort.postMessage({
|
|
224
|
-
type: Event.FRAGMENT,
|
|
225
|
-
id,
|
|
226
|
-
pieceIndex: fragment.pieceIndex,
|
|
227
|
-
buffer: fragment.buffer,
|
|
228
|
-
offset: fragment.offset,
|
|
229
|
-
length: fragment.length
|
|
230
|
-
});
|
|
231
|
-
});
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
/**
|
|
235
|
-
* Stream a byte range back as CHUNK messages.
|
|
236
|
-
*
|
|
237
|
-
* Reads through WebTorrent's own read stream — which serves already-downloaded
|
|
238
|
-
* pieces from disk and waits for the rest — and forwards it in
|
|
239
|
-
* {@link STREAM_CHUNK_BYTES} pieces, transferring ownership of each so nothing
|
|
240
|
-
* is copied across the boundary. `createSendStream` applies the backpressure,
|
|
241
|
-
* so a fast disk cannot outrun the main thread and rebuild the queue in memory.
|
|
242
|
-
*
|
|
243
|
-
* @param {object} params
|
|
244
|
-
* @param {number} params.id - Request id; CHUNK/READ_END carry it.
|
|
245
|
-
* @param {string} params.sourceKey
|
|
246
|
-
* @param {number} params.fileIndex
|
|
247
|
-
* @param {number | null} params.start - Inclusive, or null for the whole file.
|
|
248
|
-
* @param {number | null} params.end - Inclusive.
|
|
249
|
-
* @returns {Promise<void>}
|
|
250
|
-
*/
|
|
251
|
-
async function streamRange({ id, sourceKey, fileIndex, start, end, windowBytes }) {
|
|
252
|
-
const torrent = await requireTorrent(sourceKey);
|
|
253
|
-
const file = torrent.files?.[fileIndex];
|
|
254
|
-
if (!file) {
|
|
255
|
-
throw new Error(`File ${fileIndex} not found in ${sourceKey}.`);
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
const sender = createSendStream({ port: parentPort, requestId: id });
|
|
259
|
-
readsById.set(id, sender);
|
|
260
|
-
|
|
261
|
-
const rangeStart = start ?? 0;
|
|
262
|
-
const rangeEnd = end ?? file.length - 1;
|
|
263
|
-
|
|
264
|
-
let failed = false;
|
|
265
|
-
try {
|
|
266
|
-
// Positions in shared memory, not bytes: the main thread maps the same pool
|
|
267
|
-
// and reads each fragment in place, so nothing is copied and nothing is
|
|
268
|
-
// transferred. See `piece-reader.js`.
|
|
269
|
-
for await (const fragment of readFragments({
|
|
270
|
-
torrent,
|
|
271
|
-
fileIndex,
|
|
272
|
-
start: rangeStart,
|
|
273
|
-
end: rangeEnd,
|
|
274
|
-
cancellation: sender,
|
|
275
|
-
windowBytes
|
|
276
|
-
})) {
|
|
277
|
-
if (sender.isCancelled()) {
|
|
278
|
-
fragment.release();
|
|
279
|
-
break;
|
|
280
|
-
}
|
|
281
|
-
// One fragment in flight at a time. Each one holds a piece pinned, and
|
|
282
|
-
// the store guarantees only two resident pieces at its smallest budget —
|
|
283
|
-
// holding two pins while asking for a third would deadlock it against
|
|
284
|
-
// itself. The round trip costs ~100 µs against a piece worth megabytes,
|
|
285
|
-
// so there is nothing to win by overlapping them.
|
|
286
|
-
await sendFragment(id, fragment);
|
|
287
|
-
}
|
|
288
|
-
} catch (error) {
|
|
289
|
-
// The end-of-read marker means "the body is complete". Sending it after a
|
|
290
|
-
// failure told the reader the file simply ended — a truncated segment that
|
|
291
|
-
// ffmpeg reported as `Stream ends prematurely`, with the real cause thrown
|
|
292
|
-
// away. Let the error propagate instead; the command handler reports it and
|
|
293
|
-
// the main thread fails the stream.
|
|
294
|
-
failed = true;
|
|
295
|
-
throw error;
|
|
296
|
-
} finally {
|
|
297
|
-
readsById.delete(id);
|
|
298
|
-
// Any fragment still awaiting confirmation will never get one now; settling
|
|
299
|
-
// it here releases its pin rather than leaking a held slot.
|
|
300
|
-
settleFragment(id);
|
|
301
|
-
if (!failed) {
|
|
302
|
-
sender.end();
|
|
303
|
-
}
|
|
304
|
-
// Nothing else to tear down: the reader owns no stream of its own, and a
|
|
305
|
-
// cancelled read stops at its next fragment boundary because it polls the
|
|
306
|
-
// same `sender` for cancellation.
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
/**
|
|
311
|
-
* Run one command and return its result.
|
|
312
|
-
*
|
|
313
|
-
* @param {string} command
|
|
314
|
-
* @param {object} params
|
|
315
|
-
* @param {number} id
|
|
316
|
-
* @returns {Promise<unknown>}
|
|
317
|
-
*/
|
|
318
|
-
async function runCommand(command, params, id) {
|
|
319
|
-
switch (command) {
|
|
320
|
-
case Command.ADD_SOURCE: {
|
|
321
|
-
// Registered before it resolves, so anything naming this source while it
|
|
322
|
-
// is being added waits for it instead of being told it does not exist.
|
|
323
|
-
// Reusing the same promise for a repeated add also collapses two callers
|
|
324
|
-
// racing to open the same torrent into one.
|
|
325
|
-
sourceRecipes.set(params.sourceKey, {
|
|
326
|
-
sourceType: params.sourceType,
|
|
327
|
-
source: params.source
|
|
328
|
-
});
|
|
329
|
-
let pending = torrentsByKey.get(params.sourceKey);
|
|
330
|
-
if (!pending) {
|
|
331
|
-
pending = pool.getTorrent(params.sourceType, params.source);
|
|
332
|
-
torrentsByKey.set(params.sourceKey, pending);
|
|
333
|
-
// A failed add must not be remembered, or every later attempt at this
|
|
334
|
-
// source replays the same failure. The handler also marks the rejection
|
|
335
|
-
// as observed, so it cannot surface as an unhandled one.
|
|
336
|
-
pending.catch(() => {
|
|
337
|
-
if (torrentsByKey.get(params.sourceKey) === pending) {
|
|
338
|
-
torrentsByKey.delete(params.sourceKey);
|
|
339
|
-
}
|
|
340
|
-
});
|
|
341
|
-
}
|
|
342
|
-
const torrent = await pending;
|
|
343
|
-
return {
|
|
344
|
-
infoHash: torrent.infoHash,
|
|
345
|
-
name: torrent.name,
|
|
346
|
-
// Files cross as plain data; the objects stay here.
|
|
347
|
-
files: (torrent.files ?? []).map((file, index) => ({
|
|
348
|
-
index,
|
|
349
|
-
name: file.name,
|
|
350
|
-
path: file.path,
|
|
351
|
-
length: file.length
|
|
352
|
-
}))
|
|
353
|
-
};
|
|
354
|
-
}
|
|
355
|
-
|
|
356
|
-
case Command.LIST_FILES: {
|
|
357
|
-
const torrent = await requireTorrent(params.sourceKey);
|
|
358
|
-
return (torrent.files ?? []).map((file, index) => ({
|
|
359
|
-
index,
|
|
360
|
-
name: file.name,
|
|
361
|
-
path: file.path,
|
|
362
|
-
length: file.length
|
|
363
|
-
}));
|
|
364
|
-
}
|
|
365
|
-
|
|
366
|
-
case Command.HELD_TORRENTS: {
|
|
367
|
-
// Which films this proxy has right now, and how much of each. Answered
|
|
368
|
-
// from the live client rather than from the main thread's map of
|
|
369
|
-
// stand-ins, which is only cleared on shutdown and would name films this
|
|
370
|
-
// proxy let go of hours ago.
|
|
371
|
-
const held = [];
|
|
372
|
-
for (const torrent of pool.client?.torrents ?? []) {
|
|
373
|
-
const infoHash = String(torrent?.infoHash ?? "");
|
|
374
|
-
if (!infoHash) {
|
|
375
|
-
continue;
|
|
376
|
-
}
|
|
377
|
-
held.push({
|
|
378
|
-
infoHash,
|
|
379
|
-
// A viewer sent here for a film nobody has downloaded any of gains
|
|
380
|
-
// nothing, so the share is reported and the decision is made where
|
|
381
|
-
// the viewer is.
|
|
382
|
-
progress: Number.isFinite(torrent?.progress) ? torrent.progress : 0,
|
|
383
|
-
bytes: Number.isFinite(torrent?.downloaded) ? torrent.downloaded : 0
|
|
384
|
-
});
|
|
385
|
-
}
|
|
386
|
-
return { held };
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
case Command.TORRENT_TOTALS: {
|
|
390
|
-
// Downloaded and uploaded are counted apart: hashing every downloaded
|
|
391
|
-
// byte is work of a different order from sending one back to the swarm,
|
|
392
|
-
// and adding them would price both at whatever the mixture happened to
|
|
393
|
-
// be.
|
|
394
|
-
let downloaded = 0;
|
|
395
|
-
let uploaded = 0;
|
|
396
|
-
for (const torrent of pool.client?.torrents ?? []) {
|
|
397
|
-
const gotBytes = Number(torrent?.downloaded);
|
|
398
|
-
const sentBytes = Number(torrent?.uploaded);
|
|
399
|
-
downloaded += Number.isFinite(gotBytes) ? gotBytes : 0;
|
|
400
|
-
uploaded += Number.isFinite(sentBytes) ? sentBytes : 0;
|
|
401
|
-
}
|
|
402
|
-
return { downloaded, uploaded };
|
|
403
|
-
}
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
case Command.FILL_FILE: {
|
|
407
|
-
const torrent = await requireTorrent(params.sourceKey);
|
|
408
|
-
return {
|
|
409
|
-
started: fillFileInBackground(torrent, params.fileIndex, params.sourceKey)
|
|
410
|
-
};
|
|
411
|
-
}
|
|
412
|
-
|
|
413
|
-
case Command.FILL_TORRENT: {
|
|
414
|
-
const torrent = await requireTorrent(params.sourceKey);
|
|
415
|
-
return { started: pool.fillTorrentAsCapacityAllows(torrent) };
|
|
416
|
-
}
|
|
417
|
-
|
|
418
|
-
case Command.WARM_POSITION: {
|
|
419
|
-
const torrent = await requireTorrent(params.sourceKey);
|
|
420
|
-
return {
|
|
421
|
-
started: await warmResumePosition(
|
|
422
|
-
torrent,
|
|
423
|
-
params.fileIndex,
|
|
424
|
-
params.sourceKey,
|
|
425
|
-
params.positionSeconds,
|
|
426
|
-
{
|
|
427
|
-
// Told by the main thread, which is where what a file states about
|
|
428
|
-
// itself is read.
|
|
429
|
-
durationSeconds: params.durationSeconds,
|
|
430
|
-
fetchRegion: (start, bytes) =>
|
|
431
|
-
pool.prefetchFileRegion(torrent, params.fileIndex, start, bytes)
|
|
432
|
-
}
|
|
433
|
-
)
|
|
434
|
-
};
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
case Command.FILE_STATS: {
|
|
439
|
-
const torrent = await requireTorrent(params.sourceKey);
|
|
440
|
-
const stats = pool.getFileStats(torrent, params.fileIndex, {
|
|
441
|
-
resumeAnchorByteStart: params.resumeAnchorByteStart ?? null
|
|
442
|
-
});
|
|
443
|
-
// What this file's own interruptions demand, measured by the reader in
|
|
444
|
-
// this thread. It travels with the stats because the caller asking for
|
|
445
|
-
// them is the one that has to decide with them — the browser's smallest
|
|
446
|
-
// safe buffer, and the speed a quality step must sustain. Null until a
|
|
447
|
-
// second interruption has been seen: one wait shows no interval, and an
|
|
448
|
-
// interval invented from one point is exactly what this work removes.
|
|
449
|
-
const file = Array.isArray(torrent?.files) ? torrent.files[params.fileIndex] : null;
|
|
450
|
-
return {
|
|
451
|
-
...stats,
|
|
452
|
-
supply: supplyFiguresFor(torrent?.infoHash, file?.name, params.segmentSeconds ?? 4)
|
|
453
|
-
};
|
|
454
|
-
}
|
|
455
|
-
|
|
456
|
-
case Command.PRIORITY_MAP: {
|
|
457
|
-
// A MAP WITH NOTHING IN IT MUST NOT BRING A TORRENT BACK. It is what is
|
|
458
|
-
// said when the last viewer of a file leaves, which is also when the
|
|
459
|
-
// torrent may be on its way out — and `requireTorrent` rebuilds a dead
|
|
460
|
-
// handle from its recipe, so asking that way would re-add a torrent in
|
|
461
|
-
// order to be told that nothing is wanted of it.
|
|
462
|
-
const zones = Array.isArray(params.zones) ? params.zones : [];
|
|
463
|
-
const torrent = zones.length === 0
|
|
464
|
-
? await knownTorrent(params.sourceKey)
|
|
465
|
-
: await requireTorrent(params.sourceKey);
|
|
466
|
-
if (torrent) {
|
|
467
|
-
pool.applyPriorityMap(torrent, params.fileIndex, zones, params.durationSeconds);
|
|
468
|
-
}
|
|
469
|
-
return true;
|
|
470
|
-
}
|
|
471
|
-
|
|
472
|
-
case Command.PRIORITIZE: {
|
|
473
|
-
const torrent = await requireTorrent(params.sourceKey);
|
|
474
|
-
pool.prioritizeByteRange(torrent, params.fileIndex, params.byteStart, params.windowBytes, {
|
|
475
|
-
wholeFileRead: params.wholeFileRead === true
|
|
476
|
-
});
|
|
477
|
-
return true;
|
|
478
|
-
}
|
|
479
|
-
|
|
480
|
-
case Command.PREFETCH_EDGES: {
|
|
481
|
-
const torrent = await requireTorrent(params.sourceKey);
|
|
482
|
-
return pool.prefetchFileEdges(torrent, params.fileIndex, params.options ?? {});
|
|
483
|
-
}
|
|
484
|
-
|
|
485
|
-
case Command.READ_RANGE: {
|
|
486
|
-
// Streams its own reply; the caller's promise resolves once the body has
|
|
487
|
-
// been fully sent, which is what lets the client await completion.
|
|
488
|
-
await streamRange({
|
|
489
|
-
id,
|
|
490
|
-
sourceKey: params.sourceKey,
|
|
491
|
-
fileIndex: params.fileIndex,
|
|
492
|
-
start: params.start ?? null,
|
|
493
|
-
end: params.end ?? null,
|
|
494
|
-
windowBytes: params.windowBytes
|
|
495
|
-
});
|
|
496
|
-
return true;
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
case Command.CANCEL_READ: {
|
|
500
|
-
readsById.get(params.readId)?.cancel();
|
|
501
|
-
// A cancelled read will never have its outstanding fragment confirmed, so
|
|
502
|
-
// wake it here — otherwise it waits forever with a piece pinned.
|
|
503
|
-
settleFragment(params.readId);
|
|
504
|
-
return true;
|
|
505
|
-
}
|
|
506
|
-
|
|
507
|
-
case Command.HELD_RANGES: {
|
|
508
|
-
const torrent = await requireTorrent(params.sourceKey);
|
|
509
|
-
return { ranges: heldRangesOf(torrent, params.fileIndex) };
|
|
510
|
-
}
|
|
511
|
-
|
|
512
|
-
case Command.READ_HELD: {
|
|
513
|
-
const torrent = await requireTorrent(params.sourceKey);
|
|
514
|
-
const bytes = await readHeldBytes(torrent, params.fileIndex, params.start, params.end, logger);
|
|
515
|
-
return { bytes };
|
|
516
|
-
}
|
|
517
|
-
|
|
518
|
-
case Command.SPILL_ALLOWANCE: {
|
|
519
|
-
// The disk has one owner and it is on the main thread, where the segments
|
|
520
|
-
// are. What arrives is this thread’s whole share; the stores divide it
|
|
521
|
-
// between themselves.
|
|
522
|
-
return reviseSpillBudgets(Number(params.bytes));
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
case Command.MEMORY_ALLOWANCE: {
|
|
526
|
-
// Memory has one owner, on the main thread, and it is the same owner the
|
|
527
|
-
// disk has — the two resources trade, so they cannot be divided apart.
|
|
528
|
-
reviseStoreBudgets(Number(params.bytes));
|
|
529
|
-
return memoryClaim();
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
case Command.WHOLE_FILES_ALLOWANCE: {
|
|
533
|
-
// Whole files are held on this thread and the disk has one owner, on the
|
|
534
|
-
// other. What arrives is their share; what goes back is what they hold
|
|
535
|
-
// after it, which is what the owner divides by next time.
|
|
536
|
-
const after = await completedFiles.allow(Number(params.bytes));
|
|
537
|
-
return after.bytes;
|
|
538
|
-
}
|
|
539
|
-
|
|
540
|
-
case Command.DESTROY_ALL: {
|
|
541
|
-
torrentsByKey.clear();
|
|
542
|
-
sourceRecipes.clear();
|
|
543
|
-
await pool.destroyAll();
|
|
544
|
-
return true;
|
|
545
|
-
}
|
|
546
|
-
|
|
547
|
-
default:
|
|
548
|
-
throw new Error(`Unknown torrent-worker command: ${command}`);
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
parentPort.on("message", async (message) => {
|
|
553
|
-
// Chunk acknowledgements are not commands — they release backpressure on an
|
|
554
|
-
// in-flight read.
|
|
555
|
-
if (message?.type === Event.CHUNK_ACK) {
|
|
556
|
-
readsById.get(message.id)?.ack();
|
|
557
|
-
return;
|
|
558
|
-
}
|
|
559
|
-
|
|
560
|
-
// The main thread has finished reading a fragment out of shared memory, so
|
|
561
|
-
// its piece may be unpinned and the read may continue.
|
|
562
|
-
if (message?.type === Event.FRAGMENT_DONE) {
|
|
563
|
-
settleFragment(message.id);
|
|
564
|
-
return;
|
|
565
|
-
}
|
|
566
|
-
|
|
567
|
-
const { command, id, params } = message ?? {};
|
|
568
|
-
try {
|
|
569
|
-
const result = await runCommand(command, params ?? {}, id);
|
|
570
|
-
parentPort.postMessage({ type: Event.RESULT, id, result });
|
|
571
|
-
} catch (error) {
|
|
572
|
-
parentPort.postMessage({ type: Event.ERROR, id, error: error?.message ?? String(error) });
|
|
573
|
-
}
|
|
574
|
-
});
|
|
575
|
-
|
|
576
|
-
/**
|
|
577
|
-
* How often the piece store reports what it has been doing.
|
|
578
|
-
*
|
|
579
|
-
* The store decides whether a read costs nothing or costs a disk trip, and
|
|
580
|
-
* until 2.9.75 nothing about it reached the log — a field oddity would have had
|
|
581
|
-
* no evidence to work from. Reported only when something changed, so an idle
|
|
582
|
-
* proxy stays quiet.
|
|
583
|
-
*/
|
|
584
|
-
// The torrent worker's OWN isolate, reported from inside it. The piece pool is
|
|
585
|
-
// a `SharedArrayBuffer` allocated here, so the main thread's counters cannot
|
|
586
|
-
// see it however carefully they are read — which is half the reason 650 MB of a
|
|
587
|
-
// 893 MB process had no explanation on 2026-08-28 (roadmap item 2).
|
|
588
|
-
// A second between readings, a minute between lines unless the heap moved by
|
|
589
|
-
// 25 MB, and a heap snapshot of THIS isolate on every new high-water above
|
|
590
|
-
// 400 MB. Three deaths — 2026-08-30 14:00 and 23:19, and
|
|
591
|
-
// 2026-08-31 13:27 — went from a 30 MB heap to the 2240 MB ceiling inside one
|
|
592
|
-
// sixty-second gap, and the only snapshots ever written were of the main
|
|
593
|
-
// isolate, whose heap is 26 MB. So the isolate that dies has never once been
|
|
594
|
-
// looked at (roadmap item 2, `research/worker-heap-oom-2026-08-31.md`).
|
|
595
|
-
startMemoryReport({
|
|
596
|
-
log,
|
|
597
|
-
readStores: collectStoreStats,
|
|
598
|
-
// Beside the isolate's own figures, and on the SAME line, because the
|
|
599
|
-
// question they answer together is whether the off-heap mass is buffers this
|
|
600
|
-
// thread still refers to or buffers the collector has not reached yet. On
|
|
601
|
-
// separate timers the two were up to a minute apart and could not be
|
|
602
|
-
// compared at all (roadmap item 2).
|
|
603
|
-
readExtra: describePieceBuffers,
|
|
604
|
-
scope: "thread",
|
|
605
|
-
label: "torrent worker",
|
|
606
|
-
intervalMs: WORKER_MEMORY_SAMPLE_MS,
|
|
607
|
-
quietMs: 60_000,
|
|
608
|
-
changeBytes: 25 * 1024 * 1024,
|
|
609
|
-
snapshotDir: workerData?.stateDir || undefined,
|
|
610
|
-
snapshotFloorBytes: 400 * 1024 * 1024,
|
|
611
|
-
snapshotGrowthBytes: 400 * 1024 * 1024,
|
|
612
|
-
keepSnapshots: 3
|
|
613
|
-
});
|
|
614
|
-
|
|
615
|
-
const STORE_REPORT_INTERVAL_MS = 60_000;
|
|
616
|
-
|
|
617
|
-
/** Last reported reserve, so an unchanged one stays silent. */
|
|
618
|
-
let lastClaimsWithdrawn = 0;
|
|
619
|
-
let lastReserveBytes = 0;
|
|
620
|
-
|
|
621
|
-
/** Last reported figures per store, so unchanged ones stay silent. */
|
|
622
|
-
const lastReported = new Map();
|
|
623
|
-
|
|
624
|
-
setInterval(() => {
|
|
625
|
-
// What the machine can spare NOW, not what it could spare when each store was
|
|
626
|
-
// created. With per-piece buffers a lowered ceiling is honoured immediately:
|
|
627
|
-
// excess pieces are evicted to disk and their memory is reclaimable.
|
|
628
|
-
// What the machine has been seen to need for everything that is not us. It
|
|
629
|
-
// starts at nothing and grows only on evidence, so it is worth saying when it
|
|
630
|
-
// moves — it is the one term of the budget that comes from observation of
|
|
631
|
-
// other processes rather than from our own readers.
|
|
632
|
-
const reserveBefore = machineReserveBytes();
|
|
633
|
-
for (const revised of reviseStoreBudgets()) {
|
|
634
|
-
if (revised.evicted > 0) {
|
|
635
|
-
log(
|
|
636
|
-
`piece-store "${revised.name.slice(0, 40)}": allowance is now ` +
|
|
637
|
-
`${Math.round(revised.ceilingBytes / 1048576)}MB, evicted ${revised.evicted} piece(s) to meet it` +
|
|
638
|
-
(revised.releasedBlocks > 0 ? `, gave back ${revised.releasedBlocks} block(s) of memory` : "") +
|
|
639
|
-
` — now ${Math.round(revised.committedBytes / 1048576)}MB committed`
|
|
640
|
-
);
|
|
641
|
-
} else if (revised.belowActiveDemand) {
|
|
642
|
-
log(
|
|
643
|
-
`piece-store "${revised.name.slice(0, 40)}": the machine's share is smaller than active read demand, ` +
|
|
644
|
-
`so the allowance is held at ${Math.round(revised.ceilingBytes / 1048576)}MB ` +
|
|
645
|
-
"— a store cannot finish a read if it cannot hold the pieces that read is using"
|
|
646
|
-
);
|
|
647
|
-
} else if (revised.committedBytes > revised.ceilingBytes) {
|
|
648
|
-
log(
|
|
649
|
-
`piece-store "${revised.name.slice(0, 40)}": allowance is now ` +
|
|
650
|
-
`${Math.round(revised.ceilingBytes / 1048576)}MB and ` +
|
|
651
|
-
`${Math.round(revised.committedBytes / 1048576)}MB is committed — ` +
|
|
652
|
-
`all resident pieces are pinned, cannot shrink yet`
|
|
653
|
-
);
|
|
654
|
-
}
|
|
655
|
-
}
|
|
656
|
-
// WHAT THE ANNOUNCEMENTS ACTUALLY DID. The stores count every piece they
|
|
657
|
-
// stopped being able to produce; this counts the ones where the library did
|
|
658
|
-
// still hold a claim and it was taken back. The GAP between the two is the
|
|
659
|
-
// reading: announcements far above withdrawals mean the stores are mostly
|
|
660
|
-
// dropping pieces that were never completed, and a withdrawal count stuck at
|
|
661
|
-
// zero while announcements climb means the mechanism is not reaching the
|
|
662
|
-
// torrent at all.
|
|
663
|
-
if (pool.claimsWithdrawn !== lastClaimsWithdrawn) {
|
|
664
|
-
lastClaimsWithdrawn = pool.claimsWithdrawn;
|
|
665
|
-
log(
|
|
666
|
-
`torrent-pool: ${pool.claimsWithdrawn} piece claim(s) withdrawn — pieces this proxy ` +
|
|
667
|
-
"had dropped and has now told the swarm it needs again"
|
|
668
|
-
);
|
|
669
|
-
}
|
|
670
|
-
const reserveNow = machineReserveBytes();
|
|
671
|
-
if (reserveNow !== reserveBefore || reserveNow !== lastReserveBytes) {
|
|
672
|
-
lastReserveBytes = reserveNow;
|
|
673
|
-
log(
|
|
674
|
-
`piece-store: leaving ${Math.round(reserveNow / 1048576)}MB for everything else on this ` +
|
|
675
|
-
"machine — the largest fall in available memory this process has seen and did not cause"
|
|
676
|
-
);
|
|
677
|
-
}
|
|
678
|
-
for (const stats of collectStoreStats()) {
|
|
679
|
-
const signature =
|
|
680
|
-
`${stats.fromMemory}/${stats.fromDisk}/${stats.spills}/${stats.revivals}/` +
|
|
681
|
-
`${stats.blockedByPins}/${stats.evictedOnRevise}/${stats.spillFailures}/` +
|
|
682
|
-
`${stats.evictedProtected}/${stats.demand?.unionPieces ?? 0}/${stats.admittedToDisk}/` +
|
|
683
|
-
`${stats.blocksAllocated}/${stats.blocksFree}/${stats.blocksReleased}/${stats.withdrawn}`;
|
|
684
|
-
if (lastReported.get(stats.name) === signature) {
|
|
685
|
-
continue;
|
|
686
|
-
}
|
|
687
|
-
lastReported.set(stats.name, signature);
|
|
688
|
-
|
|
689
|
-
const reads = stats.fromMemory + stats.fromDisk;
|
|
690
|
-
const fromMemoryShare = reads > 0 ? ((stats.fromMemory / reads) * 100).toFixed(1) : "—";
|
|
691
|
-
log(
|
|
692
|
-
`piece-store "${stats.name.slice(0, 40)}": resident=${stats.resident}/${stats.capacity} ` +
|
|
693
|
-
`(${Math.round((stats.residentBytes || 0) / 1048576)}MB of ` +
|
|
694
|
-
`${Math.round((stats.budgetBytes || 0) / 1048576)}MB allowed) ` +
|
|
695
|
-
`committed=${Math.round((stats.committedBytes || 0) / 1048576)}MB ` +
|
|
696
|
-
`blocks=${stats.blocksAllocated} (${stats.blocksFree} spare) ` +
|
|
697
|
-
`on-disk=${Math.round((stats.spilledBytes || 0) / 1048576)}MB ` +
|
|
698
|
-
`pinned=${stats.pinned} spilled=${stats.spilled} reads=${reads} (${fromMemoryShare}% from memory) ` +
|
|
699
|
-
`spills=${stats.spills} revivals=${stats.revivals}` +
|
|
700
|
-
(stats.blockedByPins > 0 ? ` no-block-for=${stats.blockedByPins}` : "") +
|
|
701
|
-
(stats.admittedWithoutSlot > 0 ? ` to-disk-for-want-of-memory=${stats.admittedWithoutSlot}` : "") +
|
|
702
|
-
(stats.stillMs > 1000 ? ` nothing-moved-for=${Math.round(stats.stillMs / 1000)}s` : "") +
|
|
703
|
-
(stats.evictedOnRevise > 0 ? ` evictedOnRevise=${stats.evictedOnRevise}` : "") +
|
|
704
|
-
// How many pieces this store stopped being able to produce and said so,
|
|
705
|
-
// which is what makes the eviction's own bargain checkable: every one of
|
|
706
|
-
// these is a piece the torrent has been told to fetch again if it is ever
|
|
707
|
-
// wanted. A session where this climbs while reads keep succeeding is the
|
|
708
|
-
// bargain working; before 2026-09-12 the figure did not exist and the
|
|
709
|
-
// claim was never withdrawn at all.
|
|
710
|
-
(stats.withdrawn > 0 ? ` withdrawn=${stats.withdrawn}` : "") +
|
|
711
|
-
(stats.spillFailures > 0 ? ` spill-failures=${stats.spillFailures}` : "") +
|
|
712
|
-
(stats.outstanding > 0 ? ` outstanding=${stats.outstanding}` : "")
|
|
713
|
-
);
|
|
714
|
-
// Why it spills, on its own line because it is a different question from
|
|
715
|
-
// how much it holds. Three facts, and between them they say whether the
|
|
716
|
-
// thrashing is a policy to fix or arithmetic to accept: what the readers
|
|
717
|
-
// together are asking to keep against what the store may hold; how many
|
|
718
|
-
// evictions had to take a piece a reader had declared it wants; and how
|
|
719
|
-
// long a piece stayed on disk before it was wanted back. On 2026-09-02 a
|
|
720
|
-
// session did 6565 spills and 7575 revivals with 53.6% of reads served
|
|
721
|
-
// from memory, and nothing recorded which of the three was the cause
|
|
722
|
-
// (roadmap item 9).
|
|
723
|
-
const demand = stats.demand;
|
|
724
|
-
if (demand && demand.readers > 0) {
|
|
725
|
-
const age = stats.revivalAgeMedianMs;
|
|
726
|
-
log(
|
|
727
|
-
`piece-store "${stats.name.slice(0, 40)}" demand: ${demand.readers} reader(s) want ` +
|
|
728
|
-
`${demand.unionPieces} piece(s) of ${demand.capacity} the store may hold ` +
|
|
729
|
-
`(widest window ${demand.widestPieces})` +
|
|
730
|
-
// NAMED, because the count read as five encoders on a session that had
|
|
731
|
-
// two: a "reader" is whoever declared a range, and four of the five
|
|
732
|
-
// were zones of the priority map. Choosing between narrowing the
|
|
733
|
-
// windows and raising the allowance was guesswork without this.
|
|
734
|
-
` [${demand.names.join(" ")}]` +
|
|
735
|
-
(stats.evictedProtected > 0
|
|
736
|
-
? `; ${stats.evictedProtected} of ${stats.spills} eviction(s) took a piece a reader had declared`
|
|
737
|
-
: "; no eviction has taken a declared piece") +
|
|
738
|
-
(stats.evictedWithDistance > 0
|
|
739
|
-
? `, a victim lay ${(stats.evictedDistanceSum / stats.evictedWithDistance).toFixed(1)} ` +
|
|
740
|
-
"piece(s) from the nearest window on average"
|
|
741
|
-
: "") +
|
|
742
|
-
(age === null
|
|
743
|
-
? "; nothing has come back from disk yet"
|
|
744
|
-
: `; a revived piece had been on disk ${(age / 1000).toFixed(1)}s (median of ` +
|
|
745
|
-
`${stats.revivalAgeSamples}, ${stats.revivedWithinFiveSeconds} of them within 5s)`) +
|
|
746
|
-
`; of ${stats.admittedInsideWindow + stats.admittedOutsideWindow} piece(s) admitted ` +
|
|
747
|
-
`${stats.admittedOutsideWindow} were in nobody's window, ${stats.admittedToDisk} of those ` +
|
|
748
|
-
"went straight to disk" +
|
|
749
|
-
`; ${stats.blocksAllocated} block(s) of memory exist, ${stats.blocksFree} of them spare` +
|
|
750
|
-
(stats.reuseGapMs === null
|
|
751
|
-
? ", none re-used yet"
|
|
752
|
-
: `, a block waits up to ${(stats.reuseGapMs / 1000).toFixed(1)}s before it is wanted again`) +
|
|
753
|
-
`, ${stats.blocksReleased} given back` +
|
|
754
|
-
(stats.spillsSkipped > 0
|
|
755
|
-
? `; ${stats.spillsSkipped} of ${stats.spills} eviction(s) needed no write, the disk already had them`
|
|
756
|
-
: "") +
|
|
757
|
-
(stats.returnedWhilePinned > 0
|
|
758
|
-
? `; ${stats.returnedWhilePinned} BLOCK(S) WERE RECYCLED WHILE STILL BEING READ`
|
|
759
|
-
: "")
|
|
760
|
-
);
|
|
761
|
-
}
|
|
762
|
-
}
|
|
763
|
-
}, STORE_REPORT_INTERVAL_MS).unref();
|
|
764
|
-
|
|
765
|
-
/**
|
|
766
|
-
* The blocks of piece memory this thread has allocated against the pieces the
|
|
767
|
-
* stores hold in them.
|
|
768
|
-
*
|
|
769
|
-
* Not per store: the collector is per thread, and the question is about the
|
|
770
|
-
* thread. With a pool one block serves many pieces, so a number of allocations
|
|
771
|
-
* that keeps climbing while the stores hold a steady number of pieces means
|
|
772
|
-
* blocks are being made and thrown away instead of re-used.
|
|
773
|
-
*
|
|
774
|
-
* @returns {string}
|
|
775
|
-
*/
|
|
776
|
-
function describePieceBuffers() {
|
|
777
|
-
const collection = pieceBufferCollection();
|
|
778
|
-
if (collection.released === 0) {
|
|
779
|
-
return "";
|
|
780
|
-
}
|
|
781
|
-
const alive = collection.released - collection.collected;
|
|
782
|
-
const held = collectStoreStats().reduce((sum, stats) => sum + (stats.resident || 0), 0);
|
|
783
|
-
return (
|
|
784
|
-
`memory blocks ${collection.released} allocated, ${collection.collected} collected, ` +
|
|
785
|
-
`${alive} still alive against ${held} piece(s) the stores hold`
|
|
786
|
-
);
|
|
787
|
-
}
|
|
788
|
-
|
|
789
|
-
/**
|
|
790
|
-
* Say that pieces of the actively-read files of one torrent have arrived.
|
|
791
|
-
*
|
|
792
|
-
* It used to WALK them — reading each file's new subtitle clusters and pushing
|
|
793
|
-
* the cues it found. That put the cue reading, and with it a container parse,
|
|
794
|
-
* in the thread that owns the swarm; the walk is on the main thread now and
|
|
795
|
-
* this announces the one fact only this thread can know.
|
|
796
|
-
*
|
|
797
|
-
* Which files: the ones anything is stated for — a viewer's own picture and
|
|
798
|
-
* soundtrack through the priority map, and the ends of a file that is open. It
|
|
799
|
-
* replaced a count of readers, which said the same thing by keeping a second
|
|
800
|
-
* copy of it.
|
|
801
|
-
*
|
|
802
|
-
* @param {string} sourceKey
|
|
803
|
-
* @param {object} torrent
|
|
804
|
-
* @returns {void}
|
|
805
|
-
*/
|
|
806
|
-
function announceArrivals(sourceKey, torrent) {
|
|
807
|
-
const fileIndexes = [...demandFor(torrent).register.files()];
|
|
808
|
-
if (fileIndexes.length === 0) {
|
|
809
|
-
return;
|
|
810
|
-
}
|
|
811
|
-
parentPort.postMessage({ type: Event.PIECES_ARRIVED, sourceKey, fileIndexes });
|
|
812
|
-
}
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
/**
|
|
816
|
-
* Torrents already wired to warm their subtitle cues the moment a piece
|
|
817
|
-
* verifies, so the same torrent is not listened to twice.
|
|
818
|
-
*
|
|
819
|
-
* @type {WeakSet<object>}
|
|
820
|
-
*/
|
|
821
|
-
const arrivalsWired = new WeakSet();
|
|
822
|
-
|
|
823
|
-
/**
|
|
824
|
-
* A piece becoming readable is the actual event a cue can be pulled from —
|
|
825
|
-
* "downloaded", not "about to be encoded or copied": what a viewer reaches is
|
|
826
|
-
* decided by the read window ahead of the playhead, not by which of the two
|
|
827
|
-
* paths a segment takes, and the piece exists (and is worth reading for
|
|
828
|
-
* subtitles) whichever one that is. `verified` is WebTorrent's own signal for
|
|
829
|
-
* exactly that instant, set at the same place the bitfield itself is (`
|
|
830
|
-
* _markVerified`), so nothing here is guessing at readiness a different way.
|
|
831
|
-
*
|
|
832
|
-
* @param {string} sourceKey
|
|
833
|
-
* @param {object} torrent
|
|
834
|
-
* @returns {void}
|
|
835
|
-
*/
|
|
836
|
-
function ensureArrivalsWired(sourceKey, torrent) {
|
|
837
|
-
if (arrivalsWired.has(torrent)) {
|
|
838
|
-
return;
|
|
839
|
-
}
|
|
840
|
-
arrivalsWired.add(torrent);
|
|
841
|
-
torrent.on("verified", () => announceArrivals(sourceKey, torrent));
|
|
842
|
-
}
|
|
843
|
-
|
|
844
|
-
/**
|
|
845
|
-
* How often arrivals are announced regardless, as a fallback beside the
|
|
846
|
-
* per-piece `verified` listener above — it catches a listener attached after
|
|
847
|
-
* pieces had already verified, and anything the event path might otherwise
|
|
848
|
-
* miss. Cheap: the walk it wakes skips clusters it has already read.
|
|
849
|
-
*/
|
|
850
|
-
const ARRIVAL_ANNOUNCE_INTERVAL_MS = 3_000;
|
|
851
|
-
|
|
852
|
-
setInterval(() => {
|
|
853
|
-
for (const [sourceKey, torrent] of pool.torrents) {
|
|
854
|
-
ensureArrivalsWired(sourceKey, torrent);
|
|
855
|
-
announceArrivals(sourceKey, torrent);
|
|
856
|
-
}
|
|
857
|
-
}, ARRIVAL_ANNOUNCE_INTERVAL_MS).unref();
|
|
858
|
-
|
|
859
|
-
/**
|
|
860
|
-
* Files this proxy has downloaded whole. One directory, two readers of it: this
|
|
861
|
-
* thread writes them, the main thread serves them without asking anybody.
|
|
862
|
-
*/
|
|
863
|
-
const completedFiles = new CompletedFiles({ root: completedFilesRoot() });
|
|
864
|
-
|
|
865
|
-
/**
|
|
866
|
-
* Which torrent a set of files belongs to.
|
|
867
|
-
*
|
|
868
|
-
* The piece store hands its own files back without knowing what they are; this
|
|
869
|
-
* thread does know, and a file carries its torrent.
|
|
870
|
-
*
|
|
871
|
-
* @param {object[]} files
|
|
872
|
-
* @returns {string}
|
|
873
|
-
*/
|
|
874
|
-
const infoHashOf = (files) => String(files?.[0]?._torrent?.infoHash ?? "").toLowerCase();
|
|
875
|
-
|
|
876
|
-
// WHERE A PIECE COMES FROM WHEN NEITHER TIER HAS IT. Handed to every store this
|
|
877
|
-
// pool builds, so a film already assembled into a file is read from that file:
|
|
878
|
-
// which is what lets its spilled copy be dropped as the duplicate it has become,
|
|
879
|
-
// and what lets a torrent be destroyed and added again without fetching a byte.
|
|
880
|
-
pool.buildStoresWith({
|
|
881
|
-
readPieceElsewhere: ({ index, pieceLength, length, files }) =>
|
|
882
|
-
pieceFromWholeFiles({
|
|
883
|
-
index,
|
|
884
|
-
pieceLength,
|
|
885
|
-
length,
|
|
886
|
-
files,
|
|
887
|
-
wholeFileAt: (fileIndex) => completedFiles.find(infoHashOf(files), fileIndex)
|
|
888
|
-
}),
|
|
889
|
-
isPieceElsewhere: ({ index, pieceLength, length, files }) =>
|
|
890
|
-
pieceIsInWholeFiles({
|
|
891
|
-
index,
|
|
892
|
-
pieceLength,
|
|
893
|
-
length,
|
|
894
|
-
files,
|
|
895
|
-
wholeFileAt: (fileIndex) => completedFiles.find(infoHashOf(files), fileIndex)
|
|
896
|
-
})
|
|
897
|
-
});
|
|
898
|
-
void completedFiles.adopt(() => null).then((adopted) => {
|
|
899
|
-
if (adopted > 0) {
|
|
900
|
-
logger.info(
|
|
901
|
-
`whole files: took up ${adopted} file(s) a previous life left in ${completedFiles.root}`
|
|
902
|
-
);
|
|
903
|
-
}
|
|
904
|
-
});
|
|
905
|
-
|
|
906
|
-
/**
|
|
907
|
-
* How often whole files are looked for.
|
|
908
|
-
*
|
|
909
|
-
* The work itself is one pass over the files of each torrent asking a boolean
|
|
910
|
-
* the library already keeps; writing one out happens at most once per file,
|
|
911
|
-
* ever.
|
|
912
|
-
*/
|
|
913
|
-
const WHOLE_FILE_SWEEP_MS = 10_000;
|
|
914
|
-
|
|
915
|
-
/** Files being written out right now, so a sweep does not start a second one. */
|
|
916
|
-
const beingKept = new Set();
|
|
917
|
-
|
|
918
|
-
/**
|
|
919
|
-
* Keep every file that is now whole, and let go of a torrent that has nothing
|
|
920
|
-
* left to fetch.
|
|
921
|
-
*
|
|
922
|
-
* The instruction this serves, 2026-09-11: as soon as a torrent is fully
|
|
923
|
-
* downloaded, downloading stops, the torrent is deleted, and what was
|
|
924
|
-
* downloaded stays for as long as it is wanted.
|
|
925
|
-
*
|
|
926
|
-
* `file.done` is the library's own answer to "is every piece of this file
|
|
927
|
-
* here", and `torrent.done` to "is that true of every file". The second is a
|
|
928
|
-
* strong condition and will not fire for a season pack of which one episode is
|
|
929
|
-
* watched — nothing fetches the other four — and that is right: the instruction
|
|
930
|
-
* is about a torrent downloaded WHOLE.
|
|
931
|
-
*
|
|
932
|
-
* @returns {Promise<void>}
|
|
933
|
-
*/
|
|
934
|
-
async function keepWholeFiles() {
|
|
935
|
-
for (const [sourceKey, torrent] of [...pool.torrents]) {
|
|
936
|
-
// Lower case, because that is what the source key carries and the main
|
|
937
|
-
// thread looks these up by the key alone.
|
|
938
|
-
const infoHash = String(torrent?.infoHash ?? "").toLowerCase();
|
|
939
|
-
if (!infoHash || !Array.isArray(torrent.files)) {
|
|
940
|
-
continue;
|
|
941
|
-
}
|
|
942
|
-
// What anybody wants of this torrent. A file something is stated for is a
|
|
943
|
-
// file somebody may be reading, and this is the same list the reader counts
|
|
944
|
-
// used to give.
|
|
945
|
-
const wanted = new Set(
|
|
946
|
-
demandFor(torrent).register.windows()
|
|
947
|
-
.filter((window) =>
|
|
948
|
-
!String(window.claimant).startsWith("file-edges:") &&
|
|
949
|
-
!String(window.claimant).startsWith("torrent-fill:") &&
|
|
950
|
-
!String(window.claimant).startsWith("background-fill:")
|
|
951
|
-
)
|
|
952
|
-
.map((window) => window.fileIndex)
|
|
953
|
-
);
|
|
954
|
-
for (const [fileIndex, file] of torrent.files.entries()) {
|
|
955
|
-
const key = `${infoHash}/${fileIndex}`;
|
|
956
|
-
if (file?.done !== true || completedFiles.find(infoHash, fileIndex) || beingKept.has(key)) {
|
|
957
|
-
continue;
|
|
958
|
-
}
|
|
959
|
-
// NOT WHILE ANYBODY WANTS IT. Writing a film out is a read of the whole
|
|
960
|
-
// of it and a write of the whole of it — a gigabyte and a half on the
|
|
961
|
-
// file this was measured against — and doing that beside a viewer takes
|
|
962
|
-
// the disk and the piece store from them for nothing they asked for. The
|
|
963
|
-
// file is complete; it will still be complete when they leave.
|
|
964
|
-
if (wanted.has(fileIndex)) {
|
|
965
|
-
continue;
|
|
966
|
-
}
|
|
967
|
-
beingKept.add(key);
|
|
968
|
-
try {
|
|
969
|
-
const kept = await completedFiles.keep({
|
|
970
|
-
infoHash,
|
|
971
|
-
fileIndex,
|
|
972
|
-
length: file.length,
|
|
973
|
-
name: file.name,
|
|
974
|
-
open: () => file.createReadStream()
|
|
975
|
-
});
|
|
976
|
-
if (kept) {
|
|
977
|
-
logger.info(
|
|
978
|
-
`whole files: kept "${file.name}" (${Math.round(kept.length / 1048576)}MB) — ` +
|
|
979
|
-
"it is a file now, and reading it needs no torrent"
|
|
980
|
-
);
|
|
981
|
-
// The pieces it was built from are a second copy of the same bytes.
|
|
982
|
-
// Nothing is lost by dropping them: a read that wants one of them is
|
|
983
|
-
// answered from the file.
|
|
984
|
-
const store = findSharedStore(torrent);
|
|
985
|
-
const dropped = store?.dropDuplicatesHeldElsewhere?.() ?? 0;
|
|
986
|
-
if (dropped > 0) {
|
|
987
|
-
logger.info(
|
|
988
|
-
`whole files: dropped ${dropped} spilled piece(s) of "${file.name}" — ` +
|
|
989
|
-
"the film was on this disk twice and is not any more"
|
|
990
|
-
);
|
|
991
|
-
}
|
|
992
|
-
parentPort.postMessage({
|
|
993
|
-
type: Event.FILE_COMPLETE,
|
|
994
|
-
infoHash,
|
|
995
|
-
fileIndex,
|
|
996
|
-
path: kept.path,
|
|
997
|
-
length: kept.length,
|
|
998
|
-
name: kept.name
|
|
999
|
-
});
|
|
1000
|
-
}
|
|
1001
|
-
} catch (error) {
|
|
1002
|
-
logger.warn(`whole files: could not keep "${file?.name}": ${error?.message ?? error}`);
|
|
1003
|
-
} finally {
|
|
1004
|
-
beingKept.delete(key);
|
|
1005
|
-
}
|
|
1006
|
-
}
|
|
1007
|
-
// EVERYTHING, WITHOUT EXCEPTION, AND EVERY BYTE OF IT A FILE ON DISK. The
|
|
1008
|
-
// torrent has no job left: there is nothing to fetch, and this proxy does
|
|
1009
|
-
// not seed what nobody is watching.
|
|
1010
|
-
//
|
|
1011
|
-
// Safe to destroy only because a piece can now be read out of those files:
|
|
1012
|
-
// a path that asks for this source again adds the torrent back, and what it
|
|
1013
|
-
// verifies it reads from the files rather than from the swarm. Not while
|
|
1014
|
-
// anybody is reading it, for the same reason the writing above waits.
|
|
1015
|
-
const isWhole =
|
|
1016
|
-
torrent.done === true &&
|
|
1017
|
-
torrent.files.every((unused, fileIndex) => completedFiles.find(infoHash, fileIndex) !== null);
|
|
1018
|
-
if (isWhole && wanted.size === 0) {
|
|
1019
|
-
logger.info(
|
|
1020
|
-
`whole files: "${torrent.name}" is downloaded whole and saved — removing the torrent, keeping the files`
|
|
1021
|
-
);
|
|
1022
|
-
// THE RECIPE STAYS, and so does the entry that leads to it. Everything
|
|
1023
|
-
// that asks this thread about a source — the track table, the media
|
|
1024
|
-
// info, the keyframe table, the stats the browser polls — comes through
|
|
1025
|
-
// `requireTorrent`, which adds a torrent back when the one it holds is
|
|
1026
|
-
// no longer usable. Deleting the entry here would turn a viewer
|
|
1027
|
-
// returning to this film into `Unknown source`, which is a worse failure
|
|
1028
|
-
// than the one this removal is for.
|
|
1029
|
-
//
|
|
1030
|
-
// What the torrent finds when it comes back is the whole files: its
|
|
1031
|
-
// store reads pieces from them, so it fetches nothing. It is added with
|
|
1032
|
-
// verification skipped, and that is not a shortcut — the file was
|
|
1033
|
-
// written out of pieces this client had already hashed, and its size was
|
|
1034
|
-
// checked against what the torrent says. Re-hashing a gigabyte and a
|
|
1035
|
-
// half to learn what we wrote down is a minute of a viewer's time for
|
|
1036
|
-
// nothing.
|
|
1037
|
-
pool.remove(torrent, "downloaded-whole");
|
|
1038
|
-
pool.addWholeSource(sourceKey);
|
|
1039
|
-
}
|
|
1040
|
-
}
|
|
1041
|
-
}
|
|
1042
|
-
|
|
1043
|
-
setInterval(() => {
|
|
1044
|
-
void keepWholeFiles();
|
|
1045
|
-
}, WHOLE_FILE_SWEEP_MS).unref();
|
|
1046
|
-
|
|
1047
|
-
/**
|
|
1048
|
-
* Keeps this thread alive ON PURPOSE — the one interval left accounted for
|
|
1049
|
-
* (no `.unref()`).
|
|
1050
|
-
*
|
|
1051
|
-
* Every other recurring handle here is unref'd, upload is disabled by
|
|
1052
|
-
* default, and idle peer connections close about half a minute after the
|
|
1053
|
-
* traffic stops — so once nothing is being read, every handle can be gone
|
|
1054
|
-
* at once, the event loop drains, and this thread ends BY ITSELF. Node then
|
|
1055
|
-
* tears the isolate down, that teardown touches memory some native module
|
|
1056
|
-
* has already freed, and the fault (SIGSEGV inside `uv_timer_stop`, reached
|
|
1057
|
-
* through `PerIsolatePlatformData::Shutdown`) kills the whole process at
|
|
1058
|
-
* once — HTTP server, tunnel, data channels — before any JS handler runs.
|
|
1059
|
-
* Field evidence: two deaths on 2026-08-22 (15:50:12 and 16:12:21 UTC),
|
|
1060
|
-
* each ~35 s after the last byte of traffic, identical core dumps;
|
|
1061
|
-
* same crash family as the utp-native faults of 2026-08-18..21
|
|
1062
|
-
* (`research/worker-thread-drain-crash-2026-08-22.md`).
|
|
1063
|
-
*
|
|
1064
|
-
* An empty repeating interval costs nothing, keeps the loop from draining
|
|
1065
|
-
* while the process lives, and thereby keeps that teardown path — and the
|
|
1066
|
-
* corrupted structure inside it — unreachable, whichever module is guilty.
|
|
1067
|
-
*/
|
|
1068
|
-
const WORKER_KEEPALIVE_INTERVAL_MS = 5_000;
|
|
1069
|
-
|
|
1070
|
-
setInterval(() => {
|
|
1071
|
-
void process.uptime();
|
|
1072
|
-
}, WORKER_KEEPALIVE_INTERVAL_MS);
|
|
1073
|
-
|
|
1074
|
-
/**
|
|
1075
|
-
* Say why this thread is ending, from inside it, before anything is torn down.
|
|
1076
|
-
*
|
|
1077
|
-
* The crash of 2026-08-27 15:40 is `node::worker::Worker::Run()` returning and
|
|
1078
|
-
* node faulting as it closes what was left on this loop. The parent DOES watch
|
|
1079
|
-
* for an unexpected exit and has a line ready for it — but that line never
|
|
1080
|
-
* printed, because the fault happens during this thread's own teardown, before
|
|
1081
|
-
* the parent's `exit` event is delivered. So the one reading that would name
|
|
1082
|
-
* the cause was being eaten by the failure it was meant to explain.
|
|
1083
|
-
*
|
|
1084
|
-
* These handlers run first, synchronously, and write through the same channel
|
|
1085
|
-
* as every other line here. `beforeExit` means the loop drained despite the
|
|
1086
|
-
* keepalive above; `exit` means the thread is going whatever the reason. The
|
|
1087
|
-
* list of what was still holding the loop open is the part that says which.
|
|
1088
|
-
*/
|
|
1089
|
-
process.on("beforeExit", (code) => {
|
|
1090
|
-
log(
|
|
1091
|
-
`thread is about to end because the event loop drained (code ${code}) — ` +
|
|
1092
|
-
`still open: ${describeActiveResources()}`
|
|
1093
|
-
);
|
|
1094
|
-
});
|
|
1095
|
-
|
|
1096
|
-
process.on("exit", (code) => {
|
|
1097
|
-
log(`thread ending with code ${code} — still open: ${describeActiveResources()}`);
|
|
1098
|
-
});
|
|
1099
|
-
|
|
1100
|
-
process.on("uncaughtException", (error) => {
|
|
1101
|
-
log(`thread hit an uncaught error: ${error?.stack ?? error}`);
|
|
1102
|
-
});
|
|
1103
|
-
|
|
1104
|
-
process.on("unhandledRejection", (reason) => {
|
|
1105
|
-
log(`thread hit an unhandled rejection: ${reason?.stack ?? reason}`);
|
|
1106
|
-
});
|
|
1107
|
-
|
|
1108
|
-
/**
|
|
1109
|
-
* What is still holding this thread's event loop open, as node names it.
|
|
1110
|
-
*
|
|
1111
|
-
* @returns {string} A tally per resource kind, or why it could not be read.
|
|
1112
|
-
*/
|
|
1113
|
-
function describeActiveResources() {
|
|
1114
|
-
try {
|
|
1115
|
-
const names = process.getActiveResourcesInfo?.() ?? [];
|
|
1116
|
-
if (names.length === 0) {
|
|
1117
|
-
return "nothing";
|
|
1118
|
-
}
|
|
1119
|
-
const tally = new Map();
|
|
1120
|
-
for (const name of names) {
|
|
1121
|
-
tally.set(name, (tally.get(name) ?? 0) + 1);
|
|
1122
|
-
}
|
|
1123
|
-
return [...tally].map(([name, count]) => `${name}x${count}`).join(" ");
|
|
1124
|
-
} catch (error) {
|
|
1125
|
-
return `unreadable (${error?.message ?? error})`;
|
|
1126
|
-
}
|
|
1127
|
-
}
|
|
1128
|
-
|
|
1129
|
-
log("torrent worker started");
|
|
1
|
+
/**
|
|
2
|
+
* @file The torrent thread: WebTorrent and nothing else.
|
|
3
|
+
*
|
|
4
|
+
* Everything that made the main thread unresponsive lives here now — peer
|
|
5
|
+
* connections, buffer concatenation, piece bookkeeping, garbage collection from
|
|
6
|
+
* all of it. The main thread keeps only what owes a viewer a prompt answer.
|
|
7
|
+
*
|
|
8
|
+
* This file deliberately holds no HTTP, no session logic and no knowledge of
|
|
9
|
+
* HLS: it answers the commands in `protocol.js` and streams bytes back. That
|
|
10
|
+
* boundary is what keeps the split honest — anything added here will compete
|
|
11
|
+
* with the torrent for this thread, which is exactly the problem being solved.
|
|
12
|
+
*
|
|
13
|
+
* The existing `TorrentPool` is reused wholesale rather than reimplemented. It
|
|
14
|
+
* already carries the parts that took field failures to get right — idle
|
|
15
|
+
* removal, the global disk cap with LRU eviction, seek-aware piece
|
|
16
|
+
* prioritisation, adaptive upload — and none of that changes by moving threads.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
// MUST stay first: it redirects `webrtc-polyfill` to a JavaScript WebRTC stack
|
|
20
|
+
// before WebTorrent can reach the native one. Two isolates using
|
|
21
|
+
// node-datachannel at once abort the process, and the torrent's wss trackers
|
|
22
|
+
// create peer connections of their own.
|
|
23
|
+
import { isUsableTorrentHandle } from "./handle-state.js";
|
|
24
|
+
import "./install-webrtc-shim.js";
|
|
25
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
26
|
+
import { createSendStream } from "./channel.js";
|
|
27
|
+
import { readFragments, supplyFiguresFor } from "./piece-reader.js";
|
|
28
|
+
import {
|
|
29
|
+
warmResumePosition
|
|
30
|
+
} from "./resume-warm.js";
|
|
31
|
+
import { fillFileInBackground } from "./background-fill.js";
|
|
32
|
+
import { demandFor } from "../download/registry.js";
|
|
33
|
+
import { heldRangesOf, readHeldBytes } from "./held-bytes.js";
|
|
34
|
+
import { CompletedFiles, completedFilesRoot } from "../../storage/files/CompletedFiles.js";
|
|
35
|
+
import { pieceFromWholeFiles, pieceIsInWholeFiles } from "../../storage/files/piece-from-whole-file.js";
|
|
36
|
+
import { Command, Event } from "./protocol.js";
|
|
37
|
+
import { startMemoryReport, WORKER_MEMORY_SAMPLE_MS } from "../../storage/memory-report.js";
|
|
38
|
+
import { forwardLogsTo, logger } from "../../../utils/logger.js";
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Everything this thread logs goes to the main thread, which owns the file.
|
|
42
|
+
*
|
|
43
|
+
* Two threads cannot both write it — they would race on the rotation and could
|
|
44
|
+
* interleave mid-line — so there is one writer and this is how everyone else
|
|
45
|
+
* reaches it. Set before anything else runs, because a line written before this
|
|
46
|
+
* point reaches the console only, and the console is destroyed by every
|
|
47
|
+
* release.
|
|
48
|
+
*
|
|
49
|
+
* Until 2026-09-02 only the `log` function below took this route. Modules that
|
|
50
|
+
* called `logger.*` directly — the piece reader, the torrent pool, the
|
|
51
|
+
* background fill, the container track reader, the subtitle walk — wrote into a
|
|
52
|
+
* copy of the logger that had no file, and every one of their lines was lost:
|
|
53
|
+
* measured over a whole 49 938-line file, not one of them was in it.
|
|
54
|
+
*/
|
|
55
|
+
forwardLogsTo((level, message) => {
|
|
56
|
+
// THE LEVEL TRAVELS TOO. It was dropped here, so every line this thread wrote
|
|
57
|
+
// — a warning about a spill that failed, an error about a torrent that went
|
|
58
|
+
// away — arrived on the other side as information and was coloured and
|
|
59
|
+
// recorded as such.
|
|
60
|
+
parentPort.postMessage({ type: Event.LOG, level, message });
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
// Imported dynamically, and that is load-bearing: static imports are RESOLVED
|
|
64
|
+
// during linking, before any module body runs, so a statically imported pool
|
|
65
|
+
// would drag in WebTorrent — and with it the real `webrtc-polyfill` — before
|
|
66
|
+
// the hook above had a chance to register. Verified the hard way: with a static
|
|
67
|
+
// import the process still aborted, and the stack named the genuine polyfill.
|
|
68
|
+
const { TorrentPool, resolveDhtBootstrap } = await import("../torrent-pool.js");
|
|
69
|
+
const { SharedPieceStore, collectStoreStats, findSharedStore, machineReserveBytes, memoryClaim, pieceBufferCollection, reviseSpillBudgets, reviseStoreBudgets } =
|
|
70
|
+
await import("../../storage/piece-store/shared-piece-store.js");
|
|
71
|
+
|
|
72
|
+
// Resolved before the client exists, because the client builds its DHT in its
|
|
73
|
+
// own constructor and the addresses have to be in hand by then. Awaiting here
|
|
74
|
+
// costs the few milliseconds of a DNS answer, once, on a thread that has not
|
|
75
|
+
// been asked for anything yet.
|
|
76
|
+
const dhtBootstrap = await resolveDhtBootstrap();
|
|
77
|
+
|
|
78
|
+
const pool = new TorrentPool({
|
|
79
|
+
memoryBytes: workerData?.memoryBytes,
|
|
80
|
+
dhtBootstrap,
|
|
81
|
+
pieceStore: SharedPieceStore
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
/** Torrents by sourceKey — the main thread names them, this thread owns them. */
|
|
85
|
+
const torrentsByKey = new Map();
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* How each source was named when it was added, so a torrent that has since been
|
|
89
|
+
* destroyed can be added again. Kept separately from {@link torrentsByKey}
|
|
90
|
+
* because that map holds the promise, not the recipe.
|
|
91
|
+
*
|
|
92
|
+
* @type {Map<string, { sourceType: string, source: string }>}
|
|
93
|
+
*/
|
|
94
|
+
const sourceRecipes = new Map();
|
|
95
|
+
/** In-flight reads, so a cancel can stop one mid-body. */
|
|
96
|
+
const readsById = new Map();
|
|
97
|
+
|
|
98
|
+
/**
|
|
99
|
+
* Shorthand for this file. The same path as `logger.info` anywhere else in the
|
|
100
|
+
* thread — kept only because it reads better at the hundred call sites here.
|
|
101
|
+
*
|
|
102
|
+
* @param {string} message
|
|
103
|
+
* @returns {void}
|
|
104
|
+
*/
|
|
105
|
+
function log(message) {
|
|
106
|
+
logger.info(message);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* The torrent for a sourceKey, waiting for it if it is still being added.
|
|
111
|
+
*
|
|
112
|
+
* The map holds a PROMISE, registered the moment the add begins rather than
|
|
113
|
+
* when it finishes. That distinction is the whole fix: adding a magnet takes as
|
|
114
|
+
* long as its metadata does — seconds to tens of seconds — and until 2.9.77
|
|
115
|
+
* everything naming that source in the meantime was told `Unknown source`,
|
|
116
|
+
* which is false. The source exists; it is not ready. Reproduced with a magnet
|
|
117
|
+
* nobody seeds: stats, the file listing and a read all failed instantly while
|
|
118
|
+
* the add was still in flight, which on the loading screen shows up as no
|
|
119
|
+
* peers, no progress, and a plan request that fails before the torrent has had
|
|
120
|
+
* a chance to start.
|
|
121
|
+
*
|
|
122
|
+
* A source that was never added still throws, which is the honest answer.
|
|
123
|
+
*
|
|
124
|
+
* @param {string} sourceKey
|
|
125
|
+
* @returns {Promise<import("webtorrent").Torrent>}
|
|
126
|
+
*/
|
|
127
|
+
/**
|
|
128
|
+
* The torrent for a sourceKey IF there is one, and never one that has to be
|
|
129
|
+
* built to answer.
|
|
130
|
+
*
|
|
131
|
+
* The other half of the pair above, for the questions that are about something
|
|
132
|
+
* going away: a departure is not a reason to add anything, and the answer
|
|
133
|
+
* "there is nothing here" is a complete answer to them.
|
|
134
|
+
*
|
|
135
|
+
* @param {string} sourceKey
|
|
136
|
+
* @returns {Promise<import("webtorrent").Torrent | null>}
|
|
137
|
+
*/
|
|
138
|
+
async function knownTorrent(sourceKey) {
|
|
139
|
+
const pending = torrentsByKey.get(sourceKey);
|
|
140
|
+
if (!pending) {
|
|
141
|
+
return null;
|
|
142
|
+
}
|
|
143
|
+
const torrent = await pending.catch(() => null);
|
|
144
|
+
return isUsableTorrentHandle(torrent) ? torrent : null;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
async function requireTorrent(sourceKey) {
|
|
148
|
+
const pending = torrentsByKey.get(sourceKey);
|
|
149
|
+
if (!pending) {
|
|
150
|
+
throw new Error(`Unknown source ${sourceKey}.`);
|
|
151
|
+
}
|
|
152
|
+
const torrent = await pending;
|
|
153
|
+
if (isUsableTorrentHandle(torrent)) {
|
|
154
|
+
return torrent;
|
|
155
|
+
}
|
|
156
|
+
// The pool destroys a torrent that has gone unread for a quarter of an hour,
|
|
157
|
+
// and under disk pressure. It clears its OWN map when it does; this one it
|
|
158
|
+
// knows nothing about, so the promise here went on resolving to a corpse: a
|
|
159
|
+
// destroyed torrent keeps its object but loses its files. Every later session
|
|
160
|
+
// for that source then failed the same way — the plan and the codec probe
|
|
161
|
+
// answered from cache in milliseconds, nothing waited for metadata because
|
|
162
|
+
// everything believed the torrent was known, and ffmpeg's first read died on
|
|
163
|
+
// `File N not found` 130 ms in, after which the session answered 500 for
|
|
164
|
+
// ever. Measured 2026-08-06 on two sessions in a row, both from a phone,
|
|
165
|
+
// which is what made it look like a mobile problem.
|
|
166
|
+
const recipe = sourceRecipes.get(sourceKey);
|
|
167
|
+
if (!recipe) {
|
|
168
|
+
torrentsByKey.delete(sourceKey);
|
|
169
|
+
throw new Error(`Source ${sourceKey} is gone and cannot be re-added.`);
|
|
170
|
+
}
|
|
171
|
+
const revived = pool.getTorrent(recipe.sourceType, recipe.source);
|
|
172
|
+
torrentsByKey.set(sourceKey, revived);
|
|
173
|
+
revived.catch(() => {
|
|
174
|
+
if (torrentsByKey.get(sourceKey) === revived) {
|
|
175
|
+
torrentsByKey.delete(sourceKey);
|
|
176
|
+
}
|
|
177
|
+
});
|
|
178
|
+
return revived;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Fragments waiting for the main thread to say it has finished reading them,
|
|
184
|
+
* keyed by request id. One per read, because only one fragment is in flight.
|
|
185
|
+
*
|
|
186
|
+
* @type {Map<number, () => void>}
|
|
187
|
+
*/
|
|
188
|
+
const fragmentWaiters = new Map();
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Wake a read that is waiting for a fragment to be confirmed.
|
|
192
|
+
*
|
|
193
|
+
* Used both by the confirmation itself and by cancellation — a cancelled read
|
|
194
|
+
* will never be confirmed, and without this it would wait forever holding a pin.
|
|
195
|
+
*
|
|
196
|
+
* @param {number} id
|
|
197
|
+
* @returns {void}
|
|
198
|
+
*/
|
|
199
|
+
function settleFragment(id) {
|
|
200
|
+
const done = fragmentWaiters.get(id);
|
|
201
|
+
if (done) {
|
|
202
|
+
fragmentWaiters.delete(id);
|
|
203
|
+
done();
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
/**
|
|
208
|
+
* Send one fragment's position and wait until the main thread is done with it.
|
|
209
|
+
*
|
|
210
|
+
* The pin is dropped only after the confirmation, because until then the other
|
|
211
|
+
* thread may still be reading those exact bytes.
|
|
212
|
+
*
|
|
213
|
+
* @param {number} id
|
|
214
|
+
* @param {import("./piece-reader.js").PieceFragment} fragment
|
|
215
|
+
* @returns {Promise<void>}
|
|
216
|
+
*/
|
|
217
|
+
function sendFragment(id, fragment) {
|
|
218
|
+
return new Promise((resolve) => {
|
|
219
|
+
fragmentWaiters.set(id, () => {
|
|
220
|
+
fragment.release();
|
|
221
|
+
resolve();
|
|
222
|
+
});
|
|
223
|
+
parentPort.postMessage({
|
|
224
|
+
type: Event.FRAGMENT,
|
|
225
|
+
id,
|
|
226
|
+
pieceIndex: fragment.pieceIndex,
|
|
227
|
+
buffer: fragment.buffer,
|
|
228
|
+
offset: fragment.offset,
|
|
229
|
+
length: fragment.length
|
|
230
|
+
});
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Stream a byte range back as CHUNK messages.
|
|
236
|
+
*
|
|
237
|
+
* Reads through WebTorrent's own read stream — which serves already-downloaded
|
|
238
|
+
* pieces from disk and waits for the rest — and forwards it in
|
|
239
|
+
* {@link STREAM_CHUNK_BYTES} pieces, transferring ownership of each so nothing
|
|
240
|
+
* is copied across the boundary. `createSendStream` applies the backpressure,
|
|
241
|
+
* so a fast disk cannot outrun the main thread and rebuild the queue in memory.
|
|
242
|
+
*
|
|
243
|
+
* @param {object} params
|
|
244
|
+
* @param {number} params.id - Request id; CHUNK/READ_END carry it.
|
|
245
|
+
* @param {string} params.sourceKey
|
|
246
|
+
* @param {number} params.fileIndex
|
|
247
|
+
* @param {number | null} params.start - Inclusive, or null for the whole file.
|
|
248
|
+
* @param {number | null} params.end - Inclusive.
|
|
249
|
+
* @returns {Promise<void>}
|
|
250
|
+
*/
|
|
251
|
+
async function streamRange({ id, sourceKey, fileIndex, start, end, windowBytes }) {
|
|
252
|
+
const torrent = await requireTorrent(sourceKey);
|
|
253
|
+
const file = torrent.files?.[fileIndex];
|
|
254
|
+
if (!file) {
|
|
255
|
+
throw new Error(`File ${fileIndex} not found in ${sourceKey}.`);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const sender = createSendStream({ port: parentPort, requestId: id });
|
|
259
|
+
readsById.set(id, sender);
|
|
260
|
+
|
|
261
|
+
const rangeStart = start ?? 0;
|
|
262
|
+
const rangeEnd = end ?? file.length - 1;
|
|
263
|
+
|
|
264
|
+
let failed = false;
|
|
265
|
+
try {
|
|
266
|
+
// Positions in shared memory, not bytes: the main thread maps the same pool
|
|
267
|
+
// and reads each fragment in place, so nothing is copied and nothing is
|
|
268
|
+
// transferred. See `piece-reader.js`.
|
|
269
|
+
for await (const fragment of readFragments({
|
|
270
|
+
torrent,
|
|
271
|
+
fileIndex,
|
|
272
|
+
start: rangeStart,
|
|
273
|
+
end: rangeEnd,
|
|
274
|
+
cancellation: sender,
|
|
275
|
+
windowBytes
|
|
276
|
+
})) {
|
|
277
|
+
if (sender.isCancelled()) {
|
|
278
|
+
fragment.release();
|
|
279
|
+
break;
|
|
280
|
+
}
|
|
281
|
+
// One fragment in flight at a time. Each one holds a piece pinned, and
|
|
282
|
+
// the store guarantees only two resident pieces at its smallest budget —
|
|
283
|
+
// holding two pins while asking for a third would deadlock it against
|
|
284
|
+
// itself. The round trip costs ~100 µs against a piece worth megabytes,
|
|
285
|
+
// so there is nothing to win by overlapping them.
|
|
286
|
+
await sendFragment(id, fragment);
|
|
287
|
+
}
|
|
288
|
+
} catch (error) {
|
|
289
|
+
// The end-of-read marker means "the body is complete". Sending it after a
|
|
290
|
+
// failure told the reader the file simply ended — a truncated segment that
|
|
291
|
+
// ffmpeg reported as `Stream ends prematurely`, with the real cause thrown
|
|
292
|
+
// away. Let the error propagate instead; the command handler reports it and
|
|
293
|
+
// the main thread fails the stream.
|
|
294
|
+
failed = true;
|
|
295
|
+
throw error;
|
|
296
|
+
} finally {
|
|
297
|
+
readsById.delete(id);
|
|
298
|
+
// Any fragment still awaiting confirmation will never get one now; settling
|
|
299
|
+
// it here releases its pin rather than leaking a held slot.
|
|
300
|
+
settleFragment(id);
|
|
301
|
+
if (!failed) {
|
|
302
|
+
sender.end();
|
|
303
|
+
}
|
|
304
|
+
// Nothing else to tear down: the reader owns no stream of its own, and a
|
|
305
|
+
// cancelled read stops at its next fragment boundary because it polls the
|
|
306
|
+
// same `sender` for cancellation.
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Run one command and return its result.
|
|
312
|
+
*
|
|
313
|
+
* @param {string} command
|
|
314
|
+
* @param {object} params
|
|
315
|
+
* @param {number} id
|
|
316
|
+
* @returns {Promise<unknown>}
|
|
317
|
+
*/
|
|
318
|
+
async function runCommand(command, params, id) {
|
|
319
|
+
switch (command) {
|
|
320
|
+
case Command.ADD_SOURCE: {
|
|
321
|
+
// Registered before it resolves, so anything naming this source while it
|
|
322
|
+
// is being added waits for it instead of being told it does not exist.
|
|
323
|
+
// Reusing the same promise for a repeated add also collapses two callers
|
|
324
|
+
// racing to open the same torrent into one.
|
|
325
|
+
sourceRecipes.set(params.sourceKey, {
|
|
326
|
+
sourceType: params.sourceType,
|
|
327
|
+
source: params.source
|
|
328
|
+
});
|
|
329
|
+
let pending = torrentsByKey.get(params.sourceKey);
|
|
330
|
+
if (!pending) {
|
|
331
|
+
pending = pool.getTorrent(params.sourceType, params.source);
|
|
332
|
+
torrentsByKey.set(params.sourceKey, pending);
|
|
333
|
+
// A failed add must not be remembered, or every later attempt at this
|
|
334
|
+
// source replays the same failure. The handler also marks the rejection
|
|
335
|
+
// as observed, so it cannot surface as an unhandled one.
|
|
336
|
+
pending.catch(() => {
|
|
337
|
+
if (torrentsByKey.get(params.sourceKey) === pending) {
|
|
338
|
+
torrentsByKey.delete(params.sourceKey);
|
|
339
|
+
}
|
|
340
|
+
});
|
|
341
|
+
}
|
|
342
|
+
const torrent = await pending;
|
|
343
|
+
return {
|
|
344
|
+
infoHash: torrent.infoHash,
|
|
345
|
+
name: torrent.name,
|
|
346
|
+
// Files cross as plain data; the objects stay here.
|
|
347
|
+
files: (torrent.files ?? []).map((file, index) => ({
|
|
348
|
+
index,
|
|
349
|
+
name: file.name,
|
|
350
|
+
path: file.path,
|
|
351
|
+
length: file.length
|
|
352
|
+
}))
|
|
353
|
+
};
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
case Command.LIST_FILES: {
|
|
357
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
358
|
+
return (torrent.files ?? []).map((file, index) => ({
|
|
359
|
+
index,
|
|
360
|
+
name: file.name,
|
|
361
|
+
path: file.path,
|
|
362
|
+
length: file.length
|
|
363
|
+
}));
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
case Command.HELD_TORRENTS: {
|
|
367
|
+
// Which films this proxy has right now, and how much of each. Answered
|
|
368
|
+
// from the live client rather than from the main thread's map of
|
|
369
|
+
// stand-ins, which is only cleared on shutdown and would name films this
|
|
370
|
+
// proxy let go of hours ago.
|
|
371
|
+
const held = [];
|
|
372
|
+
for (const torrent of pool.client?.torrents ?? []) {
|
|
373
|
+
const infoHash = String(torrent?.infoHash ?? "");
|
|
374
|
+
if (!infoHash) {
|
|
375
|
+
continue;
|
|
376
|
+
}
|
|
377
|
+
held.push({
|
|
378
|
+
infoHash,
|
|
379
|
+
// A viewer sent here for a film nobody has downloaded any of gains
|
|
380
|
+
// nothing, so the share is reported and the decision is made where
|
|
381
|
+
// the viewer is.
|
|
382
|
+
progress: Number.isFinite(torrent?.progress) ? torrent.progress : 0,
|
|
383
|
+
bytes: Number.isFinite(torrent?.downloaded) ? torrent.downloaded : 0
|
|
384
|
+
});
|
|
385
|
+
}
|
|
386
|
+
return { held };
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
case Command.TORRENT_TOTALS: {
|
|
390
|
+
// Downloaded and uploaded are counted apart: hashing every downloaded
|
|
391
|
+
// byte is work of a different order from sending one back to the swarm,
|
|
392
|
+
// and adding them would price both at whatever the mixture happened to
|
|
393
|
+
// be.
|
|
394
|
+
let downloaded = 0;
|
|
395
|
+
let uploaded = 0;
|
|
396
|
+
for (const torrent of pool.client?.torrents ?? []) {
|
|
397
|
+
const gotBytes = Number(torrent?.downloaded);
|
|
398
|
+
const sentBytes = Number(torrent?.uploaded);
|
|
399
|
+
downloaded += Number.isFinite(gotBytes) ? gotBytes : 0;
|
|
400
|
+
uploaded += Number.isFinite(sentBytes) ? sentBytes : 0;
|
|
401
|
+
}
|
|
402
|
+
return { downloaded, uploaded };
|
|
403
|
+
}
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
case Command.FILL_FILE: {
|
|
407
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
408
|
+
return {
|
|
409
|
+
started: fillFileInBackground(torrent, params.fileIndex, params.sourceKey)
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
|
|
413
|
+
case Command.FILL_TORRENT: {
|
|
414
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
415
|
+
return { started: pool.fillTorrentAsCapacityAllows(torrent) };
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
case Command.WARM_POSITION: {
|
|
419
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
420
|
+
return {
|
|
421
|
+
started: await warmResumePosition(
|
|
422
|
+
torrent,
|
|
423
|
+
params.fileIndex,
|
|
424
|
+
params.sourceKey,
|
|
425
|
+
params.positionSeconds,
|
|
426
|
+
{
|
|
427
|
+
// Told by the main thread, which is where what a file states about
|
|
428
|
+
// itself is read.
|
|
429
|
+
durationSeconds: params.durationSeconds,
|
|
430
|
+
fetchRegion: (start, bytes) =>
|
|
431
|
+
pool.prefetchFileRegion(torrent, params.fileIndex, start, bytes)
|
|
432
|
+
}
|
|
433
|
+
)
|
|
434
|
+
};
|
|
435
|
+
}
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
case Command.FILE_STATS: {
|
|
439
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
440
|
+
const stats = pool.getFileStats(torrent, params.fileIndex, {
|
|
441
|
+
resumeAnchorByteStart: params.resumeAnchorByteStart ?? null
|
|
442
|
+
});
|
|
443
|
+
// What this file's own interruptions demand, measured by the reader in
|
|
444
|
+
// this thread. It travels with the stats because the caller asking for
|
|
445
|
+
// them is the one that has to decide with them — the browser's smallest
|
|
446
|
+
// safe buffer, and the speed a quality step must sustain. Null until a
|
|
447
|
+
// second interruption has been seen: one wait shows no interval, and an
|
|
448
|
+
// interval invented from one point is exactly what this work removes.
|
|
449
|
+
const file = Array.isArray(torrent?.files) ? torrent.files[params.fileIndex] : null;
|
|
450
|
+
return {
|
|
451
|
+
...stats,
|
|
452
|
+
supply: supplyFiguresFor(torrent?.infoHash, file?.name, params.segmentSeconds ?? 4)
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
case Command.PRIORITY_MAP: {
|
|
457
|
+
// A MAP WITH NOTHING IN IT MUST NOT BRING A TORRENT BACK. It is what is
|
|
458
|
+
// said when the last viewer of a file leaves, which is also when the
|
|
459
|
+
// torrent may be on its way out — and `requireTorrent` rebuilds a dead
|
|
460
|
+
// handle from its recipe, so asking that way would re-add a torrent in
|
|
461
|
+
// order to be told that nothing is wanted of it.
|
|
462
|
+
const zones = Array.isArray(params.zones) ? params.zones : [];
|
|
463
|
+
const torrent = zones.length === 0
|
|
464
|
+
? await knownTorrent(params.sourceKey)
|
|
465
|
+
: await requireTorrent(params.sourceKey);
|
|
466
|
+
if (torrent) {
|
|
467
|
+
pool.applyPriorityMap(torrent, params.fileIndex, zones, params.durationSeconds);
|
|
468
|
+
}
|
|
469
|
+
return true;
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
case Command.PRIORITIZE: {
|
|
473
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
474
|
+
pool.prioritizeByteRange(torrent, params.fileIndex, params.byteStart, params.windowBytes, {
|
|
475
|
+
wholeFileRead: params.wholeFileRead === true
|
|
476
|
+
});
|
|
477
|
+
return true;
|
|
478
|
+
}
|
|
479
|
+
|
|
480
|
+
case Command.PREFETCH_EDGES: {
|
|
481
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
482
|
+
return pool.prefetchFileEdges(torrent, params.fileIndex, params.options ?? {});
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
case Command.READ_RANGE: {
|
|
486
|
+
// Streams its own reply; the caller's promise resolves once the body has
|
|
487
|
+
// been fully sent, which is what lets the client await completion.
|
|
488
|
+
await streamRange({
|
|
489
|
+
id,
|
|
490
|
+
sourceKey: params.sourceKey,
|
|
491
|
+
fileIndex: params.fileIndex,
|
|
492
|
+
start: params.start ?? null,
|
|
493
|
+
end: params.end ?? null,
|
|
494
|
+
windowBytes: params.windowBytes
|
|
495
|
+
});
|
|
496
|
+
return true;
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
case Command.CANCEL_READ: {
|
|
500
|
+
readsById.get(params.readId)?.cancel();
|
|
501
|
+
// A cancelled read will never have its outstanding fragment confirmed, so
|
|
502
|
+
// wake it here — otherwise it waits forever with a piece pinned.
|
|
503
|
+
settleFragment(params.readId);
|
|
504
|
+
return true;
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
case Command.HELD_RANGES: {
|
|
508
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
509
|
+
return { ranges: heldRangesOf(torrent, params.fileIndex) };
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
case Command.READ_HELD: {
|
|
513
|
+
const torrent = await requireTorrent(params.sourceKey);
|
|
514
|
+
const bytes = await readHeldBytes(torrent, params.fileIndex, params.start, params.end, logger);
|
|
515
|
+
return { bytes };
|
|
516
|
+
}
|
|
517
|
+
|
|
518
|
+
case Command.SPILL_ALLOWANCE: {
|
|
519
|
+
// The disk has one owner and it is on the main thread, where the segments
|
|
520
|
+
// are. What arrives is this thread’s whole share; the stores divide it
|
|
521
|
+
// between themselves.
|
|
522
|
+
return reviseSpillBudgets(Number(params.bytes));
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
case Command.MEMORY_ALLOWANCE: {
|
|
526
|
+
// Memory has one owner, on the main thread, and it is the same owner the
|
|
527
|
+
// disk has — the two resources trade, so they cannot be divided apart.
|
|
528
|
+
reviseStoreBudgets(Number(params.bytes));
|
|
529
|
+
return memoryClaim();
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
case Command.WHOLE_FILES_ALLOWANCE: {
|
|
533
|
+
// Whole files are held on this thread and the disk has one owner, on the
|
|
534
|
+
// other. What arrives is their share; what goes back is what they hold
|
|
535
|
+
// after it, which is what the owner divides by next time.
|
|
536
|
+
const after = await completedFiles.allow(Number(params.bytes));
|
|
537
|
+
return after.bytes;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
case Command.DESTROY_ALL: {
|
|
541
|
+
torrentsByKey.clear();
|
|
542
|
+
sourceRecipes.clear();
|
|
543
|
+
await pool.destroyAll();
|
|
544
|
+
return true;
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
default:
|
|
548
|
+
throw new Error(`Unknown torrent-worker command: ${command}`);
|
|
549
|
+
}
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
parentPort.on("message", async (message) => {
|
|
553
|
+
// Chunk acknowledgements are not commands — they release backpressure on an
|
|
554
|
+
// in-flight read.
|
|
555
|
+
if (message?.type === Event.CHUNK_ACK) {
|
|
556
|
+
readsById.get(message.id)?.ack();
|
|
557
|
+
return;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
// The main thread has finished reading a fragment out of shared memory, so
|
|
561
|
+
// its piece may be unpinned and the read may continue.
|
|
562
|
+
if (message?.type === Event.FRAGMENT_DONE) {
|
|
563
|
+
settleFragment(message.id);
|
|
564
|
+
return;
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
const { command, id, params } = message ?? {};
|
|
568
|
+
try {
|
|
569
|
+
const result = await runCommand(command, params ?? {}, id);
|
|
570
|
+
parentPort.postMessage({ type: Event.RESULT, id, result });
|
|
571
|
+
} catch (error) {
|
|
572
|
+
parentPort.postMessage({ type: Event.ERROR, id, error: error?.message ?? String(error) });
|
|
573
|
+
}
|
|
574
|
+
});
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* How often the piece store reports what it has been doing.
|
|
578
|
+
*
|
|
579
|
+
* The store decides whether a read costs nothing or costs a disk trip, and
|
|
580
|
+
* until 2.9.75 nothing about it reached the log — a field oddity would have had
|
|
581
|
+
* no evidence to work from. Reported only when something changed, so an idle
|
|
582
|
+
* proxy stays quiet.
|
|
583
|
+
*/
|
|
584
|
+
// The torrent worker's OWN isolate, reported from inside it. The piece pool is
|
|
585
|
+
// a `SharedArrayBuffer` allocated here, so the main thread's counters cannot
|
|
586
|
+
// see it however carefully they are read — which is half the reason 650 MB of a
|
|
587
|
+
// 893 MB process had no explanation on 2026-08-28 (roadmap item 2).
|
|
588
|
+
// A second between readings, a minute between lines unless the heap moved by
|
|
589
|
+
// 25 MB, and a heap snapshot of THIS isolate on every new high-water above
|
|
590
|
+
// 400 MB. Three deaths — 2026-08-30 14:00 and 23:19, and
|
|
591
|
+
// 2026-08-31 13:27 — went from a 30 MB heap to the 2240 MB ceiling inside one
|
|
592
|
+
// sixty-second gap, and the only snapshots ever written were of the main
|
|
593
|
+
// isolate, whose heap is 26 MB. So the isolate that dies has never once been
|
|
594
|
+
// looked at (roadmap item 2, `research/worker-heap-oom-2026-08-31.md`).
|
|
595
|
+
startMemoryReport({
|
|
596
|
+
log,
|
|
597
|
+
readStores: collectStoreStats,
|
|
598
|
+
// Beside the isolate's own figures, and on the SAME line, because the
|
|
599
|
+
// question they answer together is whether the off-heap mass is buffers this
|
|
600
|
+
// thread still refers to or buffers the collector has not reached yet. On
|
|
601
|
+
// separate timers the two were up to a minute apart and could not be
|
|
602
|
+
// compared at all (roadmap item 2).
|
|
603
|
+
readExtra: describePieceBuffers,
|
|
604
|
+
scope: "thread",
|
|
605
|
+
label: "torrent worker",
|
|
606
|
+
intervalMs: WORKER_MEMORY_SAMPLE_MS,
|
|
607
|
+
quietMs: 60_000,
|
|
608
|
+
changeBytes: 25 * 1024 * 1024,
|
|
609
|
+
snapshotDir: workerData?.stateDir || undefined,
|
|
610
|
+
snapshotFloorBytes: 400 * 1024 * 1024,
|
|
611
|
+
snapshotGrowthBytes: 400 * 1024 * 1024,
|
|
612
|
+
keepSnapshots: 3
|
|
613
|
+
});
|
|
614
|
+
|
|
615
|
+
const STORE_REPORT_INTERVAL_MS = 60_000;
|
|
616
|
+
|
|
617
|
+
/** Last reported reserve, so an unchanged one stays silent. */
|
|
618
|
+
let lastClaimsWithdrawn = 0;
|
|
619
|
+
let lastReserveBytes = 0;
|
|
620
|
+
|
|
621
|
+
/** Last reported figures per store, so unchanged ones stay silent. */
|
|
622
|
+
const lastReported = new Map();
|
|
623
|
+
|
|
624
|
+
setInterval(() => {
|
|
625
|
+
// What the machine can spare NOW, not what it could spare when each store was
|
|
626
|
+
// created. With per-piece buffers a lowered ceiling is honoured immediately:
|
|
627
|
+
// excess pieces are evicted to disk and their memory is reclaimable.
|
|
628
|
+
// What the machine has been seen to need for everything that is not us. It
|
|
629
|
+
// starts at nothing and grows only on evidence, so it is worth saying when it
|
|
630
|
+
// moves — it is the one term of the budget that comes from observation of
|
|
631
|
+
// other processes rather than from our own readers.
|
|
632
|
+
const reserveBefore = machineReserveBytes();
|
|
633
|
+
for (const revised of reviseStoreBudgets()) {
|
|
634
|
+
if (revised.evicted > 0) {
|
|
635
|
+
log(
|
|
636
|
+
`piece-store "${revised.name.slice(0, 40)}": allowance is now ` +
|
|
637
|
+
`${Math.round(revised.ceilingBytes / 1048576)}MB, evicted ${revised.evicted} piece(s) to meet it` +
|
|
638
|
+
(revised.releasedBlocks > 0 ? `, gave back ${revised.releasedBlocks} block(s) of memory` : "") +
|
|
639
|
+
` — now ${Math.round(revised.committedBytes / 1048576)}MB committed`
|
|
640
|
+
);
|
|
641
|
+
} else if (revised.belowActiveDemand) {
|
|
642
|
+
log(
|
|
643
|
+
`piece-store "${revised.name.slice(0, 40)}": the machine's share is smaller than active read demand, ` +
|
|
644
|
+
`so the allowance is held at ${Math.round(revised.ceilingBytes / 1048576)}MB ` +
|
|
645
|
+
"— a store cannot finish a read if it cannot hold the pieces that read is using"
|
|
646
|
+
);
|
|
647
|
+
} else if (revised.committedBytes > revised.ceilingBytes) {
|
|
648
|
+
log(
|
|
649
|
+
`piece-store "${revised.name.slice(0, 40)}": allowance is now ` +
|
|
650
|
+
`${Math.round(revised.ceilingBytes / 1048576)}MB and ` +
|
|
651
|
+
`${Math.round(revised.committedBytes / 1048576)}MB is committed — ` +
|
|
652
|
+
`all resident pieces are pinned, cannot shrink yet`
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
}
|
|
656
|
+
// WHAT THE ANNOUNCEMENTS ACTUALLY DID. The stores count every piece they
|
|
657
|
+
// stopped being able to produce; this counts the ones where the library did
|
|
658
|
+
// still hold a claim and it was taken back. The GAP between the two is the
|
|
659
|
+
// reading: announcements far above withdrawals mean the stores are mostly
|
|
660
|
+
// dropping pieces that were never completed, and a withdrawal count stuck at
|
|
661
|
+
// zero while announcements climb means the mechanism is not reaching the
|
|
662
|
+
// torrent at all.
|
|
663
|
+
if (pool.claimsWithdrawn !== lastClaimsWithdrawn) {
|
|
664
|
+
lastClaimsWithdrawn = pool.claimsWithdrawn;
|
|
665
|
+
log(
|
|
666
|
+
`torrent-pool: ${pool.claimsWithdrawn} piece claim(s) withdrawn — pieces this proxy ` +
|
|
667
|
+
"had dropped and has now told the swarm it needs again"
|
|
668
|
+
);
|
|
669
|
+
}
|
|
670
|
+
const reserveNow = machineReserveBytes();
|
|
671
|
+
if (reserveNow !== reserveBefore || reserveNow !== lastReserveBytes) {
|
|
672
|
+
lastReserveBytes = reserveNow;
|
|
673
|
+
log(
|
|
674
|
+
`piece-store: leaving ${Math.round(reserveNow / 1048576)}MB for everything else on this ` +
|
|
675
|
+
"machine — the largest fall in available memory this process has seen and did not cause"
|
|
676
|
+
);
|
|
677
|
+
}
|
|
678
|
+
for (const stats of collectStoreStats()) {
|
|
679
|
+
const signature =
|
|
680
|
+
`${stats.fromMemory}/${stats.fromDisk}/${stats.spills}/${stats.revivals}/` +
|
|
681
|
+
`${stats.blockedByPins}/${stats.evictedOnRevise}/${stats.spillFailures}/` +
|
|
682
|
+
`${stats.evictedProtected}/${stats.demand?.unionPieces ?? 0}/${stats.admittedToDisk}/` +
|
|
683
|
+
`${stats.blocksAllocated}/${stats.blocksFree}/${stats.blocksReleased}/${stats.withdrawn}`;
|
|
684
|
+
if (lastReported.get(stats.name) === signature) {
|
|
685
|
+
continue;
|
|
686
|
+
}
|
|
687
|
+
lastReported.set(stats.name, signature);
|
|
688
|
+
|
|
689
|
+
const reads = stats.fromMemory + stats.fromDisk;
|
|
690
|
+
const fromMemoryShare = reads > 0 ? ((stats.fromMemory / reads) * 100).toFixed(1) : "—";
|
|
691
|
+
log(
|
|
692
|
+
`piece-store "${stats.name.slice(0, 40)}": resident=${stats.resident}/${stats.capacity} ` +
|
|
693
|
+
`(${Math.round((stats.residentBytes || 0) / 1048576)}MB of ` +
|
|
694
|
+
`${Math.round((stats.budgetBytes || 0) / 1048576)}MB allowed) ` +
|
|
695
|
+
`committed=${Math.round((stats.committedBytes || 0) / 1048576)}MB ` +
|
|
696
|
+
`blocks=${stats.blocksAllocated} (${stats.blocksFree} spare) ` +
|
|
697
|
+
`on-disk=${Math.round((stats.spilledBytes || 0) / 1048576)}MB ` +
|
|
698
|
+
`pinned=${stats.pinned} spilled=${stats.spilled} reads=${reads} (${fromMemoryShare}% from memory) ` +
|
|
699
|
+
`spills=${stats.spills} revivals=${stats.revivals}` +
|
|
700
|
+
(stats.blockedByPins > 0 ? ` no-block-for=${stats.blockedByPins}` : "") +
|
|
701
|
+
(stats.admittedWithoutSlot > 0 ? ` to-disk-for-want-of-memory=${stats.admittedWithoutSlot}` : "") +
|
|
702
|
+
(stats.stillMs > 1000 ? ` nothing-moved-for=${Math.round(stats.stillMs / 1000)}s` : "") +
|
|
703
|
+
(stats.evictedOnRevise > 0 ? ` evictedOnRevise=${stats.evictedOnRevise}` : "") +
|
|
704
|
+
// How many pieces this store stopped being able to produce and said so,
|
|
705
|
+
// which is what makes the eviction's own bargain checkable: every one of
|
|
706
|
+
// these is a piece the torrent has been told to fetch again if it is ever
|
|
707
|
+
// wanted. A session where this climbs while reads keep succeeding is the
|
|
708
|
+
// bargain working; before 2026-09-12 the figure did not exist and the
|
|
709
|
+
// claim was never withdrawn at all.
|
|
710
|
+
(stats.withdrawn > 0 ? ` withdrawn=${stats.withdrawn}` : "") +
|
|
711
|
+
(stats.spillFailures > 0 ? ` spill-failures=${stats.spillFailures}` : "") +
|
|
712
|
+
(stats.outstanding > 0 ? ` outstanding=${stats.outstanding}` : "")
|
|
713
|
+
);
|
|
714
|
+
// Why it spills, on its own line because it is a different question from
|
|
715
|
+
// how much it holds. Three facts, and between them they say whether the
|
|
716
|
+
// thrashing is a policy to fix or arithmetic to accept: what the readers
|
|
717
|
+
// together are asking to keep against what the store may hold; how many
|
|
718
|
+
// evictions had to take a piece a reader had declared it wants; and how
|
|
719
|
+
// long a piece stayed on disk before it was wanted back. On 2026-09-02 a
|
|
720
|
+
// session did 6565 spills and 7575 revivals with 53.6% of reads served
|
|
721
|
+
// from memory, and nothing recorded which of the three was the cause
|
|
722
|
+
// (roadmap item 9).
|
|
723
|
+
const demand = stats.demand;
|
|
724
|
+
if (demand && demand.readers > 0) {
|
|
725
|
+
const age = stats.revivalAgeMedianMs;
|
|
726
|
+
log(
|
|
727
|
+
`piece-store "${stats.name.slice(0, 40)}" demand: ${demand.readers} reader(s) want ` +
|
|
728
|
+
`${demand.unionPieces} piece(s) of ${demand.capacity} the store may hold ` +
|
|
729
|
+
`(widest window ${demand.widestPieces})` +
|
|
730
|
+
// NAMED, because the count read as five encoders on a session that had
|
|
731
|
+
// two: a "reader" is whoever declared a range, and four of the five
|
|
732
|
+
// were zones of the priority map. Choosing between narrowing the
|
|
733
|
+
// windows and raising the allowance was guesswork without this.
|
|
734
|
+
` [${demand.names.join(" ")}]` +
|
|
735
|
+
(stats.evictedProtected > 0
|
|
736
|
+
? `; ${stats.evictedProtected} of ${stats.spills} eviction(s) took a piece a reader had declared`
|
|
737
|
+
: "; no eviction has taken a declared piece") +
|
|
738
|
+
(stats.evictedWithDistance > 0
|
|
739
|
+
? `, a victim lay ${(stats.evictedDistanceSum / stats.evictedWithDistance).toFixed(1)} ` +
|
|
740
|
+
"piece(s) from the nearest window on average"
|
|
741
|
+
: "") +
|
|
742
|
+
(age === null
|
|
743
|
+
? "; nothing has come back from disk yet"
|
|
744
|
+
: `; a revived piece had been on disk ${(age / 1000).toFixed(1)}s (median of ` +
|
|
745
|
+
`${stats.revivalAgeSamples}, ${stats.revivedWithinFiveSeconds} of them within 5s)`) +
|
|
746
|
+
`; of ${stats.admittedInsideWindow + stats.admittedOutsideWindow} piece(s) admitted ` +
|
|
747
|
+
`${stats.admittedOutsideWindow} were in nobody's window, ${stats.admittedToDisk} of those ` +
|
|
748
|
+
"went straight to disk" +
|
|
749
|
+
`; ${stats.blocksAllocated} block(s) of memory exist, ${stats.blocksFree} of them spare` +
|
|
750
|
+
(stats.reuseGapMs === null
|
|
751
|
+
? ", none re-used yet"
|
|
752
|
+
: `, a block waits up to ${(stats.reuseGapMs / 1000).toFixed(1)}s before it is wanted again`) +
|
|
753
|
+
`, ${stats.blocksReleased} given back` +
|
|
754
|
+
(stats.spillsSkipped > 0
|
|
755
|
+
? `; ${stats.spillsSkipped} of ${stats.spills} eviction(s) needed no write, the disk already had them`
|
|
756
|
+
: "") +
|
|
757
|
+
(stats.returnedWhilePinned > 0
|
|
758
|
+
? `; ${stats.returnedWhilePinned} BLOCK(S) WERE RECYCLED WHILE STILL BEING READ`
|
|
759
|
+
: "")
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
}, STORE_REPORT_INTERVAL_MS).unref();
|
|
764
|
+
|
|
765
|
+
/**
|
|
766
|
+
* The blocks of piece memory this thread has allocated against the pieces the
|
|
767
|
+
* stores hold in them.
|
|
768
|
+
*
|
|
769
|
+
* Not per store: the collector is per thread, and the question is about the
|
|
770
|
+
* thread. With a pool one block serves many pieces, so a number of allocations
|
|
771
|
+
* that keeps climbing while the stores hold a steady number of pieces means
|
|
772
|
+
* blocks are being made and thrown away instead of re-used.
|
|
773
|
+
*
|
|
774
|
+
* @returns {string}
|
|
775
|
+
*/
|
|
776
|
+
function describePieceBuffers() {
|
|
777
|
+
const collection = pieceBufferCollection();
|
|
778
|
+
if (collection.released === 0) {
|
|
779
|
+
return "";
|
|
780
|
+
}
|
|
781
|
+
const alive = collection.released - collection.collected;
|
|
782
|
+
const held = collectStoreStats().reduce((sum, stats) => sum + (stats.resident || 0), 0);
|
|
783
|
+
return (
|
|
784
|
+
`memory blocks ${collection.released} allocated, ${collection.collected} collected, ` +
|
|
785
|
+
`${alive} still alive against ${held} piece(s) the stores hold`
|
|
786
|
+
);
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
/**
|
|
790
|
+
* Say that pieces of the actively-read files of one torrent have arrived.
|
|
791
|
+
*
|
|
792
|
+
* It used to WALK them — reading each file's new subtitle clusters and pushing
|
|
793
|
+
* the cues it found. That put the cue reading, and with it a container parse,
|
|
794
|
+
* in the thread that owns the swarm; the walk is on the main thread now and
|
|
795
|
+
* this announces the one fact only this thread can know.
|
|
796
|
+
*
|
|
797
|
+
* Which files: the ones anything is stated for — a viewer's own picture and
|
|
798
|
+
* soundtrack through the priority map, and the ends of a file that is open. It
|
|
799
|
+
* replaced a count of readers, which said the same thing by keeping a second
|
|
800
|
+
* copy of it.
|
|
801
|
+
*
|
|
802
|
+
* @param {string} sourceKey
|
|
803
|
+
* @param {object} torrent
|
|
804
|
+
* @returns {void}
|
|
805
|
+
*/
|
|
806
|
+
function announceArrivals(sourceKey, torrent) {
|
|
807
|
+
const fileIndexes = [...demandFor(torrent).register.files()];
|
|
808
|
+
if (fileIndexes.length === 0) {
|
|
809
|
+
return;
|
|
810
|
+
}
|
|
811
|
+
parentPort.postMessage({ type: Event.PIECES_ARRIVED, sourceKey, fileIndexes });
|
|
812
|
+
}
|
|
813
|
+
|
|
814
|
+
|
|
815
|
+
/**
|
|
816
|
+
* Torrents already wired to warm their subtitle cues the moment a piece
|
|
817
|
+
* verifies, so the same torrent is not listened to twice.
|
|
818
|
+
*
|
|
819
|
+
* @type {WeakSet<object>}
|
|
820
|
+
*/
|
|
821
|
+
const arrivalsWired = new WeakSet();
|
|
822
|
+
|
|
823
|
+
/**
|
|
824
|
+
* A piece becoming readable is the actual event a cue can be pulled from —
|
|
825
|
+
* "downloaded", not "about to be encoded or copied": what a viewer reaches is
|
|
826
|
+
* decided by the read window ahead of the playhead, not by which of the two
|
|
827
|
+
* paths a segment takes, and the piece exists (and is worth reading for
|
|
828
|
+
* subtitles) whichever one that is. `verified` is WebTorrent's own signal for
|
|
829
|
+
* exactly that instant, set at the same place the bitfield itself is (`
|
|
830
|
+
* _markVerified`), so nothing here is guessing at readiness a different way.
|
|
831
|
+
*
|
|
832
|
+
* @param {string} sourceKey
|
|
833
|
+
* @param {object} torrent
|
|
834
|
+
* @returns {void}
|
|
835
|
+
*/
|
|
836
|
+
function ensureArrivalsWired(sourceKey, torrent) {
|
|
837
|
+
if (arrivalsWired.has(torrent)) {
|
|
838
|
+
return;
|
|
839
|
+
}
|
|
840
|
+
arrivalsWired.add(torrent);
|
|
841
|
+
torrent.on("verified", () => announceArrivals(sourceKey, torrent));
|
|
842
|
+
}
|
|
843
|
+
|
|
844
|
+
/**
|
|
845
|
+
* How often arrivals are announced regardless, as a fallback beside the
|
|
846
|
+
* per-piece `verified` listener above — it catches a listener attached after
|
|
847
|
+
* pieces had already verified, and anything the event path might otherwise
|
|
848
|
+
* miss. Cheap: the walk it wakes skips clusters it has already read.
|
|
849
|
+
*/
|
|
850
|
+
const ARRIVAL_ANNOUNCE_INTERVAL_MS = 3_000;
|
|
851
|
+
|
|
852
|
+
setInterval(() => {
|
|
853
|
+
for (const [sourceKey, torrent] of pool.torrents) {
|
|
854
|
+
ensureArrivalsWired(sourceKey, torrent);
|
|
855
|
+
announceArrivals(sourceKey, torrent);
|
|
856
|
+
}
|
|
857
|
+
}, ARRIVAL_ANNOUNCE_INTERVAL_MS).unref();
|
|
858
|
+
|
|
859
|
+
/**
|
|
860
|
+
* Files this proxy has downloaded whole. One directory, two readers of it: this
|
|
861
|
+
* thread writes them, the main thread serves them without asking anybody.
|
|
862
|
+
*/
|
|
863
|
+
const completedFiles = new CompletedFiles({ root: completedFilesRoot() });
|
|
864
|
+
|
|
865
|
+
/**
|
|
866
|
+
* Which torrent a set of files belongs to.
|
|
867
|
+
*
|
|
868
|
+
* The piece store hands its own files back without knowing what they are; this
|
|
869
|
+
* thread does know, and a file carries its torrent.
|
|
870
|
+
*
|
|
871
|
+
* @param {object[]} files
|
|
872
|
+
* @returns {string}
|
|
873
|
+
*/
|
|
874
|
+
const infoHashOf = (files) => String(files?.[0]?._torrent?.infoHash ?? "").toLowerCase();
|
|
875
|
+
|
|
876
|
+
// WHERE A PIECE COMES FROM WHEN NEITHER TIER HAS IT. Handed to every store this
|
|
877
|
+
// pool builds, so a film already assembled into a file is read from that file:
|
|
878
|
+
// which is what lets its spilled copy be dropped as the duplicate it has become,
|
|
879
|
+
// and what lets a torrent be destroyed and added again without fetching a byte.
|
|
880
|
+
pool.buildStoresWith({
|
|
881
|
+
readPieceElsewhere: ({ index, pieceLength, length, files }) =>
|
|
882
|
+
pieceFromWholeFiles({
|
|
883
|
+
index,
|
|
884
|
+
pieceLength,
|
|
885
|
+
length,
|
|
886
|
+
files,
|
|
887
|
+
wholeFileAt: (fileIndex) => completedFiles.find(infoHashOf(files), fileIndex)
|
|
888
|
+
}),
|
|
889
|
+
isPieceElsewhere: ({ index, pieceLength, length, files }) =>
|
|
890
|
+
pieceIsInWholeFiles({
|
|
891
|
+
index,
|
|
892
|
+
pieceLength,
|
|
893
|
+
length,
|
|
894
|
+
files,
|
|
895
|
+
wholeFileAt: (fileIndex) => completedFiles.find(infoHashOf(files), fileIndex)
|
|
896
|
+
})
|
|
897
|
+
});
|
|
898
|
+
void completedFiles.adopt(() => null).then((adopted) => {
|
|
899
|
+
if (adopted > 0) {
|
|
900
|
+
logger.info(
|
|
901
|
+
`whole files: took up ${adopted} file(s) a previous life left in ${completedFiles.root}`
|
|
902
|
+
);
|
|
903
|
+
}
|
|
904
|
+
});
|
|
905
|
+
|
|
906
|
+
/**
|
|
907
|
+
* How often whole files are looked for.
|
|
908
|
+
*
|
|
909
|
+
* The work itself is one pass over the files of each torrent asking a boolean
|
|
910
|
+
* the library already keeps; writing one out happens at most once per file,
|
|
911
|
+
* ever.
|
|
912
|
+
*/
|
|
913
|
+
const WHOLE_FILE_SWEEP_MS = 10_000;
|
|
914
|
+
|
|
915
|
+
/** Files being written out right now, so a sweep does not start a second one. */
|
|
916
|
+
const beingKept = new Set();
|
|
917
|
+
|
|
918
|
+
/**
|
|
919
|
+
* Keep every file that is now whole, and let go of a torrent that has nothing
|
|
920
|
+
* left to fetch.
|
|
921
|
+
*
|
|
922
|
+
* The instruction this serves, 2026-09-11: as soon as a torrent is fully
|
|
923
|
+
* downloaded, downloading stops, the torrent is deleted, and what was
|
|
924
|
+
* downloaded stays for as long as it is wanted.
|
|
925
|
+
*
|
|
926
|
+
* `file.done` is the library's own answer to "is every piece of this file
|
|
927
|
+
* here", and `torrent.done` to "is that true of every file". The second is a
|
|
928
|
+
* strong condition and will not fire for a season pack of which one episode is
|
|
929
|
+
* watched — nothing fetches the other four — and that is right: the instruction
|
|
930
|
+
* is about a torrent downloaded WHOLE.
|
|
931
|
+
*
|
|
932
|
+
* @returns {Promise<void>}
|
|
933
|
+
*/
|
|
934
|
+
async function keepWholeFiles() {
|
|
935
|
+
for (const [sourceKey, torrent] of [...pool.torrents]) {
|
|
936
|
+
// Lower case, because that is what the source key carries and the main
|
|
937
|
+
// thread looks these up by the key alone.
|
|
938
|
+
const infoHash = String(torrent?.infoHash ?? "").toLowerCase();
|
|
939
|
+
if (!infoHash || !Array.isArray(torrent.files)) {
|
|
940
|
+
continue;
|
|
941
|
+
}
|
|
942
|
+
// What anybody wants of this torrent. A file something is stated for is a
|
|
943
|
+
// file somebody may be reading, and this is the same list the reader counts
|
|
944
|
+
// used to give.
|
|
945
|
+
const wanted = new Set(
|
|
946
|
+
demandFor(torrent).register.windows()
|
|
947
|
+
.filter((window) =>
|
|
948
|
+
!String(window.claimant).startsWith("file-edges:") &&
|
|
949
|
+
!String(window.claimant).startsWith("torrent-fill:") &&
|
|
950
|
+
!String(window.claimant).startsWith("background-fill:")
|
|
951
|
+
)
|
|
952
|
+
.map((window) => window.fileIndex)
|
|
953
|
+
);
|
|
954
|
+
for (const [fileIndex, file] of torrent.files.entries()) {
|
|
955
|
+
const key = `${infoHash}/${fileIndex}`;
|
|
956
|
+
if (file?.done !== true || completedFiles.find(infoHash, fileIndex) || beingKept.has(key)) {
|
|
957
|
+
continue;
|
|
958
|
+
}
|
|
959
|
+
// NOT WHILE ANYBODY WANTS IT. Writing a film out is a read of the whole
|
|
960
|
+
// of it and a write of the whole of it — a gigabyte and a half on the
|
|
961
|
+
// file this was measured against — and doing that beside a viewer takes
|
|
962
|
+
// the disk and the piece store from them for nothing they asked for. The
|
|
963
|
+
// file is complete; it will still be complete when they leave.
|
|
964
|
+
if (wanted.has(fileIndex)) {
|
|
965
|
+
continue;
|
|
966
|
+
}
|
|
967
|
+
beingKept.add(key);
|
|
968
|
+
try {
|
|
969
|
+
const kept = await completedFiles.keep({
|
|
970
|
+
infoHash,
|
|
971
|
+
fileIndex,
|
|
972
|
+
length: file.length,
|
|
973
|
+
name: file.name,
|
|
974
|
+
open: () => file.createReadStream()
|
|
975
|
+
});
|
|
976
|
+
if (kept) {
|
|
977
|
+
logger.info(
|
|
978
|
+
`whole files: kept "${file.name}" (${Math.round(kept.length / 1048576)}MB) — ` +
|
|
979
|
+
"it is a file now, and reading it needs no torrent"
|
|
980
|
+
);
|
|
981
|
+
// The pieces it was built from are a second copy of the same bytes.
|
|
982
|
+
// Nothing is lost by dropping them: a read that wants one of them is
|
|
983
|
+
// answered from the file.
|
|
984
|
+
const store = findSharedStore(torrent);
|
|
985
|
+
const dropped = store?.dropDuplicatesHeldElsewhere?.() ?? 0;
|
|
986
|
+
if (dropped > 0) {
|
|
987
|
+
logger.info(
|
|
988
|
+
`whole files: dropped ${dropped} spilled piece(s) of "${file.name}" — ` +
|
|
989
|
+
"the film was on this disk twice and is not any more"
|
|
990
|
+
);
|
|
991
|
+
}
|
|
992
|
+
parentPort.postMessage({
|
|
993
|
+
type: Event.FILE_COMPLETE,
|
|
994
|
+
infoHash,
|
|
995
|
+
fileIndex,
|
|
996
|
+
path: kept.path,
|
|
997
|
+
length: kept.length,
|
|
998
|
+
name: kept.name
|
|
999
|
+
});
|
|
1000
|
+
}
|
|
1001
|
+
} catch (error) {
|
|
1002
|
+
logger.warn(`whole files: could not keep "${file?.name}": ${error?.message ?? error}`);
|
|
1003
|
+
} finally {
|
|
1004
|
+
beingKept.delete(key);
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
// EVERYTHING, WITHOUT EXCEPTION, AND EVERY BYTE OF IT A FILE ON DISK. The
|
|
1008
|
+
// torrent has no job left: there is nothing to fetch, and this proxy does
|
|
1009
|
+
// not seed what nobody is watching.
|
|
1010
|
+
//
|
|
1011
|
+
// Safe to destroy only because a piece can now be read out of those files:
|
|
1012
|
+
// a path that asks for this source again adds the torrent back, and what it
|
|
1013
|
+
// verifies it reads from the files rather than from the swarm. Not while
|
|
1014
|
+
// anybody is reading it, for the same reason the writing above waits.
|
|
1015
|
+
const isWhole =
|
|
1016
|
+
torrent.done === true &&
|
|
1017
|
+
torrent.files.every((unused, fileIndex) => completedFiles.find(infoHash, fileIndex) !== null);
|
|
1018
|
+
if (isWhole && wanted.size === 0) {
|
|
1019
|
+
logger.info(
|
|
1020
|
+
`whole files: "${torrent.name}" is downloaded whole and saved — removing the torrent, keeping the files`
|
|
1021
|
+
);
|
|
1022
|
+
// THE RECIPE STAYS, and so does the entry that leads to it. Everything
|
|
1023
|
+
// that asks this thread about a source — the track table, the media
|
|
1024
|
+
// info, the keyframe table, the stats the browser polls — comes through
|
|
1025
|
+
// `requireTorrent`, which adds a torrent back when the one it holds is
|
|
1026
|
+
// no longer usable. Deleting the entry here would turn a viewer
|
|
1027
|
+
// returning to this film into `Unknown source`, which is a worse failure
|
|
1028
|
+
// than the one this removal is for.
|
|
1029
|
+
//
|
|
1030
|
+
// What the torrent finds when it comes back is the whole files: its
|
|
1031
|
+
// store reads pieces from them, so it fetches nothing. It is added with
|
|
1032
|
+
// verification skipped, and that is not a shortcut — the file was
|
|
1033
|
+
// written out of pieces this client had already hashed, and its size was
|
|
1034
|
+
// checked against what the torrent says. Re-hashing a gigabyte and a
|
|
1035
|
+
// half to learn what we wrote down is a minute of a viewer's time for
|
|
1036
|
+
// nothing.
|
|
1037
|
+
pool.remove(torrent, "downloaded-whole");
|
|
1038
|
+
pool.addWholeSource(sourceKey);
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
}
|
|
1042
|
+
|
|
1043
|
+
setInterval(() => {
|
|
1044
|
+
void keepWholeFiles();
|
|
1045
|
+
}, WHOLE_FILE_SWEEP_MS).unref();
|
|
1046
|
+
|
|
1047
|
+
/**
|
|
1048
|
+
* Keeps this thread alive ON PURPOSE — the one interval left accounted for
|
|
1049
|
+
* (no `.unref()`).
|
|
1050
|
+
*
|
|
1051
|
+
* Every other recurring handle here is unref'd, upload is disabled by
|
|
1052
|
+
* default, and idle peer connections close about half a minute after the
|
|
1053
|
+
* traffic stops — so once nothing is being read, every handle can be gone
|
|
1054
|
+
* at once, the event loop drains, and this thread ends BY ITSELF. Node then
|
|
1055
|
+
* tears the isolate down, that teardown touches memory some native module
|
|
1056
|
+
* has already freed, and the fault (SIGSEGV inside `uv_timer_stop`, reached
|
|
1057
|
+
* through `PerIsolatePlatformData::Shutdown`) kills the whole process at
|
|
1058
|
+
* once — HTTP server, tunnel, data channels — before any JS handler runs.
|
|
1059
|
+
* Field evidence: two deaths on 2026-08-22 (15:50:12 and 16:12:21 UTC),
|
|
1060
|
+
* each ~35 s after the last byte of traffic, identical core dumps;
|
|
1061
|
+
* same crash family as the utp-native faults of 2026-08-18..21
|
|
1062
|
+
* (`research/worker-thread-drain-crash-2026-08-22.md`).
|
|
1063
|
+
*
|
|
1064
|
+
* An empty repeating interval costs nothing, keeps the loop from draining
|
|
1065
|
+
* while the process lives, and thereby keeps that teardown path — and the
|
|
1066
|
+
* corrupted structure inside it — unreachable, whichever module is guilty.
|
|
1067
|
+
*/
|
|
1068
|
+
const WORKER_KEEPALIVE_INTERVAL_MS = 5_000;
|
|
1069
|
+
|
|
1070
|
+
setInterval(() => {
|
|
1071
|
+
void process.uptime();
|
|
1072
|
+
}, WORKER_KEEPALIVE_INTERVAL_MS);
|
|
1073
|
+
|
|
1074
|
+
/**
|
|
1075
|
+
* Say why this thread is ending, from inside it, before anything is torn down.
|
|
1076
|
+
*
|
|
1077
|
+
* The crash of 2026-08-27 15:40 is `node::worker::Worker::Run()` returning and
|
|
1078
|
+
* node faulting as it closes what was left on this loop. The parent DOES watch
|
|
1079
|
+
* for an unexpected exit and has a line ready for it — but that line never
|
|
1080
|
+
* printed, because the fault happens during this thread's own teardown, before
|
|
1081
|
+
* the parent's `exit` event is delivered. So the one reading that would name
|
|
1082
|
+
* the cause was being eaten by the failure it was meant to explain.
|
|
1083
|
+
*
|
|
1084
|
+
* These handlers run first, synchronously, and write through the same channel
|
|
1085
|
+
* as every other line here. `beforeExit` means the loop drained despite the
|
|
1086
|
+
* keepalive above; `exit` means the thread is going whatever the reason. The
|
|
1087
|
+
* list of what was still holding the loop open is the part that says which.
|
|
1088
|
+
*/
|
|
1089
|
+
process.on("beforeExit", (code) => {
|
|
1090
|
+
log(
|
|
1091
|
+
`thread is about to end because the event loop drained (code ${code}) — ` +
|
|
1092
|
+
`still open: ${describeActiveResources()}`
|
|
1093
|
+
);
|
|
1094
|
+
});
|
|
1095
|
+
|
|
1096
|
+
process.on("exit", (code) => {
|
|
1097
|
+
log(`thread ending with code ${code} — still open: ${describeActiveResources()}`);
|
|
1098
|
+
});
|
|
1099
|
+
|
|
1100
|
+
process.on("uncaughtException", (error) => {
|
|
1101
|
+
log(`thread hit an uncaught error: ${error?.stack ?? error}`);
|
|
1102
|
+
});
|
|
1103
|
+
|
|
1104
|
+
process.on("unhandledRejection", (reason) => {
|
|
1105
|
+
log(`thread hit an unhandled rejection: ${reason?.stack ?? reason}`);
|
|
1106
|
+
});
|
|
1107
|
+
|
|
1108
|
+
/**
|
|
1109
|
+
* What is still holding this thread's event loop open, as node names it.
|
|
1110
|
+
*
|
|
1111
|
+
* @returns {string} A tally per resource kind, or why it could not be read.
|
|
1112
|
+
*/
|
|
1113
|
+
function describeActiveResources() {
|
|
1114
|
+
try {
|
|
1115
|
+
const names = process.getActiveResourcesInfo?.() ?? [];
|
|
1116
|
+
if (names.length === 0) {
|
|
1117
|
+
return "nothing";
|
|
1118
|
+
}
|
|
1119
|
+
const tally = new Map();
|
|
1120
|
+
for (const name of names) {
|
|
1121
|
+
tally.set(name, (tally.get(name) ?? 0) + 1);
|
|
1122
|
+
}
|
|
1123
|
+
return [...tally].map(([name, count]) => `${name}x${count}`).join(" ");
|
|
1124
|
+
} catch (error) {
|
|
1125
|
+
return `unreadable (${error?.message ?? error})`;
|
|
1126
|
+
}
|
|
1127
|
+
}
|
|
1128
|
+
|
|
1129
|
+
log("torrent worker started");
|