@torrent-tv/proxy 2.88.4 → 2.88.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/opsx/apply.md +155 -155
- package/.claude/commands/opsx/archive.md +160 -160
- package/.claude/commands/opsx/explore.md +174 -174
- package/.claude/commands/opsx/propose.md +109 -109
- package/.claude/commands/opsx/sync.md +143 -143
- package/.claude/skills/openspec-apply-change/SKILL.md +159 -159
- package/.claude/skills/openspec-archive-change/SKILL.md +117 -117
- package/.claude/skills/openspec-explore/SKILL.md +289 -289
- package/.claude/skills/openspec-propose/SKILL.md +113 -113
- package/.claude/skills/openspec-sync-specs/SKILL.md +147 -147
- package/.dockerignore +5 -5
- package/AGENTS.md +322 -322
- package/CHANGELOG.md +2269 -2258
- package/Dockerfile +30 -30
- package/LICENSE +16 -16
- package/README.md +369 -369
- package/assets/calibration/NOTICE.md +100 -100
- package/assets/calibration/audio/NOTICE.md +19 -0
- package/assets/calibration/audio/aac.m4a +0 -0
- package/assets/calibration/audio/ac3.ac3 +0 -0
- package/assets/calibration/audio/alac.m4a +0 -0
- package/assets/calibration/audio/dts.dts +0 -0
- package/assets/calibration/audio/eac3.eac3 +0 -0
- package/assets/calibration/audio/flac.flac +0 -0
- package/assets/calibration/audio/manifest.json +98 -0
- package/assets/calibration/audio/mp2.mp2 +0 -0
- package/assets/calibration/audio/mp3.mp3 +0 -0
- package/assets/calibration/audio/opus.opus +0 -0
- package/assets/calibration/audio/pcm.wav +0 -0
- package/assets/calibration/audio/truehd.thd +0 -0
- package/assets/calibration/audio/vorbis.ogg +0 -0
- package/assets/diagnostics/sctpstate.gdb +31 -31
- package/bin/cli.js +684 -684
- package/bin/thread-pool.js +30 -30
- package/biome.json +412 -411
- package/docs/browser-proxy-contract.md +155 -152
- package/docs/container-architecture.md +218 -218
- package/docs/download-architecture.md +261 -261
- package/docs/encode-architecture.md +729 -729
- package/docs/encode-run-state.md +96 -96
- package/docs/logs.md +97 -97
- package/docs/playback-start-forecast.md +56 -0
- package/docs/socks5-egress-research.md +114 -114
- package/docs/storage-architecture.md +267 -267
- package/knip.json +14 -14
- package/openspec/changes/adaptive-bitrate/design.md +92 -92
- package/openspec/changes/adaptive-bitrate/proposal.md +56 -56
- package/openspec/changes/adaptive-bitrate/specs/adaptive-bitrate/spec.md +51 -51
- package/openspec/changes/adaptive-bitrate/tasks.md +40 -40
- package/openspec/changes/binary-distribution/.openspec.yaml +2 -2
- package/openspec/changes/binary-distribution/proposal.md +53 -53
- package/openspec/changes/chunked-request-bodies/design.md +81 -81
- package/openspec/changes/chunked-request-bodies/proposal.md +53 -53
- package/openspec/changes/chunked-request-bodies/specs/chunked-request-bodies/spec.md +30 -30
- package/openspec/changes/chunked-request-bodies/tasks.md +40 -40
- package/openspec/changes/cold-start/design.md +98 -98
- package/openspec/changes/cold-start/proposal.md +62 -62
- package/openspec/changes/cold-start/specs/cold-start/spec.md +47 -47
- package/openspec/changes/cold-start/tasks.md +48 -48
- package/openspec/changes/disk-cap/.openspec.yaml +2 -2
- package/openspec/changes/disk-cap/proposal.md +37 -37
- package/openspec/changes/disk-cap/specs/disk-cap/spec.md +25 -25
- package/openspec/changes/disk-cap/tasks.md +19 -19
- package/openspec/changes/proxy-observability/.openspec.yaml +2 -2
- package/openspec/changes/proxy-observability/design.md +38 -38
- package/openspec/changes/proxy-observability/proposal.md +49 -49
- package/openspec/changes/proxy-observability/specs/observability/spec.md +33 -33
- package/openspec/changes/proxy-observability/tasks.md +21 -21
- package/openspec/changes/seek-debounce/design.md +118 -118
- package/openspec/changes/seek-debounce/proposal.md +59 -59
- package/openspec/changes/seek-debounce/specs/seek-debounce/spec.md +41 -41
- package/openspec/changes/seek-debounce/tasks.md +33 -33
- package/openspec/changes/subtitle-language/.openspec.yaml +2 -2
- package/openspec/changes/subtitle-language/proposal.md +50 -50
- package/openspec/changes/subtitle-language/specs/subtitle-language/spec.md +34 -34
- package/openspec/changes/subtitle-language/tasks.md +26 -26
- package/openspec/changes/track-selection/.openspec.yaml +2 -2
- package/openspec/changes/track-selection/design.md +47 -47
- package/openspec/changes/track-selection/proposal.md +45 -45
- package/openspec/changes/track-selection/specs/track-selection/spec.md +41 -41
- package/openspec/changes/track-selection/tasks.md +20 -20
- package/openspec/changes/transcode-quality/.openspec.yaml +2 -2
- package/openspec/changes/transcode-quality/proposal.md +51 -51
- package/openspec/changes/transcode-quality/specs/transcode-quality/spec.md +139 -139
- package/openspec/changes/transcode-quality/tasks.md +63 -63
- package/openspec/config.yaml +38 -38
- package/package.json +1 -1
- package/public/.well-known/appspecific/com.chrome.devtools.json +1 -1
- package/research/delivery-wedge-stand-ladder-2026-09-13.md +329 -329
- package/research/double-spawn-2026-09-10.md +171 -171
- package/research/handover-2026-09-13.md +209 -209
- package/research/handover-reader-claims-removal-2026-09-12.md +166 -166
- package/research/piece-withdrawn-but-still-claimed-2026-09-12.md +175 -175
- package/research/preset-benchmark-size-scaling-2026-08-22.md +55 -55
- package/research/priority-map-is-the-truth-2026-09-12.md +225 -225
- package/routes/api/client-logs/post.js +86 -86
- package/routes/api/delivery-sink/get.js +72 -72
- package/routes/api/link-probe/get.js +51 -51
- package/routes/api/playback-plan/audio-tracks/post.js +29 -29
- package/routes/api/playback-plan/post.js +53 -53
- package/routes/api/sources/post.js +36 -36
- package/routes/api/sources/stats/get.js +132 -132
- package/routes/api/sources/warm/post.js +217 -217
- package/routes/api/subtitles/get.js +230 -230
- package/routes/api/transcode-sessions/fragment-far/post.js +60 -60
- package/routes/api/transcode-sessions/net-report/post.js +138 -138
- package/routes/api/transcode-sessions/post.js +178 -178
- package/routes/api/transcode-sessions/progress/get.js +27 -27
- package/routes/api/transcode-sessions/release/post.js +41 -41
- package/routes/api/transcode-sessions/seek/post.js +63 -63
- package/routes/health/get.js +13 -13
- package/routes/healthz/get.js +13 -13
- package/routes/stream/get.js +380 -380
- package/routes/transcode/audio-file/get.js +64 -64
- package/routes/transcode/audio-warm/get.js +95 -95
- package/routes/transcode/session-file/get.js +52 -52
- package/routes/transcode/variant-file/get.js +95 -95
- package/routes/transcode/variant-warm/get.js +84 -84
- package/scripts/build-audio-samples.mjs +31 -0
- package/scripts/render-run-graph.js +173 -173
- package/server.js +730 -725
- package/services/encode/CoverageMap.js +428 -428
- package/services/encode/CushionReport.js +345 -345
- package/services/encode/EncodeAdmission.js +376 -376
- package/services/encode/EncodeOrchestrator.js +1217 -1217
- package/services/encode/EncodePlan.js +1366 -1366
- package/services/encode/EncodeRun.js +813 -813
- package/services/encode/EncodeRuns.js +1239 -1235
- package/services/encode/Encoder.js +142 -142
- package/services/encode/EncoderSelection.js +38 -38
- package/services/encode/LocalObservations.js +359 -359
- package/services/encode/NvencEncoder.js +50 -50
- package/services/encode/OutputOpening.js +783 -783
- package/services/encode/OutputTimes.js +579 -579
- package/services/encode/QsvEncoder.js +58 -58
- package/services/encode/Renditions.js +2282 -2282
- package/services/encode/RunProgress.js +84 -84
- package/services/encode/SegmentDemand.js +95 -95
- package/services/encode/SoftwareEncoder.js +121 -121
- package/services/encode/V4l2m2mEncoder.js +53 -53
- package/services/encode/VaapiEncoder.js +66 -66
- package/services/encode/args.js +418 -418
- package/services/encode/audio-calibration.js +67 -0
- package/services/encode/audio-work-rate.js +15 -0
- package/services/encode/available-share.js +89 -89
- package/services/encode/calibration.js +361 -361
- package/services/encode/contention.js +128 -128
- package/services/encode/decode-cost-fit.js +363 -363
- package/services/encode/encode-exit.js +114 -114
- package/services/encode/encode-run-state.js +551 -551
- package/services/encode/encoder-readings.js +45 -45
- package/services/encode/fingerprint.js +322 -322
- package/services/encode/host-load.js +174 -174
- package/services/encode/hwaccel.js +1894 -1894
- package/services/encode/index.js +9 -9
- package/services/encode/learned-median.js +90 -90
- package/services/encode/output/EncodedOutput.js +57 -57
- package/services/encode/output/Output.js +131 -131
- package/services/encode/output/OutputCatalog.js +414 -414
- package/services/encode/output/OutputSpec.js +390 -390
- package/services/encode/output/Timeline.js +308 -308
- package/services/encode/output/cut-grid.js +107 -107
- package/services/encode/output/index.js +24 -20
- package/services/encode/output/ladder.js +32 -32
- package/services/encode/output/playlists.js +284 -284
- package/services/encode/output/rates.js +125 -125
- package/services/encode/output-key-format.js +21 -21
- package/services/encode/quality/EncodeCost.js +1167 -1150
- package/services/encode/quality/HostLoad.js +525 -525
- package/services/encode/quality/HostTimings.js +249 -249
- package/services/encode/quality/OutputQualityState.js +28 -28
- package/services/encode/quality/QualityController.js +935 -935
- package/services/encode/quality/QualityOffer.js +416 -416
- package/services/encode/quality/drain-threat.js +47 -47
- package/services/encode/quality/link-budget.js +349 -349
- package/services/encode/quality/output-format.js +319 -319
- package/services/encode/quality/serving-output.js +216 -216
- package/services/encode/quality/visible-rung.js +57 -57
- package/services/encode/run-budget.js +97 -97
- package/services/encode/run-command.js +755 -754
- package/services/encode/run-costs.js +155 -155
- package/services/encode/run-inputs.js +42 -42
- package/services/encode/segment-formats/fmp4.js +458 -456
- package/services/encode/segment-formats/index.js +85 -85
- package/services/encode/segment-formats/init-compat.js +324 -324
- package/services/encode/segment-formats/mp4-boxes.js +12 -1
- package/services/encode/segment-formats/mpegts.js +134 -134
- package/services/encode/start-stop-cost.js +178 -178
- package/services/encode/throughput.js +97 -97
- package/services/encode/torrent-cost.js +88 -88
- package/services/identity/installation-id.js +38 -38
- package/services/media/ContainerOrchestrator.js +149 -149
- package/services/media/KeyframeTables.js +316 -316
- package/services/media/SourceFile.js +298 -298
- package/services/media/SubtitleCues.js +520 -520
- package/services/media/SubtitleOrchestrator.js +167 -167
- package/services/media/audio-inventory.js +299 -298
- package/services/media/container/AviContainer.js +266 -266
- package/services/media/container/Container.js +609 -609
- package/services/media/container/ContainerFactory.js +56 -56
- package/services/media/container/KeyframeTable.js +272 -272
- package/services/media/container/MatroskaContainer.js +1494 -1494
- package/services/media/container/Mp4Container.js +1344 -1344
- package/services/media/container/ebml-reader.js +159 -159
- package/services/media/ffmpeg-banner.js +260 -260
- package/services/media/keyframe-probe.js +157 -157
- package/services/media/media-info-probe.js +93 -93
- package/services/media/playback-planner.js +931 -931
- package/services/media/tracks/AudioTrack.js +143 -143
- package/services/media/tracks/ContainerTrack.js +75 -75
- package/services/media/tracks/ImageSubtitleTrack.js +19 -19
- package/services/media/tracks/SubtitleTrack.js +38 -38
- package/services/media/tracks/TextSubtitleTrack.js +287 -287
- package/services/media/tracks/VideoTrack.js +33 -33
- package/services/media/tracks/audio-bound.js +233 -233
- package/services/media/tracks/index.js +15 -15
- package/services/media/tracks/language-detect.js +174 -174
- package/services/server/OutputLifecycle.js +418 -418
- package/services/server/SegmentServing.js +1150 -1150
- package/services/server/ViewerRequests.js +786 -760
- package/services/server/WaitLedger.js +142 -142
- package/services/server/controllers/PlaybackController.js +40 -40
- package/services/server/controllers/SubtitleController.js +109 -109
- package/services/server/http-responses.js +60 -60
- package/services/server/playback-readiness.js +516 -708
- package/services/server/transcode-session-files.js +325 -325
- package/services/server/wire-outputs.js +1020 -995
- package/services/storage/Diagnostics.js +202 -202
- package/services/storage/MachineBudget.js +235 -235
- package/services/storage/allowance.js +107 -107
- package/services/storage/core-dumps.js +106 -106
- package/services/storage/files/CompletedFiles.js +391 -391
- package/services/storage/files/piece-from-whole-file.js +118 -118
- package/services/storage/free.js +71 -71
- package/services/storage/keep.js +48 -48
- package/services/storage/machine-memory.js +56 -56
- package/services/storage/memory-report.js +575 -575
- package/services/storage/piece-store/piece-disk-store.js +539 -539
- package/services/storage/piece-store/piece-lru.js +495 -495
- package/services/storage/piece-store/shared-piece-store.js +1937 -1931
- package/services/storage/returns.js +103 -103
- package/services/storage/segment-store/SegmentStore.js +1255 -1240
- package/services/storage/wire.js +175 -175
- package/services/torrent/demand/DemandRegister.js +202 -202
- package/services/torrent/demand/Urgency.js +137 -137
- package/services/torrent/demand/Window.js +118 -118
- package/services/torrent/demand/index.js +8 -8
- package/services/torrent/demand/pieces.js +140 -140
- package/services/torrent/download/SwarmSelection.js +386 -386
- package/services/torrent/download/registry.js +102 -102
- package/services/torrent/download/withdraw-claim.js +80 -80
- package/services/torrent/files.js +455 -455
- package/services/torrent/naming.js +619 -619
- package/services/torrent/piece-store-of.js +29 -29
- package/services/torrent/supply-margin.js +262 -262
- package/services/torrent/torrent-pool.js +2688 -2675
- package/services/torrent/worker/background-fill.js +213 -213
- package/services/torrent/worker/channel.js +271 -271
- package/services/torrent/worker/client.js +765 -765
- package/services/torrent/worker/fastest-wires.js +342 -342
- package/services/torrent/worker/handle-state.js +23 -23
- package/services/torrent/worker/held-bytes.js +155 -155
- package/services/torrent/worker/install-webrtc-shim.js +32 -32
- package/services/torrent/worker/piece-reader.js +1078 -1078
- package/services/torrent/worker/pool-adapter.js +371 -371
- package/services/torrent/worker/protocol.js +184 -184
- package/services/torrent/worker/resume-warm.js +107 -107
- package/services/torrent/worker/webrtc-shim.js +219 -219
- package/services/torrent/worker/worker.js +1129 -1129
- package/services/transport/data-channel-handler.js +1369 -1369
- package/services/transport/delivery-probe.js +842 -842
- package/services/transport/health-collector.js +53 -53
- package/services/transport/nat-classifier.js +231 -231
- package/services/transport/packet-witness.js +808 -808
- package/services/transport/port-mapper.js +235 -235
- package/services/transport/registry-api.js +81 -81
- package/services/transport/tunnel-client.js +447 -447
- package/services/transport/usrsctp-state.js +175 -175
- package/services/transport/webrtc-manager.js +484 -484
- package/services/viewer/Assignments.js +340 -340
- package/services/viewer/PriorityMap.js +390 -390
- package/services/viewer/PriorityOrchestrator.js +325 -325
- package/services/viewer/Viewer.js +691 -691
- package/services/viewer/Viewers.js +368 -368
- package/services/viewer/active-output.js +37 -37
- package/services/viewer/audio-start.js +46 -46
- package/services/viewer/buffer-trend.js +81 -81
- package/services/viewer/choices.js +647 -647
- package/services/viewer/positions.js +126 -126
- package/services/viewer/report-intake.js +46 -46
- package/store/source-registry.js +71 -71
- package/test/asset-paths.test.js +28 -28
- package/test/audio-calibration.test.js +23 -0
- package/test/audio-inventory.test.js +212 -212
- package/test/audio-track-bitrate.test.js +40 -40
- package/test/auto-quality-step.test.js +707 -707
- package/test/available-share.test.js +76 -76
- package/test/background-fill.test.js +190 -190
- package/test/client-log-file.test.js +110 -110
- package/test/completed-files.test.js +115 -115
- package/test/concurrent-cost.test.js +148 -148
- package/test/container-media-info.test.js +228 -228
- package/test/contention.test.js +76 -76
- package/test/core-dumps.test.js +57 -57
- package/test/coverage-follows-the-disk.test.js +228 -228
- package/test/coverage-map.test.js +195 -195
- package/test/cue-push-carries-what-the-walk-found.test.js +58 -58
- package/test/cut-times-timeline.test.js +65 -65
- package/test/cuts-follow-published-grid.test.js +109 -109
- package/test/data-channel-frame.test.js +63 -63
- package/test/data-channel-viewer-lifecycle.test.js +92 -92
- package/test/dead-torrent-handle.test.js +52 -52
- package/test/declared-rates.test.js +116 -116
- package/test/declared-tracks.test.js +43 -43
- package/test/decode-cost-fit.test.js +202 -202
- package/test/decode-cost.test.js +462 -462
- package/test/delivery-probe.test.js +393 -393
- package/test/demand-register.test.js +206 -206
- package/test/dht-bootstrap.test.js +72 -72
- package/test/drain-threat.test.js +65 -65
- package/test/duplicate-add.test.js +48 -48
- package/test/encode-admission.test.js +146 -146
- package/test/encode-cost.test.js +283 -283
- package/test/encode-exit.test.js +111 -111
- package/test/encode-orchestrator.test.js +419 -419
- package/test/encode-plan-viewers.test.js +718 -718
- package/test/encode-plan.test.js +545 -545
- package/test/encode-run-state.test.js +314 -314
- package/test/encode-run.test.js +162 -162
- package/test/encode-slope.test.js +46 -46
- package/test/encoded-output.test.js +39 -39
- package/test/encoder-kinds.test.js +122 -122
- package/test/encoder-readings.test.js +46 -46
- package/test/every-claimant-is-told-its-share.test.js +152 -152
- package/test/failed-encoder-still-serves-what-exists.test.js +354 -354
- package/test/failed-start-breaker.test.js +202 -202
- package/test/fastest-wires.test.js +143 -143
- package/test/ffmpeg-stream-counts.test.js +78 -78
- package/test/file-edges.test.js +191 -191
- package/test/fmp4-presentation-clock.test.js +162 -147
- package/test/fmp4-self-contained.test.js +80 -80
- package/test/fmp4-timestamps.test.js +140 -140
- package/test/fmp4-track-completeness.test.js +41 -41
- package/test/format-keyed-outputs.test.js +78 -78
- package/test/health-metrics.test.js +38 -38
- package/test/held-request-width.test.js +157 -157
- package/test/helpers/encode-run.js +136 -136
- package/test/helpers/held-file.js +29 -29
- package/test/helpers/manager.js +57 -57
- package/test/helpers/output-spec.js +63 -63
- package/test/host-load.test.js +68 -68
- package/test/index-accuracy-is-about-the-file.test.js +128 -128
- package/test/init-compat.test.js +219 -219
- package/test/input-lost-quiets-the-plan.test.js +261 -261
- package/test/input-unavailable.test.js +48 -48
- package/test/installation-id.test.js +39 -39
- package/test/keeping-period.test.js +83 -83
- package/test/keyframe-index-accuracy.test.js +188 -188
- package/test/keyframe-table-budget.test.js +102 -102
- package/test/keyframes-belong-to-the-file.test.js +249 -249
- package/test/learned-median.test.js +37 -37
- package/test/letting-go-asks-who-is-watching.test.js +191 -191
- package/test/limit-row-by-area.test.js +60 -60
- package/test/link-budget.test.js +112 -112
- package/test/link-probe.test.js +56 -56
- package/test/local-encoding-data.test.js +273 -273
- package/test/logger-repeats.test.js +134 -134
- package/test/look-ahead-coverage.test.js +56 -56
- package/test/manager-surface.test.js +292 -292
- package/test/matroska-cues-track.test.js +192 -192
- package/test/memory-budget.test.js +390 -390
- package/test/move-cost.test.js +216 -216
- package/test/mp4-subtitles.test.js +173 -173
- package/test/one-authority.test.js +285 -285
- package/test/one-budget.test.js +149 -149
- package/test/one-output-one-session.test.js +273 -270
- package/test/one-resource-divided.test.js +185 -185
- package/test/one-serving-output-per-height.test.js +458 -458
- package/test/orchestrator-wired.test.js +206 -206
- package/test/output-catalog.test.js +39 -39
- package/test/output-shape.test.js +21 -21
- package/test/output-spec.test.js +251 -251
- package/test/output-speed.test.js +88 -88
- package/test/outputs-being-prepared.test.js +40 -40
- package/test/packet-witness-ring.test.js +236 -236
- package/test/packet-witness.test.js +148 -148
- package/test/piece-disk-store.test.js +401 -401
- package/test/piece-from-whole-file.test.js +129 -129
- package/test/piece-lru.test.js +319 -319
- package/test/piece-reader.test.js +162 -162
- package/test/piece-selection-offset.test.js +85 -85
- package/test/piece-store-eviction.test.js +595 -595
- package/test/piece-store-never-refuses.test.js +153 -153
- package/test/piece-store-reservations.test.js +403 -403
- package/test/piece-store-slow-disk.test.js +159 -159
- package/test/piece-tail.test.js +97 -97
- package/test/plan-host-timings.test.js +68 -68
- package/test/plan-is-steady.test.js +225 -225
- package/test/playback-readiness.test.js +483 -453
- package/test/playback-start-regressions.test.js +99 -99
- package/test/priority-map-download.test.js +174 -174
- package/test/priority-map-emptied.test.js +204 -204
- package/test/priority-map-per-output.test.js +215 -215
- package/test/priority-map.test.js +163 -163
- package/test/probe-wedge-certainty.test.js +60 -60
- package/test/produced-copy-choice.test.js +213 -213
- package/test/published-timeline.test.js +83 -83
- package/test/quality-offer.test.js +162 -162
- package/test/quality-variants.test.js +1257 -1257
- package/test/rate-control-in-the-output.test.js +180 -180
- package/test/read-survives-withdrawal.test.js +164 -164
- package/test/read-window.test.js +365 -365
- package/test/resume-warm.test.js +39 -39
- package/test/run-graph-drift.test.js +37 -37
- package/test/run-inputs.test.js +80 -80
- package/test/run-intervals.test.js +100 -100
- package/test/run-position-follows-published-grid.test.js +89 -89
- package/test/run-progress.test.js +65 -65
- package/test/same-height-switch.test.js +664 -664
- package/test/seek-frees-the-viewers-requests.test.js +217 -217
- package/test/seek-landing.test.js +109 -109
- package/test/seek-target-not-superseded.test.js +195 -195
- package/test/segment-closed-on-both-branches.test.js +96 -96
- package/test/segment-cut-times.test.js +63 -63
- package/test/segment-demand.test.js +80 -80
- package/test/segment-hold-supersede.test.js +91 -91
- package/test/segment-judged-by-its-own-header.test.js +221 -221
- package/test/segment-serve-wiring.test.js +475 -475
- package/test/segment-store-eviction.test.js +247 -247
- package/test/segment-store.test.js +352 -323
- package/test/segment-true-start.test.js +86 -86
- package/test/segments-are-shared.test.js +176 -176
- package/test/send-chunk-bytes.test.js +86 -86
- package/test/serving-existing-output.test.js +105 -105
- package/test/serving-output.test.js +102 -102
- package/test/shared-piece-store.test.js +310 -310
- package/test/sidecar-naming.test.js +142 -142
- package/test/soundtrack-rate-bound.test.js +289 -289
- package/test/source-file.test.js +133 -133
- package/test/source-registry.test.js +18 -18
- package/test/split-a-stretch.test.js +139 -139
- package/test/stale-request-after-seek.test.js +158 -158
- package/test/startup-readings.test.js +124 -124
- package/test/stream-route.test.js +209 -209
- package/test/subtitle-cue-framing.test.js +200 -200
- package/test/subtitle-cue-source.test.js +102 -102
- package/test/subtitle-cue-walk.test.js +370 -370
- package/test/subtitle-cursor.test.js +63 -63
- package/test/subtitle-defaults.test.js +118 -118
- package/test/subtitle-language.test.js +252 -252
- package/test/subtitle-track-numbering.test.js +386 -386
- package/test/supply-margin.test.js +177 -177
- package/test/swarm-follows-readers.test.js +174 -174
- package/test/swarm-reach.test.js +130 -130
- package/test/swarm-selection.test.js +220 -220
- package/test/tail-duplication.test.js +167 -167
- package/test/the-channel-has-both-ends.test.js +91 -91
- package/test/the-torrent-thread-serves-bytes.test.js +107 -107
- package/test/thread-pool.test.js +53 -53
- package/test/timeline.test.js +84 -84
- package/test/torrent-contents.test.js +235 -235
- package/test/torrent-cost.test.js +71 -71
- package/test/torrent-files.test.js +221 -221
- package/test/torrent-naming.test.js +255 -255
- package/test/tracks-begin-together.test.js +209 -209
- package/test/tunnel-renewal.test.js +167 -167
- package/test/two-viewers-one-picture.test.js +504 -504
- package/test/upload-hurry.test.js +186 -186
- package/test/usrsctp-state.test.js +131 -131
- package/test/video-facts.test.js +102 -102
- package/test/viewer-assignments.test.js +143 -143
- package/test/viewer-outputs.test.js +285 -285
- package/test/viewer-presence.test.js +116 -116
- package/test/viewer-requests-admission.test.js +89 -89
- package/test/viewer-states.test.js +189 -189
- package/test/viewer-subtitle-subscription.test.js +96 -96
- package/test/viewer.test.js +139 -139
- package/test/viewing-generations.test.js +390 -390
- package/test/visible-rung.test.js +37 -37
- package/test/wait-ledger.test.js +100 -100
- package/test/webrtc-shim.test.js +149 -149
- package/test/wedge-certainty.test.js +131 -131
- package/test/withdraw-piece-claim.test.js +212 -212
- package/test/worker-channel.test.js +158 -158
- package/utils/client-log-file.js +151 -151
- package/utils/logger.js +312 -312
- package/utils/parse-range.js +25 -25
- package/utils/perf.js +59 -59
- package/utils/torrent-source-key.js +38 -38
|
@@ -1,539 +1,539 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @file Where downloaded pieces live once memory cannot hold them.
|
|
3
|
-
*
|
|
4
|
-
* The second tier of the piece store, and — unlike what it replaces — an owner
|
|
5
|
-
* of what it takes: it knows how many bytes it holds, it is told how many it
|
|
6
|
-
* may hold, and when it is over that it gives disk back.
|
|
7
|
-
*
|
|
8
|
-
* WHAT IT REPLACED, because the difference is the whole of the design. `DiskTier`
|
|
9
|
-
* wrote every piece into ONE sparse file at `index * chunkLength` and answered
|
|
10
|
-
* `forget(index)` by dropping the number from a set. The bytes stayed: a sparse
|
|
11
|
-
* file's blocks are returned only by hole punching, which Node exposes no
|
|
12
|
-
* binding for, so nothing this process could do returned a single block before
|
|
13
|
-
* the whole file was removed. Measured 2026-08-31: a store holding 312-424 MB
|
|
14
|
-
* of pieces had written **14 400 MB** to that file in fifty minutes, and free
|
|
15
|
-
* space on the host fell by every megabyte of it until the session ended. On a
|
|
16
|
-
* Home Assistant install with a 32 GB card that is the card.
|
|
17
|
-
*
|
|
18
|
-
* A PIECE IS A FILE, and that is what makes the ceiling real. Removing a file
|
|
19
|
-
* returns exactly its blocks, needs no binding this runtime lacks, and makes
|
|
20
|
-
* the unit of eviction the same as the unit of storage — so the order pieces
|
|
21
|
-
* leave in is the order we choose rather than the order they happen to lie in.
|
|
22
|
-
* The read that the single file was chosen for is unaffected: it is still one
|
|
23
|
-
* `read` into a buffer the caller already owns, which is what the 22.08 ms →
|
|
24
|
-
* 7.63 ms measurement on the field host was about. What it adds is an `open`
|
|
25
|
-
* per read, tens of microseconds against those milliseconds.
|
|
26
|
-
*
|
|
27
|
-
* Nothing here decides what the allowance should be. It is told, because the
|
|
28
|
-
* disk is one and this store is not its only user — the segments an encoder
|
|
29
|
-
* produces are on it too — and a ceiling that one of two users sets for itself
|
|
30
|
-
* is not a ceiling.
|
|
31
|
-
*/
|
|
32
|
-
|
|
33
|
-
import fs from "node:fs/promises";
|
|
34
|
-
import { readdirSync, statSync } from "node:fs";
|
|
35
|
-
import path from "node:path";
|
|
36
|
-
|
|
37
|
-
/**
|
|
38
|
-
* How a removal is asked for.
|
|
39
|
-
*
|
|
40
|
-
* Windows keeps a deleted file's name reserved until the last handle on it is
|
|
41
|
-
* closed, and answers `rm` of the name — or `rmdir` of the directory holding it
|
|
42
|
-
* — with EPERM until then. These are Node's own documented options for exactly
|
|
43
|
-
* that, not a wait invented here; on POSIX they never come into play.
|
|
44
|
-
*/
|
|
45
|
-
const REMOVAL = { force: true, maxRetries: 10, retryDelay: 20 };
|
|
46
|
-
|
|
47
|
-
/**
|
|
48
|
-
* One torrent's pieces on disk.
|
|
49
|
-
*
|
|
50
|
-
* Pieces are numbered by the torrent, so a file is named by its number. The
|
|
51
|
-
* directory is the store: nothing else writes into it, and destroying the store
|
|
52
|
-
* removes it whole.
|
|
53
|
-
*/
|
|
54
|
-
export class PieceDiskStore {
|
|
55
|
-
#directory;
|
|
56
|
-
|
|
57
|
-
/** Piece index → its length on disk. @type {Map<number, number>} */
|
|
58
|
-
#stored = new Map();
|
|
59
|
-
|
|
60
|
-
/** Piece index → when it was last written or read. @type {Map<number, number>} */
|
|
61
|
-
#touched = new Map();
|
|
62
|
-
|
|
63
|
-
/** Pieces being read right now, which eviction leaves alone. @type {Map<number, number>} */
|
|
64
|
-
#reading = new Map();
|
|
65
|
-
|
|
66
|
-
/** Removals still in flight, by piece. @type {Map<number, Promise<unknown>>} */
|
|
67
|
-
#removing = new Map();
|
|
68
|
-
|
|
69
|
-
/** What this store may hold, or null while nobody has said. @type {number | null} */
|
|
70
|
-
#allowanceBytes = null;
|
|
71
|
-
|
|
72
|
-
#evictions = 0;
|
|
73
|
-
|
|
74
|
-
/** How many were thrown away for being behind every reader. */
|
|
75
|
-
#behind = 0;
|
|
76
|
-
|
|
77
|
-
#bytes = 0;
|
|
78
|
-
|
|
79
|
-
#now;
|
|
80
|
-
|
|
81
|
-
/** Where the live readers stand, from whoever holds that fact. @type {() => number[]} */
|
|
82
|
-
#readHeads;
|
|
83
|
-
|
|
84
|
-
/**
|
|
85
|
-
* Said whenever this tier stops holding a piece, whatever the reason.
|
|
86
|
-
*
|
|
87
|
-
* It states THIS tier's own fact and nothing more — not whether the bytes can
|
|
88
|
-
* still be had elsewhere, which this class has no way of knowing. Whoever
|
|
89
|
-
* listens decides what the loss means.
|
|
90
|
-
*
|
|
91
|
-
* @type {(index: number) => void}
|
|
92
|
-
*/
|
|
93
|
-
#onForgotten;
|
|
94
|
-
|
|
95
|
-
/**
|
|
96
|
-
* @param {object} params
|
|
97
|
-
* @param {string} params.directory - Where this torrent's pieces live.
|
|
98
|
-
* @param {string} params.name - A name unique to the torrent; it becomes the
|
|
99
|
-
* directory inside `directory`.
|
|
100
|
-
* @param {number | null} [params.allowanceBytes] - What it may hold. Null
|
|
101
|
-
* means nobody has said yet, and nothing is evicted until somebody does.
|
|
102
|
-
* @param {() => number} [params.now]
|
|
103
|
-
*/
|
|
104
|
-
constructor({
|
|
105
|
-
directory,
|
|
106
|
-
name,
|
|
107
|
-
allowanceBytes = null,
|
|
108
|
-
now = Date.now,
|
|
109
|
-
readHeads = () => [],
|
|
110
|
-
onForgotten = () => undefined
|
|
111
|
-
}) {
|
|
112
|
-
this.#directory = path.join(directory, name);
|
|
113
|
-
this.#allowanceBytes = Number.isFinite(allowanceBytes) && allowanceBytes >= 0 ? allowanceBytes : null;
|
|
114
|
-
this.#now = now;
|
|
115
|
-
this.#readHeads = typeof readHeads === "function" ? readHeads : () => [];
|
|
116
|
-
this.#onForgotten = typeof onForgotten === "function" ? onForgotten : () => undefined;
|
|
117
|
-
this.#adoptWhatIsAlreadyHere();
|
|
118
|
-
}
|
|
119
|
-
|
|
120
|
-
/**
|
|
121
|
-
* Take up the pieces a previous life of this torrent left in this directory.
|
|
122
|
-
*
|
|
123
|
-
* The directory is the torrent's own, so what is in it belongs to it — and
|
|
124
|
-
* without this, nothing ever reads those files again: a torrent that is torn
|
|
125
|
-
* down and added back gets a store whose index starts empty, answers "not on
|
|
126
|
-
* disk" for every piece it in fact has, and downloads the film a second time
|
|
127
|
-
* while the first copy sits beside it. That is what a torrent destroyed by an
|
|
128
|
-
* error left behind until 2026-09-11, and what the pool's own restart leaves
|
|
129
|
-
* behind every time.
|
|
130
|
-
*
|
|
131
|
-
* Read once, synchronously, because `has()` is answered synchronously and the
|
|
132
|
-
* torrent asks it immediately — a piece reported missing while a scan is
|
|
133
|
-
* still running is a piece fetched again. One directory listing per torrent.
|
|
134
|
-
*
|
|
135
|
-
* Correctness is not taken on trust: the torrent hashes every piece it means
|
|
136
|
-
* to use, so a file here that does not match is refused by the layer above
|
|
137
|
-
* and downloaded again.
|
|
138
|
-
*
|
|
139
|
-
* @returns {void}
|
|
140
|
-
*/
|
|
141
|
-
#adoptWhatIsAlreadyHere() {
|
|
142
|
-
let entries = [];
|
|
143
|
-
try {
|
|
144
|
-
entries = readdirSync(this.#directory, { withFileTypes: true });
|
|
145
|
-
} catch {
|
|
146
|
-
// No directory yet: this torrent is new here, which is the ordinary case.
|
|
147
|
-
return;
|
|
148
|
-
}
|
|
149
|
-
const born = this.#now();
|
|
150
|
-
for (const entry of entries) {
|
|
151
|
-
if (!entry.isFile() || !entry.name.endsWith(".piece")) {
|
|
152
|
-
continue;
|
|
153
|
-
}
|
|
154
|
-
const index = Number.parseInt(entry.name.slice(0, -".piece".length), 10);
|
|
155
|
-
if (!Number.isInteger(index) || index < 0) {
|
|
156
|
-
continue;
|
|
157
|
-
}
|
|
158
|
-
try {
|
|
159
|
-
const { size } = statSync(path.join(this.#directory, entry.name));
|
|
160
|
-
if (size <= 0) {
|
|
161
|
-
continue;
|
|
162
|
-
}
|
|
163
|
-
this.#stored.set(index, size);
|
|
164
|
-
this.#touched.set(index, born);
|
|
165
|
-
this.#bytes += size;
|
|
166
|
-
} catch {
|
|
167
|
-
// Gone between the listing and the reading: not ours to worry about.
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
/** Where this store's pieces live, for logging and cleanup. */
|
|
173
|
-
get path() {
|
|
174
|
-
return this.#directory;
|
|
175
|
-
}
|
|
176
|
-
|
|
177
|
-
/**
|
|
178
|
-
* Every piece this tier holds, as numbers.
|
|
179
|
-
*
|
|
180
|
-
* @returns {number[]}
|
|
181
|
-
*/
|
|
182
|
-
indexes() {
|
|
183
|
-
return [...this.#stored.keys()];
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
/** How many pieces are on disk. */
|
|
187
|
-
get size() {
|
|
188
|
-
return this.#stored.size;
|
|
189
|
-
}
|
|
190
|
-
|
|
191
|
-
/** What those pieces weigh. */
|
|
192
|
-
get bytes() {
|
|
193
|
-
return this.#bytes;
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
/** What it may hold, or null while nobody has said. */
|
|
197
|
-
get allowanceBytes() {
|
|
198
|
-
return this.#allowanceBytes;
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/**
|
|
202
|
-
* Say what it may hold from now on.
|
|
203
|
-
*
|
|
204
|
-
* Lowering it does not free anything by itself: what is already written stays
|
|
205
|
-
* until the next write needs room. A store that is over its allowance and
|
|
206
|
-
* never written to again is holding disk nobody has asked for, which is the
|
|
207
|
-
* same bargain memory makes.
|
|
208
|
-
*
|
|
209
|
-
* @param {number | null} bytes
|
|
210
|
-
* @returns {number | null} What it may hold now.
|
|
211
|
-
*/
|
|
212
|
-
reviseAllowance(bytes) {
|
|
213
|
-
this.#allowanceBytes = Number.isFinite(bytes) && bytes >= 0 ? bytes : null;
|
|
214
|
-
return this.#allowanceBytes;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
/**
|
|
218
|
-
* @param {number} index
|
|
219
|
-
* @returns {boolean}
|
|
220
|
-
*/
|
|
221
|
-
has(index) {
|
|
222
|
-
return this.#stored.has(index);
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
/**
|
|
226
|
-
* Throw away what no reader will ask for again, without waiting for the disk
|
|
227
|
-
* to be short.
|
|
228
|
-
*
|
|
229
|
-
* THE SECOND RULE, and it answers a different question from the ceiling.
|
|
230
|
-
* Material nobody needs should not sit on somebody's disk merely because
|
|
231
|
-
* there is room for it — the ceiling here is a share of what is free, and on
|
|
232
|
-
* a roomy host that is tens of gigabytes against a measured growth of 14 400
|
|
233
|
-
* MB in one viewing, so the ceiling alone never binds and nothing is ever
|
|
234
|
-
* removed until the torrent itself goes.
|
|
235
|
-
*
|
|
236
|
-
* A piece BEHIND every read head has been read and will not be read again
|
|
237
|
-
* unless somebody seeks back — and a seek back re-downloads it, which is the
|
|
238
|
-
* bargain this tier already makes when it drops a piece for room. Nothing is
|
|
239
|
-
* thrown away while any reader might still reach it.
|
|
240
|
-
*
|
|
241
|
-
* With no reader at all nothing is removed: a store between reads is not a
|
|
242
|
-
* store nobody wants, and the torrent going idle is what empties it whole.
|
|
243
|
-
*
|
|
244
|
-
* @param {number[]} readHeads - The first piece each live reader still wants.
|
|
245
|
-
* @returns {number} How many pieces were thrown away.
|
|
246
|
-
*/
|
|
247
|
-
forgetBehind(readHeads) {
|
|
248
|
-
const heads = (readHeads ?? []).filter((at) => Number.isInteger(at));
|
|
249
|
-
if (heads.length === 0) {
|
|
250
|
-
return 0;
|
|
251
|
-
}
|
|
252
|
-
const earliest = Math.min(...heads);
|
|
253
|
-
let removed = 0;
|
|
254
|
-
for (const index of [...this.#stored.keys()]) {
|
|
255
|
-
if (index < earliest && !this.#reading.has(index)) {
|
|
256
|
-
this.forget(index);
|
|
257
|
-
this.#behind += 1;
|
|
258
|
-
removed += 1;
|
|
259
|
-
}
|
|
260
|
-
}
|
|
261
|
-
return removed;
|
|
262
|
-
}
|
|
263
|
-
|
|
264
|
-
/**
|
|
265
|
-
* Write a piece out, making room for it first.
|
|
266
|
-
*
|
|
267
|
-
* @param {number} index
|
|
268
|
-
* @param {Uint8Array} bytes
|
|
269
|
-
* @returns {Promise<void>}
|
|
270
|
-
*/
|
|
271
|
-
async write(index, bytes) {
|
|
272
|
-
// A piece thrown away and wanted again before its file has gone. Windows
|
|
273
|
-
// holds a deleted-but-still-open file in a pending state and answers the
|
|
274
|
-
// next `open` of that name with EPERM, so writing it again has to wait for
|
|
275
|
-
// the removal to finish. On POSIX the wait costs a resolved promise.
|
|
276
|
-
await this.#removing.get(index);
|
|
277
|
-
await this.#ensureDirectory();
|
|
278
|
-
await this.#makeRoomFor(bytes.length, index);
|
|
279
|
-
await fs.writeFile(this.#pathOf(index), bytes);
|
|
280
|
-
if (!this.#stored.has(index)) {
|
|
281
|
-
this.#bytes += bytes.length;
|
|
282
|
-
} else {
|
|
283
|
-
this.#bytes += bytes.length - (this.#stored.get(index) ?? 0);
|
|
284
|
-
}
|
|
285
|
-
this.#stored.set(index, bytes.length);
|
|
286
|
-
this.#touched.set(index, this.#now());
|
|
287
|
-
}
|
|
288
|
-
|
|
289
|
-
/**
|
|
290
|
-
* Read a piece back into a buffer the caller already owns.
|
|
291
|
-
*
|
|
292
|
-
* PART of a piece, when the caller asks for one. A peer asks for 16 KB at a
|
|
293
|
-
* time and a piece here is megabytes, so reading the whole of it to answer
|
|
294
|
-
* one request is the difference between 16 KB and 4 MB off the disk — field
|
|
295
|
-
* 2026-09-11: 63 416 reads of which 21.6 % came from memory, 49 696 pieces
|
|
296
|
-
* revived whole, to serve an upload capped at 512 KB/s.
|
|
297
|
-
*
|
|
298
|
-
* @param {number} index
|
|
299
|
-
* @param {Uint8Array} target - Destination; its length is what gets read.
|
|
300
|
-
* @param {number} [at] - Offset within the piece to start at.
|
|
301
|
-
* @returns {Promise<number>} Bytes read.
|
|
302
|
-
*/
|
|
303
|
-
hold(index) {
|
|
304
|
-
if (!this.#stored.has(index)) {
|
|
305
|
-
return null;
|
|
306
|
-
}
|
|
307
|
-
this.#reading.set(index, (this.#reading.get(index) ?? 0) + 1);
|
|
308
|
-
let held = true;
|
|
309
|
-
return () => {
|
|
310
|
-
if (!held) {
|
|
311
|
-
return;
|
|
312
|
-
}
|
|
313
|
-
held = false;
|
|
314
|
-
const outstanding = (this.#reading.get(index) ?? 1) - 1;
|
|
315
|
-
if (outstanding > 0) {
|
|
316
|
-
this.#reading.set(index, outstanding);
|
|
317
|
-
} else {
|
|
318
|
-
this.#reading.delete(index);
|
|
319
|
-
}
|
|
320
|
-
};
|
|
321
|
-
}
|
|
322
|
-
|
|
323
|
-
async read(index, target, at = 0) {
|
|
324
|
-
if (!this.#stored.has(index)) {
|
|
325
|
-
throw new Error(`Piece ${index} is not on disk.`);
|
|
326
|
-
}
|
|
327
|
-
// While this runs, eviction leaves the piece alone. Removing a file that is
|
|
328
|
-
// open is safe on POSIX and the read would finish — but the open itself
|
|
329
|
-
// happens below, and between the check above and that open a file removed
|
|
330
|
-
// is a read that fails for a piece the caller was told is there.
|
|
331
|
-
this.#reading.set(index, (this.#reading.get(index) ?? 0) + 1);
|
|
332
|
-
let handle = null;
|
|
333
|
-
try {
|
|
334
|
-
handle = await fs.open(this.#pathOf(index), "r");
|
|
335
|
-
const { bytesRead } = await handle.read(target, 0, target.length, Math.max(0, at));
|
|
336
|
-
this.#touched.set(index, this.#now());
|
|
337
|
-
return bytesRead;
|
|
338
|
-
} finally {
|
|
339
|
-
await handle?.close().catch(() => undefined);
|
|
340
|
-
const outstanding = (this.#reading.get(index) ?? 1) - 1;
|
|
341
|
-
if (outstanding > 0) {
|
|
342
|
-
this.#reading.set(index, outstanding);
|
|
343
|
-
} else {
|
|
344
|
-
this.#reading.delete(index);
|
|
345
|
-
}
|
|
346
|
-
}
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
/**
|
|
350
|
-
* Forget a piece and give its disk back.
|
|
351
|
-
*
|
|
352
|
-
* The removal itself is left to run: the caller's contract is synchronous —
|
|
353
|
-
* after this returns, the store no longer has the piece — and the blocks come
|
|
354
|
-
* back a moment later. A failure to remove leaves a file nobody will read;
|
|
355
|
-
* the directory goes whole when the store is destroyed.
|
|
356
|
-
*
|
|
357
|
-
* @param {number} index
|
|
358
|
-
* @returns {void}
|
|
359
|
-
*/
|
|
360
|
-
forget(index) {
|
|
361
|
-
if (this.#reading.has(index)) {
|
|
362
|
-
return;
|
|
363
|
-
}
|
|
364
|
-
const length = this.#stored.get(index);
|
|
365
|
-
if (length === undefined) {
|
|
366
|
-
return;
|
|
367
|
-
}
|
|
368
|
-
this.#stored.delete(index);
|
|
369
|
-
this.#touched.delete(index);
|
|
370
|
-
this.#bytes -= length;
|
|
371
|
-
// Kept track of, because a removal still in flight holds the directory: on
|
|
372
|
-
// Windows `rmdir` refuses while any handle inside is open, so `destroy`
|
|
373
|
-
// waits for these before removing the directory. On Linux it would succeed
|
|
374
|
-
// and the removals would then fail silently against a directory that is
|
|
375
|
-
// gone — tidy either way, and correct on both.
|
|
376
|
-
const removal = fs.rm(this.#pathOf(index), REMOVAL)
|
|
377
|
-
.catch(() => undefined)
|
|
378
|
-
.finally(() => {
|
|
379
|
-
if (this.#removing.get(index) === removal) {
|
|
380
|
-
this.#removing.delete(index);
|
|
381
|
-
}
|
|
382
|
-
});
|
|
383
|
-
this.#removing.set(index, removal);
|
|
384
|
-
// SAID AFTER THE FACT IS TRUE, so a listener that asks this tier whether it
|
|
385
|
-
// holds the piece is told the same thing this method has just made so.
|
|
386
|
-
this.#onForgotten(index);
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
/**
|
|
390
|
-
* What it holds, what it may hold, and what it has had to throw away.
|
|
391
|
-
*
|
|
392
|
-
* @returns {{ pieces: number, bytes: number, allowanceBytes: number | null, evictions: number, behind: number }}
|
|
393
|
-
*/
|
|
394
|
-
stats() {
|
|
395
|
-
return {
|
|
396
|
-
pieces: this.#stored.size,
|
|
397
|
-
bytes: this.#bytes,
|
|
398
|
-
allowanceBytes: this.#allowanceBytes,
|
|
399
|
-
evictions: this.#evictions,
|
|
400
|
-
behind: this.#behind
|
|
401
|
-
};
|
|
402
|
-
}
|
|
403
|
-
|
|
404
|
-
/**
|
|
405
|
-
* Close it, leaving its contents in place.
|
|
406
|
-
*
|
|
407
|
-
* There is no handle to close — a read opens and closes its own — so this
|
|
408
|
-
* exists for the caller's lifecycle and does nothing else.
|
|
409
|
-
*
|
|
410
|
-
* @returns {Promise<void>}
|
|
411
|
-
*/
|
|
412
|
-
async close() {
|
|
413
|
-
// Nothing to release: a read opens and closes its own handle.
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
/**
|
|
417
|
-
* Close and remove everything.
|
|
418
|
-
*
|
|
419
|
-
* @returns {Promise<void>}
|
|
420
|
-
*/
|
|
421
|
-
async destroy() {
|
|
422
|
-
await this.close();
|
|
423
|
-
await this.settled();
|
|
424
|
-
this.#stored.clear();
|
|
425
|
-
this.#touched.clear();
|
|
426
|
-
this.#bytes = 0;
|
|
427
|
-
await fs.rm(this.#directory, { ...REMOVAL, recursive: true });
|
|
428
|
-
}
|
|
429
|
-
|
|
430
|
-
/**
|
|
431
|
-
* Wait for every removal this store has started.
|
|
432
|
-
*
|
|
433
|
-
* Removing a piece answers at once and frees the disk a moment later; this is
|
|
434
|
-
* how a caller that needs the disk back NOW — a test, or `destroy` — waits for
|
|
435
|
-
* it without making `forget` asynchronous for everybody else.
|
|
436
|
-
*
|
|
437
|
-
* @returns {Promise<void>}
|
|
438
|
-
*/
|
|
439
|
-
async settled() {
|
|
440
|
-
while (this.#removing.size > 0) {
|
|
441
|
-
await Promise.all([...this.#removing.values()]);
|
|
442
|
-
}
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
/**
|
|
446
|
-
* @param {number} index
|
|
447
|
-
* @returns {string}
|
|
448
|
-
*/
|
|
449
|
-
#pathOf(index) {
|
|
450
|
-
return path.join(this.#directory, `${index}.piece`);
|
|
451
|
-
}
|
|
452
|
-
|
|
453
|
-
/**
|
|
454
|
-
* Make sure the directory is there, before every write.
|
|
455
|
-
*
|
|
456
|
-
* Deliberately not remembered. A remembered "it exists" is a statement about
|
|
457
|
-
* the past: this store's own `destroy` removes the directory, a startup sweep
|
|
458
|
-
* removes what a killed process left, and an operator clearing a full disk
|
|
459
|
-
* removes anything. Each of those turns the memory into a lie and every write
|
|
460
|
-
* after it into `ENOENT`. Creating a directory that already exists costs tens
|
|
461
|
-
* of microseconds against a write measured in milliseconds.
|
|
462
|
-
*
|
|
463
|
-
* @returns {Promise<void>}
|
|
464
|
-
*/
|
|
465
|
-
async #ensureDirectory() {
|
|
466
|
-
await fs.mkdir(this.#directory, { recursive: true });
|
|
467
|
-
}
|
|
468
|
-
|
|
469
|
-
/**
|
|
470
|
-
* Throw away the least recently used pieces until one more will fit.
|
|
471
|
-
*
|
|
472
|
-
* Least recently used first, and for the reason the memory tier uses the same
|
|
473
|
-
* order: what nobody has read for the longest is what a viewer is least
|
|
474
|
-
* likely to want next. A piece being read now is never a victim, and neither
|
|
475
|
-
* is the piece about to be written.
|
|
476
|
-
*
|
|
477
|
-
* A piece thrown away is not lost, only un-had: the store answers `has` with
|
|
478
|
-
* false, the read that wanted it gets nothing, and the torrent fetches it
|
|
479
|
-
* again. That is the same bargain the memory tier makes when it spills.
|
|
480
|
-
*
|
|
481
|
-
* @param {number} incomingBytes
|
|
482
|
-
* @param {number} incomingIndex
|
|
483
|
-
* @returns {Promise<void>}
|
|
484
|
-
*/
|
|
485
|
-
async #makeRoomFor(incomingBytes, incomingIndex) {
|
|
486
|
-
if (this.#allowanceBytes === null) {
|
|
487
|
-
return;
|
|
488
|
-
}
|
|
489
|
-
const already = this.#stored.get(incomingIndex) ?? 0;
|
|
490
|
-
while (this.#bytes - already + incomingBytes > this.#allowanceBytes) {
|
|
491
|
-
const victim = this.#leastRecentlyUsed(incomingIndex);
|
|
492
|
-
if (victim === null) {
|
|
493
|
-
// Everything left is either being read or is the piece coming in. The
|
|
494
|
-
// write goes ahead: refusing it would lose a piece the swarm has
|
|
495
|
-
// already paid for, and the next write finds the readers gone.
|
|
496
|
-
return;
|
|
497
|
-
}
|
|
498
|
-
this.forget(victim);
|
|
499
|
-
this.#evictions += 1;
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
|
|
503
|
-
/**
|
|
504
|
-
* @param {number} except
|
|
505
|
-
* @returns {number | null}
|
|
506
|
-
*/
|
|
507
|
-
#leastRecentlyUsed(except) {
|
|
508
|
-
// WHERE THE READERS STAND DECIDES, and last use only settles ties.
|
|
509
|
-
//
|
|
510
|
-
// What lies behind every read head has been read and will not be read again
|
|
511
|
-
// unless somebody seeks back, so it goes before anything ahead of them,
|
|
512
|
-
// furthest behind first. It is the order the segments are given one layer
|
|
513
|
-
// up, and the order the priority map states, read from the other end.
|
|
514
|
-
// Without the heads there is nothing to order by and last use is all that
|
|
515
|
-
// is left — which is what this was, and what said nothing about what
|
|
516
|
-
// anybody is about to read.
|
|
517
|
-
const heads = this.#readHeads().filter((at) => Number.isInteger(at));
|
|
518
|
-
const earliest = heads.length > 0 ? Math.min(...heads) : null;
|
|
519
|
-
let victim = null;
|
|
520
|
-
let worst = null;
|
|
521
|
-
for (const [index, at] of this.#touched) {
|
|
522
|
-
if (index === except || this.#reading.has(index)) {
|
|
523
|
-
continue;
|
|
524
|
-
}
|
|
525
|
-
const behind = earliest !== null && index < earliest;
|
|
526
|
-
const score = { behind, distance: behind ? earliest - index : 0, at };
|
|
527
|
-
if (
|
|
528
|
-
worst === null
|
|
529
|
-
|| (score.behind && !worst.behind)
|
|
530
|
-
|| (score.behind === worst.behind && score.distance > worst.distance)
|
|
531
|
-
|| (score.behind === worst.behind && score.distance === worst.distance && score.at < worst.at)
|
|
532
|
-
) {
|
|
533
|
-
worst = score;
|
|
534
|
-
victim = index;
|
|
535
|
-
}
|
|
536
|
-
}
|
|
537
|
-
return victim;
|
|
538
|
-
}
|
|
539
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* @file Where downloaded pieces live once memory cannot hold them.
|
|
3
|
+
*
|
|
4
|
+
* The second tier of the piece store, and — unlike what it replaces — an owner
|
|
5
|
+
* of what it takes: it knows how many bytes it holds, it is told how many it
|
|
6
|
+
* may hold, and when it is over that it gives disk back.
|
|
7
|
+
*
|
|
8
|
+
* WHAT IT REPLACED, because the difference is the whole of the design. `DiskTier`
|
|
9
|
+
* wrote every piece into ONE sparse file at `index * chunkLength` and answered
|
|
10
|
+
* `forget(index)` by dropping the number from a set. The bytes stayed: a sparse
|
|
11
|
+
* file's blocks are returned only by hole punching, which Node exposes no
|
|
12
|
+
* binding for, so nothing this process could do returned a single block before
|
|
13
|
+
* the whole file was removed. Measured 2026-08-31: a store holding 312-424 MB
|
|
14
|
+
* of pieces had written **14 400 MB** to that file in fifty minutes, and free
|
|
15
|
+
* space on the host fell by every megabyte of it until the session ended. On a
|
|
16
|
+
* Home Assistant install with a 32 GB card that is the card.
|
|
17
|
+
*
|
|
18
|
+
* A PIECE IS A FILE, and that is what makes the ceiling real. Removing a file
|
|
19
|
+
* returns exactly its blocks, needs no binding this runtime lacks, and makes
|
|
20
|
+
* the unit of eviction the same as the unit of storage — so the order pieces
|
|
21
|
+
* leave in is the order we choose rather than the order they happen to lie in.
|
|
22
|
+
* The read that the single file was chosen for is unaffected: it is still one
|
|
23
|
+
* `read` into a buffer the caller already owns, which is what the 22.08 ms →
|
|
24
|
+
* 7.63 ms measurement on the field host was about. What it adds is an `open`
|
|
25
|
+
* per read, tens of microseconds against those milliseconds.
|
|
26
|
+
*
|
|
27
|
+
* Nothing here decides what the allowance should be. It is told, because the
|
|
28
|
+
* disk is one and this store is not its only user — the segments an encoder
|
|
29
|
+
* produces are on it too — and a ceiling that one of two users sets for itself
|
|
30
|
+
* is not a ceiling.
|
|
31
|
+
*/
|
|
32
|
+
|
|
33
|
+
import fs from "node:fs/promises";
|
|
34
|
+
import { readdirSync, statSync } from "node:fs";
|
|
35
|
+
import path from "node:path";
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* How a removal is asked for.
|
|
39
|
+
*
|
|
40
|
+
* Windows keeps a deleted file's name reserved until the last handle on it is
|
|
41
|
+
* closed, and answers `rm` of the name — or `rmdir` of the directory holding it
|
|
42
|
+
* — with EPERM until then. These are Node's own documented options for exactly
|
|
43
|
+
* that, not a wait invented here; on POSIX they never come into play.
|
|
44
|
+
*/
|
|
45
|
+
const REMOVAL = { force: true, maxRetries: 10, retryDelay: 20 };
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* One torrent's pieces on disk.
|
|
49
|
+
*
|
|
50
|
+
* Pieces are numbered by the torrent, so a file is named by its number. The
|
|
51
|
+
* directory is the store: nothing else writes into it, and destroying the store
|
|
52
|
+
* removes it whole.
|
|
53
|
+
*/
|
|
54
|
+
export class PieceDiskStore {
|
|
55
|
+
#directory;
|
|
56
|
+
|
|
57
|
+
/** Piece index → its length on disk. @type {Map<number, number>} */
|
|
58
|
+
#stored = new Map();
|
|
59
|
+
|
|
60
|
+
/** Piece index → when it was last written or read. @type {Map<number, number>} */
|
|
61
|
+
#touched = new Map();
|
|
62
|
+
|
|
63
|
+
/** Pieces being read right now, which eviction leaves alone. @type {Map<number, number>} */
|
|
64
|
+
#reading = new Map();
|
|
65
|
+
|
|
66
|
+
/** Removals still in flight, by piece. @type {Map<number, Promise<unknown>>} */
|
|
67
|
+
#removing = new Map();
|
|
68
|
+
|
|
69
|
+
/** What this store may hold, or null while nobody has said. @type {number | null} */
|
|
70
|
+
#allowanceBytes = null;
|
|
71
|
+
|
|
72
|
+
#evictions = 0;
|
|
73
|
+
|
|
74
|
+
/** How many were thrown away for being behind every reader. */
|
|
75
|
+
#behind = 0;
|
|
76
|
+
|
|
77
|
+
#bytes = 0;
|
|
78
|
+
|
|
79
|
+
#now;
|
|
80
|
+
|
|
81
|
+
/** Where the live readers stand, from whoever holds that fact. @type {() => number[]} */
|
|
82
|
+
#readHeads;
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Said whenever this tier stops holding a piece, whatever the reason.
|
|
86
|
+
*
|
|
87
|
+
* It states THIS tier's own fact and nothing more — not whether the bytes can
|
|
88
|
+
* still be had elsewhere, which this class has no way of knowing. Whoever
|
|
89
|
+
* listens decides what the loss means.
|
|
90
|
+
*
|
|
91
|
+
* @type {(index: number) => void}
|
|
92
|
+
*/
|
|
93
|
+
#onForgotten;
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* @param {object} params
|
|
97
|
+
* @param {string} params.directory - Where this torrent's pieces live.
|
|
98
|
+
* @param {string} params.name - A name unique to the torrent; it becomes the
|
|
99
|
+
* directory inside `directory`.
|
|
100
|
+
* @param {number | null} [params.allowanceBytes] - What it may hold. Null
|
|
101
|
+
* means nobody has said yet, and nothing is evicted until somebody does.
|
|
102
|
+
* @param {() => number} [params.now]
|
|
103
|
+
*/
|
|
104
|
+
constructor({
|
|
105
|
+
directory,
|
|
106
|
+
name,
|
|
107
|
+
allowanceBytes = null,
|
|
108
|
+
now = Date.now,
|
|
109
|
+
readHeads = () => [],
|
|
110
|
+
onForgotten = () => undefined
|
|
111
|
+
}) {
|
|
112
|
+
this.#directory = path.join(directory, name);
|
|
113
|
+
this.#allowanceBytes = Number.isFinite(allowanceBytes) && allowanceBytes >= 0 ? allowanceBytes : null;
|
|
114
|
+
this.#now = now;
|
|
115
|
+
this.#readHeads = typeof readHeads === "function" ? readHeads : () => [];
|
|
116
|
+
this.#onForgotten = typeof onForgotten === "function" ? onForgotten : () => undefined;
|
|
117
|
+
this.#adoptWhatIsAlreadyHere();
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/**
|
|
121
|
+
* Take up the pieces a previous life of this torrent left in this directory.
|
|
122
|
+
*
|
|
123
|
+
* The directory is the torrent's own, so what is in it belongs to it — and
|
|
124
|
+
* without this, nothing ever reads those files again: a torrent that is torn
|
|
125
|
+
* down and added back gets a store whose index starts empty, answers "not on
|
|
126
|
+
* disk" for every piece it in fact has, and downloads the film a second time
|
|
127
|
+
* while the first copy sits beside it. That is what a torrent destroyed by an
|
|
128
|
+
* error left behind until 2026-09-11, and what the pool's own restart leaves
|
|
129
|
+
* behind every time.
|
|
130
|
+
*
|
|
131
|
+
* Read once, synchronously, because `has()` is answered synchronously and the
|
|
132
|
+
* torrent asks it immediately — a piece reported missing while a scan is
|
|
133
|
+
* still running is a piece fetched again. One directory listing per torrent.
|
|
134
|
+
*
|
|
135
|
+
* Correctness is not taken on trust: the torrent hashes every piece it means
|
|
136
|
+
* to use, so a file here that does not match is refused by the layer above
|
|
137
|
+
* and downloaded again.
|
|
138
|
+
*
|
|
139
|
+
* @returns {void}
|
|
140
|
+
*/
|
|
141
|
+
#adoptWhatIsAlreadyHere() {
|
|
142
|
+
let entries = [];
|
|
143
|
+
try {
|
|
144
|
+
entries = readdirSync(this.#directory, { withFileTypes: true });
|
|
145
|
+
} catch {
|
|
146
|
+
// No directory yet: this torrent is new here, which is the ordinary case.
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
const born = this.#now();
|
|
150
|
+
for (const entry of entries) {
|
|
151
|
+
if (!entry.isFile() || !entry.name.endsWith(".piece")) {
|
|
152
|
+
continue;
|
|
153
|
+
}
|
|
154
|
+
const index = Number.parseInt(entry.name.slice(0, -".piece".length), 10);
|
|
155
|
+
if (!Number.isInteger(index) || index < 0) {
|
|
156
|
+
continue;
|
|
157
|
+
}
|
|
158
|
+
try {
|
|
159
|
+
const { size } = statSync(path.join(this.#directory, entry.name));
|
|
160
|
+
if (size <= 0) {
|
|
161
|
+
continue;
|
|
162
|
+
}
|
|
163
|
+
this.#stored.set(index, size);
|
|
164
|
+
this.#touched.set(index, born);
|
|
165
|
+
this.#bytes += size;
|
|
166
|
+
} catch {
|
|
167
|
+
// Gone between the listing and the reading: not ours to worry about.
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** Where this store's pieces live, for logging and cleanup. */
|
|
173
|
+
get path() {
|
|
174
|
+
return this.#directory;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Every piece this tier holds, as numbers.
|
|
179
|
+
*
|
|
180
|
+
* @returns {number[]}
|
|
181
|
+
*/
|
|
182
|
+
indexes() {
|
|
183
|
+
return [...this.#stored.keys()];
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** How many pieces are on disk. */
|
|
187
|
+
get size() {
|
|
188
|
+
return this.#stored.size;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/** What those pieces weigh. */
|
|
192
|
+
get bytes() {
|
|
193
|
+
return this.#bytes;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/** What it may hold, or null while nobody has said. */
|
|
197
|
+
get allowanceBytes() {
|
|
198
|
+
return this.#allowanceBytes;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* Say what it may hold from now on.
|
|
203
|
+
*
|
|
204
|
+
* Lowering it does not free anything by itself: what is already written stays
|
|
205
|
+
* until the next write needs room. A store that is over its allowance and
|
|
206
|
+
* never written to again is holding disk nobody has asked for, which is the
|
|
207
|
+
* same bargain memory makes.
|
|
208
|
+
*
|
|
209
|
+
* @param {number | null} bytes
|
|
210
|
+
* @returns {number | null} What it may hold now.
|
|
211
|
+
*/
|
|
212
|
+
reviseAllowance(bytes) {
|
|
213
|
+
this.#allowanceBytes = Number.isFinite(bytes) && bytes >= 0 ? bytes : null;
|
|
214
|
+
return this.#allowanceBytes;
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* @param {number} index
|
|
219
|
+
* @returns {boolean}
|
|
220
|
+
*/
|
|
221
|
+
has(index) {
|
|
222
|
+
return this.#stored.has(index);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Throw away what no reader will ask for again, without waiting for the disk
|
|
227
|
+
* to be short.
|
|
228
|
+
*
|
|
229
|
+
* THE SECOND RULE, and it answers a different question from the ceiling.
|
|
230
|
+
* Material nobody needs should not sit on somebody's disk merely because
|
|
231
|
+
* there is room for it — the ceiling here is a share of what is free, and on
|
|
232
|
+
* a roomy host that is tens of gigabytes against a measured growth of 14 400
|
|
233
|
+
* MB in one viewing, so the ceiling alone never binds and nothing is ever
|
|
234
|
+
* removed until the torrent itself goes.
|
|
235
|
+
*
|
|
236
|
+
* A piece BEHIND every read head has been read and will not be read again
|
|
237
|
+
* unless somebody seeks back — and a seek back re-downloads it, which is the
|
|
238
|
+
* bargain this tier already makes when it drops a piece for room. Nothing is
|
|
239
|
+
* thrown away while any reader might still reach it.
|
|
240
|
+
*
|
|
241
|
+
* With no reader at all nothing is removed: a store between reads is not a
|
|
242
|
+
* store nobody wants, and the torrent going idle is what empties it whole.
|
|
243
|
+
*
|
|
244
|
+
* @param {number[]} readHeads - The first piece each live reader still wants.
|
|
245
|
+
* @returns {number} How many pieces were thrown away.
|
|
246
|
+
*/
|
|
247
|
+
forgetBehind(readHeads) {
|
|
248
|
+
const heads = (readHeads ?? []).filter((at) => Number.isInteger(at));
|
|
249
|
+
if (heads.length === 0) {
|
|
250
|
+
return 0;
|
|
251
|
+
}
|
|
252
|
+
const earliest = Math.min(...heads);
|
|
253
|
+
let removed = 0;
|
|
254
|
+
for (const index of [...this.#stored.keys()]) {
|
|
255
|
+
if (index < earliest && !this.#reading.has(index)) {
|
|
256
|
+
this.forget(index);
|
|
257
|
+
this.#behind += 1;
|
|
258
|
+
removed += 1;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return removed;
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Write a piece out, making room for it first.
|
|
266
|
+
*
|
|
267
|
+
* @param {number} index
|
|
268
|
+
* @param {Uint8Array} bytes
|
|
269
|
+
* @returns {Promise<void>}
|
|
270
|
+
*/
|
|
271
|
+
async write(index, bytes) {
|
|
272
|
+
// A piece thrown away and wanted again before its file has gone. Windows
|
|
273
|
+
// holds a deleted-but-still-open file in a pending state and answers the
|
|
274
|
+
// next `open` of that name with EPERM, so writing it again has to wait for
|
|
275
|
+
// the removal to finish. On POSIX the wait costs a resolved promise.
|
|
276
|
+
await this.#removing.get(index);
|
|
277
|
+
await this.#ensureDirectory();
|
|
278
|
+
await this.#makeRoomFor(bytes.length, index);
|
|
279
|
+
await fs.writeFile(this.#pathOf(index), bytes);
|
|
280
|
+
if (!this.#stored.has(index)) {
|
|
281
|
+
this.#bytes += bytes.length;
|
|
282
|
+
} else {
|
|
283
|
+
this.#bytes += bytes.length - (this.#stored.get(index) ?? 0);
|
|
284
|
+
}
|
|
285
|
+
this.#stored.set(index, bytes.length);
|
|
286
|
+
this.#touched.set(index, this.#now());
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
/**
|
|
290
|
+
* Read a piece back into a buffer the caller already owns.
|
|
291
|
+
*
|
|
292
|
+
* PART of a piece, when the caller asks for one. A peer asks for 16 KB at a
|
|
293
|
+
* time and a piece here is megabytes, so reading the whole of it to answer
|
|
294
|
+
* one request is the difference between 16 KB and 4 MB off the disk — field
|
|
295
|
+
* 2026-09-11: 63 416 reads of which 21.6 % came from memory, 49 696 pieces
|
|
296
|
+
* revived whole, to serve an upload capped at 512 KB/s.
|
|
297
|
+
*
|
|
298
|
+
* @param {number} index
|
|
299
|
+
* @param {Uint8Array} target - Destination; its length is what gets read.
|
|
300
|
+
* @param {number} [at] - Offset within the piece to start at.
|
|
301
|
+
* @returns {Promise<number>} Bytes read.
|
|
302
|
+
*/
|
|
303
|
+
hold(index) {
|
|
304
|
+
if (!this.#stored.has(index)) {
|
|
305
|
+
return null;
|
|
306
|
+
}
|
|
307
|
+
this.#reading.set(index, (this.#reading.get(index) ?? 0) + 1);
|
|
308
|
+
let held = true;
|
|
309
|
+
return () => {
|
|
310
|
+
if (!held) {
|
|
311
|
+
return;
|
|
312
|
+
}
|
|
313
|
+
held = false;
|
|
314
|
+
const outstanding = (this.#reading.get(index) ?? 1) - 1;
|
|
315
|
+
if (outstanding > 0) {
|
|
316
|
+
this.#reading.set(index, outstanding);
|
|
317
|
+
} else {
|
|
318
|
+
this.#reading.delete(index);
|
|
319
|
+
}
|
|
320
|
+
};
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
async read(index, target, at = 0) {
|
|
324
|
+
if (!this.#stored.has(index)) {
|
|
325
|
+
throw new Error(`Piece ${index} is not on disk.`);
|
|
326
|
+
}
|
|
327
|
+
// While this runs, eviction leaves the piece alone. Removing a file that is
|
|
328
|
+
// open is safe on POSIX and the read would finish — but the open itself
|
|
329
|
+
// happens below, and between the check above and that open a file removed
|
|
330
|
+
// is a read that fails for a piece the caller was told is there.
|
|
331
|
+
this.#reading.set(index, (this.#reading.get(index) ?? 0) + 1);
|
|
332
|
+
let handle = null;
|
|
333
|
+
try {
|
|
334
|
+
handle = await fs.open(this.#pathOf(index), "r");
|
|
335
|
+
const { bytesRead } = await handle.read(target, 0, target.length, Math.max(0, at));
|
|
336
|
+
this.#touched.set(index, this.#now());
|
|
337
|
+
return bytesRead;
|
|
338
|
+
} finally {
|
|
339
|
+
await handle?.close().catch(() => undefined);
|
|
340
|
+
const outstanding = (this.#reading.get(index) ?? 1) - 1;
|
|
341
|
+
if (outstanding > 0) {
|
|
342
|
+
this.#reading.set(index, outstanding);
|
|
343
|
+
} else {
|
|
344
|
+
this.#reading.delete(index);
|
|
345
|
+
}
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Forget a piece and give its disk back.
|
|
351
|
+
*
|
|
352
|
+
* The removal itself is left to run: the caller's contract is synchronous —
|
|
353
|
+
* after this returns, the store no longer has the piece — and the blocks come
|
|
354
|
+
* back a moment later. A failure to remove leaves a file nobody will read;
|
|
355
|
+
* the directory goes whole when the store is destroyed.
|
|
356
|
+
*
|
|
357
|
+
* @param {number} index
|
|
358
|
+
* @returns {void}
|
|
359
|
+
*/
|
|
360
|
+
forget(index) {
|
|
361
|
+
if (this.#reading.has(index)) {
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
const length = this.#stored.get(index);
|
|
365
|
+
if (length === undefined) {
|
|
366
|
+
return;
|
|
367
|
+
}
|
|
368
|
+
this.#stored.delete(index);
|
|
369
|
+
this.#touched.delete(index);
|
|
370
|
+
this.#bytes -= length;
|
|
371
|
+
// Kept track of, because a removal still in flight holds the directory: on
|
|
372
|
+
// Windows `rmdir` refuses while any handle inside is open, so `destroy`
|
|
373
|
+
// waits for these before removing the directory. On Linux it would succeed
|
|
374
|
+
// and the removals would then fail silently against a directory that is
|
|
375
|
+
// gone — tidy either way, and correct on both.
|
|
376
|
+
const removal = fs.rm(this.#pathOf(index), REMOVAL)
|
|
377
|
+
.catch(() => undefined)
|
|
378
|
+
.finally(() => {
|
|
379
|
+
if (this.#removing.get(index) === removal) {
|
|
380
|
+
this.#removing.delete(index);
|
|
381
|
+
}
|
|
382
|
+
});
|
|
383
|
+
this.#removing.set(index, removal);
|
|
384
|
+
// SAID AFTER THE FACT IS TRUE, so a listener that asks this tier whether it
|
|
385
|
+
// holds the piece is told the same thing this method has just made so.
|
|
386
|
+
this.#onForgotten(index);
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/**
|
|
390
|
+
* What it holds, what it may hold, and what it has had to throw away.
|
|
391
|
+
*
|
|
392
|
+
* @returns {{ pieces: number, bytes: number, allowanceBytes: number | null, evictions: number, behind: number }}
|
|
393
|
+
*/
|
|
394
|
+
stats() {
|
|
395
|
+
return {
|
|
396
|
+
pieces: this.#stored.size,
|
|
397
|
+
bytes: this.#bytes,
|
|
398
|
+
allowanceBytes: this.#allowanceBytes,
|
|
399
|
+
evictions: this.#evictions,
|
|
400
|
+
behind: this.#behind
|
|
401
|
+
};
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
/**
|
|
405
|
+
* Close it, leaving its contents in place.
|
|
406
|
+
*
|
|
407
|
+
* There is no handle to close — a read opens and closes its own — so this
|
|
408
|
+
* exists for the caller's lifecycle and does nothing else.
|
|
409
|
+
*
|
|
410
|
+
* @returns {Promise<void>}
|
|
411
|
+
*/
|
|
412
|
+
async close() {
|
|
413
|
+
// Nothing to release: a read opens and closes its own handle.
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Close and remove everything.
|
|
418
|
+
*
|
|
419
|
+
* @returns {Promise<void>}
|
|
420
|
+
*/
|
|
421
|
+
async destroy() {
|
|
422
|
+
await this.close();
|
|
423
|
+
await this.settled();
|
|
424
|
+
this.#stored.clear();
|
|
425
|
+
this.#touched.clear();
|
|
426
|
+
this.#bytes = 0;
|
|
427
|
+
await fs.rm(this.#directory, { ...REMOVAL, recursive: true });
|
|
428
|
+
}
|
|
429
|
+
|
|
430
|
+
/**
|
|
431
|
+
* Wait for every removal this store has started.
|
|
432
|
+
*
|
|
433
|
+
* Removing a piece answers at once and frees the disk a moment later; this is
|
|
434
|
+
* how a caller that needs the disk back NOW — a test, or `destroy` — waits for
|
|
435
|
+
* it without making `forget` asynchronous for everybody else.
|
|
436
|
+
*
|
|
437
|
+
* @returns {Promise<void>}
|
|
438
|
+
*/
|
|
439
|
+
async settled() {
|
|
440
|
+
while (this.#removing.size > 0) {
|
|
441
|
+
await Promise.all([...this.#removing.values()]);
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
/**
|
|
446
|
+
* @param {number} index
|
|
447
|
+
* @returns {string}
|
|
448
|
+
*/
|
|
449
|
+
#pathOf(index) {
|
|
450
|
+
return path.join(this.#directory, `${index}.piece`);
|
|
451
|
+
}
|
|
452
|
+
|
|
453
|
+
/**
|
|
454
|
+
* Make sure the directory is there, before every write.
|
|
455
|
+
*
|
|
456
|
+
* Deliberately not remembered. A remembered "it exists" is a statement about
|
|
457
|
+
* the past: this store's own `destroy` removes the directory, a startup sweep
|
|
458
|
+
* removes what a killed process left, and an operator clearing a full disk
|
|
459
|
+
* removes anything. Each of those turns the memory into a lie and every write
|
|
460
|
+
* after it into `ENOENT`. Creating a directory that already exists costs tens
|
|
461
|
+
* of microseconds against a write measured in milliseconds.
|
|
462
|
+
*
|
|
463
|
+
* @returns {Promise<void>}
|
|
464
|
+
*/
|
|
465
|
+
async #ensureDirectory() {
|
|
466
|
+
await fs.mkdir(this.#directory, { recursive: true });
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
/**
|
|
470
|
+
* Throw away the least recently used pieces until one more will fit.
|
|
471
|
+
*
|
|
472
|
+
* Least recently used first, and for the reason the memory tier uses the same
|
|
473
|
+
* order: what nobody has read for the longest is what a viewer is least
|
|
474
|
+
* likely to want next. A piece being read now is never a victim, and neither
|
|
475
|
+
* is the piece about to be written.
|
|
476
|
+
*
|
|
477
|
+
* A piece thrown away is not lost, only un-had: the store answers `has` with
|
|
478
|
+
* false, the read that wanted it gets nothing, and the torrent fetches it
|
|
479
|
+
* again. That is the same bargain the memory tier makes when it spills.
|
|
480
|
+
*
|
|
481
|
+
* @param {number} incomingBytes
|
|
482
|
+
* @param {number} incomingIndex
|
|
483
|
+
* @returns {Promise<void>}
|
|
484
|
+
*/
|
|
485
|
+
async #makeRoomFor(incomingBytes, incomingIndex) {
|
|
486
|
+
if (this.#allowanceBytes === null) {
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
const already = this.#stored.get(incomingIndex) ?? 0;
|
|
490
|
+
while (this.#bytes - already + incomingBytes > this.#allowanceBytes) {
|
|
491
|
+
const victim = this.#leastRecentlyUsed(incomingIndex);
|
|
492
|
+
if (victim === null) {
|
|
493
|
+
// Everything left is either being read or is the piece coming in. The
|
|
494
|
+
// write goes ahead: refusing it would lose a piece the swarm has
|
|
495
|
+
// already paid for, and the next write finds the readers gone.
|
|
496
|
+
return;
|
|
497
|
+
}
|
|
498
|
+
this.forget(victim);
|
|
499
|
+
this.#evictions += 1;
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
/**
|
|
504
|
+
* @param {number} except
|
|
505
|
+
* @returns {number | null}
|
|
506
|
+
*/
|
|
507
|
+
#leastRecentlyUsed(except) {
|
|
508
|
+
// WHERE THE READERS STAND DECIDES, and last use only settles ties.
|
|
509
|
+
//
|
|
510
|
+
// What lies behind every read head has been read and will not be read again
|
|
511
|
+
// unless somebody seeks back, so it goes before anything ahead of them,
|
|
512
|
+
// furthest behind first. It is the order the segments are given one layer
|
|
513
|
+
// up, and the order the priority map states, read from the other end.
|
|
514
|
+
// Without the heads there is nothing to order by and last use is all that
|
|
515
|
+
// is left — which is what this was, and what said nothing about what
|
|
516
|
+
// anybody is about to read.
|
|
517
|
+
const heads = this.#readHeads().filter((at) => Number.isInteger(at));
|
|
518
|
+
const earliest = heads.length > 0 ? Math.min(...heads) : null;
|
|
519
|
+
let victim = null;
|
|
520
|
+
let worst = null;
|
|
521
|
+
for (const [index, at] of this.#touched) {
|
|
522
|
+
if (index === except || this.#reading.has(index)) {
|
|
523
|
+
continue;
|
|
524
|
+
}
|
|
525
|
+
const behind = earliest !== null && index < earliest;
|
|
526
|
+
const score = { behind, distance: behind ? earliest - index : 0, at };
|
|
527
|
+
if (
|
|
528
|
+
worst === null
|
|
529
|
+
|| (score.behind && !worst.behind)
|
|
530
|
+
|| (score.behind === worst.behind && score.distance > worst.distance)
|
|
531
|
+
|| (score.behind === worst.behind && score.distance === worst.distance && score.at < worst.at)
|
|
532
|
+
) {
|
|
533
|
+
worst = score;
|
|
534
|
+
victim = index;
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
return victim;
|
|
538
|
+
}
|
|
539
|
+
}
|