@torrent-tv/proxy 2.86.0 → 2.87.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +2138 -1915
- package/CLAUDE.md +318 -197
- package/assets/calibration/NOTICE.md +2 -2
- package/bin/cli.js +684 -616
- package/biome.json +409 -233
- package/docs/container-architecture.md +218 -213
- package/docs/download-architecture.md +7 -7
- package/docs/encode-architecture.md +287 -11
- package/docs/encode-run-state.md +2 -2
- package/docs/logs.md +97 -97
- package/docs/{disk-architecture.md → storage-architecture.md} +107 -1
- package/openspec/changes/adaptive-bitrate/design.md +2 -2
- package/openspec/changes/adaptive-bitrate/proposal.md +1 -1
- package/openspec/changes/adaptive-bitrate/tasks.md +1 -1
- package/openspec/changes/chunked-request-bodies/design.md +1 -1
- package/openspec/changes/chunked-request-bodies/proposal.md +1 -1
- package/openspec/changes/chunked-request-bodies/tasks.md +1 -1
- package/openspec/changes/cold-start/design.md +1 -1
- package/openspec/changes/disk-cap/proposal.md +1 -1
- package/openspec/changes/proxy-observability/proposal.md +2 -2
- package/openspec/changes/track-selection/proposal.md +1 -1
- package/openspec/changes/transcode-quality/proposal.md +1 -1
- package/package.json +2 -2
- package/research/delivery-wedge-stand-ladder-2026-09-13.md +329 -329
- package/research/double-spawn-2026-09-10.md +171 -171
- package/routes/api/playback-plan/post.js +2 -2
- package/routes/api/sources/files/get.js +1 -1
- package/routes/api/sources/stats/get.js +1 -1
- package/routes/api/sources/warm/post.js +10 -3
- package/routes/api/subtitles/get.js +230 -230
- package/routes/api/transcode-sessions/fragment-far/post.js +3 -3
- package/routes/api/transcode-sessions/net-report/post.js +129 -94
- package/routes/api/transcode-sessions/post.js +160 -139
- package/routes/api/transcode-sessions/progress/get.js +3 -3
- package/routes/api/transcode-sessions/release/post.js +3 -3
- package/routes/api/transcode-sessions/seek/post.js +12 -3
- package/routes/stream/get.js +2 -2
- package/routes/transcode/audio-file/get.js +15 -6
- package/routes/transcode/audio-warm/get.js +8 -5
- package/routes/transcode/session-file/get.js +429 -268
- package/routes/transcode/variant-file/get.js +47 -7
- package/routes/transcode/variant-warm/get.js +18 -6
- package/scripts/render-run-graph.js +1 -1
- package/server.js +720 -465
- package/services/encode/CoverageMap.js +428 -428
- package/services/encode/CushionReport.js +345 -0
- package/services/encode/EncodeAdmission.js +332 -0
- package/services/{orchestrators → encode}/EncodeOrchestrator.js +285 -17
- package/services/encode/EncodeRun.js +16 -9
- package/services/encode/EncodeRuns.js +1219 -0
- package/services/encode/Encoder.js +18 -2
- package/services/encode/EncoderSelection.js +38 -0
- package/services/encode/LocalObservations.js +359 -0
- package/services/encode/NvencEncoder.js +5 -0
- package/services/encode/OutputOpening.js +745 -0
- package/services/encode/OutputTimes.js +579 -0
- package/services/encode/Renditions.js +2162 -0
- package/services/encode/RunProgress.js +84 -0
- package/services/encode/SoftwareEncoder.js +12 -7
- package/services/encode/args.js +402 -186
- package/services/encode/calibration.js +361 -0
- package/services/encode/contention.js +128 -128
- package/services/encode/encode-run-state.js +16 -20
- package/services/encode/fingerprint.js +322 -0
- package/services/{hwaccel.js → encode/hwaccel.js} +82 -42
- package/services/{learned-median.js → encode/learned-median.js} +11 -0
- package/services/encode/output/EncodedOutput.js +57 -0
- package/services/encode/output/Output.js +131 -0
- package/services/{output/LiveOutputs.js → encode/output/OutputCatalog.js} +141 -40
- package/services/encode/output/OutputSpec.js +390 -0
- package/services/{output → encode/output}/Timeline.js +12 -37
- package/services/encode/output/index.js +20 -0
- package/services/encode/output/ladder.js +32 -0
- package/services/{output → encode/output}/playlists.js +3 -0
- package/services/encode/output-key-format.js +21 -0
- package/services/encode/quality/EncodeCost.js +1083 -0
- package/services/encode/quality/HostLoad.js +525 -0
- package/services/encode/quality/HostTimings.js +249 -0
- package/services/encode/quality/OutputQualityState.js +28 -0
- package/services/encode/quality/QualityController.js +931 -0
- package/services/encode/quality/QualityOffer.js +416 -0
- package/services/encode/quality/drain-threat.js +47 -0
- package/services/encode/quality/link-budget.js +254 -0
- package/services/encode/quality/output-format.js +266 -0
- package/services/encode/quality/serving-output.js +216 -0
- package/services/encode/quality/visible-rung.js +57 -0
- package/services/encode/run-command.js +23 -20
- package/services/encode/run-inputs.js +42 -0
- package/services/{segment-formats → encode/segment-formats}/fmp4.js +63 -0
- package/services/{segment-formats → encode/segment-formats}/index.js +3 -3
- package/services/encode/segment-formats/init-compat.js +324 -0
- package/services/{segment-formats → encode/segment-formats}/mp4-boxes.js +8 -3
- package/services/encode/throughput.js +97 -0
- package/services/{orchestrators → media}/ContainerOrchestrator.js +45 -7
- package/services/media/KeyframeTables.js +316 -0
- package/services/{source → media}/SourceFile.js +7 -55
- package/services/{torrent-worker/subtitle-cues.js → media/SubtitleCues.js} +103 -132
- package/services/media/SubtitleOrchestrator.js +167 -0
- package/services/{audio-inventory.js → media/audio-inventory.js} +7 -1
- package/services/{container → media/container}/AviContainer.js +266 -266
- package/services/{container → media/container}/Container.js +596 -601
- package/services/media/container/ContainerFactory.js +56 -0
- package/services/media/container/KeyframeTable.js +272 -0
- package/services/{container → media/container}/MatroskaContainer.js +1485 -1485
- package/services/{container → media/container}/Mp4Container.js +1244 -1244
- package/services/{container → media/container}/SubtitleFileContainer.js +322 -322
- package/services/media/keyframe-probe.js +157 -0
- package/services/media/media-info-probe.js +93 -0
- package/services/{playback-planner.js → media/playback-planner.js} +82 -39
- package/services/{tracks → media/tracks}/AudioTrack.js +131 -131
- package/services/{tracks → media/tracks}/language-detect.js +174 -174
- package/services/server/OutputLifecycle.js +418 -0
- package/services/server/SegmentServing.js +1146 -0
- package/services/server/ViewerRequests.js +522 -0
- package/services/server/controllers/PlaybackController.js +36 -0
- package/services/{controllers → server/controllers}/SubtitleController.js +17 -33
- package/services/server/wire-outputs.js +958 -0
- package/services/storage/Diagnostics.js +202 -0
- package/services/storage/MachineBudget.js +235 -0
- package/services/{piece-store → storage}/allowance.js +107 -107
- package/services/{core-dumps.js → storage/core-dumps.js} +1 -1
- package/services/{files → storage/files}/CompletedFiles.js +116 -1
- package/services/{disk → storage}/free.js +33 -0
- package/services/storage/machine-memory.js +56 -0
- package/services/{memory-report.js → storage/memory-report.js} +575 -596
- package/services/{piece-store → storage/piece-store}/piece-disk-store.js +0 -7
- package/services/{piece-store → storage/piece-store}/shared-piece-store.js +28 -21
- package/services/{encode → storage/segment-store}/SegmentStore.js +395 -8
- package/services/storage/wire.js +175 -0
- package/services/{download → torrent/download}/SwarmSelection.js +2 -2
- package/services/{download → torrent/download}/registry.js +2 -2
- package/services/torrent/files.js +1 -1
- package/services/torrent/piece-store-of.js +29 -0
- package/services/{torrent-pool.js → torrent/torrent-pool.js} +26 -119
- package/services/{torrent-worker → torrent/worker}/background-fill.js +2 -2
- package/services/{torrent-worker → torrent/worker}/channel.js +271 -250
- package/services/{torrent-worker → torrent/worker}/client.js +77 -58
- package/services/torrent/worker/held-bytes.js +155 -0
- package/services/{torrent-worker → torrent/worker}/install-webrtc-shim.js +1 -1
- package/services/{torrent-worker → torrent/worker}/piece-reader.js +21 -30
- package/services/{torrent-worker → torrent/worker}/pool-adapter.js +101 -137
- package/services/{torrent-worker → torrent/worker}/protocol.js +26 -30
- package/services/torrent/worker/resume-warm.js +107 -0
- package/services/{torrent-worker → torrent/worker}/webrtc-shim.js +2 -2
- package/services/{torrent-worker → torrent/worker}/worker.js +68 -159
- package/services/{data-channel-handler.js → transport/data-channel-handler.js} +1356 -1356
- package/services/{delivery-probe.js → transport/delivery-probe.js} +842 -842
- package/services/{health-collector.js → transport/health-collector.js} +6 -35
- package/services/{packet-witness.js → transport/packet-witness.js} +808 -784
- package/services/{port-mapper.js → transport/port-mapper.js} +1 -1
- package/services/{usrsctp-state.js → transport/usrsctp-state.js} +1 -1
- package/services/{webrtc-manager.js → transport/webrtc-manager.js} +1 -1
- package/services/viewer/Assignments.js +340 -0
- package/services/{priority → viewer}/PriorityMap.js +390 -390
- package/services/{priority → viewer}/PriorityOrchestrator.js +325 -320
- package/services/viewer/Viewer.js +657 -483
- package/services/viewer/Viewers.js +368 -257
- package/services/viewer/active-output.js +37 -54
- package/services/viewer/audio-start.js +4 -3
- package/services/viewer/buffer-trend.js +81 -0
- package/services/viewer/choices.js +647 -0
- package/services/viewer/positions.js +126 -120
- package/services/viewer/report-intake.js +46 -45
- package/test/asset-paths.test.js +28 -0
- package/test/audio-inventory.test.js +3 -3
- package/test/audio-track-bitrate.test.js +40 -0
- package/test/auto-quality-step.test.js +707 -515
- package/test/available-share.test.js +1 -1
- package/test/background-fill.test.js +1 -1
- package/test/client-log-file.test.js +110 -110
- package/test/completed-files.test.js +1 -1
- package/test/concurrent-cost.test.js +13 -3
- package/test/container-media-info.test.js +3 -3
- package/test/core-dumps.test.js +1 -1
- package/test/coverage-follows-the-disk.test.js +40 -8
- package/test/cue-push-carries-what-the-walk-found.test.js +58 -0
- package/test/cut-times-timeline.test.js +2 -2
- package/test/cuts-follow-published-grid.test.js +9 -8
- package/test/data-channel-frame.test.js +1 -1
- package/test/dead-torrent-handle.test.js +1 -1
- package/test/declared-rates.test.js +2 -2
- package/test/declared-tracks.test.js +2 -2
- package/test/decode-cost-fit.test.js +1 -1
- package/test/decode-cost.test.js +34 -25
- package/test/delivery-probe.test.js +1 -1
- package/test/demand-register.test.js +4 -4
- package/test/dht-bootstrap.test.js +1 -1
- package/test/drain-threat.test.js +65 -0
- package/test/duplicate-add.test.js +1 -1
- package/test/encode-admission.test.js +146 -0
- package/test/encode-cost.test.js +199 -10
- package/test/encode-orchestrator.test.js +39 -1
- package/test/encode-plan-viewers.test.js +2 -2
- package/test/encode-run.test.js +2 -2
- package/test/encode-slope.test.js +1 -1
- package/test/encoded-output.test.js +39 -0
- package/test/encoder-readings.test.js +1 -1
- package/test/every-claimant-is-told-its-share.test.js +152 -0
- package/test/failed-encoder-still-serves-what-exists.test.js +354 -0
- package/test/failed-start-breaker.test.js +81 -30
- package/test/fastest-wires.test.js +1 -1
- package/test/ffmpeg-stream-counts.test.js +1 -1
- package/test/file-edges.test.js +3 -3
- package/test/fmp4-self-contained.test.js +1 -1
- package/test/fmp4-timestamps.test.js +1 -1
- package/test/fmp4-track-completeness.test.js +1 -1
- package/test/format-keyed-outputs.test.js +78 -0
- package/test/health-metrics.test.js +3 -2
- package/test/held-request-width.test.js +22 -23
- package/test/helpers/encode-run.js +136 -136
- package/test/helpers/held-file.js +29 -0
- package/test/helpers/manager.js +57 -0
- package/test/helpers/output-spec.js +63 -0
- package/test/host-load.test.js +1 -1
- package/test/index-accuracy-is-about-the-file.test.js +128 -0
- package/test/init-compat.test.js +219 -0
- package/test/input-lost-quiets-the-plan.test.js +1 -1
- package/test/input-unavailable.test.js +1 -1
- package/test/keeping-period.test.js +4 -4
- package/test/keyframe-index-accuracy.test.js +97 -45
- package/test/keyframe-table-budget.test.js +28 -33
- package/test/keyframes-belong-to-the-file.test.js +174 -57
- package/test/learned-median.test.js +1 -1
- package/test/letting-go-asks-who-is-watching.test.js +191 -0
- package/test/limit-row-by-area.test.js +60 -0
- package/test/link-budget.test.js +111 -0
- package/test/link-probe.test.js +56 -56
- package/test/local-encoding-data.test.js +273 -0
- package/test/look-ahead-coverage.test.js +1 -1
- package/test/manager-surface.test.js +294 -0
- package/test/matroska-blocks.test.js +0 -0
- package/test/matroska-cues-track.test.js +1 -1
- package/test/memory-budget.test.js +2 -2
- package/test/mp4-composition-times.test.js +0 -0
- package/test/mp4-subtitles.test.js +1 -1
- package/test/one-authority.test.js +45 -41
- package/test/one-budget.test.js +149 -0
- package/test/one-output-one-session.test.js +204 -0
- package/test/{disk-space.test.js → one-resource-divided.test.js} +30 -10
- package/test/one-serving-output-per-height.test.js +458 -0
- package/test/orchestrator-wired.test.js +46 -45
- package/test/output-catalog.test.js +39 -0
- package/test/output-shape.test.js +4 -51
- package/test/output-spec.test.js +251 -157
- package/test/output-speed.test.js +17 -15
- package/test/outputs-being-prepared.test.js +40 -0
- package/test/packet-witness-ring.test.js +2 -2
- package/test/packet-witness.test.js +1 -1
- package/test/piece-disk-store.test.js +2 -2
- package/test/piece-from-whole-file.test.js +1 -1
- package/test/piece-lru.test.js +1 -1
- package/test/piece-reader.test.js +2 -2
- package/test/piece-store-eviction.test.js +1 -1
- package/test/piece-store-never-refuses.test.js +1 -1
- package/test/piece-store-reservations.test.js +1 -1
- package/test/piece-store-slow-disk.test.js +1 -1
- package/test/piece-tail.test.js +1 -1
- package/test/plan-host-timings.test.js +1 -1
- package/test/plan-is-steady.test.js +225 -225
- package/test/priority-map-download.test.js +4 -4
- package/test/priority-map-emptied.test.js +204 -205
- package/test/priority-map-per-output.test.js +215 -209
- package/test/priority-map.test.js +1 -1
- package/test/probe-wedge-certainty.test.js +1 -1
- package/test/produced-copy-choice.test.js +15 -60
- package/test/quality-offer.test.js +162 -0
- package/test/quality-variants.test.js +1229 -1167
- package/test/rate-control-in-the-output.test.js +180 -0
- package/test/read-survives-withdrawal.test.js +2 -2
- package/test/read-window.test.js +10 -3
- package/test/resume-warm.test.js +1 -1
- package/test/run-inputs.test.js +80 -0
- package/test/run-position-follows-published-grid.test.js +13 -11
- package/test/run-progress.test.js +65 -0
- package/test/same-height-switch.test.js +664 -0
- package/test/seek-frees-the-viewers-requests.test.js +217 -0
- package/test/seek-landing.test.js +13 -13
- package/test/seek-target-not-superseded.test.js +28 -29
- package/test/segment-closed-on-both-branches.test.js +96 -0
- package/test/segment-cut-times.test.js +1 -1
- package/test/segment-hold-supersede.test.js +28 -35
- package/test/segment-judged-by-its-own-header.test.js +221 -0
- package/test/segment-serve-wiring.test.js +130 -39
- package/test/segment-store-eviction.test.js +22 -7
- package/test/segment-store.test.js +98 -14
- package/test/segment-true-start.test.js +1 -1
- package/test/segments-are-shared.test.js +17 -16
- package/test/send-chunk-bytes.test.js +1 -1
- package/test/serving-existing-output.test.js +102 -0
- package/test/serving-output.test.js +102 -0
- package/test/shared-piece-store.test.js +1 -1
- package/test/source-file.test.js +1 -1
- package/test/stale-request-after-seek.test.js +17 -21
- package/test/startup-readings.test.js +10 -5
- package/test/subtitle-cue-framing.test.js +5 -5
- package/test/subtitle-cue-source.test.js +36 -38
- package/test/subtitle-cue-walk.test.js +11 -10
- package/test/subtitle-defaults.test.js +1 -1
- package/test/subtitle-language.test.js +4 -4
- package/test/subtitle-track-numbering.test.js +5 -4
- package/test/supply-margin.test.js +1 -1
- package/test/swarm-follows-readers.test.js +3 -3
- package/test/swarm-reach.test.js +1 -1
- package/test/swarm-selection.test.js +3 -3
- package/test/tail-duplication.test.js +1 -1
- package/test/the-channel-has-both-ends.test.js +91 -0
- package/test/the-torrent-thread-serves-bytes.test.js +107 -0
- package/test/thread-pool.test.js +3 -3
- package/test/timeline.test.js +1 -12
- package/test/torrent-cost.test.js +1 -1
- package/test/tracks-begin-together.test.js +38 -24
- package/test/tunnel-renewal.test.js +1 -1
- package/test/two-viewers-one-picture.test.js +120 -87
- package/test/upload-hurry.test.js +3 -3
- package/test/usrsctp-state.test.js +1 -1
- package/test/video-facts.test.js +1 -1
- package/test/viewer-assignments.test.js +143 -0
- package/test/viewer-outputs.test.js +60 -52
- package/test/viewer-presence.test.js +22 -25
- package/test/viewer-requests-admission.test.js +89 -0
- package/test/viewer-states.test.js +189 -202
- package/test/viewer-subtitle-subscription.test.js +1 -1
- package/test/viewer.test.js +139 -136
- package/test/viewing-generations.test.js +390 -0
- package/test/visible-rung.test.js +37 -0
- package/test/wait-ledger.test.js +1 -1
- package/test/webrtc-shim.test.js +3 -3
- package/test/wedge-certainty.test.js +2 -2
- package/test/withdraw-piece-claim.test.js +2 -2
- package/test/worker-channel.test.js +1 -31
- package/utils/client-log-file.js +151 -151
- package/scripts/read-container-index.mjs +0 -108
- package/services/container/ContainerFactory.js +0 -122
- package/services/controllers/PlaybackController.js +0 -33
- package/services/disk/DiskSpace.js +0 -150
- package/services/disk/wire.js +0 -61
- package/services/hls-session-manager.js +0 -9608
- package/services/orchestrators/SubtitleOrchestrator.js +0 -160
- package/services/output/Output.js +0 -94
- package/services/output/OutputSpec.js +0 -202
- package/services/output/index.js +0 -1
- package/services/output/ladder.js +0 -26
- package/services/produced-index.js +0 -222
- package/services/quality/EncodeCost.js +0 -555
- package/services/torrent-worker/container-tracks.js +0 -414
- package/services/viewer/link-readings.js +0 -56
- package/test/produced-index.test.js +0 -142
- /package/{services → bin}/thread-pool.js +0 -0
- /package/services/{available-share.js → encode/available-share.js} +0 -0
- /package/services/{decode-cost-fit.js → encode/decode-cost-fit.js} +0 -0
- /package/services/{encoder-readings.js → encode/encoder-readings.js} +0 -0
- /package/services/{host-load.js → encode/host-load.js} +0 -0
- /package/services/{output → encode/output}/cut-grid.js +0 -0
- /package/services/{output → encode/output}/rates.js +0 -0
- /package/services/{segment-formats → encode/segment-formats}/mpegts.js +0 -0
- /package/services/{torrent-cost.js → encode/torrent-cost.js} +0 -0
- /package/services/{container → media/container}/ebml-reader.js +0 -0
- /package/services/{ffmpeg-banner.js → media/ffmpeg-banner.js} +0 -0
- /package/services/{tracks → media/tracks}/ContainerTrack.js +0 -0
- /package/services/{tracks → media/tracks}/ImageSubtitleTrack.js +0 -0
- /package/services/{tracks → media/tracks}/SubtitleTrack.js +0 -0
- /package/services/{tracks → media/tracks}/TextSubtitleTrack.js +0 -0
- /package/services/{tracks → media/tracks}/VideoTrack.js +0 -0
- /package/services/{tracks → media/tracks}/index.js +0 -0
- /package/services/{files → storage/files}/piece-from-whole-file.js +0 -0
- /package/services/{disk → storage}/keep.js +0 -0
- /package/services/{piece-store → storage/piece-store}/piece-lru.js +0 -0
- /package/services/{disk → storage}/returns.js +0 -0
- /package/services/{demand → torrent/demand}/DemandRegister.js +0 -0
- /package/services/{demand → torrent/demand}/Urgency.js +0 -0
- /package/services/{demand → torrent/demand}/Window.js +0 -0
- /package/services/{demand → torrent/demand}/index.js +0 -0
- /package/services/{demand → torrent/demand}/pieces.js +0 -0
- /package/services/{download → torrent/download}/withdraw-claim.js +0 -0
- /package/services/{supply-margin.js → torrent/supply-margin.js} +0 -0
- /package/services/{torrent-source-key.js → torrent/torrent-source-key.js} +0 -0
- /package/services/{torrent-worker → torrent/worker}/fastest-wires.js +0 -0
- /package/services/{torrent-worker → torrent/worker}/handle-state.js +0 -0
- /package/services/{nat-classifier.js → transport/nat-classifier.js} +0 -0
- /package/services/{registry-api.js → transport/registry-api.js} +0 -0
- /package/services/{tunnel-client.js → transport/tunnel-client.js} +0 -0
- /package/services/{priority → viewer}/WaitLedger.js +0 -0
|
@@ -0,0 +1,1219 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The encoders of this proxy, from the plan's decision to the end of the process.
|
|
3
|
+
*
|
|
4
|
+
* The plan (`EncodePlan`, through `EncodeOrchestrator`) decides where a run goes;
|
|
5
|
+
* this builds it with the arguments its output needs, follows what it produces,
|
|
6
|
+
* says how far it got, and accounts for how it ended. Nothing here decides where
|
|
7
|
+
* an encoder should be.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import path from "node:path";
|
|
11
|
+
import { spawn } from "node:child_process";
|
|
12
|
+
import { contentionPenalty } from "./contention.js";
|
|
13
|
+
import { ENCODE_RUN_STATE, liveRunsOf, runStateOf, wireState } from "./encode-run-state.js";
|
|
14
|
+
import { ENCODE_EXIT } from "./encode-exit.js";
|
|
15
|
+
import { EncodeRun } from "./EncodeRun.js";
|
|
16
|
+
import { computeOutputDimensions } from "./args.js";
|
|
17
|
+
import { buildRunCommand, trueStartOf } from "./run-command.js";
|
|
18
|
+
// How far the accounting of a backward restart looks for work about to be done
|
|
19
|
+
// twice. It runs on the restart path and a session an hour in has thousands of
|
|
20
|
+
// segments; the figure is for a comparison, not an inventory.
|
|
21
|
+
const BACKWARD_RESTART_SCAN_SEGMENTS = 300;
|
|
22
|
+
const PROGRESS_LOG_INTERVAL_MS = 5_000;
|
|
23
|
+
/**
|
|
24
|
+
* Format a seconds value as `HH:MM:SS`, or `"n/a"` if not finite.
|
|
25
|
+
*
|
|
26
|
+
* @param {number} seconds
|
|
27
|
+
* @returns {string}
|
|
28
|
+
*/
|
|
29
|
+
export function formatSeconds(seconds) {
|
|
30
|
+
if (!Number.isFinite(seconds) || seconds < 0) {
|
|
31
|
+
return "n/a";
|
|
32
|
+
}
|
|
33
|
+
const total = Math.floor(seconds);
|
|
34
|
+
const hours = Math.floor(total / 3600);
|
|
35
|
+
const minutes = Math.floor((total % 3600) / 60);
|
|
36
|
+
const rest = total % 60;
|
|
37
|
+
return `${String(hours).padStart(2, "0")}:${String(minutes).padStart(2, "0")}:${String(rest).padStart(2, "0")}`;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Where this session's encoding begins: the earliest number any live run of it
|
|
41
|
+
* was given.
|
|
42
|
+
*
|
|
43
|
+
* @param {object[] | Set<object>} runs
|
|
44
|
+
* @returns {number | null}
|
|
45
|
+
*/
|
|
46
|
+
export function earliestRunStart(runs) {
|
|
47
|
+
const live = liveRunsOf(runs);
|
|
48
|
+
return live.length > 0 ? live[0].from : null;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* The live run of this session that was given this number, if any.
|
|
52
|
+
*
|
|
53
|
+
* A run with an explicit stretch owns the whole of it. One WITHOUT an end owns
|
|
54
|
+
* only as far as it has actually got: claiming the rest of the film would make
|
|
55
|
+
* it the owner of every number in front of it, including ones another run was
|
|
56
|
+
* expressly given.
|
|
57
|
+
*
|
|
58
|
+
* @param {object[] | Set<object>} runs
|
|
59
|
+
* @param {number} index
|
|
60
|
+
* @returns {object | null}
|
|
61
|
+
*/
|
|
62
|
+
function ownRunMaking(runs, index) {
|
|
63
|
+
let answer = null;
|
|
64
|
+
for (const run of liveRunsOf(runs)) {
|
|
65
|
+
if (run.from > index) {
|
|
66
|
+
break; // Ordered by start, so nothing further can hold this number.
|
|
67
|
+
}
|
|
68
|
+
if (Number.isInteger(run.to) && run.to >= run.from && index > run.to) {
|
|
69
|
+
continue; // Its stretch ends before this number.
|
|
70
|
+
}
|
|
71
|
+
answer = run;
|
|
72
|
+
}
|
|
73
|
+
return answer;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Whether an encoder run died because its INPUT went away, rather than because
|
|
78
|
+
* of anything about the encode itself.
|
|
79
|
+
*
|
|
80
|
+
* These are the messages the read path and ffmpeg's HTTP client produce when
|
|
81
|
+
* the torrent is gone, being re-added, or has no data for the range yet — all
|
|
82
|
+
* of them temporary by nature: the source can be added again and the pieces
|
|
83
|
+
* fetched again.
|
|
84
|
+
*
|
|
85
|
+
* @param {string} message
|
|
86
|
+
* @returns {boolean}
|
|
87
|
+
*/
|
|
88
|
+
export function isInputUnavailable(message) {
|
|
89
|
+
const text = typeof message === "string" ? message : "";
|
|
90
|
+
return (
|
|
91
|
+
/Error reading HTTP response/i.test(text) ||
|
|
92
|
+
/not found in (?:magnet|torrent):/i.test(text) ||
|
|
93
|
+
/Unknown source/i.test(text) ||
|
|
94
|
+
/is gone and cannot be re-added/i.test(text) ||
|
|
95
|
+
/Read error at pos/i.test(text) ||
|
|
96
|
+
/Server returned 5\d\d/i.test(text) ||
|
|
97
|
+
/Input\/output error/i.test(text) ||
|
|
98
|
+
/Connection reset by peer/i.test(text) ||
|
|
99
|
+
/End of file/i.test(text)
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* The ffmpeg command as one readable line.
|
|
105
|
+
*
|
|
106
|
+
* Everything is shown as passed except the list of cut times, which is one
|
|
107
|
+
* value per segment — 830 of them on a two-hour film, about 7 KB of log for a
|
|
108
|
+
* single run, repeated on every restart. The count and the two ends say
|
|
109
|
+
* everything the list is ever consulted for: whether cutting was explicit at
|
|
110
|
+
* all, how far it reaches, and where it starts.
|
|
111
|
+
*
|
|
112
|
+
* @param {string[]} args
|
|
113
|
+
* @returns {string}
|
|
114
|
+
*/
|
|
115
|
+
export function describeFfmpegArgs(args) {
|
|
116
|
+
const parts = [];
|
|
117
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
118
|
+
const value = args[index];
|
|
119
|
+
// Both take the same list, and a keyframe-grid variant passes it twice.
|
|
120
|
+
// `-force_key_frames` also takes an expression, which is short and is left
|
|
121
|
+
// alone — only a list is folded.
|
|
122
|
+
if (
|
|
123
|
+
(value === "-segment_times" || value === "-force_key_frames") &&
|
|
124
|
+
typeof args[index + 1] === "string" &&
|
|
125
|
+
args[index + 1].includes(",")
|
|
126
|
+
) {
|
|
127
|
+
const times = args[index + 1].split(",");
|
|
128
|
+
parts.push(value, `<${times.length} cuts ${times[0]}..${times[times.length - 1]}>`);
|
|
129
|
+
index += 1;
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
parts.push(value);
|
|
133
|
+
}
|
|
134
|
+
return parts.join(" ");
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export class EncodeRuns {
|
|
138
|
+
#inputState = new WeakMap();
|
|
139
|
+
/** What this reads and asks of the rest of the proxy, and nothing else. @type {object} */
|
|
140
|
+
#host;
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Whether a re-decision of what encoders should exist is already queued for
|
|
144
|
+
* the end of this turn. See `planEncodersSoon`.
|
|
145
|
+
* @type {boolean}
|
|
146
|
+
*/
|
|
147
|
+
#planScheduled = false;
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* @param {object} host - `logger`, `viewerSecondsOn`, `noteRunStarting`, `inputOf`, `producedNumbers`, `servesAudioSeparately`, `disposeSession`, `contentionPenalties`, `encodeCost`, `encodeOrchestrator`, `encoders`, `ffmpegBin`, `outputTimes`, `outputs`, `priority`, `segmentDurationSec`, `segmentStore`, `videoEncoder`
|
|
151
|
+
*/
|
|
152
|
+
constructor(host) {
|
|
153
|
+
this.#host = host;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
#inputStateFor(output) {
|
|
157
|
+
let state = this.#inputState.get(output);
|
|
158
|
+
if (!state) {
|
|
159
|
+
state = {
|
|
160
|
+
readWindowBytes: 0,
|
|
161
|
+
inputRetryCount: 0,
|
|
162
|
+
backwardRestarts: { count: 0, segmentsBack: 0, worstBack: 0, remade: 0 },
|
|
163
|
+
firstWantedAt: new Map(),
|
|
164
|
+
/** Bytes the swarm has delivered to this output's own input reads. */
|
|
165
|
+
inputBytes: 0,
|
|
166
|
+
/** The mismatch between the held init and the run's size last said, so it is said once. */
|
|
167
|
+
initSizeSaid: ""
|
|
168
|
+
};
|
|
169
|
+
this.#inputState.set(output, state);
|
|
170
|
+
}
|
|
171
|
+
return state;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
setReadWindow(output, bytes) {
|
|
175
|
+
this.#inputStateFor(output).readWindowBytes = Number.isFinite(bytes) ? bytes : 0;
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
readWindowFor(output) {
|
|
179
|
+
return this.#inputStateFor(output).readWindowBytes;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
noteWanted(output, index, at = Date.now()) {
|
|
183
|
+
const wanted = this.#inputStateFor(output).firstWantedAt;
|
|
184
|
+
if (!wanted.has(index)) wanted.set(index, at);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
resetInputRetry(output) {
|
|
188
|
+
this.#inputStateFor(output).inputRetryCount = 0;
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
/**
|
|
192
|
+
* Count bytes the swarm has delivered to one output's own input read.
|
|
193
|
+
*
|
|
194
|
+
* Called by the `/stream` route for every fragment it writes to an encoder.
|
|
195
|
+
* Cheap on purpose — one addition, no clock, no log — because it runs per
|
|
196
|
+
* fragment on the path that feeds ffmpeg.
|
|
197
|
+
*
|
|
198
|
+
* @param {string} outputId
|
|
199
|
+
* @param {number} bytes
|
|
200
|
+
* @returns {void}
|
|
201
|
+
*/
|
|
202
|
+
noteInputBytes(outputId, bytes) {
|
|
203
|
+
if (!outputId || !(bytes > 0)) {
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
const output = this.#host.outputs.get(outputId);
|
|
207
|
+
if (output) {
|
|
208
|
+
this.#inputStateFor(output).inputBytes += bytes;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* @param {object} output
|
|
214
|
+
* @returns {number}
|
|
215
|
+
*/
|
|
216
|
+
inputBytesOf(output) {
|
|
217
|
+
return output ? this.#inputStateFor(output).inputBytes : 0;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
runsOf(output) {
|
|
221
|
+
return output ? this.#host.encodeOrchestrator.runsOn(output.outputKey) : [];
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
/**
|
|
225
|
+
* Where the earliest live encoder of this output began, or null with none.
|
|
226
|
+
*
|
|
227
|
+
* @param {object} output
|
|
228
|
+
* @returns {number | null}
|
|
229
|
+
*/
|
|
230
|
+
earliestStartOf(output) {
|
|
231
|
+
return earliestRunStart(this.runsOf(output));
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
isLive(output) {
|
|
235
|
+
return Boolean(output && this.#host.outputs.get(output.id) === output);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
liveRunsOf(output) {
|
|
239
|
+
return liveRunsOf(this.runsOf(output));
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
runStateOf(output) {
|
|
243
|
+
return output ? this.#host.encodeOrchestrator.stateOf(output.outputKey) : runStateOf([]);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
lastErrorOf(output) {
|
|
247
|
+
return output ? this.#host.encodeOrchestrator.errorOf(output.outputKey) : "";
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Whether this output's encoding has failed for good.
|
|
252
|
+
*
|
|
253
|
+
* @param {object} output
|
|
254
|
+
* @returns {boolean}
|
|
255
|
+
*/
|
|
256
|
+
hasFailed(output) {
|
|
257
|
+
return this.runStateOf(output) === ENCODE_RUN_STATE.ENDED_FAILED;
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Whether this output's input went away and is being fetched again.
|
|
262
|
+
*
|
|
263
|
+
* @param {object} output
|
|
264
|
+
* @returns {boolean}
|
|
265
|
+
*/
|
|
266
|
+
isWaitingForInput(output) {
|
|
267
|
+
return this.runStateOf(output) === ENCODE_RUN_STATE.RETRY_WAIT;
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/**
|
|
271
|
+
* The run state in the words the browser is told.
|
|
272
|
+
*
|
|
273
|
+
* @param {object} output
|
|
274
|
+
* @returns {string}
|
|
275
|
+
*/
|
|
276
|
+
wireStateOf(output) {
|
|
277
|
+
return wireState(this.runStateOf(output));
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* What went wrong, when this output's encoding has failed; empty otherwise.
|
|
282
|
+
*
|
|
283
|
+
* @param {object} output
|
|
284
|
+
* @returns {string}
|
|
285
|
+
*/
|
|
286
|
+
failureOf(output) {
|
|
287
|
+
return this.hasFailed(output) ? this.lastErrorOf(output) : "";
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
progressOf(output, index = null) {
|
|
291
|
+
if (!output) {
|
|
292
|
+
return null;
|
|
293
|
+
}
|
|
294
|
+
const at = Number.isInteger(index)
|
|
295
|
+
? index
|
|
296
|
+
: this.#host.outputTimes.segmentIndexForTime(output, this.#host.viewerSecondsOn(output));
|
|
297
|
+
const progress = this.#host.encodeOrchestrator.progressOf(output.outputKey, at);
|
|
298
|
+
if (progress) {
|
|
299
|
+
return progress;
|
|
300
|
+
}
|
|
301
|
+
const start = this.#host.viewerSecondsOn(output);
|
|
302
|
+
const total = Number(output.file?.durationSeconds);
|
|
303
|
+
return {
|
|
304
|
+
processedSeconds: start,
|
|
305
|
+
startPositionSeconds: start,
|
|
306
|
+
totalSeconds: Number.isFinite(total) && total > 0 ? total : null,
|
|
307
|
+
percent: 0,
|
|
308
|
+
remainingSeconds: Number.isFinite(total) && total > 0 ? Math.max(0, total - start) : null,
|
|
309
|
+
speed: "",
|
|
310
|
+
updatedAt: this.#host.outputs.startedAt(output) ?? Date.now()
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
planEncodersNow() {
|
|
315
|
+
const byOutput = this.#outputsWithSessions();
|
|
316
|
+
for (const [address, sessions] of byOutput) {
|
|
317
|
+
// From the TIMELINE, which is where how a file is cut has lived since
|
|
318
|
+
// 2.76.0. Read off the session it left, this was `undefined` on every
|
|
319
|
+
// session ever made: the map then held no length, and the walk that
|
|
320
|
+
// gives a run its end ran to MAX_SAFE_INTEGER — the main thread spun at
|
|
321
|
+
// 100% and the proxy answered nothing, measured on the addon host
|
|
322
|
+
// 2026-09-05 with the stack read out of the live process.
|
|
323
|
+
const segmentCount = Number(sessions[0].timeline?.segmentCount) || 0;
|
|
324
|
+
if (segmentCount > 0) {
|
|
325
|
+
this.#host.encodeOrchestrator.setSegmentCount(address, segmentCount);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
// THE MAP IS BUILT ONCE, IN ITS OWN LAYER, AND BOTH SIDES READ THAT ONE.
|
|
329
|
+
//
|
|
330
|
+
// It used to be built twice: once here, per viewer, converted and stated to
|
|
331
|
+
// the encoding as a window each, and once again inside `publishFor` for the
|
|
332
|
+
// downloading. Two answers to one question, and the encoding's copy carried
|
|
333
|
+
// the viewer's NAME as the key of a claim — against the rule that the
|
|
334
|
+
// encoding and the viewer are not connected at all.
|
|
335
|
+
this.#host.priority.publishFor({
|
|
336
|
+
sessionGroups: byOutput.values(),
|
|
337
|
+
});
|
|
338
|
+
// FROM THE VIEWERS OF THAT OUTPUT, not from the viewers of the film.
|
|
339
|
+
//
|
|
340
|
+
// Both scopes are right for what asks them: the swarm is asked for bytes of
|
|
341
|
+
// a FILE, which every output of it reads, and encoders are placed per
|
|
342
|
+
// OUTPUT, which a person watching 480p wants nothing of at 1080p. Handed the
|
|
343
|
+
// film's map, the plan wanted an encoder on every output of it, and what
|
|
344
|
+
// stopped the unwatched ones was this class killing them by its own
|
|
345
|
+
// judgement — while the viewer's own move announced itself and had the plan
|
|
346
|
+
// start them again.
|
|
347
|
+
//
|
|
348
|
+
// Converted into each output's own numbering, because two outputs of one
|
|
349
|
+
// film are cut independently and the same second is a different number in
|
|
350
|
+
// each: 454 pieces against 401 on the field file.
|
|
351
|
+
for (const [address, sessions] of byOutput) {
|
|
352
|
+
const timeline = sessions[0].timeline;
|
|
353
|
+
this.#host.encodeOrchestrator.notePriorityMap(
|
|
354
|
+
address,
|
|
355
|
+
timeline?.inSegments?.(
|
|
356
|
+
this.#host.priority.mapForOutput(address),
|
|
357
|
+
Number(timeline?.segmentCount) || 0
|
|
358
|
+
) ?? []
|
|
359
|
+
);
|
|
360
|
+
}
|
|
361
|
+
this.#host.encodeOrchestrator.reconcile();
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
/**
|
|
365
|
+
* Re-decide what encoders should exist, once, after the change that is being
|
|
366
|
+
* made now.
|
|
367
|
+
*
|
|
368
|
+
* COALESCED, NOT DELAYED. One turn of the event loop may carry several
|
|
369
|
+
* changes — a viewer arrives and is placed and states their soundtrack — and
|
|
370
|
+
* each of them is a reason to re-decide, while re-deciding three times in a
|
|
371
|
+
* row would give the same answer three times and could act on a half-built
|
|
372
|
+
* state. So the decision is taken once, after the current turn, and no
|
|
373
|
+
* interval is involved: there is nothing to choose and nothing to tune.
|
|
374
|
+
*
|
|
375
|
+
* @returns {void}
|
|
376
|
+
*/
|
|
377
|
+
planEncodersSoon() {
|
|
378
|
+
if (this.#planScheduled) {
|
|
379
|
+
return;
|
|
380
|
+
}
|
|
381
|
+
this.#planScheduled = true;
|
|
382
|
+
queueMicrotask(() => {
|
|
383
|
+
this.#planScheduled = false;
|
|
384
|
+
try {
|
|
385
|
+
this.planEncodersNow();
|
|
386
|
+
} catch (error) {
|
|
387
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
388
|
+
this.#host.logger.warn(`transcode could not re-plan encoders: ${message}`);
|
|
389
|
+
}
|
|
390
|
+
});
|
|
391
|
+
}
|
|
392
|
+
|
|
393
|
+
/**
|
|
394
|
+
* The live sessions grouped by the output they produce.
|
|
395
|
+
*
|
|
396
|
+
* @returns {Map<string, HlsSession[]>}
|
|
397
|
+
*/
|
|
398
|
+
#outputsWithSessions() {
|
|
399
|
+
/** @type {Map<string, HlsSession[]>} */
|
|
400
|
+
const byOutput = new Map();
|
|
401
|
+
for (const session of this.#host.outputs.values()) {
|
|
402
|
+
const address = session.outputKey ?? "";
|
|
403
|
+
if (!address) {
|
|
404
|
+
continue;
|
|
405
|
+
}
|
|
406
|
+
byOutput.set(address, [...(byOutput.get(address) ?? []), session]);
|
|
407
|
+
}
|
|
408
|
+
return byOutput;
|
|
409
|
+
}
|
|
410
|
+
|
|
411
|
+
forgetEncodingOfGone(output) {
|
|
412
|
+
if (this.#host.outputs.outputsOn(output.outputKey).length === 0) {
|
|
413
|
+
this.#host.encodeOrchestrator.forgetOutput(output.outputKey);
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/**
|
|
418
|
+
* Make an encoder for this output, beginning at this segment.
|
|
419
|
+
*
|
|
420
|
+
* Another run of the SAME session, not another session. A run at another
|
|
421
|
+
* position is not a different output — the material, the tracks, the grid and
|
|
422
|
+
* the box are all the same, and only where it begins differs, which changes
|
|
423
|
+
* no byte of what is produced. Both write into the output's one directory and
|
|
424
|
+
* either viewer is served whatever either has made.
|
|
425
|
+
*
|
|
426
|
+
* @param {string} address
|
|
427
|
+
* @param {number} from
|
|
428
|
+
* @param {number} to
|
|
429
|
+
* @param {string} because - The plan's own words for why it placed this one.
|
|
430
|
+
* @returns {object | null}
|
|
431
|
+
*/
|
|
432
|
+
makeRunAt(address, from, to, because) {
|
|
433
|
+
let base = null;
|
|
434
|
+
for (const session of this.#host.outputs.values()) {
|
|
435
|
+
if (session.outputKey === address) {
|
|
436
|
+
base = session;
|
|
437
|
+
break;
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
if (!base) {
|
|
441
|
+
return null;
|
|
442
|
+
}
|
|
443
|
+
// A position that has already failed to start, this many times running,
|
|
444
|
+
// will fail again: nothing about it has changed between the attempts, which
|
|
445
|
+
// is exactly why the attempts keep taking the same fraction of a second and
|
|
446
|
+
// ending the same way.
|
|
447
|
+
//
|
|
448
|
+
// The plan is arithmetic over coverage and demand, and neither of them
|
|
449
|
+
// knows that a process refused to start — so without asking here, the plan
|
|
450
|
+
// commands the same start, the run ends, the ended stretch goes back to the
|
|
451
|
+
// map, and the plan commands it again. Measured 2026-09-05 with ffmpeg
|
|
452
|
+
// absent: fifty passes of the plan in the time a probe took to notice, as
|
|
453
|
+
// fast as spawning could fail. The five-second timer this decision used to
|
|
454
|
+
// sit on hid that, at the price of hiding it in the field too — the loop
|
|
455
|
+
// seen there ran for sixteen minutes and read as "a restart every five
|
|
456
|
+
// seconds".
|
|
457
|
+
//
|
|
458
|
+
// A DIFFERENT position is unaffected and gets its own budget, and the count
|
|
459
|
+
// resets the moment a run at this one does real work.
|
|
460
|
+
if (!this.#host.encodeOrchestrator.mayStartAt(address, from)) {
|
|
461
|
+
return null;
|
|
462
|
+
}
|
|
463
|
+
// The encoder is built and handed back in this same call. Nothing here
|
|
464
|
+
// waits: the one thing this path used to wait for was the death of the run
|
|
465
|
+
// it was replacing, and that killing is gone. Answering with nothing while
|
|
466
|
+
// the encoder was built behind the answer is what let the same stretch be
|
|
467
|
+
// started over and over — 684 starts in 482 seconds of field 2026-09-05.
|
|
468
|
+
try {
|
|
469
|
+
return this.#startEncodeRun(base, from, because, { to });
|
|
470
|
+
} catch (error) {
|
|
471
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
472
|
+
this.#host.logger.warn(`transcode could not start a run at #${from} of ${address}: ${message}`);
|
|
473
|
+
return null;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
/**
|
|
478
|
+
* How many encoders this machine can afford on one output at once.
|
|
479
|
+
*
|
|
480
|
+
* Measured rather than chosen, and it is the same arithmetic that decides
|
|
481
|
+
* which quality steps are offered: what a second job costs here is not
|
|
482
|
+
* assumed to be nothing. Field 2026-09-03 on the addon host, `testsrc2`
|
|
483
|
+
* through libx264 `ultrafast` — at 854x480 one run makes 7.12x and two make
|
|
484
|
+
* 4.20x and 4.16x, so both stay far above realtime and a second is
|
|
485
|
+
* affordable; at 1920x1080 one makes 1.96x and two make 0.99x and 0.98x, so
|
|
486
|
+
* the machine is full at one.
|
|
487
|
+
*
|
|
488
|
+
* @param {string} address
|
|
489
|
+
* @returns {number}
|
|
490
|
+
*/
|
|
491
|
+
maxRunsForOutput(address) {
|
|
492
|
+
const alone = this.#host.encodeCost.speedForOutput(address);
|
|
493
|
+
if (!(alone > 0)) {
|
|
494
|
+
// The host measured nothing at all, which is a broken startup rather than
|
|
495
|
+
// a state to plan around. One encoder is what it keeps.
|
|
496
|
+
return 1;
|
|
497
|
+
}
|
|
498
|
+
let affordable = 1;
|
|
499
|
+
for (;;) {
|
|
500
|
+
const { penalty, measured, from } = contentionPenalty(affordable, this.#host.contentionPenalties);
|
|
501
|
+
// An UNMEASURED penalty is 1, and that is the honest answer to "what does
|
|
502
|
+
// a second job cost" only in the sense that nothing has been measured —
|
|
503
|
+
// it is not a statement that a second job is free. Measured on the addon
|
|
504
|
+
// host it is 1.70x at 854x480 and 1.98x at 1920x1080, so taking 1 would
|
|
505
|
+
// let a machine that cannot hold two runs start two. Where nothing has
|
|
506
|
+
// been measured, one is what it has.
|
|
507
|
+
//
|
|
508
|
+
// `from` is the concurrency the reading came from, and past the measured
|
|
509
|
+
// range `contentionPenalty` HOLDS the largest reading rather than continue
|
|
510
|
+
// a curve two points cannot describe. Held is the right answer to "what
|
|
511
|
+
// does this cost", and the wrong one to "may I start another": it would
|
|
512
|
+
// price a fifth encoder at what a second was measured to cost, and a fast
|
|
513
|
+
// host would keep dividing until some chosen ceiling stopped it. So the
|
|
514
|
+
// ladder stops where the measurements stop — the bound on how many
|
|
515
|
+
// encoders may run is measured, like the budget itself, and there is no
|
|
516
|
+
// constant here to overrule it.
|
|
517
|
+
if (!measured || from !== affordable || !(alone / penalty >= 1)) {
|
|
518
|
+
break;
|
|
519
|
+
}
|
|
520
|
+
affordable += 1;
|
|
521
|
+
}
|
|
522
|
+
return affordable;
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
/**
|
|
526
|
+
* A segment this run made has just been served.
|
|
527
|
+
*
|
|
528
|
+
* The one event whose raising is guarded by the state, and the guard is a
|
|
529
|
+
* READ of the state rather than a second copy of it: "something has been
|
|
530
|
+
* produced" is a level, not an edge, so without the guard every served
|
|
531
|
+
* segment would raise it and the log would fill with refusals.
|
|
532
|
+
*
|
|
533
|
+
* Whose file it is decides the rest, and the directory answers that outright:
|
|
534
|
+
* runs write into one each, and serving searches them newest-first, so a
|
|
535
|
+
* segment left by an EARLIER run is served routinely. Comparing indices
|
|
536
|
+
* instead would have been fooled by the ordinary case of a backward seek —
|
|
537
|
+
* the new run starts at #10, the old one left #50 on disk, and #50 is above
|
|
538
|
+
* the new run's start index while saying nothing about it.
|
|
539
|
+
*
|
|
540
|
+
* @param {HlsSession} session
|
|
541
|
+
* @param {string} filePath - Where the served segment was actually found.
|
|
542
|
+
* @returns {void}
|
|
543
|
+
*/
|
|
544
|
+
noteRunProducedSegment(session, filePath) {
|
|
545
|
+
if (typeof filePath !== "string" || path.dirname(filePath) !== this.#host.segmentStore.pathFor(session.outputKey ?? "")) {
|
|
546
|
+
return;
|
|
547
|
+
}
|
|
548
|
+
const index = session.segmentFormat.segmentIndexFromName(path.basename(filePath));
|
|
549
|
+
if (index === null) {
|
|
550
|
+
return;
|
|
551
|
+
}
|
|
552
|
+
// Whichever run was given this number. An output has as many runs as the
|
|
553
|
+
// machine affords, and a segment moves the one it belongs to out of
|
|
554
|
+
// starting — not whichever happens to be listed first.
|
|
555
|
+
ownRunMaking(this.runsOf(session), index)?.noteProduced(index);
|
|
556
|
+
}
|
|
557
|
+
|
|
558
|
+
/**
|
|
559
|
+
* Say so when a run is about to encode a picture the init segment already in
|
|
560
|
+
* the player's hands does not describe.
|
|
561
|
+
*
|
|
562
|
+
* This is the whole class of fault named in one line, and it exists because
|
|
563
|
+
* the fault is otherwise SILENT: no layer reports an error, the encoder is
|
|
564
|
+
* healthy, segments are served in milliseconds, and what the viewer gets is
|
|
565
|
+
* either a band of macroblock garbage or a picture that never appears. The
|
|
566
|
+
* shape is the one 2.48.0 uses for the TIME a run begins at — a run states
|
|
567
|
+
* where it really landed, and a disagreement with what was published is
|
|
568
|
+
* named rather than left to be inferred from a browser's own reading.
|
|
569
|
+
*
|
|
570
|
+
* The size the init describes is read from the init's own bytes, not taken
|
|
571
|
+
* from our record of what the encoder was told, because those two disagreeing
|
|
572
|
+
* IS the fault. The size the run will produce is computed by the same
|
|
573
|
+
* arithmetic the scale filter performs, so a source smaller than the target
|
|
574
|
+
* box is not reported as a disagreement.
|
|
575
|
+
*
|
|
576
|
+
* Said once per distinct pair of sizes: a run repeated at the same wrong size
|
|
577
|
+
* has nothing further to say.
|
|
578
|
+
*
|
|
579
|
+
* @param {HlsSession} session
|
|
580
|
+
* @returns {void}
|
|
581
|
+
*/
|
|
582
|
+
#warnIfRunLeavesTheInitBehind(session) {
|
|
583
|
+
const initBytes = this.#host.segmentStore.initOf(session.outputKey ?? "");
|
|
584
|
+
if (!session.spec.transcodesVideo || !initBytes || initBytes.length === 0) {
|
|
585
|
+
return; // nothing served yet, or nothing being encoded
|
|
586
|
+
}
|
|
587
|
+
const format = session.segmentFormat;
|
|
588
|
+
if (typeof format?.initVideoSize !== "function") {
|
|
589
|
+
return; // a self-describing container (MPEG-TS) cannot have this fault
|
|
590
|
+
}
|
|
591
|
+
const described = format.initVideoSize(initBytes);
|
|
592
|
+
if (!described) {
|
|
593
|
+
return;
|
|
594
|
+
}
|
|
595
|
+
if (!(session.output.encodeWidth > 0) || !(session.output.encodeHeight > 0)) {
|
|
596
|
+
return; // the run keeps the encoder's own default box; nothing was told
|
|
597
|
+
}
|
|
598
|
+
const producing = computeOutputDimensions(
|
|
599
|
+
session.output.encodeWidth,
|
|
600
|
+
session.output.encodeHeight,
|
|
601
|
+
session.file.width,
|
|
602
|
+
session.file.height
|
|
603
|
+
);
|
|
604
|
+
if (!producing) {
|
|
605
|
+
return; // the source size is unknown, so nothing can be predicted
|
|
606
|
+
}
|
|
607
|
+
if (described.width === producing.w && described.height === producing.h) {
|
|
608
|
+
return;
|
|
609
|
+
}
|
|
610
|
+
const said = `${described.width}x${described.height}->${producing.w}x${producing.h}`;
|
|
611
|
+
const inputState = this.#inputStateFor(session);
|
|
612
|
+
if (inputState.initSizeSaid === said) {
|
|
613
|
+
return;
|
|
614
|
+
}
|
|
615
|
+
inputState.initSizeSaid = said;
|
|
616
|
+
this.#host.logger.warn(
|
|
617
|
+
`transcode ${session.id} is about to encode ${producing.w}x${producing.h} while the init segment ` +
|
|
618
|
+
`the player holds describes ${described.width}x${described.height} "${session.file.name}" — ` +
|
|
619
|
+
`every fragment of this run will be decoded against parameter sets for a picture that is not ` +
|
|
620
|
+
`being made. A change of size is a change of variant; nothing here should have moved it.`
|
|
621
|
+
);
|
|
622
|
+
}
|
|
623
|
+
|
|
624
|
+
/**
|
|
625
|
+
* How far a run starting here may work before it meets somebody else's
|
|
626
|
+
* material — read off the ONE coverage map, never worked out again here.
|
|
627
|
+
*
|
|
628
|
+
* This replaced `planRunInterval`, which was a second authority over the same
|
|
629
|
+
* question and answered it by different rules: it walked the whole track for
|
|
630
|
+
* the first free number, MOVED the start there itself, and counted every live
|
|
631
|
+
* run as claiming up to `head + look-ahead`. Measured in the field
|
|
632
|
+
* 2026-09-05: the plan commanded a start at #46, this moved it to #78 — the
|
|
633
|
+
* "run moved forward from #46 to #78" line — and the plan, seeing a run
|
|
634
|
+
* outside the window it had asked for, killed it as unwanted and commanded
|
|
635
|
+
* the same start again, 350-700ms per cycle, dozens of times, the viewer's
|
|
636
|
+
* picture stopped for 125 seconds.
|
|
637
|
+
*
|
|
638
|
+
* Where a run STARTS is the plan's decision and arrives here as an argument.
|
|
639
|
+
* All this answers is where it must stop, and the answer is a fact of the
|
|
640
|
+
* map: the free stretch from that number on. `-1` means the end of the track,
|
|
641
|
+
* which is what "no end" is written as everywhere here.
|
|
642
|
+
*
|
|
643
|
+
* @param {object} session
|
|
644
|
+
* @param {number} startIndex
|
|
645
|
+
* @param {object | null} exceptRun - The run being replaced, whose own claim
|
|
646
|
+
* is not somebody else's material.
|
|
647
|
+
* @returns {number} The last number to work through, or `-1` for the end.
|
|
648
|
+
*/
|
|
649
|
+
#runEndFrom(session, startIndex, exceptRun = null) {
|
|
650
|
+
return this.#host.encodeOrchestrator.freeStretchEnd({
|
|
651
|
+
address: session.outputKey ?? "",
|
|
652
|
+
from: startIndex,
|
|
653
|
+
exceptRun,
|
|
654
|
+
segmentCount: Number(session.timeline?.segmentCount) || 0
|
|
655
|
+
});
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
// Returns the encoder it built, or nothing when there was nothing to build.
|
|
659
|
+
// It waits for nothing: the one thing it used to wait for was the death of
|
|
660
|
+
// the run it was replacing, and that killing is gone.
|
|
661
|
+
#startEncodeRun(session, startIndex, because, ordered = null) {
|
|
662
|
+
if (!this.isLive(session)) {
|
|
663
|
+
return null;
|
|
664
|
+
}
|
|
665
|
+
// A new run starts the quality budget's reckoning afresh: a pair of
|
|
666
|
+
// readings spanning the restart would count the gap between two runs as
|
|
667
|
+
// slow encoding. Told, not written: those readings are the budget's.
|
|
668
|
+
this.#host.noteRunStarting(session);
|
|
669
|
+
// Where a restart's seconds go. A seek costs 5-8 s in the field and the
|
|
670
|
+
// recorded reason — waiting for the previous ffmpeg to exit, measured at
|
|
671
|
+
// 0.54-1.47 s — does not account for it. Before rebuilding the hottest path
|
|
672
|
+
// in the proxy on a guess, make each stage state its own cost.
|
|
673
|
+
const restartEnteredAt = Date.now();
|
|
674
|
+
// Reads where the old run began BEFORE the new one takes its place, and
|
|
675
|
+
// does not await: everything below is the restart path, which is measured
|
|
676
|
+
// in milliseconds and has been worked on twice to keep it that way.
|
|
677
|
+
this.#accountBackwardRestart(session, startIndex);
|
|
678
|
+
// STARTING AN ENCODER STOPS NOTHING. It used to stop any live run whose own
|
|
679
|
+
// start was not below this one's — a rule left over from when a session
|
|
680
|
+
// held exactly one run and "the previous one" meant "the only one". Once a
|
|
681
|
+
// an output could hold several, that rule began killing runs the plan had
|
|
682
|
+
// decided to keep: 294 stops for this reason in eight minutes of field
|
|
683
|
+
// 2026-09-05, against four starts asked for by a viewer. Who is stopped is
|
|
684
|
+
// decided in one place, and it is not this one.
|
|
685
|
+
//
|
|
686
|
+
// HOW FAR IT MAY WORK is decided by the same one place and arrives here as
|
|
687
|
+
// an argument. Reading it off the coverage map a second time was the last
|
|
688
|
+
// remaining second answer to that question: the plan computed `to`, passed
|
|
689
|
+
// it, and the parameter list did not name it.
|
|
690
|
+
const runEnd = Number.isInteger(ordered?.to)
|
|
691
|
+
? ordered.to
|
|
692
|
+
: this.#runEndFrom(session, startIndex, null);
|
|
693
|
+
// The restart backs off a segment or two from what was asked for, so the
|
|
694
|
+
// request that prompted it is recorded under a HIGHER index than the run
|
|
695
|
+
// starts at. Looking it up by the start index alone found nothing and the
|
|
696
|
+
// line never printed once.
|
|
697
|
+
let wantedAt = null;
|
|
698
|
+
const inputState = this.#inputStateFor(session);
|
|
699
|
+
for (const [index, at] of inputState.firstWantedAt) {
|
|
700
|
+
if (index >= startIndex && (wantedAt === null || at < wantedAt)) {
|
|
701
|
+
wantedAt = at;
|
|
702
|
+
}
|
|
703
|
+
}
|
|
704
|
+
if (typeof wantedAt === "number") {
|
|
705
|
+
this.#host.logger.info(
|
|
706
|
+
`transcode ${session.id} restart for #${startIndex} decided ` +
|
|
707
|
+
`${restartEnteredAt - wantedAt}ms after it was first asked for`
|
|
708
|
+
);
|
|
709
|
+
}
|
|
710
|
+
// Cleared with the run that could not answer them. These are "how long has
|
|
711
|
+
// this segment gone unanswered", and the question only means anything about
|
|
712
|
+
// the run in force: a timestamp kept from an abandoned scan probe minutes
|
|
713
|
+
// ago says a fresh request has already waited long enough, which is how the
|
|
714
|
+
// behind-head repair came to fire on the very first poll instead of waiting
|
|
715
|
+
// for the seek that should move the encoder. It also stops the map growing
|
|
716
|
+
// for the life of a session.
|
|
717
|
+
inputState.firstWantedAt.clear();
|
|
718
|
+
// WHERE THIS NUMBER REALLY BEGINS, when a produced piece has said so and
|
|
719
|
+
// the table the player holds still says otherwise.
|
|
720
|
+
//
|
|
721
|
+
// Only a soundtrack, and only because a soundtrack has no keyframes: it
|
|
722
|
+
// begins exactly where it is asked to, to within one audio frame, while the
|
|
723
|
+
// picture can begin nowhere but a real keyframe and so moves forward to the
|
|
724
|
+
// next one — up to three seconds, measured 2026-08-17. Left at the published
|
|
725
|
+
// time, the sound of one film starts up to that far from its picture after
|
|
726
|
+
// every restart, each correctly labelled with where it really is, and the
|
|
727
|
+
// viewer gets sound with no new picture for the difference.
|
|
728
|
+
//
|
|
729
|
+
// DERIVED HERE rather than handed in. It used to arrive as an argument from
|
|
730
|
+
// the one caller that knew it — the correction that had just measured it —
|
|
731
|
+
// which meant the instant was only ever right for a run that caller
|
|
732
|
+
// started. Any run the plan placed at a corrected number got the published
|
|
733
|
+
// time and landed apart again. It is a fact of the FILE's cutting, held in
|
|
734
|
+
// the live table every session of the file shares, so it is read from
|
|
735
|
+
// there.
|
|
736
|
+
const positionSecondsOverride = session.spec.carries === "audio-only"
|
|
737
|
+
? trueStartOf(session.timeline, startIndex)
|
|
738
|
+
: undefined;
|
|
739
|
+
|
|
740
|
+
// What is still read off the session for a run. The list is the measure of
|
|
741
|
+
// how far a session still is from being the three things a run is built
|
|
742
|
+
// from — the material, what is produced of it, and the stretch — and it
|
|
743
|
+
// shrinks in place as those are taken off. There is deliberately no method
|
|
744
|
+
// wrapping it: a named adapter with one caller is a thing to remember to
|
|
745
|
+
// delete, and this is a thing that disappears by being emptied.
|
|
746
|
+
const inputs = this.#host.inputOf(session);
|
|
747
|
+
const { args, safeIndex, startSeconds, cutTimes } = buildRunCommand({
|
|
748
|
+
keyframes: session.keyframes,
|
|
749
|
+
inputFile: inputs.inputFile,
|
|
750
|
+
audioFile: inputs.audioFile,
|
|
751
|
+
inputUrl: inputs.inputUrl,
|
|
752
|
+
audioInputUrl: inputs.audioInputUrl,
|
|
753
|
+
timeline: session.timeline,
|
|
754
|
+
output: session.output,
|
|
755
|
+
segmentFormat: session.segmentFormat,
|
|
756
|
+
transcodeVideo: session.spec.transcodesVideo,
|
|
757
|
+
transcodeAudio: session.spec.transcodesAudio,
|
|
758
|
+
audioOnly: session.spec.carries === "audio-only",
|
|
759
|
+
audioSeparate: this.#host.servesAudioSeparately(session),
|
|
760
|
+
audioSourceTrackIndex: session.spec.audioSourceTrackIndex,
|
|
761
|
+
rateControl: session.spec.video?.encode?.rateControl ?? null,
|
|
762
|
+
startIndex,
|
|
763
|
+
endIndex: runEnd,
|
|
764
|
+
positionSecondsOverride,
|
|
765
|
+
videoEncoder: this.#host.videoEncoder,
|
|
766
|
+
segmentDurationSec: this.#host.segmentDurationSec
|
|
767
|
+
});
|
|
768
|
+
|
|
769
|
+
// The exact command, every run. An encode failure is otherwise reported
|
|
770
|
+
// with ffmpeg's message and nothing about what it was asked to do, and the
|
|
771
|
+
// two are not always deducible from each other: 2026-08-04 a run died with
|
|
772
|
+
// "Cannot write moov atom before AC3 packets" although both muxing paths
|
|
773
|
+
// were verified to handle a copied AC-3 track on this very host, so the
|
|
774
|
+
// arguments that run actually received are the missing evidence. One line
|
|
775
|
+
// per run, and a run happens at most every few seconds.
|
|
776
|
+
// Numbered, because a burst of seeks starts several runs in one second and
|
|
777
|
+
// every line about them carries the SESSION id, which is the same for all.
|
|
778
|
+
// Without a run number the command that failed cannot be told from the two
|
|
779
|
+
// that succeeded around it — which is exactly the state the unexplained
|
|
780
|
+
// `Cannot write moov atom before AC3 packets` was found in.
|
|
781
|
+
|
|
782
|
+
// One directory for the output, and every run writes straight into it.
|
|
783
|
+
//
|
|
784
|
+
// Runs used to be kept apart by a directory each, because two of them
|
|
785
|
+
// writing one segment name at the same time produce a file belonging to
|
|
786
|
+
// neither — which was also the only reason a restart ever had to wait for
|
|
787
|
+
// its predecessor to die. Intervals remove that by construction: a run is
|
|
788
|
+
// given a stretch nobody else holds and stops at the end of it, so no two
|
|
789
|
+
// runs ever want the same number. What is left of the old reason — a run
|
|
790
|
+
// killed mid-piece leaving a partial file — is not a correctness problem
|
|
791
|
+
// either, because the store serves a segment only once its closure is
|
|
792
|
+
// proven, and it is cleared up when the run ends.
|
|
793
|
+
const run = new EncodeRun({
|
|
794
|
+
address: session.outputKey ?? session.id,
|
|
795
|
+
encoder: this.#host.videoEncoder,
|
|
796
|
+
from: safeIndex,
|
|
797
|
+
to: runEnd,
|
|
798
|
+
buildArgs: () => args,
|
|
799
|
+
argsDescribed: describeFfmpegArgs(args),
|
|
800
|
+
// Whether this run cuts at times we gave it. Decides how a segment is
|
|
801
|
+
// judged finished — see getFileStream.
|
|
802
|
+
usesExplicitCuts: Boolean(cutTimes && cutTimes.length > 0),
|
|
803
|
+
startSeconds,
|
|
804
|
+
totalSeconds: Number(session.file.durationSeconds) || null,
|
|
805
|
+
spawn: (spawnArgs) =>
|
|
806
|
+
spawn(this.#host.ffmpegBin, spawnArgs, {
|
|
807
|
+
cwd: this.#host.segmentStore.directoryFor(session.outputKey ?? ""),
|
|
808
|
+
// A fourth channel: the encoder names every piece it has CLOSED on it,
|
|
809
|
+
// which is the only proof a piece is whole.
|
|
810
|
+
stdio: ["ignore", "pipe", "pipe", "pipe"]
|
|
811
|
+
}),
|
|
812
|
+
logger: this.#host.logger,
|
|
813
|
+
// The film's last segment number, which is what tells "it reached the
|
|
814
|
+
// end" from "its input dried up": ffmpeg exits zero for both and over a
|
|
815
|
+
// torrent cannot tell them apart.
|
|
816
|
+
lastSegmentIndex: () =>
|
|
817
|
+
session.timeline?.segmentCount > 0 ? session.timeline.segmentCount - 1 : null,
|
|
818
|
+
inputUnavailable: (message) => isInputUnavailable(message),
|
|
819
|
+
onProgress: (report) => this.#noteRunProgress(session, run, report),
|
|
820
|
+
indexOfName: (name) => session.segmentFormat.segmentIndexFromName(name),
|
|
821
|
+
// Why this encoder exists, recorded with its argument list. It used to be
|
|
822
|
+
// handed to a separate `start` call; there is no separate call now.
|
|
823
|
+
because,
|
|
824
|
+
onClosed: (name) => this.#host.segmentStore.publish(session.outputKey ?? "", name, session.segmentFormat),
|
|
825
|
+
onEnded: (ended) => this.noteRunEnded(session, run, ended)
|
|
826
|
+
});
|
|
827
|
+
// THE ONE FAULT THAT IS OTHERWISE SILENT, asked before this run produces a
|
|
828
|
+
// frame. It lost its caller in a refactor on 2026-09-04 and had none until
|
|
829
|
+
// 2026-09-15 — not by a decision, which is why it is back rather than gone.
|
|
830
|
+
this.#warnIfRunLeavesTheInitBehind(session);
|
|
831
|
+
|
|
832
|
+
this.#host.logger.info(
|
|
833
|
+
`transcode ${session.id} encode-run #${safeIndex}..#${runEnd} from segment #${safeIndex} ` +
|
|
834
|
+
`(+${Date.now() - restartEnteredAt}ms since the restart was asked for) ` +
|
|
835
|
+
`(${formatSeconds(startSeconds)}) "${session.file.name}"`
|
|
836
|
+
);
|
|
837
|
+
// The four numbers a run is positioned by, said once, because their
|
|
838
|
+
// disagreement is invisible everywhere else. The two tables are printed
|
|
839
|
+
// side by side: while they differ, every cut of this run is off by the
|
|
840
|
+
// difference, and nothing downstream can tell that from a bad index.
|
|
841
|
+
const liveStart = this.#host.outputTimes.segmentStartTime(session, safeIndex);
|
|
842
|
+
this.#host.logger.info(
|
|
843
|
+
`transcode ${session.id} run #${safeIndex}..#${runEnd} positioned at ${startSeconds.toFixed(3)}s ` +
|
|
844
|
+
`for boundary #${safeIndex} (published ${startSeconds.toFixed(3)}s, ` +
|
|
845
|
+
`live ${liveStart.toFixed(3)}s, apart ${(liveStart - startSeconds).toFixed(3)}s), ` +
|
|
846
|
+
`numbering from #${safeIndex}`
|
|
847
|
+
);
|
|
848
|
+
return run;
|
|
849
|
+
}
|
|
850
|
+
|
|
851
|
+
/**
|
|
852
|
+
* Log a report from a run that still belongs to this output. The run owns the
|
|
853
|
+
* clock and has already placed its relative ffmpeg time on the source
|
|
854
|
+
* timeline.
|
|
855
|
+
*
|
|
856
|
+
* @param {HlsSession} session
|
|
857
|
+
* @param {import("./EncodeRun.js").EncodeRun} run
|
|
858
|
+
* @param {object} progress
|
|
859
|
+
* @returns {void}
|
|
860
|
+
*/
|
|
861
|
+
#noteRunProgress(session, run, progress) {
|
|
862
|
+
if (!this.isLive(session) || !this.runsOf(session).includes(run)) {
|
|
863
|
+
return;
|
|
864
|
+
}
|
|
865
|
+
if (run.progress.shouldLog(PROGRESS_LOG_INTERVAL_MS)) {
|
|
866
|
+
this.#host.logger.info(
|
|
867
|
+
`transcode ${session.id} "${session.file.name}" ${progress.percent.toFixed(1)}% ` +
|
|
868
|
+
`(${formatSeconds(progress.processedSeconds)} / ${formatSeconds(progress.totalSeconds)})` +
|
|
869
|
+
` speed=${progress.speed || "n/a"}`
|
|
870
|
+
);
|
|
871
|
+
}
|
|
872
|
+
}
|
|
873
|
+
|
|
874
|
+
/**
|
|
875
|
+
* What to do about a run that has ended.
|
|
876
|
+
*
|
|
877
|
+
* The run says how it ended and why; this decides what follows, which is a
|
|
878
|
+
* question about the SESSION and the machine rather than about the process:
|
|
879
|
+
* whether to condemn a hardware encoder, whether to wait for the input to
|
|
880
|
+
* come back, whether a position has failed often enough to stop retrying it.
|
|
881
|
+
*
|
|
882
|
+
* An ending from a run the session has already replaced is not ignored — it
|
|
883
|
+
* is simply not about this session's current run, so nothing that describes
|
|
884
|
+
* the current run is written from it. That is what the identity check used to
|
|
885
|
+
* be for, and it is one condition now instead of one at the head of every
|
|
886
|
+
* handler.
|
|
887
|
+
*
|
|
888
|
+
* @param {HlsSession} session
|
|
889
|
+
* @param {import("./EncodeRun.js").EncodeRun} run
|
|
890
|
+
* @param {import("./EncodeRun.js").RunEnded} ended
|
|
891
|
+
* @returns {void}
|
|
892
|
+
*/
|
|
893
|
+
noteRunEnded(session, run, ended) {
|
|
894
|
+
// First, and for every run whatever else follows: the stretch it held goes
|
|
895
|
+
// back to the map. A run whose claim is never released tells the plan that
|
|
896
|
+
// numbers nobody is making are being made, and nothing is ever started
|
|
897
|
+
// there again.
|
|
898
|
+
// WAS this run still the session's when it ended? Asked before it is
|
|
899
|
+
// removed, because after the removal the question always answers "no" —
|
|
900
|
+
// and it was asked after, so every line below this point was unreachable.
|
|
901
|
+
//
|
|
902
|
+
// Measured 2026-09-05 over both of the field host's log files: zero
|
|
903
|
+
// occurrences of this handler's own `encode-run #… failed:` line and zero
|
|
904
|
+
// of `fast failure at segment`, across every session this proxy has ever
|
|
905
|
+
// run. So the fallback from a failed hardware encoder to software, the
|
|
906
|
+
// retry when the torrent data goes away, the limit on retrying a position
|
|
907
|
+
// that keeps failing, and the error line naming the ffmpeg command have all
|
|
908
|
+
// been dead code — which is also why nothing ever stopped the restart loop
|
|
909
|
+
// recorded in the field the same day.
|
|
910
|
+
const wasCurrent = this.runsOf(session).includes(run);
|
|
911
|
+
this.#host.encodeOrchestrator.noteEnded(ended);
|
|
912
|
+
// What this admitted encode was seen to do, kept for the next time this
|
|
913
|
+
// host prices the same mode (roadmap item 97, step 14). Whatever the
|
|
914
|
+
// ending: a run that failed still ran at the speed it ran at, and the
|
|
915
|
+
// segments it closed still carried what they carried.
|
|
916
|
+
this.#host.observeEncodeEnded?.(session);
|
|
917
|
+
// A stretch went back to the map, so what should be running has changed.
|
|
918
|
+
// Said here rather than waited for: this is the moment it became true.
|
|
919
|
+
this.planEncodersSoon();
|
|
920
|
+
// ANYBODY ALREADY WAITING IS TOLD, and it is asked HERE because every
|
|
921
|
+
// ending passes this line while the handlers below return one by one. It
|
|
922
|
+
// used to be the last statement of this method, which the `SHORT` branch —
|
|
923
|
+
// ffmpeg exiting 0 having produced less than the playlist promises —
|
|
924
|
+
// returns before reaching. That ending is terminal (`ENDED_FAILED`), so a
|
|
925
|
+
// request arriving after it is answered with the failure at once while a
|
|
926
|
+
// wait that BEGAN before it sat out its whole deadline, or, where the page
|
|
927
|
+
// states none, until the viewer disconnected.
|
|
928
|
+
//
|
|
929
|
+
// THE CONDITION IS THE OUTPUT'S ACTUAL STATE, not this run's ending.
|
|
930
|
+
// `stateOf` answers from the LIVE runs and falls back to the last ending
|
|
931
|
+
// only when none remain, so one run of several ending in failure wakes
|
|
932
|
+
// nobody — something is still making it — and the last one does. Asked
|
|
933
|
+
// after `noteEnded` has recorded the ending, so the woken request reads the
|
|
934
|
+
// final state; and it re-reads the disk first, because a piece already made
|
|
935
|
+
// is served whatever became of the encoder.
|
|
936
|
+
if (this.hasFailed(session)) {
|
|
937
|
+
this.#host.invalidateWaits(session);
|
|
938
|
+
// A move to this output being prepared for a viewer waits for a segment
|
|
939
|
+
// that nothing will now make, and no publication will ever end that wait.
|
|
940
|
+
// This is the event that does (roadmap item 97, step 12).
|
|
941
|
+
this.#host.productionFailed(session);
|
|
942
|
+
}
|
|
943
|
+
if (!this.isLive(session)) {
|
|
944
|
+
this.forgetEncodingOfGone(session);
|
|
945
|
+
return;
|
|
946
|
+
}
|
|
947
|
+
if (!wasCurrent) {
|
|
948
|
+
// A run the session had already replaced. It has logged its own ending;
|
|
949
|
+
// nothing about the session follows from it.
|
|
950
|
+
return;
|
|
951
|
+
}
|
|
952
|
+
const lastError = this.lastErrorOf(session) || ended.because;
|
|
953
|
+
if (ended.ending === ENCODE_EXIT.STOPPED || ended.ending === ENCODE_EXIT.GONE) {
|
|
954
|
+
return;
|
|
955
|
+
}
|
|
956
|
+
if (ended.ending === ENCODE_EXIT.COMPLETE) {
|
|
957
|
+
this.#host.logger.info(`transcode ${session.id} encode-run complete "${session.file.name}"`);
|
|
958
|
+
return;
|
|
959
|
+
}
|
|
960
|
+
if (ended.ending === ENCODE_EXIT.SHORT) {
|
|
961
|
+
// ffmpeg exits 0 both when it reaches the end of the file and when its
|
|
962
|
+
// input simply stops producing bytes — over HTTP the two look identical
|
|
963
|
+
// to it. Field 2026-08-05: the torrent's download died, the read ended,
|
|
964
|
+
// and a run that had made 188 segments of 624 reported itself complete;
|
|
965
|
+
// the player then consumed what was on disk and froze on the first
|
|
966
|
+
// segment nobody was making. So the claim is checked against the playlist
|
|
967
|
+
// we published, and a run that stopped short is a FAILURE that can be
|
|
968
|
+
// restarted, not a finished file.
|
|
969
|
+
this.#host.logger.error(
|
|
970
|
+
`transcode ${session.id} encode-run #${ended.from}..#${ended.to} ended early: ` +
|
|
971
|
+
`${ended.because} "${session.file.name}"`
|
|
972
|
+
);
|
|
973
|
+
return;
|
|
974
|
+
}
|
|
975
|
+
// Runtime safety net: if a hardware encode fails, downgrade this proxy to
|
|
976
|
+
// software encoding for all sessions and restart this one, so playback is
|
|
977
|
+
// never permanently broken by a hardware/driver issue.
|
|
978
|
+
//
|
|
979
|
+
// Asked only of a genuine encoder failure. It used to be asked of every
|
|
980
|
+
// non-zero exit, so a run whose TORRENT DATA went away — which says nothing
|
|
981
|
+
// whatever about the encoder — condemned a working NVENC or QuickSync to
|
|
982
|
+
// software for the life of the process, and started an extra run at the old
|
|
983
|
+
// index while it was at it.
|
|
984
|
+
if (ended.ending === ENCODE_EXIT.FAILED && session.spec.transcodesVideo && this.#host.videoEncoder.kind !== "software") {
|
|
985
|
+
const failed = this.#host.encoders.useSoftware();
|
|
986
|
+
const failedEncoder = failed?.name ?? this.#host.videoEncoder.name;
|
|
987
|
+
this.#host.logger.warn(
|
|
988
|
+
`transcode ${session.id} hardware encoder ${failedEncoder} failed ` +
|
|
989
|
+
`(${lastError}); falling back to software libx264, and every output named by ${failedEncoder} is closed`
|
|
990
|
+
);
|
|
991
|
+
// An output IS its format, and the encoder is part of it: pieces from
|
|
992
|
+
// another encoder cannot be decoded with the header this output's viewers
|
|
993
|
+
// already hold. So the outputs made by the failed encoder end here, and
|
|
994
|
+
// the next request for that picture names a format built on the new one.
|
|
995
|
+
//
|
|
996
|
+
// THE ONE DISPOSAL THAT DOES NOT ASK WHETHER ASSIGNMENTS STILL HOLD IT, and
|
|
997
|
+
// deliberately. Every other one asks, because a repeat of a request must be
|
|
998
|
+
// answered by what answered it first. Here what answered it first has no
|
|
999
|
+
// encoder that can go on making it: holding the output keeps only a name
|
|
1000
|
+
// whose next segment nobody will produce. A response already being sent is
|
|
1001
|
+
// not cut short by this — it reads from a file already opened.
|
|
1002
|
+
for (const other of [...this.#host.outputs.values()]) {
|
|
1003
|
+
if (other.spec.video?.encode?.encoder === failedEncoder) {
|
|
1004
|
+
void this.#host.disposeSession(other.id);
|
|
1005
|
+
}
|
|
1006
|
+
}
|
|
1007
|
+
this.planEncodersSoon();
|
|
1008
|
+
return;
|
|
1009
|
+
}
|
|
1010
|
+
// Losing the INPUT is not the session failing — it is the data not being
|
|
1011
|
+
// there YET. The torrent can be re-added and re-downloaded, so the honest
|
|
1012
|
+
// answer to the viewer is "still working", not an error screen. Field
|
|
1013
|
+
// 2026-08-06: a torrent evicted mid-seek took the film with it, the run
|
|
1014
|
+
// died on `File 0 not found`, the session went terminal and answered 500 to
|
|
1015
|
+
// every request from then on — although the swarm was there and the data
|
|
1016
|
+
// would have come back in seconds. The circuit breaker below stays for what
|
|
1017
|
+
// it was built for, a target that genuinely cannot be encoded; it must not
|
|
1018
|
+
// condemn a session whose data merely went away.
|
|
1019
|
+
if (ended.ending === ENCODE_EXIT.INPUT_LOST) {
|
|
1020
|
+
const inputState = this.#inputStateFor(session);
|
|
1021
|
+
inputState.inputRetryCount += 1;
|
|
1022
|
+
this.#host.logger.warn(
|
|
1023
|
+
`transcode ${session.id} encode-run #${ended.from}..#${ended.to} lost its input ` +
|
|
1024
|
+
`(${lastError}) (attempt ${inputState.inputRetryCount})`
|
|
1025
|
+
);
|
|
1026
|
+
// HOW LONG TO WAIT IS THE PLAN'S, and this says only what happened. The
|
|
1027
|
+
// delay used to be timed here, against the dead run, which the plan never
|
|
1028
|
+
// consults — so it placed a fresh run at the same spot as fast as ffmpeg
|
|
1029
|
+
// could fail there: 2432 starts in 23 minutes in the field 2026-09-12,
|
|
1030
|
+
// against a delay that had reached its 15 s ceiling long before. The
|
|
1031
|
+
// orchestrator holds it now, beside the decision it governs.
|
|
1032
|
+
if (run.state === ENCODE_RUN_STATE.RETRY_WAIT) {
|
|
1033
|
+
run.retryDue();
|
|
1034
|
+
}
|
|
1035
|
+
return;
|
|
1036
|
+
}
|
|
1037
|
+
// A run that exits THIS fast never did real work: it failed at the start
|
|
1038
|
+
// itself — opening the input, spawning the process — rather than
|
|
1039
|
+
// mid-stream. Consecutive fast failures at the SAME position are counted so
|
|
1040
|
+
// that whoever commands a start can stop commanding one that keeps failing.
|
|
1041
|
+
//
|
|
1042
|
+
// COUNTED AT EVERY POSITION, #0 included. It used to be counted only past
|
|
1043
|
+
// #0, because it was written for seek restarts and a seek is never to the
|
|
1044
|
+
// beginning. That left the one position the plan commands FIRST with no
|
|
1045
|
+
// count at all, and the plan re-commanding a start that cannot succeed is
|
|
1046
|
+
// an unbounded loop: measured 2026-09-05, ffmpeg failing to spawn produced
|
|
1047
|
+
// fifty passes of the plan before a probe stopped it, as fast as the
|
|
1048
|
+
// failures arrived.
|
|
1049
|
+
const failure = this.#host.encodeOrchestrator.noteStartFailure(
|
|
1050
|
+
session.outputKey,
|
|
1051
|
+
ended.from,
|
|
1052
|
+
ended.livedMs
|
|
1053
|
+
);
|
|
1054
|
+
if (failure.count > 0) {
|
|
1055
|
+
this.#host.logger.warn(
|
|
1056
|
+
`transcode ${session.id} fast failure at segment #${ended.from} ` +
|
|
1057
|
+
`(${ended.livedMs}ms) — ${failure.count}/${failure.limit} consecutive`
|
|
1058
|
+
);
|
|
1059
|
+
if (failure.blocked) {
|
|
1060
|
+
this.#host.logger.error(
|
|
1061
|
+
`transcode ${session.id} will not be started at #${ended.from} again: ` +
|
|
1062
|
+
`${failure.count} starts there failed within ${failure.fastMs}ms each ` +
|
|
1063
|
+
`(${lastError}) "${session.file.name}"`
|
|
1064
|
+
);
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
1067
|
+
this.#host.logger.error(
|
|
1068
|
+
`transcode ${session.id} encode-run #${ended.from}..#${ended.to} failed: ${lastError}` +
|
|
1069
|
+
` — ${this.#describeTrackSelection(session)}` +
|
|
1070
|
+
`\n ffmpeg ${run.argsDescribed || "(command not recorded)"}`
|
|
1071
|
+
);
|
|
1072
|
+
}
|
|
1073
|
+
|
|
1074
|
+
/**
|
|
1075
|
+
* What this run asked the source for, against what the source said it has.
|
|
1076
|
+
*
|
|
1077
|
+
* Written for one failure in particular: every `-map` this proxy builds ends
|
|
1078
|
+
* in `?`, so ffmpeg drops a mapping for an absent stream without complaint,
|
|
1079
|
+
* and a run whose mappings ALL drop produces a file with nothing in it —
|
|
1080
|
+
* reported as `Output file does not contain any stream`, exit 255. Read from
|
|
1081
|
+
* the exit code alone that is indistinguishable from any other refusal. Read
|
|
1082
|
+
* beside the tracks the file actually holds it is unmistakable, and it names
|
|
1083
|
+
* which side is wrong: an audio index past the end of the list is ours, no
|
|
1084
|
+
* streams at all is the source's.
|
|
1085
|
+
*
|
|
1086
|
+
* @param {HlsSession} session
|
|
1087
|
+
* @returns {string}
|
|
1088
|
+
*/
|
|
1089
|
+
#describeTrackSelection(session) {
|
|
1090
|
+
const wanted = [];
|
|
1091
|
+
// Named exactly as the command line names them, second input included: a
|
|
1092
|
+
// refusal whose message describes a different mapping than the one that was
|
|
1093
|
+
// refused is the reading that cost a wrong diagnosis before.
|
|
1094
|
+
const inputs = this.#host.inputOf(session);
|
|
1095
|
+
const audioInput = inputs.audioInputUrl.length > 0 ? 1 : 0;
|
|
1096
|
+
const audioTrack = session.spec.audioSourceTrackIndex;
|
|
1097
|
+
if (session.spec.carries === "audio-only") {
|
|
1098
|
+
wanted.push(`audio 0:a:${audioTrack}`);
|
|
1099
|
+
} else if (this.#host.servesAudioSeparately(session)) {
|
|
1100
|
+
wanted.push("video 0:v:0");
|
|
1101
|
+
} else {
|
|
1102
|
+
wanted.push("video 0:v:0", `audio ${audioInput}:a:${audioTrack}`);
|
|
1103
|
+
}
|
|
1104
|
+
const counts = session.file.streamCounts;
|
|
1105
|
+
const held = counts
|
|
1106
|
+
? `the source holds ${counts.video} video, ${counts.audio} audio, ` +
|
|
1107
|
+
`${counts.subtitle} subtitle` +
|
|
1108
|
+
(counts.other > 0 ? `, ${counts.other} other` : "")
|
|
1109
|
+
: "what the source holds was not recorded";
|
|
1110
|
+
return `this run asked for ${wanted.join(" + ")}, and ${held}`;
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
/**
|
|
1114
|
+
* Stop this session's encoder without replacing it.
|
|
1115
|
+
*
|
|
1116
|
+
* A variant nobody is watching must not go on encoding: the host has one
|
|
1117
|
+
* encoder's worth of capacity, and the whole point of switching quality
|
|
1118
|
+
* seamlessly is that the new rung gets it. The session itself stays — its
|
|
1119
|
+
* produced segments remain servable, and switching back restarts it from
|
|
1120
|
+
* where the viewer then is.
|
|
1121
|
+
*
|
|
1122
|
+
* @param {HlsSession} session
|
|
1123
|
+
* @param {string} reason - Named in the log; a stopped encoder is otherwise
|
|
1124
|
+
* indistinguishable from one that died.
|
|
1125
|
+
* @returns {void}
|
|
1126
|
+
*/
|
|
1127
|
+
stopEncodeRun(session, reason) {
|
|
1128
|
+
const running = this.liveRunsOf(session);
|
|
1129
|
+
if (running.length === 0) {
|
|
1130
|
+
return;
|
|
1131
|
+
}
|
|
1132
|
+
// Every run this session has going. The callers all mean the session's
|
|
1133
|
+
// encoding as a whole: a rung nobody is watching any more, a session being
|
|
1134
|
+
// disposed. A run that had already finished or failed is not among them,
|
|
1135
|
+
// which is what keeps a stop from erasing how it actually ended.
|
|
1136
|
+
for (const run of running) {
|
|
1137
|
+
// The run resumes itself if it was suspended — a stopped process does not
|
|
1138
|
+
// act on SIGTERM until it is continued — records the cause, and answers
|
|
1139
|
+
// its own exit. Nothing here has to null a field so that the exit is read
|
|
1140
|
+
// correctly, because there is no shared field left to misread.
|
|
1141
|
+
run.stop(reason);
|
|
1142
|
+
// Nothing is cleared up from here. What this run left open is under its
|
|
1143
|
+
// own working name, and the encoding layer removes it when the run's
|
|
1144
|
+
// ending reaches it — one place, and it needs neither the stretch nor the
|
|
1145
|
+
// init bytes this method used to fetch to judge a file by its contents.
|
|
1146
|
+
}
|
|
1147
|
+
this.#host.logger.info(`transcode ${session.id} ${running.length} encoder(s) stopped: ${reason}`);
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
/**
|
|
1151
|
+
* What moving the encoder BACKWARDS costs, said out loud when it happens.
|
|
1152
|
+
*
|
|
1153
|
+
* Nothing already written is lost — every run keeps its own directory and
|
|
1154
|
+
* {@link SegmentStore#pathOfName} serves the union of all of them
|
|
1155
|
+
* — so the price of a restart is not the files. It is two other things, and
|
|
1156
|
+
* neither was ever counted:
|
|
1157
|
+
*
|
|
1158
|
+
* - **work done twice.** The new run begins at the target and encodes
|
|
1159
|
+
* forward through segments the old run had already finished. ffmpeg cannot
|
|
1160
|
+
* know they exist, so it makes them again.
|
|
1161
|
+
* - **the viewer in front.** While the run walks back up to where it already
|
|
1162
|
+
* was, nothing new is being made ahead of them, and their cushion drains.
|
|
1163
|
+
*
|
|
1164
|
+
* Both are what decides whether a session should be allowed a SECOND
|
|
1165
|
+
* concurrent run instead — roadmap item 64. That question cannot be answered
|
|
1166
|
+
* from taste, and this is the reading it needs: how often it happens at all,
|
|
1167
|
+
* how far back, and how much of the walk is a repeat.
|
|
1168
|
+
*
|
|
1169
|
+
* Nothing here is awaited by the caller. Everything below the call site is
|
|
1170
|
+
* the restart path, which is measured in milliseconds and has been worked on
|
|
1171
|
+
* twice to keep it that way; a reading that delays the thing it is reading
|
|
1172
|
+
* about is not a reading. The figures that MUST be taken before the new run
|
|
1173
|
+
* exists are taken synchronously, and only the file counting is left to run
|
|
1174
|
+
* on its own — against the directories that existed at this instant, so what
|
|
1175
|
+
* the new run is about to write cannot be counted as already there.
|
|
1176
|
+
*
|
|
1177
|
+
* @param {HlsSession} session
|
|
1178
|
+
* @param {number} startIndex - Where the new run will begin.
|
|
1179
|
+
* @returns {void}
|
|
1180
|
+
*/
|
|
1181
|
+
#accountBackwardRestart(session, startIndex) {
|
|
1182
|
+
const previousStart = earliestRunStart(this.runsOf(session));
|
|
1183
|
+
if (!Number.isInteger(previousStart) || !Number.isInteger(startIndex) || startIndex >= previousStart) {
|
|
1184
|
+
// A first run, or one moving forward. Neither costs anything here: a
|
|
1185
|
+
// forward restart skips material it never made.
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
const processed = Number(this.progressOf(session)?.processedSeconds);
|
|
1189
|
+
const head = Number.isFinite(processed)
|
|
1190
|
+
? Math.max(previousStart, this.#host.outputTimes.segmentIndexForTime(session, processed))
|
|
1191
|
+
: previousStart;
|
|
1192
|
+
// Bounded: a session an hour in has thousands of segments, and the count is
|
|
1193
|
+
// for a comparison, not an inventory.
|
|
1194
|
+
const last = Math.min(head, startIndex + BACKWARD_RESTART_SCAN_SEGMENTS);
|
|
1195
|
+
const accounting = this.#inputStateFor(session).backwardRestarts;
|
|
1196
|
+
accounting.count += 1;
|
|
1197
|
+
accounting.segmentsBack += previousStart - startIndex;
|
|
1198
|
+
accounting.worstBack = Math.max(accounting.worstBack, previousStart - startIndex);
|
|
1199
|
+
|
|
1200
|
+
void (async () => {
|
|
1201
|
+
let alreadyOnDisk = 0;
|
|
1202
|
+
for (const index of this.#host.producedNumbers(session)) {
|
|
1203
|
+
if (index >= startIndex && index <= last) {
|
|
1204
|
+
alreadyOnDisk += 1;
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
accounting.remade += alreadyOnDisk;
|
|
1208
|
+
this.#host.logger.info(
|
|
1209
|
+
`transcode ${session.id} moving the encoder BACK from #${previousStart} to #${startIndex} ` +
|
|
1210
|
+
`(head was #${head}): ${alreadyOnDisk} of the ${last - startIndex + 1} segment(s) it will walk through ` +
|
|
1211
|
+
`are already on disk and will be made again, and nothing is produced ahead of #${head} until it gets ` +
|
|
1212
|
+
`back there — ${accounting.count} backward restart(s) this session, worst ${accounting.worstBack} ` +
|
|
1213
|
+
`segment(s) back, ${accounting.remade} segment(s) remade in total (roadmap 64)`
|
|
1214
|
+
);
|
|
1215
|
+
})().catch(() => {
|
|
1216
|
+
// silent-ok: a reading that fails is not worth ending a restart over.
|
|
1217
|
+
});
|
|
1218
|
+
}
|
|
1219
|
+
}
|