@torrent-tv/proxy 2.91.0 → 2.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +3 -2
- package/package.json +18 -2
- package/routes/api/playback-plan/post.js +21 -4
- package/routes/api/sources/files/get.js +20 -28
- package/routes/api/sources/post.js +4 -2
- package/routes/api/sources/stats/get.js +4 -11
- package/routes/api/sources/viewer/post.js +16 -0
- package/routes/api/sources/warm/post.js +6 -215
- package/routes/api/subtitles/get.js +22 -125
- package/routes/api/subtitles/post.js +26 -0
- package/routes/api/transcode-sessions/post.js +24 -2
- package/routes/media/get.js +80 -0
- package/routes/stream/get.js +26 -94
- package/routes/transcode/audio-warm/get.js +9 -21
- package/routes/transcode/variant-warm/get.js +8 -14
- package/server.js +410 -46
- package/services/encode/AdmittedInput.js +136 -0
- package/services/encode/CushionReport.js +1 -76
- package/services/encode/EncodeInputs.js +142 -0
- package/services/encode/EncodeOrchestrator.js +4 -1
- package/services/encode/EncodeRun.js +13 -4
- package/services/encode/EncodeRuns.js +132 -52
- package/services/encode/InputFailures.js +37 -0
- package/services/encode/MatroskaInput.js +158 -0
- package/services/encode/OutputOpening.js +16 -64
- package/services/encode/Renditions.js +1 -17
- package/services/encode/admitted-command.js +60 -0
- package/services/encode/annex-b-input.js +37 -0
- package/services/encode/bit-payload.js +29 -0
- package/services/encode/output/OutputCatalog.js +1 -1
- package/services/encode/output/Timeline.js +11 -0
- package/services/encode/pcm-normalization.js +30 -0
- package/services/encode/piece-completeness.js +54 -1
- package/services/encode/quality/HostLoad.js +19 -73
- package/services/encode/quality/QualityOffer.js +1 -1
- package/services/encode/run-command.js +1 -589
- package/services/encode/segment-formats/mp4-boxes.js +3 -1
- package/services/encode/segment-formats/presentation-segment.js +114 -0
- package/services/media/ContainerOrchestrator.js +118 -48
- package/services/media/KeyframeTables.js +16 -115
- package/services/media/MediaReadRequests.js +120 -0
- package/services/media/ProbeRequests.js +135 -0
- package/services/media/SegmentInputs.js +48 -0
- package/services/media/SourceFile.js +6 -0
- package/services/media/SubtitleOrchestrator.js +190 -4
- package/services/media/await-plan.js +47 -0
- package/services/media/container/AsfContainer.js +217 -0
- package/services/media/container/AviContainer.js +343 -34
- package/services/media/container/CadencedPictureOrder.js +45 -0
- package/services/media/container/Container.js +7 -3
- package/services/media/container/ContainerFactory.js +10 -3
- package/services/media/container/FfprobeContainer.js +222 -0
- package/services/media/container/MatroskaContainer.js +72 -46
- package/services/media/container/Mp4Container.js +472 -139
- package/services/media/container/MpegPsContainer.js +281 -0
- package/services/media/container/MpegTsContainer.js +277 -0
- package/services/media/container/PacketIndex.js +270 -0
- package/services/media/container/PacketRecords.js +237 -0
- package/services/media/container/RetainedBytes.js +39 -0
- package/services/media/container/RetainedReads.js +51 -0
- package/services/media/container/ac3-frame.js +46 -0
- package/services/media/container/asf-packets.js +153 -0
- package/services/media/container/avc-picture-order.js +50 -0
- package/services/media/container/avi-open-dml.js +47 -0
- package/services/media/container/dts-frame.js +51 -0
- package/services/media/container/ebml-stream.js +12 -1
- package/services/media/container/ffprobe-record.js +45 -0
- package/services/media/container/flv-packet-ranges.js +27 -0
- package/services/media/container/following-packet-time.js +37 -0
- package/services/media/container/h264-configuration.js +183 -0
- package/services/media/container/h264-elementary-index.js +245 -0
- package/services/media/container/hevc-configuration.js +165 -0
- package/services/media/container/hevc-elementary-index.js +132 -0
- package/services/media/container/hevc-picture-order.js +66 -0
- package/services/media/container/latm-frame.js +99 -0
- package/services/media/container/matroska-packets.js +207 -0
- package/services/media/container/memory-unavailable.js +8 -0
- package/services/media/container/mp3-packet-dependencies.js +41 -0
- package/services/media/container/mp4-fragments.js +148 -0
- package/services/media/container/mpeg-audio-frame.js +19 -0
- package/services/media/container/mpeg-audio-reservoir.js +27 -0
- package/services/media/container/mpeg-elementary-index.js +196 -0
- package/services/media/container/mpeg-sequence-reader.js +69 -0
- package/services/media/container/mpeg-ts-packets.js +93 -0
- package/services/media/container/mpeg-video-configuration.js +50 -0
- package/services/media/container/mpeg4-picture-timing.js +108 -0
- package/services/media/container/pes-header.js +34 -0
- package/services/media/playback-declarations.js +57 -0
- package/services/media/playback-planner.js +72 -230
- package/services/media/probe-packets.js +61 -0
- package/services/media/read-playback-declarations.js +18 -0
- package/services/media/tracks/AudioTrack.js +9 -0
- package/services/media/tracks/TextSubtitleTrack.js +4 -1
- package/services/server/OutputLifecycle.js +2 -0
- package/services/server/SegmentServing.js +40 -33
- package/services/server/ViewerRequests.js +22 -41
- package/services/server/controllers/PlaybackController.js +4 -0
- package/services/server/controllers/SubtitleController.js +8 -29
- package/services/server/transcode-session-files.js +58 -99
- package/services/server/wire-outputs.js +58 -15
- package/services/storage/IndexMemory.js +85 -0
- package/services/storage/MachineBudget.js +27 -3
- package/services/storage/allowance.js +8 -1
- package/services/storage/piece-store/piece-disk-store.js +18 -3
- package/services/storage/piece-store/piece-lru.js +8 -4
- package/services/storage/piece-store/shared-piece-store.js +26 -11
- package/services/storage/segment-store/SegmentStore.js +49 -18
- package/services/storage/wire.js +7 -0
- package/services/torrent/Contents.js +52 -20
- package/services/torrent/demand/DemandRegister.js +13 -4
- package/services/torrent/demand/Window.js +13 -2
- package/services/torrent/demand-progress.js +25 -0
- package/services/torrent/download/DeadlineScheduler.js +167 -0
- package/services/torrent/download/FutureDownload.js +166 -0
- package/services/torrent/download/SwarmSelection.js +30 -59
- package/services/torrent/download/registry.js +45 -5
- package/services/torrent/download/withdraw-claim.js +2 -0
- package/services/torrent/torrent-pool.js +39 -600
- package/services/torrent/worker/channel.js +8 -1
- package/services/torrent/worker/client.js +74 -72
- package/services/torrent/worker/files-in-use.js +9 -0
- package/services/torrent/worker/handle-state.js +12 -0
- package/services/torrent/worker/held-bytes.js +82 -99
- package/services/torrent/worker/held-file-bytes.js +17 -0
- package/services/torrent/worker/piece-reader.js +67 -683
- package/services/torrent/worker/pool-adapter.js +37 -123
- package/services/torrent/worker/protocol.js +6 -17
- package/services/torrent/worker/whole-file-stats.js +2 -0
- package/services/torrent/worker/worker.js +86 -156
- package/services/transport/data-channel-handler.js +56 -13
- package/services/viewer/DownloadMaps.js +168 -0
- package/services/viewer/NativeSourceMap.js +33 -0
- package/services/viewer/PriorityMap.js +49 -40
- package/services/viewer/PriorityOrchestrator.js +42 -13
- package/services/viewer/SourcePreparation.js +278 -0
- package/services/viewer/Viewer.js +11 -0
- package/services/viewer/Viewers.js +109 -2
- package/services/viewer/playback-readiness.js +50 -63
- package/services/viewer/segment-demands.js +37 -0
- package/services/viewer/urgent-output-ready.js +26 -0
- package/services/encode/run-inputs.js +0 -53
- package/services/media/keyframe-probe.js +0 -157
- package/services/media/media-info-probe.js +0 -93
- package/services/torrent/worker/background-fill.js +0 -213
- package/services/torrent/worker/resume-warm.js +0 -107
|
@@ -14,6 +14,7 @@ import { waits } from "./WaitLedger.js";
|
|
|
14
14
|
import { readMachineState, readProcessCpuSeconds, readProxyCpuSeconds, readSystemCpu, shareOfMachine } from "../encode/host-load.js";
|
|
15
15
|
import { minimumBufferFrom } from "../torrent/supply-margin.js";
|
|
16
16
|
import { PriorityOrchestrator } from "../viewer/PriorityOrchestrator.js";
|
|
17
|
+
import { urgentOutputsReady } from "../viewer/urgent-output-ready.js";
|
|
17
18
|
import { softwareDescriptor } from "../encode/hwaccel.js";
|
|
18
19
|
import { resolveSegmentFormat } from "../encode/segment-formats/index.js";
|
|
19
20
|
import { Timelines } from "../encode/output/Timeline.js";
|
|
@@ -74,6 +75,7 @@ import { audioRenditionName } from "../media/audio-inventory.js";
|
|
|
74
75
|
import { Renditions } from "../encode/Renditions.js";
|
|
75
76
|
import { CushionReport, LOOKAHEAD_PAUSE_SECONDS } from "../encode/CushionReport.js";
|
|
76
77
|
import { EncodeRuns } from "../encode/EncodeRuns.js";
|
|
78
|
+
import { EncodeInputs } from "../encode/EncodeInputs.js";
|
|
77
79
|
import { OutputTimes } from "../encode/OutputTimes.js";
|
|
78
80
|
import { HostLoad } from "../encode/quality/HostLoad.js";
|
|
79
81
|
import { HostTimings } from "../encode/quality/HostTimings.js";
|
|
@@ -87,8 +89,6 @@ import { OutputSpec } from "../encode/output/OutputSpec.js";
|
|
|
87
89
|
import { compareInits } from "../encode/segment-formats/init-compat.js";
|
|
88
90
|
import { nominalKbpsFor, AUDIO_TRANSCODE_KBPS } from "../encode/args.js";
|
|
89
91
|
import { audioReadingFor } from "../encode/audio-work-rate.js";
|
|
90
|
-
import { probeInputMediaInfo } from "../media/media-info-probe.js";
|
|
91
|
-
import { probeVideoKeyframeTimes } from "../media/keyframe-probe.js";
|
|
92
92
|
import { wireMachineBudget } from "../storage/wire.js";
|
|
93
93
|
import { Returns } from "../storage/returns.js";
|
|
94
94
|
import { freeBytesFor } from "../storage/free.js";
|
|
@@ -217,7 +217,6 @@ export function wireOutputs({
|
|
|
217
217
|
getCachedMediaInfo = null,
|
|
218
218
|
getCachedAudioTracks = null,
|
|
219
219
|
getContainerMediaInfo = null,
|
|
220
|
-
fetchWholeFile = null,
|
|
221
220
|
segmentFormatId = undefined,
|
|
222
221
|
stateDir = "",
|
|
223
222
|
segmentStore = null,
|
|
@@ -228,7 +227,15 @@ export function wireOutputs({
|
|
|
228
227
|
diagnostics = null,
|
|
229
228
|
diagnosticsRoot = "",
|
|
230
229
|
memoryClaimant = null,
|
|
231
|
-
|
|
230
|
+
resolveEncodeInput = null,
|
|
231
|
+
readSourceMedia = null,
|
|
232
|
+
readMetadataActivity = null,
|
|
233
|
+
sourceInputsFor = null,
|
|
234
|
+
readEncodeRanges = null,
|
|
235
|
+
encodeInputs = null,
|
|
236
|
+
indexMemory = null,
|
|
237
|
+
budgetPolicy = null,
|
|
238
|
+
onViewerChanged = null}) {
|
|
232
239
|
const parts = {};
|
|
233
240
|
const audioDescriptionOf = (output) => (getCachedAudioTracks?.({
|
|
234
241
|
sourceKey: output.file.sourceKey, fileIndex: output.spec.grid?.fileIndex ?? output.file.fileIndex
|
|
@@ -284,6 +291,7 @@ export function wireOutputs({
|
|
|
284
291
|
parts.limitsFor = (frame) => [nominalKbpsFor(frame)];
|
|
285
292
|
// What the torrent and the proxy itself spend on this host, and how fast each watched torrent moves.
|
|
286
293
|
parts.hostLoad = new HostLoad({
|
|
294
|
+
readMetadataActivity,
|
|
287
295
|
readMachineState,
|
|
288
296
|
readProcessCpuSeconds,
|
|
289
297
|
readProxyCpuSeconds,
|
|
@@ -326,7 +334,7 @@ export function wireOutputs({
|
|
|
326
334
|
});
|
|
327
335
|
},
|
|
328
336
|
viewerSecondsOn: (output, consumerId, now) => viewerSecondsOn(parts.viewers, output, consumerId, now),
|
|
329
|
-
|
|
337
|
+
get encodeInputs() { return parts.encodeInputs; },
|
|
330
338
|
producedNumbers: (...args) => parts.serving.producedNumbers(...args),
|
|
331
339
|
servesAudioSeparately: (...args) => parts.renditions.servesAudioSeparately(...args),
|
|
332
340
|
disposeSession: (...args) => parts.lifecycle.disposeSession(...args),
|
|
@@ -352,7 +360,6 @@ export function wireOutputs({
|
|
|
352
360
|
logger,
|
|
353
361
|
producedNumbers: (...args) => parts.serving.producedNumbers(...args),
|
|
354
362
|
get encodeRuns() { return parts.encodeRuns; },
|
|
355
|
-
get fetchWholeFile() { return parts.fetchWholeFile; },
|
|
356
363
|
get getCachedAudioTracks() { return parts.getCachedAudioTracks; },
|
|
357
364
|
get hostLoad() { return parts.hostLoad; },
|
|
358
365
|
get lookaheadSeconds() { return parts.lookaheadSeconds; },
|
|
@@ -523,8 +530,8 @@ export function wireOutputs({
|
|
|
523
530
|
// carrying here: the only figure an encoder with no bound of its own has.
|
|
524
531
|
observedPeakMbps: (spec) => parts.localObservations?.peakMbps(spec) ?? null,
|
|
525
532
|
limitsFor: (frame) => parts.limitsFor(frame),
|
|
526
|
-
|
|
527
|
-
|
|
533
|
+
readSourceMedia,
|
|
534
|
+
get encodeInputs() { return parts.encodeInputs; },
|
|
528
535
|
get decodeCostModel() { return parts.decodeCostModel; },
|
|
529
536
|
get enabled() { return parts.enabled; },
|
|
530
537
|
get encodeCost() { return parts.encodeCost; },
|
|
@@ -565,6 +572,7 @@ export function wireOutputs({
|
|
|
565
572
|
// When this viewer's playback can start and run to the end: the viewer
|
|
566
573
|
// component's forecast, which models the viewer's browser.
|
|
567
574
|
playbackReadiness: { predict: predictPlaybackReadiness, forecastRate, RateTrend },
|
|
575
|
+
sourceInputsFor: (output, index) => parts.sourceInputsFor?.(output, index),
|
|
568
576
|
encodeSpeedReadingOf: (output) => parts.encodeCost.latestSpeedReadingOf(output),
|
|
569
577
|
projectedEncodeSpeedOf: (output) => parts.encodeCost.projectedSpeedOf(output),
|
|
570
578
|
audioDescriptionOf,
|
|
@@ -615,6 +623,7 @@ export function wireOutputs({
|
|
|
615
623
|
});
|
|
616
624
|
parts.enabled = Boolean(enabled);
|
|
617
625
|
parts.ffmpegBin = ffmpegBin;
|
|
626
|
+
parts.sourceInputsFor = typeof sourceInputsFor === "function" ? sourceInputsFor : null;
|
|
618
627
|
parts.keyframeTables = keyframeTables;
|
|
619
628
|
// Where measurements about this host are kept between runs. Empty means
|
|
620
629
|
// beside the installed proxy; a deployment with somewhere persistent to
|
|
@@ -637,12 +646,6 @@ export function wireOutputs({
|
|
|
637
646
|
// it meant reading a header this proxy had already read.
|
|
638
647
|
parts.getContainerMediaInfo =
|
|
639
648
|
typeof getContainerMediaInfo === "function" ? getContainerMediaInfo : null;
|
|
640
|
-
// Fetch one whole file of a source, as a bounded read rather than a
|
|
641
|
-
// selection. Used to pull a soundtrack that ships beside the picture onto
|
|
642
|
-
// the disk while the swarm has capacity to spare — see
|
|
643
|
-
// `#fetchSpareSoundtracks`. Optional: a proxy wired without it simply reads
|
|
644
|
-
// such a soundtrack when it is played.
|
|
645
|
-
parts.fetchWholeFile = typeof fetchWholeFile === "function" ? fetchWholeFile : null;
|
|
646
649
|
// Optional async accessor for a source's live download stats, used by the
|
|
647
650
|
// realtime budget to tell a CPU limit from a download-starved input:
|
|
648
651
|
// (sourceKey, fileIndex) => Promise<{ downloadSpeed, fileLength, fileProgress } | null>.
|
|
@@ -760,7 +763,10 @@ export function wireOutputs({
|
|
|
760
763
|
// exist, because that decision reads nothing else about viewers. It used to
|
|
761
764
|
// be re-taken on a five-second timer instead, which made a just-created
|
|
762
765
|
// output wait up to five seconds before anything noticed it had a viewer.
|
|
763
|
-
parts.viewers = new Viewers({ onChange: () =>
|
|
766
|
+
parts.viewers = new Viewers({ onChange: () => {
|
|
767
|
+
parts.encodeRuns.planEncodersSoon();
|
|
768
|
+
onViewerChanged?.();
|
|
769
|
+
} });
|
|
764
770
|
// The outputs that exist, and every question about them: the picture a step
|
|
765
771
|
// belongs to, the steps, the soundtracks, the height a step is named by.
|
|
766
772
|
parts.outputs = new OutputCatalog({
|
|
@@ -863,6 +869,19 @@ export function wireOutputs({
|
|
|
863
869
|
// a step supersedes belongs to the film's shape. Neither layer is handed
|
|
864
870
|
// the other — one gets a plain id, the other is read for one field.
|
|
865
871
|
watchedBy: (session, viewer) => !parts.outputs.supersededBy(session, viewer.activeVariantId ?? null),
|
|
872
|
+
urgentReadyFor: (session, viewer, seconds, now) => {
|
|
873
|
+
if (parts.subtitleReadyFor?.(viewer) === false) return false;
|
|
874
|
+
return urgentOutputsReady({
|
|
875
|
+
sourceKey: session.file.sourceKey, fileIndex: session.file.fileIndex,
|
|
876
|
+
durationSeconds: session.file.durationSeconds, atSeconds: viewer.positionSeconds(now) ?? 0, seconds,
|
|
877
|
+
outputs: parts.outputs.values(),
|
|
878
|
+
consumed: (output) => parts.viewers.forOutput(output).has(viewer.id) &&
|
|
879
|
+
!parts.outputs.supersededBy(output, viewer.activeVariantId ?? null),
|
|
880
|
+
segmentIndex: (output, at) => parts.outputTimes.segmentIndexForTime(output, at),
|
|
881
|
+
segmentStart: (output, index) => parts.outputTimes.segmentStartTime(output, index),
|
|
882
|
+
closed: (key, index) => parts.segmentStore.isClosed(key, index)
|
|
883
|
+
});
|
|
884
|
+
},
|
|
866
885
|
allowanceFor: (session) => minimumBufferFrom({
|
|
867
886
|
segmentSeconds: parts.segmentDurationSec,
|
|
868
887
|
worstSupplyWaitSec: parts.hostLoad.supplyFor(session.file)?.worstWaitSec
|
|
@@ -910,6 +929,9 @@ export function wireOutputs({
|
|
|
910
929
|
availability: () => parts.hostLoad.hostAvailability ?? null
|
|
911
930
|
});
|
|
912
931
|
parts.encodeOrchestrator = new EncodeOrchestrator({
|
|
932
|
+
inputDemandChanged: (address, windows) => {
|
|
933
|
+
for (const output of parts.outputs.outputsOn(address)) parts.encodeInputs?.retain(output, windows);
|
|
934
|
+
},
|
|
913
935
|
admission: parts.admission,
|
|
914
936
|
// What the viewers actually waited for, by band. The ledger is the
|
|
915
937
|
// priority layer's; the encoding is handed a way to ask it.
|
|
@@ -951,6 +973,25 @@ export function wireOutputs({
|
|
|
951
973
|
// one term of the keeping period that is guessed rather than measured, and
|
|
952
974
|
// the only place it can be measured from.
|
|
953
975
|
parts.returns = new Returns();
|
|
976
|
+
parts.encodeInputs = encodeInputs ?? (typeof resolveEncodeInput === "function" ? new EncodeInputs({
|
|
977
|
+
resolve: resolveEncodeInput,
|
|
978
|
+
readRanges: readEncodeRanges,
|
|
979
|
+
reviseBudget: () => parts.machineBudget.revise(),
|
|
980
|
+
capacity: () => parts.machineBudget.capacityOf("memory"),
|
|
981
|
+
urgent: (output, index) => parts.encodeOrchestrator.wantedSegmentsOn(output.outputKey).some(window =>
|
|
982
|
+
index >= window.from && index <= window.to && window.withinSeconds === 0),
|
|
983
|
+
changed: (output, result) => {
|
|
984
|
+
if (result.kind === "terminal") {
|
|
985
|
+
logger.warn(`encode input output=${output.outputKey} terminal=${result.reason}: ${result.message ?? ""}`);
|
|
986
|
+
parts.serving.invalidateWaits(output);
|
|
987
|
+
}
|
|
988
|
+
parts.encodeRuns.planEncodersSoon();
|
|
989
|
+
},
|
|
990
|
+
failed: (output, error) => {
|
|
991
|
+
logger.warn(`encode input output=${output.outputKey} failed: ${error.message}`);
|
|
992
|
+
parts.serving.invalidateWaits(output);
|
|
993
|
+
}
|
|
994
|
+
}) : null);
|
|
954
995
|
// One owner of the disk, and the list of what takes it lives with the owner.
|
|
955
996
|
parts.machineBudget = wireMachineBudget({
|
|
956
997
|
segmentStore: parts.segmentStore,
|
|
@@ -959,6 +1000,8 @@ export function wireOutputs({
|
|
|
959
1000
|
diagnostics: parts.diagnostics,
|
|
960
1001
|
diagnosticsRoot: parts.diagnosticsRoot,
|
|
961
1002
|
memory: parts.memoryClaimant,
|
|
1003
|
+
encodeInputs: parts.encodeInputs,
|
|
1004
|
+
indexMemory,
|
|
962
1005
|
policy: parts.budgetPolicy ?? undefined,
|
|
963
1006
|
readFree: freeBytesFor,
|
|
964
1007
|
logger
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
/** Exact binary index ownership and pending growth under the shared budget. */
|
|
2
|
+
export class IndexMemory {
|
|
3
|
+
#files = new Map();
|
|
4
|
+
#held = 0;
|
|
5
|
+
#packetBytes = 0;
|
|
6
|
+
#allowed = 0;
|
|
7
|
+
#revise;
|
|
8
|
+
#changed;
|
|
9
|
+
#revisionQueued = false;
|
|
10
|
+
|
|
11
|
+
constructor({ reviseBudget, changed }) {
|
|
12
|
+
this.#revise = reviseBudget;
|
|
13
|
+
this.#changed = changed;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
held() { return this.#held; }
|
|
17
|
+
packetBytes() { return this.#packetBytes; }
|
|
18
|
+
wanted() {
|
|
19
|
+
return this.#held + [...this.#files.values()].reduce((sum, file) => sum + Math.max(0, file.needed - file.held), 0);
|
|
20
|
+
}
|
|
21
|
+
required() { return this.wanted(); }
|
|
22
|
+
|
|
23
|
+
allow(bytes) {
|
|
24
|
+
const allowed = Math.max(0, bytes);
|
|
25
|
+
if (allowed === this.#allowed) return;
|
|
26
|
+
this.#allowed = allowed;
|
|
27
|
+
this.#changed?.();
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
#requestRevision() {
|
|
31
|
+
if (this.#revisionQueued) return;
|
|
32
|
+
this.#revisionQueued = true;
|
|
33
|
+
queueMicrotask(() => {
|
|
34
|
+
this.#revisionQueued = false;
|
|
35
|
+
this.#revise?.();
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
forFile(sourceKey, fileIndex) {
|
|
40
|
+
const key = JSON.stringify([sourceKey, fileIndex]);
|
|
41
|
+
let file = this.#files.get(key);
|
|
42
|
+
if (file) return file.memory;
|
|
43
|
+
file = { sourceKey, held: 0, needed: 0, records: new Set(), closed: false };
|
|
44
|
+
file.memory = { forRecord: (record, kind = "packets") => {
|
|
45
|
+
file.records.add(record);
|
|
46
|
+
let disposed = false;
|
|
47
|
+
return {
|
|
48
|
+
reserve: bytes => {
|
|
49
|
+
if (file.closed || disposed) return false;
|
|
50
|
+
if (this.#held + bytes > this.#allowed) {
|
|
51
|
+
const needed = file.held + bytes;
|
|
52
|
+
if (needed > file.needed) { file.needed = needed; this.#requestRevision(); }
|
|
53
|
+
return false;
|
|
54
|
+
}
|
|
55
|
+
file.held += bytes;
|
|
56
|
+
this.#held += bytes;
|
|
57
|
+
if (kind === "packets") this.#packetBytes += bytes;
|
|
58
|
+
if (file.held >= file.needed) file.needed = 0;
|
|
59
|
+
return true;
|
|
60
|
+
},
|
|
61
|
+
release: bytes => {
|
|
62
|
+
file.held -= bytes;
|
|
63
|
+
this.#held -= bytes;
|
|
64
|
+
if (kind === "packets") this.#packetBytes -= bytes;
|
|
65
|
+
this.#requestRevision();
|
|
66
|
+
},
|
|
67
|
+
dispose: () => { disposed = true; file.records.delete(record); }
|
|
68
|
+
};
|
|
69
|
+
} };
|
|
70
|
+
this.#files.set(key, file);
|
|
71
|
+
return file.memory;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
forget(sourceKey) {
|
|
75
|
+
for (const [key, file] of this.#files) {
|
|
76
|
+
if (file.sourceKey !== sourceKey) continue;
|
|
77
|
+
file.closed = true;
|
|
78
|
+
file.needed = 0;
|
|
79
|
+
for (const record of [...file.records]) record.dispose();
|
|
80
|
+
this.#files.delete(key);
|
|
81
|
+
}
|
|
82
|
+
this.#requestRevision();
|
|
83
|
+
this.#changed?.();
|
|
84
|
+
}
|
|
85
|
+
}
|
|
@@ -75,6 +75,8 @@ import { OtherDemand, divideAllowance } from "./allowance.js";
|
|
|
75
75
|
*/
|
|
76
76
|
|
|
77
77
|
export class MachineBudget {
|
|
78
|
+
#revisionPromise = null;
|
|
79
|
+
#reviseAgain = false;
|
|
78
80
|
/** Resource name → how to read what is free, and what others have needed. */
|
|
79
81
|
#resources = new Map();
|
|
80
82
|
|
|
@@ -135,12 +137,33 @@ export class MachineBudget {
|
|
|
135
137
|
this.#claimants = this.#claimants.filter((one) => one.name !== name);
|
|
136
138
|
}
|
|
137
139
|
|
|
140
|
+
/** Total measured policy allowance, independent of its current division. */
|
|
141
|
+
capacityOf(name) {
|
|
142
|
+
const reading = this.#last.get(name);
|
|
143
|
+
return reading?.measured ? reading.allowanceBytes : null;
|
|
144
|
+
}
|
|
145
|
+
|
|
138
146
|
/**
|
|
139
147
|
* Read every resource, divide each, and tell each claimant its share.
|
|
140
148
|
*
|
|
141
149
|
* @returns {Promise<Map<string, { freeBytes: number, allowanceBytes: number, shares: object[] }>>}
|
|
142
150
|
*/
|
|
143
|
-
|
|
151
|
+
revise() {
|
|
152
|
+
if (this.#revisionPromise) {
|
|
153
|
+
this.#reviseAgain = true;
|
|
154
|
+
return this.#revisionPromise;
|
|
155
|
+
}
|
|
156
|
+
this.#revisionPromise = (async () => {
|
|
157
|
+
do {
|
|
158
|
+
this.#reviseAgain = false;
|
|
159
|
+
await this.#readRevision();
|
|
160
|
+
} while (this.#reviseAgain);
|
|
161
|
+
return this.#last;
|
|
162
|
+
})().finally(() => { this.#revisionPromise = null; });
|
|
163
|
+
return this.#revisionPromise;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
async #readRevision() {
|
|
144
167
|
for (const [name, resource] of this.#resources) {
|
|
145
168
|
const claimants = this.#claimants.filter((one) => one.resource === name);
|
|
146
169
|
if (claimants.length === 0) {
|
|
@@ -153,14 +176,15 @@ export class MachineBudget {
|
|
|
153
176
|
// there is how the spill file came to have no limit in the first place.
|
|
154
177
|
const freeBytes = Number.isFinite(free) && free !== null ? Math.max(0, free) : 0;
|
|
155
178
|
const allowanceBytes = this.#allowanceFor(name, resource, freeBytes, held);
|
|
156
|
-
const shares = divideAllowance(claimants.map((one) => Math.max(0, one.wanted())), allowanceBytes
|
|
179
|
+
const shares = divideAllowance(claimants.map((one) => Math.max(0, one.wanted())), allowanceBytes,
|
|
180
|
+
claimants.map(one => Math.max(0, one.required?.() ?? 0)));
|
|
157
181
|
const reading = claimants.map((one, position) => ({
|
|
158
182
|
name: one.name,
|
|
159
183
|
held: one.held(),
|
|
160
184
|
allowed: shares[position],
|
|
161
185
|
short: Number.isFinite(one.minimum) && shares[position] < one.minimum
|
|
162
186
|
}));
|
|
163
|
-
this.#last.set(name, { freeBytes, allowanceBytes, shares: reading });
|
|
187
|
+
this.#last.set(name, { freeBytes, allowanceBytes, measured: Number.isFinite(free) && free !== null, shares: reading });
|
|
164
188
|
for (const [position, one] of claimants.entries()) {
|
|
165
189
|
one.allow(shares[position]);
|
|
166
190
|
}
|
|
@@ -98,7 +98,14 @@ export function machineAllowanceBytes(freeBytes, heldBytes, reserveBytes) {
|
|
|
98
98
|
* @param {number} allowanceBytes
|
|
99
99
|
* @returns {number[]} What each store may hold, in the same order.
|
|
100
100
|
*/
|
|
101
|
-
export function divideAllowance(wantedBytes, allowanceBytes) {
|
|
101
|
+
export function divideAllowance(wantedBytes, allowanceBytes, requiredBytes = []) {
|
|
102
|
+
const required = wantedBytes.map((wanted, index) => Math.min(Math.max(0, wanted), Math.max(0, requiredBytes[index] ?? 0)));
|
|
103
|
+
const requiredTotal = required.reduce((sum, bytes) => sum + bytes, 0);
|
|
104
|
+
if (requiredTotal > 0) {
|
|
105
|
+
if (requiredTotal >= allowanceBytes) return divideAllowance(required, allowanceBytes);
|
|
106
|
+
const remaining = divideAllowance(wantedBytes.map((wanted, index) => Math.max(0, wanted - required[index])), allowanceBytes - requiredTotal);
|
|
107
|
+
return remaining.map((bytes, index) => bytes + required[index]);
|
|
108
|
+
}
|
|
102
109
|
const total = wantedBytes.reduce((sum, want) => sum + Math.max(0, want), 0);
|
|
103
110
|
if (total <= allowanceBytes || total === 0) {
|
|
104
111
|
return wantedBytes.map((want) => Math.max(0, want));
|
|
@@ -80,6 +80,8 @@ export class PieceDiskStore {
|
|
|
80
80
|
|
|
81
81
|
/** Where the live readers stand, from whoever holds that fact. @type {() => number[]} */
|
|
82
82
|
#readHeads;
|
|
83
|
+
#canForget;
|
|
84
|
+
#wantedAt;
|
|
83
85
|
|
|
84
86
|
/**
|
|
85
87
|
* Said whenever this tier stops holding a piece, whatever the reason.
|
|
@@ -107,12 +109,16 @@ export class PieceDiskStore {
|
|
|
107
109
|
allowanceBytes = null,
|
|
108
110
|
now = Date.now,
|
|
109
111
|
readHeads = () => [],
|
|
112
|
+
canForget = () => true,
|
|
113
|
+
wantedAt = null,
|
|
110
114
|
onForgotten = () => undefined
|
|
111
115
|
}) {
|
|
112
116
|
this.#directory = path.join(directory, name);
|
|
113
117
|
this.#allowanceBytes = Number.isFinite(allowanceBytes) && allowanceBytes >= 0 ? allowanceBytes : null;
|
|
114
118
|
this.#now = now;
|
|
115
119
|
this.#readHeads = typeof readHeads === "function" ? readHeads : () => [];
|
|
120
|
+
this.#canForget = canForget;
|
|
121
|
+
this.#wantedAt = wantedAt;
|
|
116
122
|
this.#onForgotten = typeof onForgotten === "function" ? onForgotten : () => undefined;
|
|
117
123
|
this.#adoptWhatIsAlreadyHere();
|
|
118
124
|
}
|
|
@@ -211,6 +217,7 @@ export class PieceDiskStore {
|
|
|
211
217
|
*/
|
|
212
218
|
reviseAllowance(bytes) {
|
|
213
219
|
this.#allowanceBytes = Number.isFinite(bytes) && bytes >= 0 ? bytes : null;
|
|
220
|
+
this.#makeRoomFor(0, -1);
|
|
214
221
|
return this.#allowanceBytes;
|
|
215
222
|
}
|
|
216
223
|
|
|
@@ -358,7 +365,7 @@ export class PieceDiskStore {
|
|
|
358
365
|
* @returns {void}
|
|
359
366
|
*/
|
|
360
367
|
forget(index) {
|
|
361
|
-
if (this.#reading.has(index)) {
|
|
368
|
+
if (this.#reading.has(index) || !this.#canForget(index)) {
|
|
362
369
|
return;
|
|
363
370
|
}
|
|
364
371
|
const length = this.#stored.get(index);
|
|
@@ -482,7 +489,7 @@ export class PieceDiskStore {
|
|
|
482
489
|
* @param {number} incomingIndex
|
|
483
490
|
* @returns {Promise<void>}
|
|
484
491
|
*/
|
|
485
|
-
|
|
492
|
+
#makeRoomFor(incomingBytes, incomingIndex) {
|
|
486
493
|
if (this.#allowanceBytes === null) {
|
|
487
494
|
return;
|
|
488
495
|
}
|
|
@@ -519,7 +526,15 @@ export class PieceDiskStore {
|
|
|
519
526
|
let victim = null;
|
|
520
527
|
let worst = null;
|
|
521
528
|
for (const [index, at] of this.#touched) {
|
|
522
|
-
if (index === except || this.#reading.has(index)) {
|
|
529
|
+
if (index === except || this.#reading.has(index) || !this.#canForget(index)) {
|
|
530
|
+
continue;
|
|
531
|
+
}
|
|
532
|
+
if (this.#wantedAt) {
|
|
533
|
+
const want = this.#wantedAt(index);
|
|
534
|
+
if (worst === null || want > worst.want || (want === worst.want && at < worst.at)) {
|
|
535
|
+
worst = { want, at };
|
|
536
|
+
victim = index;
|
|
537
|
+
}
|
|
523
538
|
continue;
|
|
524
539
|
}
|
|
525
540
|
const behind = earliest !== null && index < earliest;
|
|
@@ -251,8 +251,10 @@ export class PieceLru {
|
|
|
251
251
|
wantAt(index) {
|
|
252
252
|
let want = Number.POSITIVE_INFINITY;
|
|
253
253
|
for (const range of this.#protected.values()) {
|
|
254
|
-
|
|
255
|
-
|
|
254
|
+
const deadline = range.deadlineAt === undefined ? range.urgency :
|
|
255
|
+
Number.isFinite(range.deadlineAt) ? range.deadlineAt : Number.MAX_SAFE_INTEGER - range.priority;
|
|
256
|
+
if (index >= range.from && index <= range.to && deadline < want) {
|
|
257
|
+
want = deadline;
|
|
256
258
|
}
|
|
257
259
|
}
|
|
258
260
|
return want;
|
|
@@ -432,14 +434,16 @@ export class PieceLru {
|
|
|
432
434
|
* displace a stated one.
|
|
433
435
|
* @returns {void}
|
|
434
436
|
*/
|
|
435
|
-
protect(readerId, from, to, urgency) {
|
|
437
|
+
protect(readerId, from, to, urgency, deadlineAt, priority = 1) {
|
|
436
438
|
if (!Number.isInteger(from) || !Number.isInteger(to) || to < from) {
|
|
437
439
|
return;
|
|
438
440
|
}
|
|
439
441
|
this.#protected.set(readerId, {
|
|
440
442
|
from,
|
|
441
443
|
to,
|
|
442
|
-
urgency: Number.isFinite(urgency) ? Number(urgency) : Number.MAX_SAFE_INTEGER
|
|
444
|
+
urgency: Number.isFinite(urgency) ? Number(urgency) : Number.MAX_SAFE_INTEGER,
|
|
445
|
+
deadlineAt,
|
|
446
|
+
priority
|
|
443
447
|
});
|
|
444
448
|
}
|
|
445
449
|
|
|
@@ -530,6 +530,8 @@ export class SharedPieceStore {
|
|
|
530
530
|
// Where the live readers stand, so what goes first is decided by them and
|
|
531
531
|
// not by which piece happened to be touched longest ago.
|
|
532
532
|
readHeads: () => this.#lru.readHeads(),
|
|
533
|
+
canForget: (index) => !this.#lru.isPinned(index),
|
|
534
|
+
wantedAt: (index) => this.#lru.wantAt(index),
|
|
533
535
|
// EVERY WAY A PIECE LEAVES THE DISK COMES THROUGH HERE — behind the read
|
|
534
536
|
// heads, over the allowance, dropped as a duplicate, or forgotten while a
|
|
535
537
|
// spill finished. One listener instead of a list of call sites to
|
|
@@ -631,19 +633,13 @@ export class SharedPieceStore {
|
|
|
631
633
|
* @returns {{ name: string, allowanceBytes: number | null, bytes: number }}
|
|
632
634
|
*/
|
|
633
635
|
reviseSpillCeiling(allowedBytes) {
|
|
634
|
-
//
|
|
635
|
-
//
|
|
636
|
-
// somebody seeks back, and a seek back re-downloads it — the same bargain
|
|
637
|
-
// this tier makes whenever it drops a piece for room. Without it the spill
|
|
638
|
-
// is bounded only by a share of free space, which on a roomy host is tens of
|
|
639
|
-
// gigabytes against a measured growth of 14 400 MB in one viewing: the
|
|
640
|
-
// ceiling never binds and nothing is ever removed until the torrent goes.
|
|
641
|
-
const behind = this.#disk.forgetBehind(this.#lru.readHeads());
|
|
636
|
+
// Downloaded material is removed only for capacity. Playback position
|
|
637
|
+
// alone cannot invalidate bytes needed by another viewer or a later seek.
|
|
642
638
|
return {
|
|
643
639
|
name: this.#name,
|
|
644
640
|
allowanceBytes: this.#disk.reviseAllowance(allowedBytes),
|
|
645
641
|
bytes: this.#disk.bytes,
|
|
646
|
-
behind
|
|
642
|
+
behind: 0
|
|
647
643
|
};
|
|
648
644
|
}
|
|
649
645
|
|
|
@@ -816,10 +812,29 @@ export class SharedPieceStore {
|
|
|
816
812
|
|
|
817
813
|
/** Availability belongs to the store, not to a second cached bitfield. */
|
|
818
814
|
locationOf(index) {
|
|
815
|
+
if (this.#closed || !Number.isSafeInteger(index) || index < 0 ||
|
|
816
|
+
(this.#lastChunkIndex >= 0 && index > this.#lastChunkIndex)) return "missing";
|
|
819
817
|
return this.#buffers.has(index) ? "memory" : this.#evicting.has(index) ? "writing" :
|
|
820
818
|
this.#disk.has(index) ? "disk" : this.isInWholeFiles(index) ? "whole-file" : "missing";
|
|
821
819
|
}
|
|
822
820
|
|
|
821
|
+
/** Atomically acquire holds only when every requested piece is present. */
|
|
822
|
+
holdAvailable(indexes) {
|
|
823
|
+
if (this.#closed || !Array.isArray(indexes)) return null;
|
|
824
|
+
const unique = [...new Set(indexes)];
|
|
825
|
+
if (unique.some((index) => !Number.isInteger(index) || index < 0 || this.locationOf(index) === "missing")) {
|
|
826
|
+
return null;
|
|
827
|
+
}
|
|
828
|
+
for (const index of unique) this.#lru.pin(index);
|
|
829
|
+
let held = true;
|
|
830
|
+
return () => {
|
|
831
|
+
if (!held) return;
|
|
832
|
+
held = false;
|
|
833
|
+
for (const index of unique) this.#lru.unpin(index);
|
|
834
|
+
this.#noteProgress();
|
|
835
|
+
};
|
|
836
|
+
}
|
|
837
|
+
|
|
823
838
|
pin(index) {
|
|
824
839
|
this.#lru.pin(index);
|
|
825
840
|
// A piece reader pins before it waits for the bytes to arrive or return
|
|
@@ -1833,8 +1848,8 @@ export class SharedPieceStore {
|
|
|
1833
1848
|
* @param {number} [urgency]
|
|
1834
1849
|
* @returns {void}
|
|
1835
1850
|
*/
|
|
1836
|
-
protectRange(readerId, from, to, urgency) {
|
|
1837
|
-
this.#lru.protect(readerId, from, to, urgency);
|
|
1851
|
+
protectRange(readerId, from, to, urgency, deadlineAt, priority) {
|
|
1852
|
+
this.#lru.protect(readerId, from, to, urgency, deadlineAt, priority);
|
|
1838
1853
|
// A READER DECLARES ITSELF IN A MOMENT; the allowance was re-derived once a
|
|
1839
1854
|
// minute. Between the two a read met whatever the store had shrunk to while
|
|
1840
1855
|
// nobody was reading, and a claim gives up after five seconds — twelve
|